diff --git a/.claude/rules/data-grid.md b/.claude/rules/data-grid.md index 272876fd8b..e86d304b26 100644 --- a/.claude/rules/data-grid.md +++ b/.claude/rules/data-grid.md @@ -30,6 +30,7 @@ Three layers, knowledge pointing one way: an owner's adapter, then `DataGridMode - **Row height, row count, reloads, inserts and removes reach both panes in one call**, so their document heights never differ. - **Vertical scroll stays in step through bounds-change notifications**: each pane's clip moves the other with `clipView.scroll(to:)` and `reflectScrolledClipView`, only when its y differs. Anywhere else scroll with `NSView.scroll(_:)`: moving a clip's x that way leaves the header behind. - **The frozen pane forwards every wheel event to the main scroll view**, so one momentum engine moves both. +- **A `fitsRows` grid is as tall as its rows and scrolls only sideways.** It publishes `model.fittingHeight` for its host to size it, hands a vertical gesture to the enclosing scroll view (the axis is decided once per gesture), and reveals a row by scrolling that page. The Structure tab's sections use it. - **Frozen columns take at most two thirds of the grid** (`DataGridMetrics.maximumFrozenShare`). Columns past that scroll, and Freeze Up to This Column is refused in the menu. - **`DataGridTableView.scrollToVisible(_:)` widens the rect leftward by the frozen width**, so a revealed column never parks under the frozen pane. - **A column moves only within its side of the freeze line.** A freeze or unfreeze rebuilds both panes' column lists from the set, writing width and position explicitly: remove-then-add alone strands a column at the end and resets its width. @@ -65,4 +66,4 @@ Three layers, knowledge pointing one way: an owner's adapter, then `DataGridMode - **AppKit builds the AXCell tree from cell views only**, so while `DataGridAccessibility.isActive` the main table mounts a `DataGridAccessibilityCellView` (identifier `data-grid-cell`) for each on-screen cell. It draws nothing and takes no clicks. Keep that path working whenever row drawing changes. - **A label reads "Row N, column M: value"**: N is the row number on screen, page offset included, and M the display position. UI tests find cells by it (`.claude/rules/ui-tests.md`, The grid). - **The frozen pane is hidden from accessibility.** A frozen column's stand-in in the main table reports the frame the frozen pane shows it at. -- **Only the viewport mounts.** Mounting for every row starved the main thread behind a 1,000-row page (#2392); rows scrolled into view remount. +- **Only the viewport mounts.** Mounting for every row starved the main thread behind a 1,000-row page (#2392). A client's read prepares the rows it walks, off-screen ones without a stand-in, and AppKit never asks for a prepared row again. So each on-screen row still without one remounts after a scroll, a layout of the grid and a read of the table: a fitted grid grows to its rows, and a page moves a grid, with no scroll. diff --git a/.github/plugin-registry.json b/.github/plugin-registry.json index f6bf29a547..71757eddca 100644 --- a/.github/plugin-registry.json +++ b/.github/plugin-registry.json @@ -55,18 +55,27 @@ "MongoDBDriver": { "name": "MongoDB Driver" }, + "Neo4jDriverPlugin": { + "name": "Neo4j Driver" + }, "OracleDriver": { "name": "Oracle Driver" }, "ParquetExport": { "name": "Parquet Export" }, + "QdrantDriverPlugin": { + "name": "Qdrant Driver" + }, "SnowflakeDriverPlugin": { "name": "Snowflake Driver" }, "SpannerDriverPlugin": { "name": "Spanner Driver" }, + "StarRocksDriverPlugin": { + "name": "StarRocks Driver" + }, "SurrealDBDriverPlugin": { "name": "SurrealDB Driver" }, diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index f96745925a..77ded5c235 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -82,7 +82,7 @@ jobs: # install, then the meson toolchain, the -dev packages the # workspace links, the runtime pieces the test suites need (xvfb, # dbus, gnome-keyring for the Secret Service step, openssh-client - # for the SSH transport). + # for the SSH transport, glycin-loaders for a value's picture). run: | apt-get install -y --no-install-recommends \ build-essential \ @@ -100,9 +100,12 @@ jobs: libgtksourceview-5-dev \ libsqlite3-dev \ libkrb5-dev \ + liblcms2-dev \ + libseccomp-dev \ clang \ libclang-dev \ adwaita-icon-theme \ + glycin-loaders \ locales \ gnome-keyring \ openssh-client \ @@ -115,7 +118,7 @@ jobs: components: rustfmt, clippy # Meson points Cargo at _build/cargo-home and _build/cargo-target, # so those are what the cache keeps. - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: workspaces: linux -> _build/cargo-target cache-directories: linux/_build/cargo-home @@ -268,7 +271,7 @@ jobs: - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: toolchain: "1.98" - - uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29 + - uses: taiki-e/install-action@5488ac5a951645ce95062214902ac4b6bc1a1037 # v2.87.32 with: tool: cargo-deny@${{ env.CARGO_DENY_VERSION }},cargo-shear@${{ env.CARGO_SHEAR_VERSION }},cargo-about@${{ env.CARGO_ABOUT_VERSION }},zizmor@${{ env.ZIZMOR_VERSION }} - name: Advisories, bans, licenses and sources @@ -334,7 +337,7 @@ jobs: build-essential pkgconf meson ninja-build gettext \ desktop-file-utils appstream libglib2.0-bin libglib2.0-dev libxml2-utils \ libgtk-4-dev libadwaita-1-dev libgtksourceview-5-dev \ - libsqlite3-dev libkrb5-dev clang libclang-dev cargo rustc + libsqlite3-dev libkrb5-dev liblcms2-dev libseccomp-dev clang libclang-dev cargo rustc - name: POTFILES.in is current run: | git ls-files 'crates/app/src/*.rs' 'data/resources/*.ui' 'data/*.desktop.in.in' \ @@ -489,11 +492,11 @@ jobs: - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: toolchain: "1.98" - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: workspaces: linux key: ${{ matrix.package }} - - uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29 + - uses: taiki-e/install-action@5488ac5a951645ce95062214902ac4b6bc1a1037 # v2.87.32 with: tool: cargo-nextest@${{ env.CARGO_NEXTEST_VERSION }} - name: Verify docker is available @@ -564,11 +567,11 @@ jobs: - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: toolchain: "1.98" - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: workspaces: linux key: tablepro-session - - uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29 + - uses: taiki-e/install-action@5488ac5a951645ce95062214902ac4b6bc1a1037 # v2.87.32 with: tool: cargo-nextest@${{ env.CARGO_NEXTEST_VERSION }} - name: Verify docker is available @@ -626,6 +629,8 @@ jobs: libgtksourceview-5-dev \ libsqlite3-dev \ libkrb5-dev \ + liblcms2-dev \ + libseccomp-dev \ clang \ libclang-dev \ xvfb \ @@ -634,11 +639,11 @@ jobs: - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: toolchain: "1.98" - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: workspaces: linux key: tablepro-app - - uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29 + - uses: taiki-e/install-action@5488ac5a951645ce95062214902ac4b6bc1a1037 # v2.87.32 with: tool: cargo-nextest@${{ env.CARGO_NEXTEST_VERSION }} - name: Verify docker is available diff --git a/.github/workflows/release-linux.yml b/.github/workflows/release-linux.yml index 416e184d6f..83a6ed42d8 100644 --- a/.github/workflows/release-linux.yml +++ b/.github/workflows/release-linux.yml @@ -115,7 +115,7 @@ jobs: build-essential pkgconf meson ninja-build gettext \ desktop-file-utils appstream libglib2.0-bin libglib2.0-dev libxml2-utils \ libgtk-4-dev libadwaita-1-dev libgtksourceview-5-dev \ - libsqlite3-dev libkrb5-dev clang libclang-dev cargo rustc + libsqlite3-dev libkrb5-dev liblcms2-dev libseccomp-dev clang libclang-dev cargo rustc working-directory: / - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9b1b4d8bf2..bbd31b8db5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -37,7 +37,7 @@ jobs: - name: Set up Python 3.9 id: python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: "3.9" update-environment: false diff --git a/.github/workflows/repo-hygiene.yml b/.github/workflows/repo-hygiene.yml index 2ecf1f6edf..3a8f59100f 100644 --- a/.github/workflows/repo-hygiene.yml +++ b/.github/workflows/repo-hygiene.yml @@ -55,7 +55,7 @@ jobs: # runner's Python, which has PyYAML. - name: Set up Python 3.9 for the release tooling id: python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: "3.9" update-environment: false @@ -117,6 +117,10 @@ jobs: - name: Check every test target is in its test plan run: python3 apple/scripts/ci/check-test-plans.py + # Oracle reads a bare dictionary name from the current schema before the SYS synonym. + - name: Check Oracle SQL names dictionary objects through OracleDictionary + run: python3 apple/scripts/ci/check-oracle-dictionary-names.py + # The corpora run by hand against servers (scripts/probes/check-column-type-corpus.sh), so a type a driver # starts listing is caught here. - name: Check every listed column type has a corpus column diff --git a/.oss-scanner/Dockerfile b/.oss-scanner/Dockerfile index 88b83578d0..09a6864eef 100644 --- a/.oss-scanner/Dockerfile +++ b/.oss-scanner/Dockerfile @@ -12,7 +12,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates curl git \ build-essential pkgconf meson ninja-build gettext desktop-file-utils appstream libglib2.0-bin \ libxml2-utils libgtk-4-dev libadwaita-1-dev libgtksourceview-5-dev libsqlite3-dev libkrb5-dev \ - clang libclang-dev adwaita-icon-theme locales gnome-keyring openssh-client xvfb xauth dbus-daemon \ + liblcms2-dev libseccomp-dev clang libclang-dev adwaita-icon-theme glycin-loaders locales gnome-keyring \ + openssh-client xvfb xauth dbus-daemon \ && rm -rf /var/lib/apt/lists/* \ && localedef -i en_US -f UTF-8 en_US.UTF-8 diff --git a/.oss-scanner/swift/Package.swift b/.oss-scanner/swift/Package.swift index 9e02a473ad..49240392bd 100644 --- a/.oss-scanner/swift/Package.swift +++ b/.oss-scanner/swift/Package.swift @@ -14,7 +14,9 @@ let package = Package( .target(name: "TableProCoreTypes"), .target(name: "TableProDocumentPath"), .target(name: "TableProJavaScriptText"), + .target(name: "TableProDamengCore", dependencies: ["TableProSQLGrammar"]), .testTarget(name: "TableProSQLGrammarTests", dependencies: ["TableProSQLGrammar"]), - .testTarget(name: "TableProLogRedactionTests", dependencies: ["TableProLogRedaction"]) + .testTarget(name: "TableProLogRedactionTests", dependencies: ["TableProLogRedaction"]), + .testTarget(name: "TableProDamengCoreTests", dependencies: ["TableProDamengCore", "TableProSQLGrammar"]) ] ) diff --git a/.oss-scanner/threat_model.md b/.oss-scanner/threat_model.md index 9e67d73ebe..4fe2a19276 100644 --- a/.oss-scanner/threat_model.md +++ b/.oss-scanner/threat_model.md @@ -32,7 +32,7 @@ The user is trusted: the SQL they type, the connections they create, the plugins 7. Secrets: `apple/TablePro/Core/Storage/KeychainHelper.swift`, `apple/TablePro/Core/Sync/SyncRecordMapper.swift` (passwords must never reach CloudKit), `linux/crates/storage/src/secrets`, `TableProLogRedaction`, the MCP error redactor. 8. SQL the app builds from data: `apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift`, `apple/TablePro/Core/Database/FilterSQLGenerator.swift`, `SQLEscaping.swift`, `DialectQuoteHelper.swift`, each driver's `quoteIdentifier`; on Linux `linux/crates/core/src/dml`, `linux/crates/core/src/filter/validate_raw_predicate.rs` and each driver's `dialect.rs`. A crafted table or column name that escapes quoting runs SQL with the user's privileges. 9. SSH and TLS: `apple/TablePro/Core/SSH/` (libssh2, host key prompts), `apple/Packages/TableProMacCore/Sources/TableProSSHConfig`, `linux/crates/ssh` (drives the system `ssh`, so watch for option injection from connection fields), `linux/crates/net/src/tls`. -10. Rendering of cell values: the out-of-process image renderer (`apple/TablePro/CLI/ImageRenderMain.swift`), `apple/TablePro/Core/Services/Formatting/PhpSerializeParser.swift`, the MongoDB shell's JavaScriptCore runtime. +10. Rendering of cell values: the out-of-process image renderer (`apple/TablePro/CLI/ImageRenderMain.swift`), the Linux Row Details picture, which glycin decodes in a sandbox (`linux/crates/app/src/ui/data_grid/row_detail/picture.rs`), `apple/TablePro/Core/Services/Formatting/PhpSerializeParser.swift`, the MongoDB shell's JavaScriptCore runtime. Matter least: layout, theming and localization code, `linux/crates/test-fixtures`, `linux/crates/test-support`, `apple/TableProUITests`, `scripts/`, `.github/`, docs. diff --git a/README.ko.md b/README.ko.md index d066b55520..a976414762 100644 --- a/README.ko.md +++ b/README.ko.md @@ -171,4 +171,4 @@ TablePro에 기여해 주신 모든 분께 감사드립니다. ## 라이선스 -이 프로젝트는 [GNU Affero General Public License v3.0(AGPLv3)](LICENSE)에 따라 라이선스가 부여됩니다. +이 프로젝트는 [GNU Affero General Public License v3.0(AGPLv3)](LICENSE)에 따라 라이선스가 부여됩니다. TablePro 이름과 아이콘은 이 라이선스에 포함되지 않으며, 사용 방법은 [TRADEMARKS.md](TRADEMARKS.md)에 있습니다. diff --git a/README.md b/README.md index 3c07e7197e..c2ed042266 100644 --- a/README.md +++ b/README.md @@ -182,5 +182,5 @@ Want to join them? Read [CONTRIBUTING.md](CONTRIBUTING.md). ## License -This project is licensed under the [GNU Affero General Public License v3.0 (AGPLv3)](LICENSE). +This project is licensed under the [GNU Affero General Public License v3.0 (AGPLv3)](LICENSE). The TablePro name and icon are not part of that license: [TRADEMARKS.md](TRADEMARKS.md) says how you can use them. diff --git a/README.vi.md b/README.vi.md index 4ea88ccfab..3886e7a2c8 100644 --- a/README.vi.md +++ b/README.vi.md @@ -146,5 +146,5 @@ Muốn tham gia? Đọc [CONTRIBUTING.md](CONTRIBUTING.md). ## Bản quyền -Dự án này cấp phép theo [GNU Affero General Public License v3.0 (AGPLv3)](LICENSE). +Dự án này cấp phép theo [GNU Affero General Public License v3.0 (AGPLv3)](LICENSE). Tên và biểu tượng TablePro không thuộc giấy phép đó: cách dùng được ghi trong [TRADEMARKS.md](TRADEMARKS.md). diff --git a/README.zh.md b/README.zh.md index 7fb23a2c33..fd8d3c0fdd 100644 --- a/README.zh.md +++ b/README.zh.md @@ -146,5 +146,5 @@ TablePro 由用户和以下赞助者支持。[成为赞助者](https://tablepro. ## 许可证 -本项目采用 [GNU Affero General Public License v3.0 (AGPLv3)](LICENSE) 许可。 +本项目采用 [GNU Affero General Public License v3.0 (AGPLv3)](LICENSE) 许可。TablePro 名称和图标不在该许可范围内,使用方式见 [TRADEMARKS.md](TRADEMARKS.md)。 diff --git a/TRADEMARKS.md b/TRADEMARKS.md new file mode 100644 index 0000000000..35e46b4862 --- /dev/null +++ b/TRADEMARKS.md @@ -0,0 +1,24 @@ +# Trademark policy + +The code in this repository is licensed under the [AGPLv3](LICENSE). That license covers the code. It does not cover the name **TablePro** or the TablePro icon and logo, which identify the builds this project publishes. Section 7(e) of the AGPL lets a project keep those out of the license, and this file says what you can do with them. + +The aim is simple: anyone who downloads something called TablePro gets the app this project ships, not a build someone else changed. + +## What you can do + +- Build TablePro from this source for your own use, changed or not. The AGPL allows that, including a build with the license check removed. +- Redistribute an official release unchanged, for example through a package manager, under the name TablePro. +- Package the unchanged source for a Linux distribution or a package manager under the name TablePro, with only the changes packaging needs. +- Say that your work is based on TablePro, works with TablePro, or is a fork of TablePro, in plain words that do not suggest it is TablePro. +- Use the name and icon to write about TablePro, review it, teach it, or link to it. + +## What needs a different name and icon + +- A build with any other change, such as new or removed features, a removed license check, or other branding. Rename it and replace the icon before you share it. +- A product, company, domain, app store listing, or package name that people could take for TablePro or for something made by this project. +- Anything that suggests this project endorses, sponsors, or supports your work when it does not. +- Selling copies, or builds with the paid features unlocked, under the TablePro name. + +## Questions + +For anything this file does not cover, or to ask for permission, write to [hello@tablepro.app](mailto:hello@tablepro.app). diff --git a/apple/.swiftlint.yml b/apple/.swiftlint.yml index c31e1bc9cd..698a5f7bcd 100644 --- a/apple/.swiftlint.yml +++ b/apple/.swiftlint.yml @@ -214,7 +214,7 @@ custom_rules: severity: error appstorage_key_namespace: name: "AppStorage Key Namespace" - regex: '@AppStorage\(\s*"(?!com\.TablePro)(?!(?:hideExportSuccessDialog|skipSchemaPreview|structureCodeFontSize)")[^"\\]+"' + regex: '@AppStorage\(\s*"(?!com\.TablePro)(?!(?:hideExportSuccessDialog|structureCodeFontSize)")[^"\\]+"' excluded: ['.*/(Packages|Plugins)/'] message: "Route an @AppStorage key through the preferences layer, under com.TablePro." severity: error diff --git a/apple/CHANGELOG.md b/apple/CHANGELOG.md index f60114318e..7a25751fe7 100644 --- a/apple/CHANGELOG.md +++ b/apple/CHANGELOG.md @@ -9,6 +9,79 @@ Unreleased changes are files in `changelog.d/`. +## [0.83.1](https://github.com/TableProApp/TablePro/releases/tag/v0.83.1) - 2026-11-01 [CRITICAL] + +### Changed + +- With CanonicalizeHostname, a DNS alias is followed only where CanonicalizePermittedCNAMEs allows it. + +### Removed + +- DuckDB Restore no longer accepts a folder of Parquet files, only a .duckdb file. + +### Fixed + +- A table tab could reload under an open cell and drop what you were typing after a save in another window. ([#3848](https://github.com/TableProApp/TablePro/pull/3848)) +- Edits made to rows while a query or refresh was still loading new ones were discarded when it finished. ([#3848](https://github.com/TableProApp/TablePro/pull/3848)) +- Table tabs ran their query again when you switched back to them or returned to TablePro from another app. ([#3848](https://github.com/TableProApp/TablePro/pull/3848)) +- A table tab waiting for its connection could run the statement under the cursor in a query tab instead. ([#3848](https://github.com/TableProApp/TablePro/pull/3848)) +- On iOS, a search or filter with a backslash on MySQL or ClickHouse, or with `[` on SQL Server, matched the wrong rows. ([#3906](https://github.com/TableProApp/TablePro/pull/3906)) +- A DELETE with no WHERE running unconfirmed behind EXPLAIN ANALYZE, an unusual CTE, or a WHERE glued to a name. ([#3907](https://github.com/TableProApp/TablePro/pull/3907)) +- CREATE OR REPLACE TABLE, SCHEMA or DATABASE replacing data without the confirmation DROP gets. ([#3907](https://github.com/TableProApp/TablePro/pull/3907)) +- HTTP drivers such as Typesense following a server's redirect to another address with the connection's key. ([#3907](https://github.com/TableProApp/TablePro/pull/3907)) +- Spanner, BigQuery and Teradata saving a grid change they could not key to the default schema's table. ([#3907](https://github.com/TableProApp/TablePro/pull/3907)) +- Teradata showing each result one statement late after a statement hit the 20-second receive timeout. ([#3907](https://github.com/TableProApp/TablePro/pull/3907)) +- DuckDB's Open the File Read-Only being ignored for a database not named .duckdb or .ddb. ([#3907](https://github.com/TableProApp/TablePro/pull/3907)) +- T-SQL BULK INSERT and OPENROWSET, and MongoDB $_internalJs operators, passing the external file and code refusals. ([#3907](https://github.com/TableProApp/TablePro/pull/3907)) +- Bigtable saving a grid change to another row when the key starts with a combining mark. ([#3907](https://github.com/TableProApp/TablePro/pull/3907)) +- DuckDB Back Up to a .duckdb file, and DuckDB Restore, failed with "Cannot prepare multiple statements at once". +- Pull in a linked folder failing when the repository is a partial clone. + +### Security + +- A backslash in a Snowflake filter value could end its string literal and run the rest as SQL. ([#3906](https://github.com/TableProApp/TablePro/pull/3906)) +- A Firebase Database URL with an encoded slash sending your Google access token to another host. +- A PostgreSQL or SQL Server database name could send Back Up or Restore, with its password, to another server. +- A PostgreSQL type named geometry or geography that PostGIS did not create could run its owner's code as you. +- A SQL export carrying a statement of its own in a cell value, a type name or a table comment when restored. +- A Typesense path with dot segments, or a Firebase or Bigtable request behind a comment, passing the read-only check. +- A credential profile in a shared connection file reading ~/.pgpass and sending the password to a host the file named. +- A name with a quote before an accent mark could run its own SQL when you browsed, searched or exported the table. +- A remote SQLite file path with a quote and a combining accent could run commands on the SSH server. +- A write after a CTE list, a Snowflake ->> chain, or inside a SurrealDB or CockroachDB SELECT passing as a read. +- ANALYZE TABLE, EXPLAIN PRAGMA and SQLite pragma_optimize passing as reads for read-only external clients. +- An imported SSH host or user could run commands through a Match exec line in ~/.ssh/config. +- An imported SSH host with a line break could make TablePro trust a host key for another server. +- Browsing a PostgreSQL schema could run a function or operator its owner planted there, with your privileges. +- CanonicalizeFallbackLocal no was ignored, and an SSH tunnel dialed the short host name anyway. +- CanonicalizeHostname followed any DNS alias, so DNS chose the host key an SSH tunnel was checked against. +- Cassandra and ScyllaDB connections through a tunnel signing in to other cluster nodes outside the tunnel. +- Dameng connections set to Required or a Verify mode connecting without TLS. +- EXPLAIN or DESCRIBE with a parenthesised query or a quoted option passing as a read for read-only MCP clients. +- File and URL reads through DuckDB, ClickHouse, quoted function names or VACUUM INTO passing the external check. +- Git status for a linked folder running filter and fetch commands set up in that folder's own Git configuration. +- Import Commands showing the first 300 characters of Startup SQL while keeping all of it. +- MCP clients and the assistant running statements past Read Only on Dameng behind a comment the driver read differently. +- MCP clients running DROP or server API calls unasked through grid requests on BigQuery, Elasticsearch and Typesense. +- MCP clients running DROP unasked on Trino and Presto behind a statement the driver split differently from the gate. +- MCP clients running statements or changing consistency past Read Only on Cassandra and ScyllaDB with a forged comment. +- MCP clients with read access running writes through browse_table or get_table_statistics with a crafted table name. +- MCP clients, the AI assistant and AppleScript running a second statement the Read Only and destructive checks missed. +- Open Project Folder filling a project's ${VAR} references from TablePro's own environment. +- Opening a crafted table on iPhone or iPad running a second statement on PostgreSQL, Redshift or SQL Server. +- Oracle Explain reading a PLAN_TABLE or V$ views planted in the tab's schema instead of Oracle's own. +- Restoring a DuckDB folder of Parquet files ran its schema.sql with file access, so the folder could write any file. +- Restoring a MySQL or MariaDB dump could run shell commands written in the .sql file. +- Restoring a SQLite dump could run shell commands written in the .sql file. +- SQL Server EXEC of a string, sp_executesql, OPENQUERY and OPENROWSET passing as reads or without the code check. +- SQL Server Verify CA and Verify Identity on iPhone and iPad connecting without checking the certificate. +- Secrets in a pasted MongoDB connection string saved outside the Keychain and carried by exports, links and sync. +- Shared connections reusing your Entra ID token, Snowflake CLI profile, MongoDB AWS credentials or Run Ledger Plugins. +- Teradata with SSL through a tunnel sending the password to port 443 on this Mac. +- Trino and Presto sending the password or token to another host the server named in its reply. +- iPhone and iPad Safe Mode letting DESC ANALYZE of a write, OPENQUERY and state-changing calls run as reads. + + ## [0.83.0](https://github.com/TableProApp/TablePro/releases/tag/v0.83.0) - 2026-10-26 Search Database finds a value in every table, Compare Results matches two results row by row, and Generate Rows fills a table with test data. diff --git a/apple/Configs/Version-iOS.xcconfig b/apple/Configs/Version-iOS.xcconfig index 705cfb6c43..09f58d5e70 100644 --- a/apple/Configs/Version-iOS.xcconfig +++ b/apple/Configs/Version-iOS.xcconfig @@ -1,5 +1,5 @@ // Single source of truth for the iOS app version. // The app and its widget extension must ship the same values, so both read this file. -MARKETING_VERSION = 1.1 -CURRENT_PROJECT_VERSION = 23 +MARKETING_VERSION = 1.1.1 +CURRENT_PROJECT_VERSION = 24 diff --git a/apple/Libs/checksums.sha256 b/apple/Libs/checksums.sha256 index 58559a0baf..228b6a3f62 100644 --- a/apple/Libs/checksums.sha256 +++ b/apple/Libs/checksums.sha256 @@ -10,10 +10,10 @@ c47c8406cdb817e121a89ad5171f6931fd73c4bce6c73c3809e626257afd7374 Libs/libcrypto e274f77ed3feaee3d4f8f6ac297376d56a1369388f037599a90ea440390349b4 Libs/libcrypto_universal.a ec04bc233ea917de16a48b130d1467c68f5b95d469f423b7b9054d0e45bb4e16 Libs/libcrypto_x86_64.a e274f77ed3feaee3d4f8f6ac297376d56a1369388f037599a90ea440390349b4 Libs/libcrypto.a -f833d1edba062e03a5a7f7b5e40fbfe0f015c081158b8dd455d6129bf4515c12 Libs/libduckdb_arm64.a -c1aef28aa225d34bb4d8be853000f4d2640a77366ff4b2cfa61f53648290c357 Libs/libduckdb_universal.a -e971a4c84689626699482ba5e2f148345b50e53fc1aed57b6e37fbb90c85db9c Libs/libduckdb_x86_64.a -c1aef28aa225d34bb4d8be853000f4d2640a77366ff4b2cfa61f53648290c357 Libs/libduckdb.a +09b2deb312d03840dda422a6931a9918ad81c308941c774556adb1386d6ab589 Libs/libduckdb_arm64.a +9749f1be288f03cc4362b15fbc0137edd8aee75542f52f7d7ca8d563586a1e77 Libs/libduckdb_universal.a +3183445676921ea6e4ea4cb7f0689703fb3ceba3d1699b21e88a2e36f12e0e2e Libs/libduckdb_x86_64.a +9749f1be288f03cc4362b15fbc0137edd8aee75542f52f7d7ca8d563586a1e77 Libs/libduckdb.a 5017c3fecadf4955fad1db923a8358fc9943fa17e8a5d54288db6a28fc5a3aa8 Libs/libhiredis_arm64.a 58974eb81d5fefca9fb6732bd15921f1cf12a23f9a2402b1b099ec6c089f1e35 Libs/libhiredis_ssl_arm64.a 09c8cb4acb62a54c09c4f2605bbc464b93d9e2ea5b2ae89a1d2165f8adaa9b9c Libs/libhiredis_ssl_universal.a diff --git a/apple/Libs/ios/checksums.sha256 b/apple/Libs/ios/checksums.sha256 index bd457322f7..f6e64e4c7a 100644 --- a/apple/Libs/ios/checksums.sha256 +++ b/apple/Libs/ios/checksums.sha256 @@ -1,8 +1,10 @@ -840741c66e9ee257f97eb7cc16407c37960aa49faa4b110e56ccc0f8dc3a6db2 ./DuckDB.xcframework/Info.plist -c8dd688cf14559c744255d1bb3879330d6ae3523268e8904fa7b885e140030aa ./DuckDB.xcframework/ios-arm64/Headers/duckdb.h -f4073989393e739b0b5ad68c7f47f7750715d2d46e46141e81b1bc49cf782620 ./DuckDB.xcframework/ios-arm64/libduckdb-device.a -c8dd688cf14559c744255d1bb3879330d6ae3523268e8904fa7b885e140030aa ./DuckDB.xcframework/ios-arm64_x86_64-simulator/Headers/duckdb.h -75523877523153f63249a39eb4ec0d7f1bb05db3a4dbb5e6c35ee848b09afb71 ./DuckDB.xcframework/ios-arm64_x86_64-simulator/libduckdb-sim.a +fcfb2160ac0c32d40fd3241bb0ec0ec157238a5db6ef68a5d98e93a616956cf6 ./DuckDB.xcframework/Info.plist +8dbc9c71a10f0a75fcacc1a3f4f7c76093a84d232fc152222ee5b6e512ab4560 ./DuckDB.xcframework/ios-arm64-simulator/Headers/duckdb.h +a90f28cbe2662c69fc73f726832479ae7b5d043566b83eb1ac95a8ab35d3e37d ./DuckDB.xcframework/ios-arm64-simulator/Headers/duckdb_static_extensions.h +842bd80d7f54472c4161434474af155675e0d7b19c7751e1d1046d8e144a19b6 ./DuckDB.xcframework/ios-arm64-simulator/libduckdb-sim.a +8dbc9c71a10f0a75fcacc1a3f4f7c76093a84d232fc152222ee5b6e512ab4560 ./DuckDB.xcframework/ios-arm64/Headers/duckdb.h +a90f28cbe2662c69fc73f726832479ae7b5d043566b83eb1ac95a8ab35d3e37d ./DuckDB.xcframework/ios-arm64/Headers/duckdb_static_extensions.h +c667055fe35aa1e47ca8f2dd36b8328743429872022a0bb25701b593d9a7d1f0 ./DuckDB.xcframework/ios-arm64/libduckdb-device.a 0481f79db3949ba16425ffd8e15e640757a208ca05728d98cdbfb4bc617e06c3 ./FreeTDS.xcframework/Info.plist 734d8502be13513e974a621557b1a4776d9f605b5a48039e28a3597f27f704ce ./FreeTDS.xcframework/ios-arm64-simulator/Headers/sybdb.h eaa21732fa7cfb538b3bb9bcca7342b99b407d2e838c09e5f3d07398c21406a3 ./FreeTDS.xcframework/ios-arm64-simulator/Headers/sybfront.h diff --git a/apple/Libs/release b/apple/Libs/release index edf700ee23..f05581e49b 100644 --- a/apple/Libs/release +++ b/apple/Libs/release @@ -1 +1 @@ -libs-5 +libs-6 diff --git a/apple/Packages/TableProCore/Package.swift b/apple/Packages/TableProCore/Package.swift index 34aae47217..261bbc1c36 100644 --- a/apple/Packages/TableProCore/Package.swift +++ b/apple/Packages/TableProCore/Package.swift @@ -31,11 +31,14 @@ let package = Package( .library(name: "TableProKerberos", targets: ["TableProKerberos"]), .library(name: "TableProTeradataCore", targets: ["TableProTeradataCore"]), .library(name: "TableProTrinoCore", targets: ["TableProTrinoCore"]), + .library(name: "TableProDamengCore", targets: ["TableProDamengCore"]), .library(name: "TableProTLSClientIdentity", targets: ["TableProTLSClientIdentity"]), .library(name: "TableProTLSTestFixtures", targets: ["TableProTLSTestFixtures"]), .library(name: "TableProGoogleCloud", targets: ["TableProGoogleCloud"]), .library(name: "TableProSpannerCore", targets: ["TableProSpannerCore"]), .library(name: "TableProWeaviateCore", targets: ["TableProWeaviateCore"]), + .library(name: "TableProQdrantCore", targets: ["TableProQdrantCore"]), + .library(name: "TableProNeo4jCore", targets: ["TableProNeo4jCore"]), .library(name: "TableProFirebaseCore", targets: ["TableProFirebaseCore"]), .library(name: "TableProBigtableCore", targets: ["TableProBigtableCore"]), .library(name: "TableProNumberFormatting", targets: ["TableProNumberFormatting"]), @@ -105,7 +108,7 @@ let package = Package( ), .target( name: "TableProImport", - dependencies: ["TableProConnectionLibrary"], + dependencies: ["TableProConnectionLibrary", "TableProSSHTransport"], path: "Sources/TableProImport" ), .target( @@ -152,9 +155,15 @@ let package = Package( ), .target( name: "TableProTrinoCore", - dependencies: [], + dependencies: ["TableProSQLGrammar"], path: "Sources/TableProTrinoCore" ), + .target( + name: "TableProDamengCore", + dependencies: ["TableProSQLGrammar"], + path: "Sources/TableProDamengCore", + swiftSettings: pluginCoreSettings + ), .target( name: "TableProTLSClientIdentity", dependencies: [], @@ -185,6 +194,17 @@ let package = Package( dependencies: [], path: "Sources/TableProWeaviateCore" ), + .target( + name: "TableProQdrantCore", + dependencies: [], + path: "Sources/TableProQdrantCore", + swiftSettings: pluginCoreSettings + ), + .target( + name: "TableProNeo4jCore", + dependencies: [], + path: "Sources/TableProNeo4jCore" + ), .target( name: "TableProBigtableCore", dependencies: ["TableProGoogleCloud"], @@ -273,16 +293,43 @@ let package = Package( dependencies: ["TableProBigtableCore", "TableProGoogleCloud"], path: "Tests/TableProBigtableCoreTests" ), + .testTarget( + name: "TableProNeo4jCoreTests", + dependencies: ["TableProNeo4jCore"], + path: "Tests/TableProNeo4jCoreTests" + ), + .testTarget( + name: "TableProFirebaseCoreTests", + dependencies: ["TableProFirebaseCore"], + path: "Tests/TableProFirebaseCoreTests" + ), .testTarget( name: "TableProSpannerCoreTests", dependencies: ["TableProSpannerCore"], path: "Tests/TableProSpannerCoreTests" ), + .testTarget( + name: "TableProTeradataCoreTests", + dependencies: ["TableProTeradataCore"], + path: "Tests/TableProTeradataCoreTests" + ), + .testTarget( + name: "TableProQdrantCoreTests", + dependencies: ["TableProQdrantCore"], + path: "Tests/TableProQdrantCoreTests", + swiftSettings: pluginCoreSettings + ), .testTarget( name: "TableProTrinoCoreTests", - dependencies: ["TableProTrinoCore"], + dependencies: ["TableProTrinoCore", "TableProSQLGrammar"], path: "Tests/TableProTrinoCoreTests" ), + .testTarget( + name: "TableProDamengCoreTests", + dependencies: ["TableProDamengCore", "TableProSQLGrammar"], + path: "Tests/TableProDamengCoreTests", + swiftSettings: pluginCoreSettings + ), .testTarget( name: "TableProTabularIOTests", dependencies: ["TableProTabularIO", "TableProPluginKit"], diff --git a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionProvider.swift b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionProvider.swift index a190774b8c..ca5511a9f1 100644 --- a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionProvider.swift +++ b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionProvider.swift @@ -150,22 +150,25 @@ public final class SQLCompletionProvider { // A dot prefix is a table or alias first, then a schema (its tables) or a database (its schemas, or its // tables on an engine with none). The namespace fallback also covers an alias that resolves to a schema - // name parsed out of the FROM clause. + // name parsed out of the FROM clause. In a table slot it can only qualify the table being named, so a table + // that shares the name is never read for columns. if let dotPrefix = context.dotPrefix { guard let catalog else { return [] } - if let derived = context.tableReferences.first(where: { - $0.isDerived && $0.identifier.caseInsensitiveCompare(dotPrefix) == .orderedSame - }) { - let columns = await derivedColumns(of: derived) - if !columns.isEmpty { - return [columns.map { SQLCompletionItem.column($0, dataType: nil, tableName: derived.identifier) }] + if !context.expectsObjectName { + if let derived = context.tableReferences.first(where: { + $0.isDerived && $0.identifier.caseInsensitiveCompare(dotPrefix) == .orderedSame + }) { + let columns = await derivedColumns(of: derived) + if !columns.isEmpty { + return [columns.map { SQLCompletionItem.column($0, dataType: nil, tableName: derived.identifier) }] + } + } + if let tableName = await catalog.resolveAlias(dotPrefix, in: context.tableReferences) { + let schema = context.tableReferences.first { + $0.tableName.caseInsensitiveCompare(tableName) == .orderedSame + }?.schema + items = await catalog.columnCompletionItems(for: tableName, schema: schema) } - } - if let tableName = await catalog.resolveAlias(dotPrefix, in: context.tableReferences) { - let schema = context.tableReferences.first { - $0.tableName.caseInsensitiveCompare(tableName) == .orderedSame - }?.schema - items = await catalog.columnCompletionItems(for: tableName, schema: schema) } if items.isEmpty { if await catalog.isKnownSchema(dotPrefix) { @@ -493,7 +496,7 @@ public final class SQLCompletionProvider { var item = SQLCompletionItem( label: "'\(value)'", kind: .keyword, - insertText: "'\(value.replacingOccurrences(of: "'", with: "''"))'", + insertText: "'\(value.replacingOccurrences(of: "'", with: "''", options: .literal))'", detail: column, filterText: value.lowercased() ) @@ -598,13 +601,13 @@ public final class SQLCompletionProvider { // `AS 'Total'` names the column on MySQL and SQLite, but ORDER BY 'Total' sorts by a constant. private func identifierSpelling(of spelling: String, name: String) -> String { - guard spelling.hasPrefix("'") else { return spelling } + guard spelling.unicodeScalars.first == "'" else { return spelling } let isBare = name.unicodeScalars.first.map { CharacterSet.letters.contains($0) || $0 == "_" } == true && name.unicodeScalars.allSatisfy { CharacterSet.alphanumerics.contains($0) || $0 == "_" } guard !isBare || SQLKeywords.isKeyword(name) else { return name } // The analyzer reads a backslash-escaped `"` as a string, which is where a backtick quotes a name instead. let quote = grammar.isSuperset(of: [.backslashEscapesInDoubleQuotes, .backtickQuotes]) ? "`" : "\"" - return quote + name.replacingOccurrences(of: quote, with: quote + quote) + quote + return quote + name.replacingOccurrences(of: quote, with: quote + quote, options: .literal) + quote } // CTEs were bound to their own scope by the parser, so what is left to expand is a table the catalog knows. diff --git a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLContextAnalyzer.swift b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLContextAnalyzer.swift index aadeb59b74..841ac697c2 100644 --- a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLContextAnalyzer.swift +++ b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLContextAnalyzer.swift @@ -209,6 +209,7 @@ public final class SQLContextAnalyzer { private static let doubleQuote = UInt16(UnicodeScalar("\"").value) private static let backtick = UInt16(UnicodeScalar("`").value) private static let openBracket = UInt16(UnicodeScalar("[").value) + private static let closeBracket = UInt16(UnicodeScalar("]").value) private static let dollar = UInt16(UnicodeScalar("$").value) private static let hash = UInt16(UnicodeScalar("#").value) @@ -260,8 +261,9 @@ public final class SQLContextAnalyzer { // and hold any character. private static let quotedSegment = "[`\"'\\[]?\\w+[`\"'\\]]?" + // A trailing dot stays in the path, so `JOIN sales.` reads as unfinished instead of naming a table `sales`. private static let tableRefRegexes: [NSRegularExpression] = { - let path = "(\(quotedSegment)(?:\\.\(quotedSegment))*)" + let path = "(\(quotedSegment)(?:\\.\(quotedSegment))*\\.?)" let alias = "(?:\\s+(?:AS\\s+)?(\(quotedSegment)))?" let patterns = [ // MySQL's STRAIGHT_JOIN has no word boundary before JOIN. @@ -282,7 +284,7 @@ public final class SQLContextAnalyzer { ) private static let alterTableRegex = compileRegex( - "(?i)\\bALTER\\s+TABLE\\s+(\(quotedSegment)(?:\\.\(quotedSegment))*)" + "(?i)\\bALTER\\s+TABLE\\s+(\(quotedSegment)(?:\\.\(quotedSegment))*\\.?)" ) private static let derivedTableParser = DerivedTableParser() @@ -855,7 +857,13 @@ public final class SQLContextAnalyzer { append(upTo: index) start = index + 1 } - append(upTo: end) + // The name after a trailing dot is still being typed: an empty last segment, so no caller reads `sales.` as + // a table named `sales`. + if start == end, end > range.location { + segments.append(PathSegment(masked: "", name: "", isQuoted: false)) + } else { + append(upTo: end) + } return segments } @@ -1048,10 +1056,6 @@ public final class SQLContextAnalyzer { dotPrefix: String?, currentFunction: String? = nil ) -> ClauseResolution { - if dotPrefix != nil { - return ClauseResolution(clause: .select, expectsObjectName: false) - } - // Bounds the regex work on a long statement. SQLContextAnalyzerWindowingTests pins the size. let windowSize = 5_000 let nsText = textBeforeCursor as NSString @@ -1059,6 +1063,10 @@ public final class SQLContextAnalyzer { ? nsText.substring(from: nsText.length - windowSize) : textBeforeCursor + if dotPrefix != nil { + return ClauseResolution(clause: .select, expectsObjectName: qualifiesTableName(after: windowedText)) + } + // Structural contexts first: `CREATE TABLE test (id ` looks like a call to `test(` but is a column definition. let range = NSRange(location: 0, length: (windowedText as NSString).length) for (regex, clause) in Self.structuralClauseRegexes { @@ -1079,6 +1087,42 @@ public final class SQLContextAnalyzer { return ClauseResolution(clause: .unknown, expectsObjectName: false) } + // `FROM sales.` qualifies the table being named. A FROM after `IS DISTINCT` or inside a call such as + // `EXTRACT(YEAR FROM o.`, and any qualifier this cannot place, stays a table or alias. + private func qualifiesTableName(after textBeforePrefix: String) -> Bool { + let tokens = tokenize(Self.removingTrailingPath(textBeforePrefix)) + guard let keyword = tokens.last, keyword.kind == .word, + ["FROM", "JOIN", "INTO"].contains(keyword.upper) else { return false } + guard keyword.upper == "FROM" else { return true } + if tokens.count >= 2, tokens[tokens.count - 2].upper == "DISTINCT" { return false } + var open: [Int] = [] + for (index, token) in tokens.enumerated() { + if token.kind == .openParen { + open.append(index) + } else if token.kind == .closeParen, !open.isEmpty { + open.removeLast() + } + } + guard let paren = open.last, paren > 0, tokens[paren - 1].kind == .word else { return true } + return Self.groupingKeywords.contains(tokens[paren - 1].upper) + } + + // A parenthesis after one of these opens a subquery or a join group, so a FROM inside it names a table. + private static let groupingKeywords: Set = [ + "FROM", "JOIN", "LATERAL", "AS", "IN", "EXISTS", "ANY", "ALL", "SOME", "UNION", "INTERSECT", "EXCEPT" + ] + + private static func removingTrailingPath(_ text: String) -> String { + let ns = text as NSString + var start = ns.length + while start > 0 { + let ch = ns.character(at: start - 1) + guard SQLTokenBoundary.isTokenChar(ch) || ch == dot || ch == openBracket || ch == closeBracket else { break } + start -= 1 + } + return ns.substring(to: start) + } + // MARK: - Linear Clause Scan private enum ScanTokenKind { diff --git a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLIdentifierSpelling.swift b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLIdentifierSpelling.swift index 5531a4ec99..8eadaa7789 100644 --- a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLIdentifierSpelling.swift +++ b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLIdentifierSpelling.swift @@ -103,7 +103,7 @@ public struct SQLIdentifierSpelling: Sendable { public func quoted(_ name: String) -> String { let quote = String(rules.quote) - return quote + name.replacingOccurrences(of: quote, with: quote + quote) + quote + return quote + name.replacingOccurrences(of: quote, with: quote + quote, options: .literal) + quote } // A name written into a statement with nothing typed: as stored keeps the caller's own spelling. @@ -131,7 +131,7 @@ public struct SQLIdentifierSpelling: Sendable { if quoting != .none, identifier.parts.count == 1, case .catalog(let name) = identifier.parts[0] { // The user's own quote asks for the exact name: inside a closed pair it goes in raw. let quote = String(rules.quote) - let text = quoting == .enclosed ? name.replacingOccurrences(of: quote, with: quote + quote) : quoted(name) + let text = quoting == .enclosed ? name.replacingOccurrences(of: quote, with: quote + quote, options: .literal) : quoted(name) return item.recased(label: name, insertText: text) } // A qualified reference replaces the user's quotes, so it is spelled as if none were typed. diff --git a/apple/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift b/apple/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift index d96c5be4f0..bc2ce87ab9 100644 --- a/apple/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift +++ b/apple/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift @@ -15,14 +15,16 @@ public struct BigtableColumn: Hashable, Sendable, Comparable { family + ":" + BigtableBytes.display(qualifier) } + // By scalar: a qualifier that starts with a combining mark joins the colon into one Character. public init(name: String) throws { - guard let colon = name.firstIndex(of: ":"), colon != name.startIndex else { + guard let colon = name.unicodeScalars.firstIndex(of: ":"), colon != name.startIndex else { throw BigtableError.configuration(String( format: String(localized: "\"%@\" is not a column. Name it as family:qualifier."), name )) } - self.family = String(name[.. Bool { diff --git a/apple/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift b/apple/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift index 08b8966201..c05591c4f0 100644 --- a/apple/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift +++ b/apple/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift @@ -157,7 +157,7 @@ public struct BigtableRequest: Sendable, Equatable { /// Text that would not read back as one bare word goes in JSON quotes. static func token(_ text: String, reserved: String) -> String { - let plain = !text.isEmpty && !text.hasPrefix("\"") && text.unicodeScalars.allSatisfy { scalar in + let plain = !text.isEmpty && text.unicodeScalars.first != "\"" && text.unicodeScalars.allSatisfy { scalar in !CharacterSet.whitespacesAndNewlines.contains(scalar) && !reserved.unicodeScalars.contains(scalar) } if plain { return text } @@ -173,22 +173,23 @@ public struct BigtableRequest: Sendable, Equatable { static let missingTable = BigtableError.configuration(String(localized: "Name the table before the key, as in GET events/user#42.")) } -/// Reads a request line one token at a time. +// Read by scalar, as `token` writes: a key that starts with a combining mark joins the quote or slash before it +// into one Character. private struct RequestScanner { - private var text: Substring + private var text: Substring.UnicodeScalarView private var query: [String: String] = [:] init(_ text: String) { - self.text = Substring(text) + self.text = Substring(text).unicodeScalars } var isAtEnd: Bool { - text.allSatisfy(\.isWhitespace) + text.allSatisfy(\.properties.isWhitespace) } - mutating func word(stoppingAt stop: Character? = nil) -> String? { + mutating func word(stoppingAt stop: Unicode.Scalar? = nil) -> String? { skipSpace() - let word = text.prefix { !$0.isWhitespace && $0 != stop } + let word = text.prefix { !$0.properties.isWhitespace && $0 != stop } guard !word.isEmpty else { return nil } text = text.dropFirst(word.count) return String(word) @@ -196,7 +197,7 @@ private struct RequestScanner { mutating func rest() -> String { let remaining = String(text).trimmingCharacters(in: .whitespacesAndNewlines) - text = "" + text.removeAll() return remaining } @@ -204,7 +205,7 @@ private struct RequestScanner { /// no slash is a table alone, as in `SCAN events`. mutating func target(tableOnly: Bool) throws -> (table: String, key: Data?) { skipSpace() - let head = text.prefix { !$0.isWhitespace && $0 != "\"" && $0 != "?" } + let head = text.prefix { !$0.properties.isWhitespace && $0 != "\"" && $0 != "?" } let table: String if let slash = head.firstIndex(of: "/") { table = String(text[.. = [ + "tlscertificatekeyfilepassword", + "sslclientcertificatekeypassword", + "proxypassword", + ] + static let mongoKeyPassphraseOptions: Set = [ + "tlscertificatekeyfilepassword", + "sslclientcertificatekeypassword", + ] + static let mechanismPropertiesOption = "authmechanismproperties" + static let sessionTokenProperty = "AWS_SESSION_TOKEN" + + public static func isSecret(key: String, value: String) -> Bool { + if key.caseInsensitiveCompare(mongoSessionTokenFieldId) == .orderedSame { return true } + guard let option = mongoOption(in: key) else { return false } + if mongoSecretOptions.contains(option) { return true } + return option == mechanismPropertiesOption && splitSessionToken(from: value).sessionToken != nil + } + + // Mechanism properties keep everything but the token. + public static func removingSecrets(from fields: [String: String]) -> [String: String] { + var kept: [String: String] = [:] + for (key, value) in fields { + guard isSecret(key: key, value: value) else { + kept[key] = value + continue + } + if let option = mongoOption(in: key), isMongoMechanismProperties(option: option), + let remaining = splitSessionToken(from: value).remaining { + kept[key] = remaining + } + } + return kept + } + + // The lowercased option name of a `mongoParam_` field, nil for any other field. + public static func mongoOption(in key: String) -> String? { + guard key.count > mongoOptionPrefix.count, + key.lowercased().hasPrefix(mongoOptionPrefix.lowercased()) + else { return nil } + return String(key.dropFirst(mongoOptionPrefix.count)).lowercased() + } + + public static func isMongoKeyPassphrase(option: String) -> Bool { + mongoKeyPassphraseOptions.contains(option.lowercased()) + } + + public static func isMongoProxyPassword(option: String) -> Bool { + option.lowercased() == "proxypassword" + } + + public static func isMongoMechanismProperties(option: String) -> Bool { + option.lowercased() == mechanismPropertiesOption + } + + // `authMechanismProperties` is `KEY:value` pairs joined by commas; libmongoc reads keys without case. + public static func splitSessionToken(from properties: String) -> (remaining: String?, sessionToken: String?) { + var kept: [String] = [] + var token: String? + for pair in properties.split(separator: ",", omittingEmptySubsequences: true) { + let parts = pair.split(separator: ":", maxSplits: 1, omittingEmptySubsequences: false) + let name = parts[0].trimmingCharacters(in: .whitespaces) + if name.caseInsensitiveCompare(sessionTokenProperty) == .orderedSame { + let value = parts.count > 1 ? String(parts[1]) : "" + if !value.isEmpty { token = value } + } else { + kept.append(String(pair)) + } + } + return (kept.isEmpty ? nil : kept.joined(separator: ","), token) + } + + public static func joiningSessionToken(_ token: String, to properties: String?) -> String { + let remaining = properties.flatMap { splitSessionToken(from: $0).remaining } + let pair = "\(sessionTokenProperty):\(token)" + guard let remaining else { return pair } + return "\(remaining),\(pair)" + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift b/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift index d6962fdd63..7114c21ee2 100644 --- a/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift +++ b/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift @@ -36,6 +36,7 @@ public enum SQLDDLFallbackPolicy { "Elasticsearch", "Typesense", "Weaviate", + "Qdrant", "Kafka", "etcd", "Redis", @@ -46,6 +47,7 @@ public enum SQLDDLFallbackPolicy { "Firestore", "Firebase Realtime Database", "Bigtable", + "Neo4j", ] /// True when `DROP`/`TRUNCATE` built by the app is something this engine could run. diff --git a/apple/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift b/apple/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift index bb002a60ef..5f5c437839 100644 --- a/apple/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift +++ b/apple/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift @@ -14,6 +14,7 @@ public struct DatabaseType: Hashable, Codable, Sendable, RawRepresentable { public static let tidb = DatabaseType(rawValue: "TiDB") public static let databend = DatabaseType(rawValue: "Databend") public static let oceanbase = DatabaseType(rawValue: "OceanBase") + public static let starrocks = DatabaseType(rawValue: "StarRocks") public static let postgresql = DatabaseType(rawValue: "PostgreSQL") public static let sqlite = DatabaseType(rawValue: "SQLite") public static let redis = DatabaseType(rawValue: "Redis") @@ -42,15 +43,16 @@ public struct DatabaseType: Hashable, Codable, Sendable, RawRepresentable { public static let kafka = DatabaseType(rawValue: "Kafka") public static let cloudflareR2SQL = DatabaseType(rawValue: "Cloudflare R2 SQL") public static let weaviate = DatabaseType(rawValue: "Weaviate") + public static let qdrant = DatabaseType(rawValue: "Qdrant") public static let firestore = DatabaseType(rawValue: "Firestore") public static let firebaseRealtimeDatabase = DatabaseType(rawValue: "Firebase Realtime Database") public static let bigtable = DatabaseType(rawValue: "Bigtable") public static let allKnownTypes: [DatabaseType] = [ - .mysql, .mariadb, .tidb, .databend, .oceanbase, .postgresql, .sqlite, .redis, .mongodb, + .mysql, .mariadb, .tidb, .databend, .oceanbase, .starrocks, .postgresql, .sqlite, .redis, .mongodb, .clickhouse, .mssql, .oracle, .dameng, .duckdb, .cassandra, .redshift, .etcd, .cloudflareD1, .dynamodb, .bigquery, .spanner, .snowflake, .libsql, .beancount, - .surrealdb, .teradata, .trino, .kafka, .cloudflareR2SQL, .weaviate, .firestore, .firebaseRealtimeDatabase, .bigtable + .surrealdb, .teradata, .trino, .kafka, .cloudflareR2SQL, .weaviate, .qdrant, .firestore, .firebaseRealtimeDatabase, .bigtable ] /// Icon name for this database type — asset catalog name (e.g. "mysql-icon") or SF Symbol fallback @@ -61,6 +63,7 @@ public struct DatabaseType: Hashable, Codable, Sendable, RawRepresentable { case .tidb: return "tidb-icon" case .databend: return "databend-icon" case .oceanbase: return "oceanbase-icon" + case .starrocks: return "starrocks-icon" case .postgresql: return "postgresql-icon" case .redshift: return "redshift-icon" case .sqlite: return "sqlite-icon" @@ -87,6 +90,7 @@ public struct DatabaseType: Hashable, Codable, Sendable, RawRepresentable { case .kafka: return "kafka-icon" case .cloudflareR2SQL: return "cloudflare-r2-sql-icon" case .weaviate: return "weaviate-icon" + case .qdrant: return "qdrant-icon" case .firestore, .firebaseRealtimeDatabase: return "firebase-icon" case .bigtable: return "bigtable-icon" default: return "externaldrive" diff --git a/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengGrammar.swift b/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengGrammar.swift new file mode 100644 index 0000000000..490ce4e799 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengGrammar.swift @@ -0,0 +1,14 @@ +import Foundation +import TableProSQLGrammar + +enum DamengGrammar { + // The reading the app runs a Dameng session's text with: DM8's measured grammar plus the BACKSLASH_ESCAPE mode the + // driver detects at connect and reports as its session state. + static func execution(backslashEscapes: Bool) -> SQLLexicalGrammar { + let session = SQLSessionLexicalFacts( + determined: .backslashEscapesInSingleQuotes, + enabled: backslashEscapes ? .backslashEscapesInSingleQuotes : [] + ) + return SQLLexicalReadings.resolve(databaseTypeId: "Dameng", declared: nil, session: session).execution + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengPlaceholders.swift b/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengPlaceholders.swift new file mode 100644 index 0000000000..d4eda1777a --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengPlaceholders.swift @@ -0,0 +1,28 @@ +import Foundation +import TableProSQLGrammar + +public enum DamengPlaceholders { + private static let questionMark = UInt16(UnicodeScalar("?").value) + + // One more piece than there are placeholders. A `?` in a literal, a quoted name or a comment stays in its piece. + public static func pieces(of template: String, backslashEscapes: Bool) -> [String] { + let grammar = DamengGrammar.execution(backslashEscapes: backslashEscapes) + let text = template as NSString + var pieces: [String] = [] + var start = 0 + var index = 0 + while index < text.length { + if let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar) { + index = max(span.end, index + 1) + continue + } + if text.character(at: index) == questionMark { + pieces.append(text.substring(with: NSRange(location: start, length: index - start))) + start = index + 1 + } + index += 1 + } + pieces.append(text.substring(from: min(start, text.length))) + return pieces + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengScriptSplitter.swift b/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengScriptSplitter.swift new file mode 100644 index 0000000000..4911eacf04 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengScriptSplitter.swift @@ -0,0 +1,22 @@ +import Foundation +import TableProSQLGrammar + +// Cuts where the app's gates cut, so the driver never sends a piece they did not count. A PL/SQL block goes whole: +// the Dameng grammar does not delimit one, and a wrong cut sends the server a fragment. +public enum DamengScriptSplitter { + private static let blockKeywords: Set = [ + "BEGIN", "CASE", "DECLARE", "END", "IF", "LOOP" + ] + + public static func statements(in script: String, backslashEscapes: Bool) -> [String] { + let grammar = DamengGrammar.execution(backslashEscapes: backslashEscapes) + guard !holdsBlock(script, grammar: grammar) else { return [script] } + return SQLStatementScanner.allStatements(in: script, grammar: grammar) + } + + private static func holdsBlock(_ script: String, grammar: SQLLexicalGrammar) -> Bool { + SQLCodeProjection.code(of: script, grammar: grammar) + .split { !($0.isLetter || $0.isNumber || $0 == "_") } + .contains { blockKeywords.contains($0.uppercased()) } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengStatementClassifier.swift b/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengStatementClassifier.swift new file mode 100644 index 0000000000..c61ef9fed3 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengStatementClassifier.swift @@ -0,0 +1,51 @@ +import Foundation +import TableProSQLGrammar + +// A framing hint and the replay rule, never a verdict on what the user sees: the bridge converts whatever comes back. +public enum DamengStatementClassifier { + private static let resultKeywords: Set = [ + "DESC", "DESCRIBE", "EXPLAIN", "SELECT", "SHOW", "VALUES", "WITH" + ] + + // Shorter than resultKeywords: WITH and VALUES can head a write, and a replayed write may apply twice. + private static let readOnlyKeywords: Set = [ + "DESC", "DESCRIBE", "EXPLAIN", "SELECT", "SHOW" + ] + + private static let openParenthesis = UInt16(UnicodeScalar("(").value) + private static let semicolon = UInt16(UnicodeScalar(";").value) + + public static func likelyReturnsRows(_ query: String) -> Bool { + guard let keyword = firstKeyword(query) else { return false } + return resultKeywords.contains(keyword) + } + + public static func isReadOnly(_ query: String) -> Bool { + guard let keyword = firstKeyword(query) else { return false } + return readOnlyKeywords.contains(keyword) + } + + private static func firstKeyword(_ query: String) -> String? { + let text = query as NSString + let grammar = DamengGrammar.execution(backslashEscapes: false) + var index = 0 + while index < text.length { + if let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar) { + guard span.kind.isComment else { return nil } + index = max(span.end, index + 1) + continue + } + let blank = StatementBlank.blankLength(in: text, at: index) + if blank > 0 { + index += blank + continue + } + let unit = text.character(at: index) + guard unit == openParenthesis || unit == semicolon else { break } + index += 1 + } + guard index < text.length else { return nil } + let keyword = text.substring(from: index).prefix { $0.isLetter || $0 == "_" } + return keyword.isEmpty ? nil : keyword.uppercased() + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProDatabase/SQLEscaping.swift b/apple/Packages/TableProCore/Sources/TableProDatabase/SQLEscaping.swift index 4a31b07ddf..f992ca4a5e 100644 --- a/apple/Packages/TableProCore/Sources/TableProDatabase/SQLEscaping.swift +++ b/apple/Packages/TableProCore/Sources/TableProDatabase/SQLEscaping.swift @@ -3,17 +3,17 @@ import Foundation public enum SQLEscaping { public static func ansiStringLiteral(_ value: String) -> String { value - .replacingOccurrences(of: "\0", with: "") - .replacingOccurrences(of: "'", with: "''") + .replacingOccurrences(of: "\0", with: "", options: .literal) + .replacingOccurrences(of: "'", with: "''", options: .literal) } public static func backslashStringLiteral(_ value: String) -> String { value - .replacingOccurrences(of: "\\", with: "\\\\") - .replacingOccurrences(of: "\0", with: "\\0") - .replacingOccurrences(of: "\n", with: "\\n") - .replacingOccurrences(of: "\r", with: "\\r") - .replacingOccurrences(of: "\u{1a}", with: "\\Z") - .replacingOccurrences(of: "'", with: "''") + .replacingOccurrences(of: "\\", with: "\\\\", options: .literal) + .replacingOccurrences(of: "\0", with: "\\0", options: .literal) + .replacingOccurrences(of: "\n", with: "\\n", options: .literal) + .replacingOccurrences(of: "\r", with: "\\r", options: .literal) + .replacingOccurrences(of: "\u{1a}", with: "\\Z", options: .literal) + .replacingOccurrences(of: "'", with: "''", options: .literal) } } diff --git a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Config/FirebaseConnectionSettings.swift b/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Config/FirebaseConnectionSettings.swift index 84175aef2d..b76179a1c6 100644 --- a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Config/FirebaseConnectionSettings.swift +++ b/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Config/FirebaseConnectionSettings.swift @@ -113,8 +113,16 @@ public struct FirebaseConnectionSettings: Sendable, Equatable { return url } let trusted = host.hasSuffix(".firebaseio.com") || host.hasSuffix(".firebasedatabase.app") - guard scheme == "https", trusted else { throw invalidURL(text) } - return URL(string: "https://\(host)/") ?? url + guard scheme == "https", trusted, isDNSName(host), let instance = URL(string: "https://\(host)/") else { + throw invalidURL(text) + } + return instance + } + + // `URL.host` is percent-decoded, so an encoded `/`, `:`, `?`, `#` or `@` would pass the suffix check and then end + // the host early once the text is parsed again. + static func isDNSName(_ host: String) -> Bool { + host.range(of: #"^[a-z0-9-]+(\.[a-z0-9-]+)+$"#, options: .regularExpression) != nil } static func parseEmulatorHost(_ text: String?) throws -> String { diff --git a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreQuery.swift b/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreQuery.swift index 8760f9b3f7..9fc630c138 100644 --- a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreQuery.swift +++ b/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreQuery.swift @@ -268,7 +268,7 @@ public enum FirestoreQueryBuilder { if column == nameField { return column } let simple = column.range(of: "^[A-Za-z_][A-Za-z0-9_]*$", options: .regularExpression) != nil if simple { return column } - let escaped = column.replacingOccurrences(of: "\\", with: "\\\\").replacingOccurrences(of: "`", with: "\\`") + let escaped = column.replacingOccurrences(of: "\\", with: "\\\\", options: .literal).replacingOccurrences(of: "`", with: "\\`", options: .literal) return "`" + escaped + "`" } diff --git a/apple/Packages/TableProCore/Sources/TableProImport/Bundle/ConnectionBundle+Validation.swift b/apple/Packages/TableProCore/Sources/TableProImport/Bundle/ConnectionBundle+Validation.swift index c7780f3ed9..c45cbd0787 100644 --- a/apple/Packages/TableProCore/Sources/TableProImport/Bundle/ConnectionBundle+Validation.swift +++ b/apple/Packages/TableProCore/Sources/TableProImport/Bundle/ConnectionBundle+Validation.swift @@ -7,6 +7,7 @@ enum BundleViolation: Equatable, Sendable { case unresolvedCredentials(BundleRef) case cycle(BundleRef) case tooDeep(BundleRef) + case invalidSSHDestination(connection: String) /// Deeper than any folder or group the app creates, and shallow enough that a crafted file cannot /// make every query row carry a path of thousands of names. @@ -30,6 +31,8 @@ enum BundleViolation: Equatable, Sendable { String(format: String(localized: "the reference “%@” is inside itself"), ref.rawValue) case .tooDeep(let ref): String(format: String(localized: "the reference “%@” is nested too deeply"), ref.rawValue) + case .invalidSSHDestination(let connection): + String(format: String(localized: "the SSH host or user of “%@” contains characters ssh does not allow"), connection) } } @@ -51,6 +54,9 @@ enum BundleViolation: Equatable, Sendable { for refs in refLists { if let violation = identityViolation(refs) { return violation } } + if let connection = connections.first(where: { $0.settings.refusedSSHDestination != nil }) { + return .invalidSSHDestination(connection: connection.settings.name) + } let connectionRefs = Set(connections.map(\.ref)) let groupRefs = Set(groups.map(\.ref)) diff --git a/apple/Packages/TableProCore/Sources/TableProImport/Bundle/ExportableConnection.swift b/apple/Packages/TableProCore/Sources/TableProImport/Bundle/ExportableConnection.swift index 1ad029607f..178826dc3c 100644 --- a/apple/Packages/TableProCore/Sources/TableProImport/Bundle/ExportableConnection.swift +++ b/apple/Packages/TableProCore/Sources/TableProImport/Bundle/ExportableConnection.swift @@ -1,5 +1,6 @@ import Foundation import TableProConnectionLibrary +import TableProSSHTransport import UniformTypeIdentifiers public extension UTType { @@ -143,17 +144,24 @@ public extension ExportableConnection { private static let queryTimeoutSecondsRange = 0 ... Int(Int32.max) / 1_000 static let importBlockedAdditionalFieldKeys: Set = [ + "beancountrunledgerplugins", "postdisconnectscript", "preconnectscript", "pretunnelhost", "pretunnelport", "promptforpassword", + "snowflakeconnectionname", "spendpoint", "sslclientkeypassphrase", "usepgpass", ] - static let importBlockedAdditionalFieldPrefixes: Set = ["aws"] + static let importBlockedAdditionalFieldPrefixes: Set = ["aws", "entra"] + + // Values that pick a credential kept on this Mac, under keys whose other values are plain settings. + static let importBlockedAdditionalFieldValues: [String: Set] = [ + "mongoauthmechanism": ["mongodb-aws"], + ] static func isImportBlockedAdditionalFieldKey(_ key: String) -> Bool { let normalized = key.lowercased() @@ -161,12 +169,19 @@ public extension ExportableConnection { return importBlockedAdditionalFieldPrefixes.contains { normalized.hasPrefix($0) } } + static func isImportBlockedAdditionalField(key: String, value: String) -> Bool { + if isImportBlockedAdditionalFieldKey(key) { return true } + if ConnectionSecretOptions.isSecret(key: key, value: value) { return true } + let blockedValues = importBlockedAdditionalFieldValues[key.lowercased()] ?? [] + return blockedValues.contains(value.trimmingCharacters(in: .whitespaces).lowercased()) + } + static func shareableAdditionalFields( _ fields: [String: String], excluding excludedKeys: Set = [] ) -> [String: String]? { - let shareable = fields.filter { key, _ in - !excludedKeys.contains(key) && !isImportBlockedAdditionalFieldKey(key) + let shareable = fields.filter { key, value in + !excludedKeys.contains(key) && !isImportBlockedAdditionalField(key: key, value: value) } return shareable.isEmpty ? nil : shareable } @@ -191,7 +206,7 @@ public extension ExportableConnection { /// An explicit timeout that is out of range is dropped, not replaced by a legacy additional field. func sanitizedForImport() -> ExportableConnection { - var allowed = (additionalFields ?? [:]).filter { !Self.isImportBlockedAdditionalFieldKey($0.key) } + var allowed = (additionalFields ?? [:]).filter { !Self.isImportBlockedAdditionalField(key: $0.key, value: $0.value) } let legacyConnectTimeout = allowed.removeValue(forKey: Self.connectTimeoutSecondsKey).flatMap(Int.init) let legacyQueryTimeout = allowed.removeValue(forKey: Self.queryTimeoutSecondsKey).flatMap(Int.init) var copy = self @@ -243,6 +258,15 @@ public struct ExportableTunnelCommand: Codable, Sendable, Equatable { } } +public extension ExportableConnection { + // Match exec hands the host and user to /bin/sh, and the host key store is one entry per line. + var refusedSSHDestination: SSHDestinationName.Problem? { + guard let ssh = sshConfig, ssh.enabled else { return nil } + let hops = [(ssh.host, ssh.username)] + (ssh.jumpHosts ?? []).map { ($0.host, $0.username) } + return hops.lazy.compactMap { SSHDestinationName.problem(host: $0.0, user: $0.1) }.first + } +} + public struct ExportableSSHConfig: Codable, Sendable, Equatable { public var enabled: Bool public var host: String diff --git a/apple/Packages/TableProCore/Sources/TableProImport/Import/ImportPlanner.swift b/apple/Packages/TableProCore/Sources/TableProImport/Import/ImportPlanner.swift index 5cf8e769e7..aa939d542c 100644 --- a/apple/Packages/TableProCore/Sources/TableProImport/Import/ImportPlanner.swift +++ b/apple/Packages/TableProCore/Sources/TableProImport/Import/ImportPlanner.swift @@ -88,7 +88,8 @@ extension ImportPlanner { ref: profile.ref, name: profile.name, username: profile.username, - passwordMode: profile.passwordMode == .pgpass ? .pgpass : .prompt, + // A file never chooses how this Mac finds a password: pgpass would read ~/.pgpass for its host. + passwordMode: .prompt, secureFieldIds: profile.secureFieldIds )) } diff --git a/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLConnectionOptions.swift b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLConnectionOptions.swift index ac0cf7b97b..aedd33873c 100644 --- a/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLConnectionOptions.swift +++ b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLConnectionOptions.swift @@ -14,14 +14,11 @@ public struct MSSQLConnectionOptions: Sendable, Equatable { public var kerberosCachePath: String? public var kerberosServicePrincipal: String? - /// How far to check the server certificate. Set after init by the plugin from the - /// connection's SSL mode, so the existing initializer keeps its signature. - public var certificateVerification: MSSQLCertificateVerification = .none + // No default: a caller that forgets it would encrypt without checking who answers. + public var certificateVerification: MSSQLCertificateVerification public var caCertificatePath: String? - /// Microsoft Entra ID access token, sent in place of a user name and password. Set after - /// init for the same reason as `certificateVerification`. Only read when - /// `authMethod == .entra`. + // Sent in place of a user name and password, read only when `authMethod == .entra`. public var fedAuthToken: String? public static let defaultPort = 1_433 @@ -38,6 +35,7 @@ public struct MSSQLConnectionOptions: Sendable, Equatable { database: String, schema: String = MSSQLConnectionOptions.defaultSchema, encryptionLevel: MSSQLEncryptionLevel = MSSQLConnectionOptions.defaultEncryptionLevel, + certificateVerification: MSSQLCertificateVerification, applicationName: String = MSSQLConnectionOptions.defaultApplicationName, loginTimeoutSeconds: Int = MSSQLConnectionOptions.defaultLoginTimeoutSeconds, authMethod: MSSQLAuthMethod = .sqlServer, @@ -60,6 +58,7 @@ public struct MSSQLConnectionOptions: Sendable, Equatable { self.database = database self.schema = schema self.encryptionLevel = encryptionLevel + self.certificateVerification = certificateVerification self.applicationName = applicationName self.loginTimeoutSeconds = loginTimeoutSeconds } diff --git a/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLViewSources.swift b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLViewSources.swift new file mode 100644 index 0000000000..35ca9a4a2d --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLViewSources.swift @@ -0,0 +1,364 @@ +import Foundation + +// INFORMATION_SCHEMA says NO for every view; sys.dm_exec_describe_first_result_set compiles without running. +public enum MSSQLViewSources { + public enum Refusal: Equatable, Sendable { + case notUpdatable + case writesRunTriggers + case readsSeveralTables + case baseTableHidden + case couldNotCheck + } + + public enum Operation: String, CaseIterable, Sendable { + case update + case insert + case delete + } + + public struct BaseTable: Hashable, Sendable { + // Set for a table on a linked server. + public let server: String? + public let database: String? + public let schema: String + public let name: String + + public init(server: String? = nil, database: String?, schema: String, name: String) { + self.server = server + self.database = database + self.schema = schema + self.name = name + } + } + + public struct ViewColumn: Equatable, Sendable { + public let name: String + public let baseColumn: String? + public let acceptsWrites: Bool + + public init(name: String, baseColumn: String?, acceptsWrites: Bool) { + self.name = name + self.baseColumn = baseColumn + self.acceptsWrites = acceptsWrites + } + } + + public struct BaseColumn: Equatable, Sendable { + public let name: String + public let isNullable: Bool + // Nil also for a default whose text needs VIEW DEFINITION to read. + public let defaultValue: String? + public let hasDefault: Bool + public let isIdentity: Bool + public let isComputed: Bool + // A rowversion or a GENERATED ALWAYS period column. + public let isServerGenerated: Bool + + public init( + name: String, + isNullable: Bool, + defaultValue: String?, + hasDefault: Bool, + isIdentity: Bool, + isComputed: Bool, + isServerGenerated: Bool + ) { + self.name = name + self.isNullable = isNullable + self.defaultValue = defaultValue + self.hasDefault = hasDefault + self.isIdentity = isIdentity + self.isComputed = isComputed + self.isServerGenerated = isServerGenerated + } + + // A default the user cannot read still fills the column when an INSERT leaves it out. + public var assignsValue: Bool { + isIdentity || isServerGenerated || (hasDefault && defaultValue == nil) + } + + public var acceptsWrites: Bool { + !isComputed && !isIdentity && !isServerGenerated + } + } + + static let maxViewDepth = 8 + + // MARK: - The view itself + + // sys.sql_expression_dependencies needs VIEW DEFINITION; sys.sql_dependencies does not but drops recreated views. + // A schema-bound view has one sys.sql_expression_dependencies row per column; minor id 0 is the object. + public static func viewFactsQuery(schema: String, view: String) -> String { + let object = MSSQLStringLiteral.quoted(MSSQLSchemaQueries.bracketed(schema: schema, table: view)) + return """ + DECLARE @view int = OBJECT_ID(\(object)); + DECLARE @edges TABLE (referencing_id int NOT NULL, referenced_id int NOT NULL); + IF EXISTS (SELECT 1 FROM sys.views WHERE object_id = @view) + BEGIN + INSERT INTO @edges (referencing_id, referenced_id) + SELECT DISTINCT d.object_id, d.referenced_major_id + FROM sys.sql_dependencies d + JOIN sys.views r ON r.object_id = d.referenced_major_id; + IF HAS_PERMS_BY_NAME(N'sys.sql_expression_dependencies', 'OBJECT', 'SELECT') = 1 + INSERT INTO @edges (referencing_id, referenced_id) + SELECT d.referencing_id, d.referenced_id + FROM sys.sql_expression_dependencies d + JOIN sys.views r ON r.object_id = d.referenced_id + WHERE d.referenced_minor_id = 0 + AND NOT EXISTS ( + SELECT 1 + FROM @edges e + WHERE e.referencing_id = d.referencing_id AND e.referenced_id = d.referenced_id + ); + END; + WITH chain AS ( + SELECT v.object_id, 0 AS depth + FROM sys.views v + WHERE v.object_id = @view + UNION ALL + SELECT e.referenced_id, c.depth + 1 + FROM chain c + JOIN @edges e ON e.referencing_id = c.object_id + WHERE c.depth < \(maxViewDepth) + ) + SELECT + v.has_opaque_metadata, + CASE WHEN EXISTS ( + SELECT 1 + FROM chain c + JOIN sys.triggers tr ON tr.parent_id = c.object_id + WHERE tr.is_instead_of_trigger = 1 AND tr.is_disabled = 0 + ) THEN 1 ELSE 0 END AS runs_instead_of_trigger, + CASE WHEN EXISTS ( + SELECT 1 + FROM chain c + JOIN @edges e ON e.referencing_id = c.object_id + WHERE c.depth = \(maxViewDepth) + ) THEN 1 ELSE 0 END AS chain_continues + FROM sys.views v + WHERE v.object_id = @view + """ + } + + public struct ViewFacts: Equatable, Sendable { + public let hasOpaqueMetadata: Bool + public let runsInsteadOfTrigger: Bool + // Views nest deeper than the walk followed, so a trigger further down went unseen. + public let chainContinues: Bool + + public init(hasOpaqueMetadata: Bool, runsInsteadOfTrigger: Bool, chainContinues: Bool = false) { + self.hasOpaqueMetadata = hasOpaqueMetadata + self.runsInsteadOfTrigger = runsInsteadOfTrigger + self.chainContinues = chainContinues + } + + // Nil when no view of that name is visible to the user. + public init?(rows: [[String?]]) { + guard let row = rows.first else { return nil } + self.init( + hasOpaqueMetadata: row.cell(0) == "1", + runsInsteadOfTrigger: row.cell(1) == "1", + chainContinues: row.cell(2) == "1" + ) + } + + public var refusal: Refusal? { + // WITH VIEW_METADATA makes browse mode name the view in place of its base table. + if hasOpaqueMetadata { return .baseTableHidden } + if runsInsteadOfTrigger { return .writesRunTriggers } + if chainContinues { return .couldNotCheck } + return nil + } + } + + // MARK: - Browse mode + + // Mode 1 is FOR BROWSE: base table and column per column, plus hidden key and rowversion rows of every table. + public static func browseQuery(schema: String, view: String) -> String { + let select = "SELECT * FROM \(MSSQLSchemaQueries.bracketed(schema: schema, table: view))" + return """ + SELECT name, is_hidden, source_server, source_database, source_schema, source_table, source_column, + is_updateable + FROM sys.dm_exec_describe_first_result_set(\(MSSQLStringLiteral.quoted(select)), NULL, 1) + ORDER BY column_ordinal + """ + } + + public enum Source: Equatable, Sendable { + case table(BaseTable) + case refused(Refusal) + } + + public struct BrowseFacts: Equatable, Sendable { + public let columns: [ViewColumn] + public let baseTables: [BaseTable] + // View columns the probes write, one per base column, in view order. + public let writableColumns: [String] + + public init(rows: [[String?]]) { + var columns: [ViewColumn] = [] + var baseTables: [BaseTable] = [] + var writableColumns: [String] = [] + var probedBaseColumns: Set = [] + for row in rows { + // A view that no longer compiles yields error rows with no name. + guard let name = row.cell(0) else { continue } + let server = row.cell(2) + if let schema = row.cell(4), let table = row.cell(5) { + let base = BaseTable(server: server, database: row.cell(3), schema: schema, name: table) + if !baseTables.contains(base) { baseTables.append(base) } + } + guard row.cell(1) != "1" else { continue } + let baseColumn = server == nil ? row.cell(6) : nil + let acceptsWrites = row.cell(7) == "1" + columns.append(ViewColumn(name: name, baseColumn: baseColumn, acceptsWrites: acceptsWrites)) + if acceptsWrites, let baseColumn, probedBaseColumns.insert(baseColumn).inserted { + writableColumns.append(name) + } + } + self.columns = columns + self.baseTables = baseTables + self.writableColumns = writableColumns + } + + public var source: Source { + guard baseTables.count == 1, let table = baseTables.first else { + return .refused(baseTables.isEmpty ? .notUpdatable : .readsSeveralTables) + } + // Its key and columns live in another server's catalog. + return table.server == nil ? .table(table) : .refused(.baseTableHidden) + } + } + + // MARK: - Operations + + public static func probesQuery(schema: String, view: String, writableColumns: [String]) -> String { + let target = MSSQLSchemaQueries.bracketed(schema: schema, table: view) + var probes: [(Operation, String)] = [(.delete, "DELETE FROM \(target) WHERE 1 = 0")] + if let first = writableColumns.first.map(bracketedColumn) { + let list = writableColumns.map(bracketedColumn).joined(separator: ", ") + probes.append((.update, "UPDATE \(target) SET \(first) = \(first) WHERE 1 = 0")) + probes.append((.insert, "INSERT INTO \(target) (\(list)) SELECT \(list) FROM \(target) WHERE 1 = 0")) + } + let values = probes + .map { "('\($0.0.rawValue)', \(MSSQLStringLiteral.quoted($0.1)))" } + .joined(separator: ",\n ") + return """ + SELECT p.operation, r.error_number + FROM (VALUES + \(values) + ) AS p(operation, statement) + CROSS APPLY sys.dm_exec_describe_first_result_set(p.statement, NULL, 0) r + WHERE r.error_number IS NOT NULL + """ + } + + // Browse mode names a table once however often the view joins it, so a self-join shows only as 4405 on DELETE. + public static func refusal(probeRows: [[String?]]) -> Refusal? { + let joins = probeRows.contains { $0.cell(0) == Operation.delete.rawValue && $0.cell(1) == "4405" } + return joins ? .readsSeveralTables : nil + } + + public static func allowedOperations(writableColumns: [String], probeRows: [[String?]]) -> Set { + var allowed: Set = writableColumns.isEmpty ? [.delete] : Set(Operation.allCases) + for row in probeRows { + guard let operation = row.cell(0).flatMap(Operation.init(rawValue:)), + let number = row.cell(1).flatMap({ Int($0) }), + structuralErrors.contains(number) + else { continue } + allowed.remove(operation) + } + return allowed + } + + // 11501 marks any compile error. 229 and 230 come with 11529 and wait for the save, as on a table. + static let structuralErrors: Set = [ + 208, // invalid object name + 264, // a column assigned twice + 271, // computed column + 272, // timestamp column + 4_403, // aggregate, DISTINCT, GROUP BY, PIVOT + 4_405, // the statement touches several base tables + 4_406, // derived or constant column + 4_426, // UNION + 8_102, // identity column + 11_501, // the batch has compile errors + 13_537, // GENERATED ALWAYS column + ] + + // MARK: - Base table + + // Read in the base table's own database. sys.tables skips a view that browse mode stopped at. + public static func baseColumnsQuery(_ table: BaseTable) -> String { + let catalog = table.database.map { "[\(MSSQLSchemaQueries.escapeBracket($0))]." } ?? "" + let object = MSSQLStringLiteral.quoted( + catalog + MSSQLSchemaQueries.bracketed(schema: table.schema, table: table.name) + ) + return """ + SELECT + c.name, + c.is_nullable, + d.definition, + CASE WHEN c.default_object_id <> 0 THEN 1 ELSE 0 END AS has_default, + c.is_identity, + c.is_computed, + CASE + WHEN TYPE_NAME(c.system_type_id) = 'timestamp' THEN 1 + WHEN c.generated_always_type > 0 THEN 1 + ELSE 0 + END AS is_server_generated, + ic.key_ordinal + FROM \(catalog)sys.columns c + JOIN \(catalog)sys.tables t ON t.object_id = c.object_id + LEFT JOIN \(catalog)sys.default_constraints d ON d.object_id = c.default_object_id + LEFT JOIN \(catalog)sys.indexes i ON i.object_id = c.object_id AND i.is_primary_key = 1 + LEFT JOIN \(catalog)sys.index_columns ic + ON ic.object_id = i.object_id AND ic.index_id = i.index_id AND ic.column_id = c.column_id + WHERE c.object_id = OBJECT_ID(\(object)) + ORDER BY c.column_id + """ + } + + public struct BaseFacts: Equatable, Sendable { + public let keyColumns: [String] + public let columns: [BaseColumn] + + public init(keyColumns: [String], columns: [BaseColumn]) { + self.keyColumns = keyColumns + self.columns = columns + } + + // Nil when no row comes back: OBJECT_ID is NULL for a table the user has no permission on. + public init?(rows: [[String?]]) { + var columns: [BaseColumn] = [] + var keyed: [(ordinal: Int, name: String)] = [] + for row in rows { + guard let name = row.cell(0) else { continue } + columns.append(BaseColumn( + name: name, + isNullable: row.cell(1) == "1", + defaultValue: row.cell(2), + hasDefault: row.cell(3) == "1", + isIdentity: row.cell(4) == "1", + isComputed: row.cell(5) == "1", + isServerGenerated: row.cell(6) == "1" + )) + if let ordinal = row.cell(7).flatMap({ Int($0) }) { + keyed.append((ordinal, name)) + } + } + guard !columns.isEmpty else { return nil } + self.init(keyColumns: keyed.sorted { $0.ordinal < $1.ordinal }.map(\.name), columns: columns) + } + } + + private static func bracketedColumn(_ column: String) -> String { + "[\(MSSQLSchemaQueries.escapeBracket(column))]" + } +} + +private extension Array where Element == String? { + func cell(_ index: Int) -> String? { + indices.contains(index) ? self[index] : nil + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLLiteralSpelling.swift b/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLLiteralSpelling.swift index c9098218b2..3b0aaf5808 100644 --- a/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLLiteralSpelling.swift +++ b/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLLiteralSpelling.swift @@ -24,7 +24,7 @@ public enum MySQLLiteralSpelling: Equatable, Sendable { case .backslashEscapes: return mysqlEscapeStringLiteral(value) case .quoteDoubling: - return value.replacingOccurrences(of: "'", with: "''") + return value.replacingOccurrences(of: "'", with: "''", options: .literal) } } @@ -140,11 +140,12 @@ public enum MySQLLiteralSpelling: Equatable, Sendable { return scalars.count } + /// `--` opens a comment only when a space or a control character follows it, U+007F among them. private static func opensDashComment(_ scalars: [Unicode.Scalar], at index: Int) -> Bool { guard index + 1 < scalars.count, scalars[index + 1] == "-" else { return false } guard index + 2 < scalars.count else { return true } let next = scalars[index + 2] - return next == " " || next == "\t" || next == "\n" || next == "\r" + return next <= " " || next == "\u{7F}" } private static func endOfLine(in scalars: [Unicode.Scalar], from start: Int) -> Int { diff --git a/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLObjectQueries.swift b/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLObjectQueries.swift index cfb77225bf..76d48cf2c5 100644 --- a/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLObjectQueries.swift +++ b/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLObjectQueries.swift @@ -14,7 +14,7 @@ public enum MySQLObjectQueries { } public static func quoteIdentifier(_ value: String) -> String { - "`\(value.replacingOccurrences(of: "`", with: "``"))`" + "`\(value.replacingOccurrences(of: "`", with: "``", options: .literal))`" } /// The database a catalog read means. @@ -129,9 +129,9 @@ public enum MySQLObjectQueries { public static func likePattern(matching name: String) -> String { name - .replacingOccurrences(of: "\\", with: "\\\\") - .replacingOccurrences(of: "%", with: "\\%") - .replacingOccurrences(of: "_", with: "\\_") + .replacingOccurrences(of: "\\", with: "\\\\", options: .literal) + .replacingOccurrences(of: "%", with: "\\%", options: .literal) + .replacingOccurrences(of: "_", with: "\\_", options: .literal) } /// Whether `information_schema` describes this database at all, in one scalar. diff --git a/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLStatementKeyword.swift b/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLStatementKeyword.swift index 35bbf14782..1cb3bae3ff 100644 --- a/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLStatementKeyword.swift +++ b/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLStatementKeyword.swift @@ -20,7 +20,8 @@ nonisolated public func mysqlLeadingKeyword(_ query: String) -> String { while true { rest = rest.drop(while: { $0.isWhitespace }) if rest.hasPrefix("/*"), !rest.hasPrefix("/*!") { - guard let end = rest.range(of: "*/") else { return "" } + // Past the opener: its `*` closes nothing, so `/*/` is still open. + guard let end = rest.dropFirst(2).range(of: "*/") else { return "" } rest = rest[end.upperBound...] } else if rest.hasPrefix("#") || (rest.hasPrefix("--") && rest.dropFirst(2).first?.isWhitespace != false) { guard let end = rest.firstIndex(where: { $0.isNewline }) else { return "" } diff --git a/apple/Packages/TableProCore/Sources/TableProNeo4jCore/BoltMessage.swift b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/BoltMessage.swift new file mode 100644 index 0000000000..5b13e9e2e2 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/BoltMessage.swift @@ -0,0 +1,175 @@ +import Foundation + +public struct BoltVersion: Comparable, Hashable, Sendable, CustomStringConvertible { + public let major: UInt8 + public let minor: UInt8 + + public init(major: UInt8, minor: UInt8) { + self.major = major + self.minor = minor + } + + public var description: String { "\(major).\(minor)" } + + public static func < (lhs: BoltVersion, rhs: BoltVersion) -> Bool { + (lhs.major, lhs.minor) < (rhs.major, rhs.minor) + } + + public var separatesLogon: Bool { self >= BoltVersion(major: 5, minor: 1) } + public var sendsBoltAgent: Bool { self >= BoltVersion(major: 5, minor: 3) } + public var hasElementIds: Bool { major >= 5 } +} + +public enum BoltHandshake { + public static let magic: [UInt8] = [0x60, 0x60, 0xB0, 0x17] + + // Each proposal is [0, range, minor, major]: major.minor down to major.(minor - range). + public static let proposals: [UInt8] = [ + 0x00, 0x04, 0x04, 0x05, + 0x00, 0x02, 0x04, 0x04, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 + ] + + public static var request: Data { Data(magic + proposals) } + + public static func version(fromResponse bytes: [UInt8]) -> BoltVersion? { + guard bytes.count == 4, bytes != [0, 0, 0, 0] else { return nil } + return BoltVersion(major: bytes[3], minor: bytes[2]) + } +} + +public enum BoltRequest: Equatable, Sendable { + case hello(extra: [String: PackStreamValue]) + case logon(auth: [String: PackStreamValue]) + case run(query: String, parameters: [String: PackStreamValue], extra: [String: PackStreamValue]) + case pull(count: Int64) + case discard + case reset + case goodbye + + public var structure: PackStreamValue { + switch self { + case .hello(let extra): + return .structure(tag: 0x01, fields: [.map(extra)]) + case .logon(let auth): + return .structure(tag: 0x6A, fields: [.map(auth)]) + case .run(let query, let parameters, let extra): + return .structure(tag: 0x10, fields: [.string(query), .map(parameters), .map(extra)]) + case .pull(let count): + return .structure(tag: 0x3F, fields: [.map(["n": .int(count)])]) + case .discard: + return .structure(tag: 0x2F, fields: [.map(["n": .int(-1)])]) + case .reset: + return .structure(tag: 0x0F, fields: []) + case .goodbye: + return .structure(tag: 0x02, fields: []) + } + } + + public var expectsResponse: Bool { self != .goodbye } + + public var framed: Data { BoltChunker.frame(PackStreamEncoder.encode(structure)) } +} + +public enum BoltResponse: Equatable, Sendable { + case success([String: PackStreamValue]) + case record([PackStreamValue]) + case ignored + case failure(code: String, message: String) + + public static func decode(_ message: Data) throws -> BoltResponse { + guard case .structure(let tag, let fields) = try PackStreamDecoder.decode(message) else { + throw BoltProtocolError.unexpectedMessage("not a structure") + } + switch tag { + case 0x70: + return .success(fields.first?.mapValue ?? [:]) + case 0x71: + return .record(fields.first?.listValue ?? []) + case 0x7E: + return .ignored + case 0x7F: + let metadata = fields.first?.mapValue ?? [:] + let code = metadata["code"]?.stringValue ?? metadata["neo4j_code"]?.stringValue ?? "" + return .failure(code: code, message: metadata["message"]?.stringValue ?? "") + default: + throw BoltProtocolError.unexpectedMessage(String(format: "tag 0x%02X", tag)) + } + } +} + +public enum BoltProtocolError: Error, Equatable, Sendable, LocalizedError { + case unexpectedMessage(String) + case unsupportedServer + case chunkTooLarge(Int) + + public var errorDescription: String? { + switch self { + case .unexpectedMessage(let detail): + return String(format: String(localized: "The server sent a message this client does not understand (%@)."), detail) + case .unsupportedServer: + return String(localized: "The server does not speak Bolt 4.2 to 5.4. Neo4j 4.2 or later is required.") + case .chunkTooLarge(let size): + return String(format: String(localized: "The server sent a %d-byte message, which is over the limit."), size) + } + } +} + +public enum BoltChunker { + public static let maximumChunkSize = Int(UInt16.max) + + public static func frame(_ message: Data) -> Data { + var framed = Data() + framed.reserveCapacity(message.count + 4) + var offset = message.startIndex + while offset < message.endIndex { + let end = min(offset + maximumChunkSize, message.endIndex) + let size = UInt16(end - offset) + framed.append(UInt8(size >> 8)) + framed.append(UInt8(size & 0xFF)) + framed.append(message[offset ..< end]) + offset = end + } + framed.append(contentsOf: [0x00, 0x00]) + return framed + } +} + +// A zero-size chunk with nothing before it is a NOOP keep-alive, not an empty message. +public struct BoltDechunker: Sendable { + public static let maximumMessageSize = 512 * 1_024 * 1_024 + + private var buffer: [UInt8] = [] + private var cursor = 0 + private var message = Data() + + public init() {} + + public mutating func append(_ bytes: Bytes) where Bytes.Element == UInt8 { + if cursor > 0, cursor >= buffer.count / 2 { + buffer.removeFirst(cursor) + cursor = 0 + } + buffer.append(contentsOf: bytes) + } + + public mutating func nextMessage() throws -> Data? { + while buffer.count - cursor >= 2 { + let size = Int(buffer[cursor]) << 8 | Int(buffer[cursor + 1]) + if size == 0 { + cursor += 2 + guard !message.isEmpty else { continue } + defer { message = Data() } + return message + } + guard buffer.count - cursor >= 2 + size else { return nil } + message.append(contentsOf: buffer[(cursor + 2) ..< (cursor + 2 + size)]) + cursor += 2 + size + guard message.count <= Self.maximumMessageSize else { + throw BoltProtocolError.chunkTooLarge(message.count) + } + } + return nil + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jCypher.swift b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jCypher.swift new file mode 100644 index 0000000000..22d79a0760 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jCypher.swift @@ -0,0 +1,345 @@ +import Foundation + +public enum Neo4jEntityKind: String, Sendable, CaseIterable { + case node + case relationship + + public var schemaName: String { + switch self { + case .node: "Labels" + case .relationship: "Relationship Types" + } + } + + public var entityType: String { + switch self { + case .node: "NODE" + case .relationship: "RELATIONSHIP" + } + } + + public init?(schemaName: String?) { + guard let match = Self.allCases.first(where: { $0.schemaName == schemaName }) else { return nil } + self = match + } +} + +public struct Neo4jFilter: Sendable, Equatable { + public let column: String + public let op: String + public let value: String + public let secondValue: String? + public let isCaseSensitive: Bool + + public init(column: String, op: String, value: String, secondValue: String? = nil, isCaseSensitive: Bool = true) { + self.column = column + self.op = op + self.value = value + self.secondValue = secondValue + self.isCaseSensitive = isCaseSensitive + } +} + +public enum Neo4jCypherError: Error, Equatable, Sendable, LocalizedError { + case unsupportedOperator(String) + case unquotableName(String) + case incompleteRange + + public var errorDescription: String? { + switch self { + case .unsupportedOperator(let op): + return String(format: String(localized: "The %@ filter is not available for Neo4j."), op) + case .unquotableName(let name): + return String(format: String(localized: "Neo4j cannot query the name %@ because it contains \\u."), name) + case .incompleteRange: + return String(localized: "Enter both bounds to filter with BETWEEN.") + } + } +} + +public enum Neo4jCypher { + public static let idColumn = "" + public static let labelsColumn = "" + public static let startColumn = "" + public static let endColumn = "" + public static let rawFilterColumn = "__RAW__" + + // A comment keeps a browse query valid Cypher when pasted into the editor. + public static let browseMarker = "// TablePro browse: " + + // Neo4j decodes `\uXXXX` before parsing, inside backticks too, so no spelling of `\u` stays inside its quotes. + // `\uZZZZ` is an escape the server rejects; the query builders refuse such a name before it gets that far. + public static func quote(_ identifier: String) -> String { + let escaped = identifier + .replacingOccurrences(of: "\\u", with: "\\uZZZZ") + .replacingOccurrences(of: "`", with: "``") + return "`" + escaped + "`" + } + + public static func isQuotable(_ name: String) -> Bool { + !name.contains("\\u") + } + + static func requireQuotable(_ names: [String]) throws { + if let name = names.first(where: { !isQuotable($0) }) { throw Neo4jCypherError.unquotableName(name) } + } + + public static func literal(_ text: String) -> String { + let escaped = text + .replacingOccurrences(of: "\\", with: "\\\\") + .replacingOccurrences(of: "'", with: "\\'") + return "'" + escaped + "'" + } + + public static func browseKind(of query: String) -> Neo4jEntityKind? { + let trimmed = query.drop { $0.isWhitespace } + guard trimmed.hasPrefix(browseMarker) else { return nil } + let rest = trimmed.dropFirst(browseMarker.count).prefix { !$0.isNewline } + return Neo4jEntityKind(rawValue: rest.trimmingCharacters(in: .whitespaces)) + } + + public static func browsedName(in query: String) -> String? { + guard browseKind(of: query) != nil, let start = query.range(of: ":`") else { return nil } + var name = "" + var index = start.upperBound + while index < query.endIndex { + let character = query[index] + let next = query.index(after: index) + guard character == "`" else { + name.append(character) + index = next + continue + } + guard next < query.endIndex, query[next] == "`" else { return name } + name.append("`") + index = query.index(after: next) + } + return nil + } + + public static func fixedColumns(for kind: Neo4jEntityKind) -> [String] { + switch kind { + case .node: [idColumn, labelsColumn] + case .relationship: [idColumn, startColumn, endColumn] + } + } + + public static func pattern(for kind: Neo4jEntityKind, name: String) -> String { + switch kind { + case .node: "MATCH (n:\(quote(name)))" + case .relationship: "MATCH ()-[n:\(quote(name))]->()" + } + } + + public static func browseQuery( + kind: Neo4jEntityKind, + name: String, + filters: [Neo4jFilter] = [], + matchesAll: Bool = true, + sort: [(column: String, ascending: Bool)] = [], + limit: Int, + offset: Int, + usesElementId: Bool = true + ) throws -> String { + try requireQuotable([name] + sort.map(\.column)) + var lines = [browseMarker + kind.rawValue, pattern(for: kind, name: name)] + if let filter = try whereClause(filters, matchesAll: matchesAll, usesElementId: usesElementId) { + lines.append(filter) + } + lines.append("RETURN n") + if !sort.isEmpty { + let keys = sort.map { expression(for: $0.column, usesElementId: usesElementId) + ($0.ascending ? " ASC" : " DESC") } + lines.append("ORDER BY " + keys.joined(separator: ", ")) + } + lines.append("SKIP \(max(0, offset)) LIMIT \(max(0, limit))") + return lines.joined(separator: "\n") + } + + public static func exportQuery(kind: Neo4jEntityKind, name: String) throws -> String { + try requireQuotable([name]) + return [browseMarker + kind.rawValue, pattern(for: kind, name: name), "RETURN n"].joined(separator: "\n") + } + + public static func countQuery( + kind: Neo4jEntityKind, + name: String, + filters: [Neo4jFilter] = [], + matchesAll: Bool = true, + usesElementId: Bool = true + ) throws -> String { + try requireQuotable([name]) + var lines = [pattern(for: kind, name: name)] + if let filter = try whereClause(filters, matchesAll: matchesAll, usesElementId: usesElementId) { + lines.append(filter) + } + lines.append("RETURN count(n) AS count") + return lines.joined(separator: "\n") + } + + // Bounded so the Structure tab stays fast on a large graph, unlike `db.schema.nodeTypeProperties()`. + public static func propertySampleQuery(kind: Neo4jEntityKind, name: String, sampleSize: Int = 1_000) -> String { + pattern(for: kind, name: name) + + " WITH n LIMIT \(sampleSize) UNWIND keys(n) AS key" + + " RETURN key, head(collect(n[key])) AS sample ORDER BY key" + } + + public static func propertyKeysQuery(kind: Neo4jEntityKind, name: String) throws -> String { + try requireQuotable([name]) + return pattern(for: kind, name: name) + " UNWIND keys(n) AS key RETURN DISTINCT key ORDER BY key" + } + + public static func expression(for column: String, usesElementId: Bool) -> String { + let idFunction = usesElementId ? "elementId" : "id" + switch column { + case idColumn: return "\(idFunction)(n)" + case labelsColumn: return "labels(n)" + case startColumn: return "\(idFunction)(startNode(n))" + case endColumn: return "\(idFunction)(endNode(n))" + default: return "n." + quote(column) + } + } + + private static func whereClause(_ filters: [Neo4jFilter], matchesAll: Bool, usesElementId: Bool) throws -> String? { + let clauses = try filters.map { try condition(for: $0, usesElementId: usesElementId) } + guard !clauses.isEmpty else { return nil } + let joined = clauses.count == 1 + ? clauses[0] + : clauses.map { "(\($0))" }.joined(separator: matchesAll ? " AND " : " OR ") + return "WHERE " + joined + } + + private static let negations = ["!=": "=", "<>": "=", "NOT CONTAINS": "CONTAINS", "NOT IN": "IN"] + + static func condition(for filter: Neo4jFilter, usesElementId: Bool) throws -> String { + if filter.column == rawFilterColumn { return filter.value } + try requireQuotable([filter.column]) + let op = filter.op.uppercased() + if let presence = presence(op, column: filter.column, usesElementId: usesElementId) { return presence } + + let subject = Subject(filter: filter, usesElementId: usesElementId) + var predicate = try positive(negations[op] ?? op, filter: filter, subject: subject) + // `labels(n)` is a list, so a label filter matches when any one label does. + if filter.column == labelsColumn { predicate = "any(label IN labels(n) WHERE \(predicate))" } + return negations[op] == nil ? predicate : "NOT (\(predicate))" + } + + private struct Subject { + let expression: String + // An element id or a label is always a string, so it never compares as a number. + let isText: Bool + let foldsCase: Bool + + init(filter: Neo4jFilter, usesElementId: Bool) { + let isLabel = filter.column == labelsColumn + let isElementId = usesElementId && [idColumn, startColumn, endColumn].contains(filter.column) + expression = isLabel ? "label" : Neo4jCypher.expression(for: filter.column, usesElementId: usesElementId) + isText = isLabel || isElementId + foldsCase = !filter.isCaseSensitive && !isElementId + } + } + + private static func presence(_ op: String, column: String, usesElementId: Bool) -> String? { + let target = expression(for: column, usesElementId: usesElementId) + switch op { + case "IS NULL": return "\(target) IS NULL" + case "IS NOT NULL": return "\(target) IS NOT NULL" + case "IS EMPTY" where column == labelsColumn: return "size(labels(n)) = 0" + case "IS NOT EMPTY" where column == labelsColumn: return "size(labels(n)) > 0" + case "IS EMPTY": return "(\(target) IS NULL OR \(target) = '')" + case "IS NOT EMPTY": return "(\(target) IS NOT NULL AND \(target) <> '')" + default: return nil + } + } + + private static func positive(_ op: String, filter: Neo4jFilter, subject: Subject) throws -> String { + let target = subject.expression + switch op { + case "=": + return matchesAny(subject, [filter.value]) + case "IN": + return matchesAny(subject, filter.value.split(separator: ",").map { $0.trimmingCharacters(in: .whitespaces) }) + case ">", ">=", "<", "<=": + return "\(target) \(op) \(bound(filter.value, subject))" + case "BETWEEN": + let range = try bounds(of: filter) + return "\(target) >= \(bound(range.lower, subject)) AND \(target) <= \(bound(range.upper, subject))" + case "CONTAINS", "STARTS WITH", "ENDS WITH": + let needle = literal(subject.foldsCase ? filter.value.lowercased() : filter.value) + return "\(text(subject, folded: subject.foldsCase)) \(op) \(needle)" + case "REGEX": + let pattern = subject.foldsCase ? "(?i)" + filter.value : filter.value + return "\(text(subject, folded: false)) =~ \(literal(pattern))" + default: + throw Neo4jCypherError.unsupportedOperator(filter.op) + } + } + + // A property can hold any type, so text operators read it as text first: a number or a list is then a + // non-match rather than a type error that fails the whole page. + private static func text(_ subject: Subject, folded: Bool) -> String { + let text = subject.isText ? subject.expression : "toStringOrNull(\(subject.expression))" + return folded ? "toLower(\(text))" : text + } + + private static func bound(_ value: String, _ subject: Subject) -> String { + subject.isText ? literal(value) : typedLiteral(value) + } + + private static func matchesAny(_ subject: Subject, _ values: [String]) -> String { + let target = subject.expression + if subject.isText { + let items = values.map { literal(subject.foldsCase ? $0.lowercased() : $0) } + let lhs = subject.foldsCase ? "toLower(\(target))" : target + return items.count == 1 ? "\(lhs) = \(items[0])" : "\(lhs) IN [\(items.joined(separator: ", "))]" + } + if subject.foldsCase { + // Case folding applies to text only, so `30.0` still matches `= 30`. + let folded = "\(text(subject, folded: true)) IN [\(values.map { literal($0.lowercased()) }.joined(separator: ", "))]" + let typed = values.map(typedLiteral).filter { !$0.hasPrefix("'") } + guard !typed.isEmpty else { return folded } + return "(\(target) IN [\(typed.joined(separator: ", "))] OR \(folded))" + } + // A property has no declared type, so `123` also matches the text '123'. + let forms = values.flatMap(spellings) + if forms.count == 1 { return "\(target) = \(forms[0])" } + if values.count == 1 { return "(" + forms.map { "\(target) = \($0)" }.joined(separator: " OR ") + ")" } + return "\(target) IN [\(forms.joined(separator: ", "))]" + } + + private static func spellings(_ value: String) -> [String] { + let typed = typedLiteral(value) + return typed == literal(value) ? [typed] : [typed, literal(value)] + } + + // The host sends BETWEEN's value as "lower,upper" and the upper bound again on its own. + private static func bounds(of filter: Neo4jFilter) throws -> (lower: String, upper: String) { + let lower: String + let upper: String + if let second = filter.secondValue { + let suffix = "," + second + lower = filter.value.hasSuffix(suffix) ? String(filter.value.dropLast(suffix.count)) : filter.value + upper = second + } else if let comma = filter.value.firstIndex(of: ",") { + lower = String(filter.value[.. String { + if let integer = Int64(value), String(integer) == value { return value } + if value.range(of: #"^-?(0|[1-9][0-9]*)\.[0-9]+([eE][+-]?[0-9]+)?$"#, options: .regularExpression) != nil, + let number = Double(value), number.isFinite { + return value + } + switch value.lowercased() { + case "true", "false": return value.lowercased() + default: return literal(value) + } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jEntity.swift b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jEntity.swift new file mode 100644 index 0000000000..8d2a53ad0c --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jEntity.swift @@ -0,0 +1,64 @@ +import Foundation + +public enum Neo4jStructureTag { + public static let node: UInt8 = 0x4E + public static let relationship: UInt8 = 0x52 + public static let unboundRelationship: UInt8 = 0x72 + public static let path: UInt8 = 0x50 + public static let date: UInt8 = 0x44 + public static let time: UInt8 = 0x54 + public static let localTime: UInt8 = 0x74 + public static let dateTime: UInt8 = 0x49 + public static let dateTimeZoneId: UInt8 = 0x69 + public static let legacyDateTime: UInt8 = 0x46 + public static let legacyDateTimeZoneId: UInt8 = 0x66 + public static let localDateTime: UInt8 = 0x64 + public static let duration: UInt8 = 0x45 + public static let point2D: UInt8 = 0x58 + public static let point3D: UInt8 = 0x59 +} + +// Bolt 4 has no element id, so the numeric id stands in, written as text. +public struct Neo4jEntity: Equatable, Sendable { + public enum Kind: Equatable, Sendable { + case node(labels: [String]) + case relationship(type: String, startId: String, endId: String) + } + + public let elementId: String + public let kind: Kind + public let properties: [String: PackStreamValue] + + public init(elementId: String, kind: Kind, properties: [String: PackStreamValue]) { + self.elementId = elementId + self.kind = kind + self.properties = properties + } + + public init?(_ value: PackStreamValue) { + guard case .structure(let tag, let fields) = value else { return nil } + switch tag { + case Neo4jStructureTag.node where fields.count >= 3: + elementId = Self.identifier(fields, elementIdIndex: 3, idIndex: 0) + kind = .node(labels: fields[1].listValue?.compactMap(\.stringValue) ?? []) + properties = fields[2].mapValue ?? [:] + case Neo4jStructureTag.relationship where fields.count >= 5: + elementId = Self.identifier(fields, elementIdIndex: 5, idIndex: 0) + kind = .relationship( + type: fields[3].stringValue ?? "", + startId: Self.identifier(fields, elementIdIndex: 6, idIndex: 1), + endId: Self.identifier(fields, elementIdIndex: 7, idIndex: 2) + ) + properties = fields[4].mapValue ?? [:] + default: + return nil + } + } + + private static func identifier(_ fields: [PackStreamValue], elementIdIndex: Int, idIndex: Int) -> String { + if elementIdIndex < fields.count, let elementId = fields[elementIdIndex].stringValue { + return elementId + } + return fields[idIndex].intValue.map(String.init) ?? "" + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jResultTable.swift b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jResultTable.swift new file mode 100644 index 0000000000..cefdfeca6f --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jResultTable.swift @@ -0,0 +1,62 @@ +import Foundation + +public struct Neo4jResultTable: Equatable, Sendable { + public let columns: [String] + public let typeNames: [String] + public let rows: [[String?]] + + public init(columns: [String], typeNames: [String], rows: [[String?]]) { + self.columns = columns + self.typeNames = typeNames + self.rows = rows + } + + public static func records(keys: [String], records: [[PackStreamValue]]) -> Neo4jResultTable { + let typeNames = keys.indices.map { index -> String in + for record in records where index < record.count && record[index] != .null { + return Neo4jValueText.typeName(record[index]) + } + return "ANY" + } + let rows = records.map { record in + keys.indices.map { index in index < record.count ? Neo4jValueText.text(record[index]) : nil } + } + return Neo4jResultTable(columns: keys, typeNames: typeNames, rows: rows) + } + + // Known keys keep their order; keys first seen on this page follow, sorted. A stream's columns are fixed by its + // header, so it passes `addsUnseenProperties: false`. + public static func entities( + kind: Neo4jEntityKind, + records: [[PackStreamValue]], + knownProperties: [String], + addsUnseenProperties: Bool = true + ) -> Neo4jResultTable { + let entities = records.compactMap { $0.first.flatMap(Neo4jEntity.init) } + let known = Set(knownProperties) + let extra = addsUnseenProperties ? Set(entities.flatMap(\.properties.keys)).subtracting(known).sorted() : [] + let properties = knownProperties + extra + let fixed = Neo4jCypher.fixedColumns(for: kind) + + var typeNames = fixed.map { _ in "STRING" } + if kind == .node { typeNames[1] = "LIST" } + for key in properties { + let sample = entities.lazy.compactMap { $0.properties[key] }.first { $0 != .null } + typeNames.append(sample.map(Neo4jValueText.typeName) ?? "ANY") + } + + let rows = entities.map { entity -> [String?] in + var row: [String?] = [entity.elementId] + switch entity.kind { + case .node(let labels): + row.append(Neo4jValueText.text(.list(labels.map(PackStreamValue.string)))) + case .relationship(_, let startId, let endId): + row.append(startId) + row.append(endId) + } + row += properties.map { entity.properties[$0].flatMap(Neo4jValueText.text) } + return row + } + return Neo4jResultTable(columns: fixed + properties, typeNames: typeNames, rows: rows) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jUpdateCounts.swift b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jUpdateCounts.swift new file mode 100644 index 0000000000..619c215945 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jUpdateCounts.swift @@ -0,0 +1,41 @@ +import Foundation + +public struct Neo4jUpdateCounts: Equatable, Sendable { + public enum Counter: String, CaseIterable, Sendable { + case nodesCreated = "nodes-created" + case nodesDeleted = "nodes-deleted" + case relationshipsCreated = "relationships-created" + case relationshipsDeleted = "relationships-deleted" + case propertiesSet = "properties-set" + case labelsAdded = "labels-added" + case labelsRemoved = "labels-removed" + case indexesAdded = "indexes-added" + case indexesRemoved = "indexes-removed" + case constraintsAdded = "constraints-added" + case constraintsRemoved = "constraints-removed" + case systemUpdates = "system-updates" + } + + public let counters: [(counter: Counter, count: Int)] + + public init(summary: [String: PackStreamValue]) { + let stats = summary["stats"]?.mapValue ?? [:] + counters = Counter.allCases.compactMap { counter in + let count = Int(stats[counter.rawValue]?.intValue ?? 0) + return count > 0 ? (counter, count) : nil + } + } + + public func count(_ counter: Counter) -> Int { + counters.first { $0.counter == counter }?.count ?? 0 + } + + // A property or label counter counts values, not entities. + public var entitiesChanged: Int { + [.nodesCreated, .nodesDeleted, .relationshipsCreated, .relationshipsDeleted].reduce(0) { $0 + count($1) } + } + + public static func == (lhs: Self, rhs: Self) -> Bool { + lhs.counters.elementsEqual(rhs.counters) { $0.counter == $1.counter && $0.count == $1.count } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jValueText.swift b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jValueText.swift new file mode 100644 index 0000000000..4285727595 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jValueText.swift @@ -0,0 +1,305 @@ +import Foundation + +public enum Neo4jValueText { + public static func text(_ value: PackStreamValue) -> String? { + switch value { + case .null: + return nil + case .bool(let flag): + return flag ? "true" : "false" + case .int(let number): + return String(number) + case .float(let number): + return floatText(number) + case .string(let text): + return text + case .bytes(let bytes): + return "0x" + bytes.map { String(format: "%02x", $0) }.joined() + case .list, .map: + return json(jsonObject(value)) + case .structure(let tag, let fields): + if let scalar = structureText(tag: tag, fields: fields) { return scalar } + return json(jsonObject(value)) + } + } + + public static func typeName(_ value: PackStreamValue) -> String { + switch value { + case .null: return "NULL" + case .bool: return "BOOLEAN" + case .int: return "INTEGER" + case .float: return "FLOAT" + case .string: return "STRING" + case .bytes: return "BYTES" + case .list: return "LIST" + case .map: return "MAP" + case .structure(let tag, _): + switch tag { + case Neo4jStructureTag.node: return "NODE" + case Neo4jStructureTag.relationship, Neo4jStructureTag.unboundRelationship: return "RELATIONSHIP" + case Neo4jStructureTag.path: return "PATH" + case Neo4jStructureTag.date: return "DATE" + case Neo4jStructureTag.time: return "ZONED TIME" + case Neo4jStructureTag.localTime: return "LOCAL TIME" + case Neo4jStructureTag.dateTime, Neo4jStructureTag.dateTimeZoneId, + Neo4jStructureTag.legacyDateTime, Neo4jStructureTag.legacyDateTimeZoneId: + return "ZONED DATETIME" + case Neo4jStructureTag.localDateTime: return "LOCAL DATETIME" + case Neo4jStructureTag.duration: return "DURATION" + case Neo4jStructureTag.point2D, Neo4jStructureTag.point3D: return "POINT" + default: return "ANY" + } + } + } + + public static func jsonObject(_ value: PackStreamValue) -> Any { + switch value { + case .null: + return NSNull() + case .bool(let flag): + return flag + case .int(let number): + return number + case .float(let number): + return number.isFinite ? number as Any : floatText(number) + case .string(let text): + return text + case .bytes: + return text(value) ?? "" + case .list(let items): + return items.map(jsonObject) + case .map(let entries): + return entries.mapValues(jsonObject) + case .structure(let tag, let fields): + if let entity = Neo4jEntity(value) { + return entityObject(entity) + } + if tag == Neo4jStructureTag.unboundRelationship, fields.count >= 3 { + return [ + "elementId": fields.count > 3 ? fields[3].stringValue ?? "" : fields[0].intValue.map(String.init) ?? "", + "type": fields[1].stringValue ?? "", + "properties": (fields[2].mapValue ?? [:]).mapValues(jsonObject) + ] as [String: Any] + } + if tag == Neo4jStructureTag.path, fields.count == 3 { + return pathObject(fields) ?? [ + "nodes": (fields[0].listValue ?? []).map(jsonObject), + "relationships": (fields[1].listValue ?? []).map(jsonObject) + ] + } + return structureText(tag: tag, fields: fields) ?? NSNull() + } + } + + // Bolt sends a path's distinct nodes and relationships plus (relationship, node) index pairs; a positive + // 1-based relationship index walks forward, a negative one backward. + private static func pathObject(_ fields: [PackStreamValue]) -> [String: Any]? { + let nodes = (fields[0].listValue ?? []).compactMap(Neo4jEntity.init) + let relationships = fields[1].listValue ?? [] + let sequence = (fields[2].listValue ?? []).compactMap(\.intValue) + guard let start = nodes.first, sequence.count.isMultiple(of: 2) else { return nil } + + var orderedNodes = [entityObject(start)] + var orderedRelationships: [Any] = [] + var current = start + for step in stride(from: 0, to: sequence.count, by: 2) { + let relationshipIndex = Int(sequence[step].magnitude) - 1 + let nodeIndex = Int(sequence[step + 1]) + guard sequence[step] != 0, relationships.indices.contains(relationshipIndex), + nodes.indices.contains(nodeIndex), + case .structure(_, let relationshipFields) = relationships[relationshipIndex], + relationshipFields.count >= 3 + else { return nil } + let next = nodes[nodeIndex] + let forward = sequence[step] > 0 + orderedRelationships.append(entityObject(Neo4jEntity( + elementId: relationshipFields.count > 3 + ? relationshipFields[3].stringValue ?? "" + : relationshipFields[0].intValue.map(String.init) ?? "", + kind: .relationship( + type: relationshipFields[1].stringValue ?? "", + startId: forward ? current.elementId : next.elementId, + endId: forward ? next.elementId : current.elementId + ), + properties: relationshipFields[2].mapValue ?? [:] + ))) + orderedNodes.append(entityObject(next)) + current = next + } + return ["nodes": orderedNodes, "relationships": orderedRelationships] + } + + private static func entityObject(_ entity: Neo4jEntity) -> [String: Any] { + var object: [String: Any] = [ + "elementId": entity.elementId, + "properties": entity.properties.mapValues(jsonObject) + ] + switch entity.kind { + case .node(let labels): + object["labels"] = labels + case .relationship(let type, let startId, let endId): + object["type"] = type + object["startElementId"] = startId + object["endElementId"] = endId + } + return object + } + + private static func json(_ object: Any) -> String { + guard JSONSerialization.isValidJSONObject(object), + let data = try? JSONSerialization.data(withJSONObject: object, options: [.sortedKeys, .withoutEscapingSlashes]), + let text = String(data: data, encoding: .utf8) + else { return String(describing: object) } + return text + } + + static func floatText(_ number: Double) -> String { + if number.isNaN { return "NaN" } + if number.isInfinite { return number < 0 ? "-Infinity" : "Infinity" } + return String(number) + } + + // MARK: - Temporal and spatial + + static func structureText(tag: UInt8, fields: [PackStreamValue]) -> String? { + let ints = fields.map { $0.intValue } + switch tag { + case Neo4jStructureTag.date: + guard let days = ints.first ?? nil else { return nil } + return dateText(days: days) + case Neo4jStructureTag.localTime: + guard let nanos = ints.first ?? nil else { return nil } + return timeText(nanosOfDay: nanos) + case Neo4jStructureTag.time: + guard ints.count == 2, let nanos = ints[0], let offset = ints[1] else { return nil } + return timeText(nanosOfDay: nanos) + offsetText(offset) + case Neo4jStructureTag.localDateTime: + guard ints.count == 2, let seconds = ints[0], let nanos = ints[1] else { return nil } + return dateTimeText(localSeconds: seconds, nanos: nanos) + case Neo4jStructureTag.dateTime, Neo4jStructureTag.legacyDateTime: + guard ints.count == 3, let seconds = ints[0], let nanos = ints[1], let offset = ints[2] else { return nil } + let local = tag == Neo4jStructureTag.dateTime ? seconds + offset : seconds + return dateTimeText(localSeconds: local, nanos: nanos) + offsetText(offset) + case Neo4jStructureTag.dateTimeZoneId, Neo4jStructureTag.legacyDateTimeZoneId: + guard fields.count == 3, let seconds = ints[0], let nanos = ints[1], + let zoneId = fields[2].stringValue else { return nil } + let zone = TimeZone(identifier: zoneId) + let offset: Int64 + if tag == Neo4jStructureTag.dateTimeZoneId { + offset = Int64(zone?.secondsFromGMT(for: Date(timeIntervalSince1970: TimeInterval(seconds))) ?? 0) + } else { + let guess = Int64(zone?.secondsFromGMT(for: Date(timeIntervalSince1970: TimeInterval(seconds))) ?? 0) + offset = Int64(zone?.secondsFromGMT(for: Date(timeIntervalSince1970: TimeInterval(seconds - guess))) ?? 0) + } + let local = tag == Neo4jStructureTag.dateTimeZoneId ? seconds + offset : seconds + return dateTimeText(localSeconds: local, nanos: nanos) + offsetText(offset) + "[\(zoneId)]" + case Neo4jStructureTag.duration: + guard ints.count == 4, let months = ints[0], let days = ints[1], + let seconds = ints[2], let nanos = ints[3] else { return nil } + return durationText(months: months, days: days, seconds: seconds, nanos: nanos) + case Neo4jStructureTag.point2D, Neo4jStructureTag.point3D: + guard let srid = ints.first ?? nil else { return nil } + let names = ["x", "y", "z"] + let coordinates = fields.dropFirst().enumerated().map { index, field -> String in + let number: Double + switch field { + case .float(let value): number = value + case .int(let value): number = Double(value) + default: number = .nan + } + return "\(names[min(index, 2)]): \(floatText(number))" + } + return "point({srid: \(srid), " + coordinates.joined(separator: ", ") + "})" + default: + return nil + } + } + + static func dateText(days: Int64) -> String { + let (year, month, day) = civilDate(daysSinceEpoch: days) + let yearText: String + if year < 0 || year > 9_999 { + yearText = (year < 0 ? "-" : "+") + String(format: "%04lld", abs(year)) + } else { + yearText = String(format: "%04lld", year) + } + return yearText + String(format: "-%02lld-%02lld", month, day) + } + + static func timeText(nanosOfDay: Int64) -> String { + let seconds = nanosOfDay / 1_000_000_000 + let nanos = nanosOfDay % 1_000_000_000 + return String(format: "%02lld:%02lld:%02lld", seconds / 3_600, seconds / 60 % 60, seconds % 60) + + fractionText(nanos) + } + + static func dateTimeText(localSeconds: Int64, nanos: Int64) -> String { + let days = floorDivide(localSeconds, 86_400) + let secondsOfDay = localSeconds - days * 86_400 + return dateText(days: days) + "T" + timeText(nanosOfDay: secondsOfDay * 1_000_000_000 + nanos) + } + + static func offsetText(_ offsetSeconds: Int64) -> String { + guard offsetSeconds != 0 else { return "Z" } + let sign = offsetSeconds < 0 ? "-" : "+" + let magnitude = abs(offsetSeconds) + var text = sign + String(format: "%02lld:%02lld", magnitude / 3_600, magnitude / 60 % 60) + if magnitude % 60 != 0 { text += String(format: ":%02lld", magnitude % 60) } + return text + } + + static func durationText(months: Int64, days: Int64, seconds: Int64, nanos: Int64) -> String { + var text = "P" + if months / 12 != 0 { text += "\(months / 12)Y" } + if months % 12 != 0 { text += "\(months % 12)M" } + if days != 0 { text += "\(days)D" } + // Seconds and nanoseconds stay apart: their product overflows Int64 for durations Neo4j accepts. + let carry = floorDivide(nanos, 1_000_000_000) + let (total, overflow) = seconds.addingReportingOverflow(carry) + let signedSeconds = overflow ? seconds : total + var fraction = nanos - carry * 1_000_000_000 + if signedSeconds != 0 || fraction != 0 { + let negative = signedSeconds < 0 + var wholeSeconds = signedSeconds.magnitude + if negative, fraction != 0 { + wholeSeconds -= 1 + fraction = 1_000_000_000 - fraction + } + var time = "T" + if wholeSeconds / 3_600 != 0 { time += (negative ? "-" : "") + "\(wholeSeconds / 3_600)H" } + if wholeSeconds / 60 % 60 != 0 { time += (negative ? "-" : "") + "\(wholeSeconds / 60 % 60)M" } + let remainder = wholeSeconds % 60 + if remainder != 0 || fraction != 0 { + time += (negative ? "-" : "") + "\(remainder)" + fractionText(fraction) + "S" + } + text += time + } + return text == "P" ? "PT0S" : text + } + + private static func fractionText(_ nanos: Int64) -> String { + guard nanos != 0 else { return "" } + var digits = String(format: "%09lld", nanos) + while digits.hasSuffix("0") { digits.removeLast() } + return "." + digits + } + + private static func floorDivide(_ value: Int64, _ divisor: Int64) -> Int64 { + let quotient = value / divisor + return (value % divisor != 0 && (value < 0) != (divisor < 0)) ? quotient - 1 : quotient + } + + // Howard Hinnant's `civil_from_days`. + private static func civilDate(daysSinceEpoch: Int64) -> (Int64, Int64, Int64) { + let shifted = daysSinceEpoch + 719_468 + let era = floorDivide(shifted, 146_097) + let dayOfEra = shifted - era * 146_097 + let yearOfEra = (dayOfEra - dayOfEra / 1_460 + dayOfEra / 36_524 - dayOfEra / 146_096) / 365 + let dayOfYear = dayOfEra - (365 * yearOfEra + yearOfEra / 4 - yearOfEra / 100) + let monthIndex = (5 * dayOfYear + 2) / 153 + let day = dayOfYear - (153 * monthIndex + 2) / 5 + 1 + let month = monthIndex < 10 ? monthIndex + 3 : monthIndex - 9 + let year = yearOfEra + era * 400 + (month <= 2 ? 1 : 0) + return (year, month, day) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProNeo4jCore/PackStream.swift b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/PackStream.swift new file mode 100644 index 0000000000..c6874b6a1c --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/PackStream.swift @@ -0,0 +1,257 @@ +import Foundation + +public indirect enum PackStreamValue: Equatable, Sendable { + case null + case bool(Bool) + case int(Int64) + case float(Double) + case bytes(Data) + case string(String) + case list([PackStreamValue]) + case map([String: PackStreamValue]) + case structure(tag: UInt8, fields: [PackStreamValue]) + + public var stringValue: String? { + if case .string(let value) = self { return value } + return nil + } + + public var intValue: Int64? { + if case .int(let value) = self { return value } + return nil + } + + public var boolValue: Bool? { + if case .bool(let value) = self { return value } + return nil + } + + public var listValue: [PackStreamValue]? { + if case .list(let value) = self { return value } + return nil + } + + public var mapValue: [String: PackStreamValue]? { + if case .map(let value) = self { return value } + return nil + } +} + +public enum PackStreamError: Error, Equatable, Sendable { + case truncated + case unknownMarker(UInt8) + case invalidString + case nonStringKey + case nestedTooDeeply +} + +public enum PackStreamEncoder { + public static func encode(_ value: PackStreamValue) -> Data { + var data = Data() + write(value, into: &data) + return data + } + + public static func write(_ value: PackStreamValue, into data: inout Data) { + switch value { + case .null: + data.append(0xC0) + case .bool(let flag): + data.append(flag ? 0xC3 : 0xC2) + case .int(let number): + writeInt(number, into: &data) + case .float(let number): + data.append(0xC1) + appendBigEndian(number.bitPattern, into: &data) + case .bytes(let bytes): + writeHeader(count: bytes.count, tiny: nil, markers: (0xCC, 0xCD, 0xCE), into: &data) + data.append(bytes) + case .string(let text): + let utf8 = Data(text.utf8) + writeHeader(count: utf8.count, tiny: 0x80, markers: (0xD0, 0xD1, 0xD2), into: &data) + data.append(utf8) + case .list(let items): + writeHeader(count: items.count, tiny: 0x90, markers: (0xD4, 0xD5, 0xD6), into: &data) + for item in items { write(item, into: &data) } + case .map(let entries): + writeHeader(count: entries.count, tiny: 0xA0, markers: (0xD8, 0xD9, 0xDA), into: &data) + for key in entries.keys.sorted() { + write(.string(key), into: &data) + write(entries[key] ?? .null, into: &data) + } + case .structure(let tag, let fields): + precondition(fields.count < 16, "Bolt has no marker for a structure over 15 fields") + data.append(0xB0 | UInt8(fields.count)) + data.append(tag) + for field in fields { write(field, into: &data) } + } + } + + private static func writeInt(_ number: Int64, into data: inout Data) { + switch number { + case -16 ... 127: + data.append(UInt8(bitPattern: Int8(number))) + case Int64(Int8.min) ... Int64(Int8.max): + data.append(0xC8) + data.append(UInt8(bitPattern: Int8(number))) + case Int64(Int16.min) ... Int64(Int16.max): + data.append(0xC9) + appendBigEndian(UInt16(bitPattern: Int16(number)), into: &data) + case Int64(Int32.min) ... Int64(Int32.max): + data.append(0xCA) + appendBigEndian(UInt32(bitPattern: Int32(number)), into: &data) + default: + data.append(0xCB) + appendBigEndian(UInt64(bitPattern: number), into: &data) + } + } + + private static func writeHeader( + count: Int, + tiny: UInt8?, + markers: (UInt8, UInt8, UInt8), + into data: inout Data + ) { + if let tiny, count < 16 { + data.append(tiny | UInt8(count)) + } else if count <= Int(UInt8.max) { + data.append(markers.0) + data.append(UInt8(count)) + } else if count <= Int(UInt16.max) { + data.append(markers.1) + appendBigEndian(UInt16(count), into: &data) + } else { + data.append(markers.2) + appendBigEndian(UInt32(count), into: &data) + } + } + + private static func appendBigEndian(_ value: T, into data: inout Data) { + withUnsafeBytes(of: value.bigEndian) { data.append(contentsOf: $0) } + } +} + +public struct PackStreamDecoder { + // Each level is a stack frame, so a server cannot overflow the stack with a deeply nested list. + public static let maximumDepth = 64 + + private let bytes: [UInt8] + private var index = 0 + private var depth = 0 + + public init(_ data: Data) { + bytes = [UInt8](data) + } + + public var isAtEnd: Bool { index >= bytes.count } + + public static func decode(_ data: Data) throws -> PackStreamValue { + var decoder = PackStreamDecoder(data) + return try decoder.next() + } + + public mutating func next() throws -> PackStreamValue { + depth += 1 + defer { depth -= 1 } + guard depth <= Self.maximumDepth else { throw PackStreamError.nestedTooDeeply } + let marker = try byte() + switch marker { + case 0x00 ... 0x7F: + return .int(Int64(marker)) + case 0xF0 ... 0xFF: + return .int(Int64(Int8(bitPattern: marker))) + case 0x80 ... 0x8F: + return try string(count: Int(marker & 0x0F)) + case 0x90 ... 0x9F: + return try list(count: Int(marker & 0x0F)) + case 0xA0 ... 0xAF: + return try map(count: Int(marker & 0x0F)) + case 0xB0 ... 0xBF: + let tag = try byte() + var fields: [PackStreamValue] = [] + for _ in 0 ..< Int(marker & 0x0F) { fields.append(try next()) } + return .structure(tag: tag, fields: fields) + case 0xC0: + return .null + case 0xC1: + return .float(Double(bitPattern: try integer(UInt64.self))) + case 0xC2: + return .bool(false) + case 0xC3: + return .bool(true) + case 0xC8: + return .int(Int64(Int8(bitPattern: try byte()))) + case 0xC9: + return .int(Int64(Int16(bitPattern: try integer(UInt16.self)))) + case 0xCA: + return .int(Int64(Int32(bitPattern: try integer(UInt32.self)))) + case 0xCB: + return .int(Int64(bitPattern: try integer(UInt64.self))) + case 0xCC: + return .bytes(Data(try take(Int(try byte())))) + case 0xCD: + return .bytes(Data(try take(Int(try integer(UInt16.self))))) + case 0xCE: + return .bytes(Data(try take(Int(try integer(UInt32.self))))) + case 0xD0: + return try string(count: Int(try byte())) + case 0xD1: + return try string(count: Int(try integer(UInt16.self))) + case 0xD2: + return try string(count: Int(try integer(UInt32.self))) + case 0xD4: + return try list(count: Int(try byte())) + case 0xD5: + return try list(count: Int(try integer(UInt16.self))) + case 0xD6: + return try list(count: Int(try integer(UInt32.self))) + case 0xD8: + return try map(count: Int(try byte())) + case 0xD9: + return try map(count: Int(try integer(UInt16.self))) + case 0xDA: + return try map(count: Int(try integer(UInt32.self))) + default: + throw PackStreamError.unknownMarker(marker) + } + } + + private mutating func byte() throws -> UInt8 { + guard index < bytes.count else { throw PackStreamError.truncated } + defer { index += 1 } + return bytes[index] + } + + private mutating func take(_ count: Int) throws -> ArraySlice { + guard count >= 0, bytes.count - index >= count else { throw PackStreamError.truncated } + defer { index += count } + return bytes[index ..< index + count] + } + + private mutating func integer(_: T.Type) throws -> T { + try take(MemoryLayout.size).reduce(T.zero) { ($0 << 8) | T($1) } + } + + private mutating func string(count: Int) throws -> PackStreamValue { + guard let text = String(bytes: try take(count), encoding: .utf8) else { + throw PackStreamError.invalidString + } + return .string(text) + } + + private mutating func list(count: Int) throws -> PackStreamValue { + var items: [PackStreamValue] = [] + items.reserveCapacity(min(count, 1_024)) + for _ in 0 ..< count { items.append(try next()) } + return .list(items) + } + + private mutating func map(count: Int) throws -> PackStreamValue { + var entries: [String: PackStreamValue] = [:] + for _ in 0 ..< count { + guard case .string(let key) = try next() else { throw PackStreamError.nonStringKey } + entries[key] = try next() + } + return .map(entries) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/LibPQStringConformance.swift b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/LibPQStringConformance.swift index f22d445738..8310bc99c2 100644 --- a/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/LibPQStringConformance.swift +++ b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/LibPQStringConformance.swift @@ -22,9 +22,9 @@ public enum LibPQStringConformance { public static func escape(_ value: String, standardConformingStrings: Bool) -> String { let quoted = value - .replacingOccurrences(of: "\0", with: "") - .replacingOccurrences(of: "'", with: "''") + .replacingOccurrences(of: "\0", with: "", options: .literal) + .replacingOccurrences(of: "'", with: "''", options: .literal) guard !standardConformingStrings else { return quoted } - return quoted.replacingOccurrences(of: "\\", with: "\\\\") + return quoted.replacingOccurrences(of: "\\", with: "\\\\", options: .literal) } } diff --git a/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCatalogSearchPath.swift b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCatalogSearchPath.swift new file mode 100644 index 0000000000..cda64d28c8 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCatalogSearchPath.swift @@ -0,0 +1,28 @@ +import Foundation + +/// The `search_path` a driver-authored catalog statement runs under, set with `SET LOCAL` in the same query string so it +/// lasts only as long as the read. +/// +/// PostgreSQL resolves a function or operator over every schema on the path, and a candidate that takes the call's +/// argument types exactly beats a `pg_catalog` one reached through a cast, wherever the two sit on the path. A +/// `pg_total_relation_size(oid)` or an `<>(oid, integer)` in a browsed schema would otherwise run with the reader's +/// privileges, the substitution of CVE-2018-1058. `pg_dump` runs with the same narrowed path for the same reason. +/// `pg_temp` is listed last so a temporary relation never shadows a catalog; functions and operators are never looked +/// up there. +/// +/// `SET LOCAL` in the read's own query string makes the pair one implicit transaction, so the path is back as soon as +/// the read returns; inside a transaction the caller holds, the driver scopes it with a savepoint. `pg_catalog` rather +/// than an empty path: PostgreSQL 9.1 rejects `''` ("schema "" does not exist"), and `pg_catalog` deparses every user +/// relation with its schema all the same (measured identical output on 9.2, 9.3, 9.6 and 17). +nonisolated public enum PostgreSQLCatalogSearchPath { + public static let pinned = "SET LOCAL search_path = pg_catalog, pg_temp; " + + /// For a read whose output the server deparses: `format_type`, `pg_get_expr` and the `pg_get_*def` functions leave + /// out the schema of a name the path resolves, so this writes the schema's own names bare. The schema is on the + /// path, so a statement run under it calls only `pg_catalog`-qualified functions and compares only operands a + /// `pg_catalog` operator takes exactly. + public static func relative(to schema: String) -> String { + let quoted = "\"\(schema.replacingOccurrences(of: "\"", with: "\"\"", options: .literal))\"" + return "SET LOCAL search_path = pg_catalog, \(quoted), pg_temp; " + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCatalogTypeNames.swift b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCatalogTypeNames.swift index 218ec3889d..e5bf1f8b15 100644 --- a/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCatalogTypeNames.swift +++ b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCatalogTypeNames.swift @@ -105,8 +105,11 @@ public enum PostgreSQLCatalogTypeNames { /// because Redshift's catalog predates `typcategory` and the predicate agrees with it on /// every real type: `point` and `name` carry a `typelem` at a fixed length, and the one /// disagreement is the pseudo-type `_record`, which no column has. + /// + /// Sent on the user's own session under its search path, so each oid is typed: a bare integer compares oid with + /// integer, which a `=(oid, integer)` in a schema on that path takes exactly. public static func lookupQuery(oids: [UInt32]) -> String? { - let list = Set(oids).sorted().map(String.init).joined(separator: ", ") + let list = Set(oids).sorted().map { "'\($0)'::pg_catalog.oid" }.joined(separator: ", ") guard !list.isEmpty else { return nil } return """ SELECT t.oid::text, t.typname, t.typtype::text, diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConnectionSettings.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConnectionSettings.swift new file mode 100644 index 0000000000..97a914a012 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConnectionSettings.swift @@ -0,0 +1,59 @@ +import Foundation + +public enum QdrantFieldID { + public static let apiKey = "qdApiKey" + public static let skipTLSVerify = "qdSkipTLSVerify" +} + +public struct QdrantConnectionSettings: Sendable, Equatable { + public static let defaultPort = 6_333 + + public let host: String + public let port: Int + public let usesTLS: Bool + public let apiKey: String + public let skipTLSVerify: Bool + + public init(host: String, port: Int, usesTLS: Bool, apiKey: String, skipTLSVerify: Bool) { + self.host = host.trimmingCharacters(in: .whitespacesAndNewlines) + self.port = port + self.usesTLS = usesTLS + self.apiKey = apiKey.trimmingCharacters(in: .whitespacesAndNewlines) + self.skipTLSVerify = skipTLSVerify + } + + public static func parse( + host: String, + port: Int, + usesTLS: Bool, + fields: [String: String] + ) throws -> QdrantConnectionSettings { + let resolvedHost = host.trimmingCharacters(in: .whitespacesAndNewlines) + let settings = QdrantConnectionSettings( + host: resolvedHost.isEmpty ? "localhost" : resolvedHost, + port: port > 0 ? port : defaultPort, + usesTLS: usesTLS, + apiKey: fields[QdrantFieldID.apiKey] ?? "", + skipTLSVerify: fields[QdrantFieldID.skipTLSVerify] == "true" + ) + _ = try settings.baseURL() + return settings + } + + /// Qdrant reads the key from `api-key`, and also accepts it as a bearer token; the dedicated + /// header is the one both a self-hosted node and Qdrant Cloud document. + public var headers: [String: String] { + apiKey.isEmpty ? [:] : ["api-key": apiKey] + } + + public func baseURL() throws -> URL { + var components = URLComponents() + components.scheme = usesTLS ? "https" : "http" + components.host = host + components.port = port + guard let url = components.url else { + throw QdrantError.configuration(String(localized: "The host is not valid in a URL.")) + } + return url + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConsole.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConsole.swift new file mode 100644 index 0000000000..0ceee6f41e --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantConsole.swift @@ -0,0 +1,111 @@ +import Foundation + +public struct QdrantConsoleRequest: Sendable, Equatable { + public let method: String + public let path: String + public let body: String? + + public init(method: String, path: String, body: String?) { + self.method = method + self.path = path + self.body = body + } + + /// The POST endpoints that only read. Retrieving points by id is a POST to `/points` itself. + static let readSuffixes: [[String]] = [ + ["points"], ["points", "scroll"], ["points", "search"], ["points", "query"], ["points", "count"], + ["points", "recommend"], ["points", "discover"], ["points", "facet"], + ["points", "search", "batch"], ["points", "query", "batch"], ["points", "recommend", "batch"], + ["points", "discover", "batch"], ["points", "search", "groups"], ["points", "query", "groups"], + ["points", "recommend", "groups"], ["points", "search", "matrix", "pairs"], + ["points", "search", "matrix", "offsets"] + ] + + public var isRead: Bool { + if method == "GET" || method == "HEAD" { return true } + guard method == "POST" else { return false } + let route = path.split(separator: "?", maxSplits: 1).first.map(String.init) ?? path + let segments = route.split(separator: "/").map { $0.lowercased() } + guard segments.count >= 3, segments[0] == "collections" else { return false } + return Self.readSuffixes.contains { Array(segments.dropFirst(2)) == $0 } + } +} + +/// One REST request typed in the editor: `METHOD /path`, then an optional JSON body on the same +/// line or on the lines below it. +public enum QdrantConsoleParser { + static let methods: Set = ["GET", "POST", "PUT", "PATCH", "DELETE", "HEAD"] + + public static func parse(_ input: String) -> QdrantConsoleRequest? { + let trimmed = input.trimmingCharacters(in: .whitespacesAndNewlines) + let lines = trimmed.split(separator: "\n", maxSplits: 1, omittingEmptySubsequences: false) + guard let header = lines.first else { return nil } + let following = lines.count > 1 + ? String(lines[1]).trimmingCharacters(in: .whitespacesAndNewlines) + : "" + + let parts = header.split(maxSplits: 2, omittingEmptySubsequences: true, whereSeparator: \.isWhitespace) + guard parts.count >= 2 else { return nil } + let method = String(parts[0]).uppercased() + guard methods.contains(method) else { return nil } + let path = String(parts[1]) + guard path.hasPrefix("/"), !path.hasPrefix("//") else { return nil } + let inline = parts.count > 2 ? String(parts[2]).trimmingCharacters(in: .whitespacesAndNewlines) : "" + let body = inline.isEmpty ? following : inline + return QdrantConsoleRequest(method: method, path: path, body: body.isEmpty ? nil : body) + } +} + +public enum QdrantPathEncoding { + private static let allowed: CharacterSet = { + var set = CharacterSet.alphanumerics + set.insert(charactersIn: "-_.") + return set + }() + + public static func segment(_ value: String) -> String { + value.addingPercentEncoding(withAllowedCharacters: allowed) ?? value + } + + /// The path keeps its own query string (`?wait=true`): assigning it whole to `URLComponents.path` + /// would encode the `?` and send the request to a path that does not exist. + public static func resolve(_ path: String, against base: URL) -> URL? { + guard path.hasPrefix("/"), !path.hasPrefix("//"), !path.contains("://"), + var components = URLComponents(url: base, resolvingAgainstBaseURL: false), + let requested = URLComponents(string: path) + else { return nil } + components.percentEncodedPath = requested.percentEncodedPath + components.percentEncodedQuery = requested.percentEncodedQuery + return components.url + } +} + +public enum QdrantOperations { + public static let exportTag = "QDRANT_EXPORT:" + + /// The console request that removes one collection. Plain request text, so the confirmation shows + /// it verbatim and the query classifier tiers it destructive by its verb. + public static func deleteCollection(named name: String, objectType: String) -> String? { + guard objectType.uppercased() == "TABLE", !name.isEmpty else { return nil } + return "DELETE /collections/\(QdrantPathEncoding.segment(name))" + } + + /// An empty filter matches every point, which keeps the collection and its config. + public static func deleteAllPoints(collection: String) -> String? { + guard !collection.isEmpty else { return nil } + return "POST /collections/\(QdrantPathEncoding.segment(collection))/points/delete?wait=true\n{\"filter\":{}}" + } + + public static func encodeExport(collection: String) -> String { + exportTag + Data(collection.utf8).base64EncodedString() + } + + public static func decodeExport(_ query: String) -> String? { + guard query.hasPrefix(exportTag), + let data = Data(base64Encoded: String(query.dropFirst(exportTag.count))), + let collection = String(data: data, encoding: .utf8), + !collection.isEmpty + else { return nil } + return collection + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantError.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantError.swift new file mode 100644 index 0000000000..e97312d379 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantError.swift @@ -0,0 +1,54 @@ +import Foundation + +public enum QdrantError: Error, LocalizedError, Equatable, Sendable { + case configuration(String) + case notConnected + case transport(String) + case authentication(String) + case api(status: Int, message: String) + case malformedResponse(String) + case cancelled + + public var errorDescription: String? { + switch self { + case .configuration(let detail), .transport(let detail), .malformedResponse(let detail), + .authentication(let detail): + return detail + case .notConnected: + return String(localized: "Not connected to Qdrant.") + case .api(_, let message): + return message + case .cancelled: + return String(localized: "The request was cancelled.") + } + } + + public static func from(status: Int, body: Data) -> QdrantError { + let message = apiMessage(from: body) + ?? String(format: String(localized: "Qdrant returned HTTP %d."), status) + if status == 401 || status == 403 { + return .authentication(message) + } + return .api(status: status, message: message) + } + + /// Qdrant puts the reason in `status.error`; a proxy in front of it may answer with plain text. + public static func apiMessage(from body: Data) -> String? { + guard let json = try? JSONSerialization.jsonObject(with: body) else { + let text = String(data: body, encoding: .utf8)?.trimmingCharacters(in: .whitespacesAndNewlines) + return (text?.isEmpty ?? true) ? nil : text + } + guard let object = json as? [String: Any] else { return nil } + if let status = object["status"] as? [String: Any], + let error = status["error"] as? String, !error.isEmpty { + return error + } + if let error = object["error"] as? String, !error.isEmpty { + return error + } + if let message = object["message"] as? String, !message.isEmpty { + return message + } + return nil + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift new file mode 100644 index 0000000000..7fa0dc57d8 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantFilterBuilder.swift @@ -0,0 +1,246 @@ +import Foundation + +public struct QdrantFilterSpec: Sendable, Equatable { + public let column: String + public let op: String + public let value: String + public let secondValue: String? + + public init(column: String, op: String, value: String, secondValue: String? = nil) { + self.column = column + self.op = op + self.value = value + self.secondValue = secondValue + } +} + +public enum QdrantFilterError: Error, LocalizedError, Equatable { + case unsupportedOperator(String) + case missingUpperBound(column: String) + case notANumber(column: String, value: String) + case notABoolean(column: String, value: String) + case notAPointID(value: String) + case emptyList(column: String) + case comparisonNeedsNumberOrDate(column: String, op: String) + case textMatchNeedsTextIndex(column: String, op: String) + case vectorNotFilterable(column: String) + case rawFilterNotAnObject + + public var errorDescription: String? { + switch self { + case .unsupportedOperator(let op): + return String(format: String(localized: "Qdrant cannot filter with %@."), op) + case .missingUpperBound(let column): + return String(format: String(localized: "BETWEEN on %@ needs an upper bound."), column) + case .notANumber(let column, let value): + return String(format: String(localized: "%@ holds numbers, and %@ is not a number."), column, value) + case .notABoolean(let column, let value): + return String( + format: String(localized: "%@ holds booleans, so it matches only true or false, not %@."), + column, value + ) + case .notAPointID(let value): + return String( + format: String(localized: "%@ is not a point id. Qdrant ids are unsigned integers or UUIDs."), + value + ) + case .emptyList(let column): + return String(format: String(localized: "IN on %@ needs at least one value."), column) + case .comparisonNeedsNumberOrDate(let column, let op): + return String( + format: String(localized: "Qdrant compares only numbers and dates with %@, and %@ holds neither."), + op, column + ) + case .textMatchNeedsTextIndex(let column, let op): + return String( + format: String(localized: "%@ on %@ needs a full-text payload index on that field. Without one Qdrant matches nothing."), + op, column + ) + case .vectorNotFilterable(let column): + return String(format: String(localized: "Qdrant cannot filter on %@."), column) + case .rawFilterNotAnObject: + return String(localized: "A raw filter for Qdrant is a JSON condition, such as {\"key\": \"city\", \"match\": {\"value\": \"Berlin\"}}.") + } + } +} + +/// Turns the filter bar's rows into Qdrant's filter JSON. AND rows go under `must` and OR rows under +/// `should`; a negated row is a nested filter with `must_not`, which Qdrant accepts as a condition. +public enum QdrantFilterBuilder { + public static let rawFilterColumn = "__RAW__" + + public static func filter( + _ filters: [QdrantFilterSpec], + logicMode: String, + collection: QdrantCollection + ) throws -> [String: Any]? { + guard !filters.isEmpty else { return nil } + let conditions = try filters.map { try condition(for: $0, collection: collection) } + let clause = logicMode.uppercased() == "OR" ? "should" : "must" + return [clause: conditions] + } + + public static func condition(for filter: QdrantFilterSpec, collection: QdrantCollection) throws -> [String: Any] { + if filter.column == rawFilterColumn { + return try raw(filter.value) + } + guard !QdrantSchema.isVectorColumn(filter.column) else { + throw QdrantFilterError.vectorNotFilterable(column: filter.column) + } + let op = filter.op.uppercased() + if filter.column == QdrantSchema.idColumn { + return try idCondition(op: op, filter: filter) + } + let type = collection.fieldType(of: filter.column) ?? .json + let column = QdrantSchema.payloadKey(forColumn: filter.column) + + switch op { + case "=": + return try equality(column: column, value: filter.value, type: type) + case "!=", "<>": + return negated(try equality(column: column, value: filter.value, type: type)) + case ">", ">=", "<", "<=": + return try range(column: column, type: type, op: op, bounds: [rangeKey(op): filter.value]) + case "BETWEEN": + guard let upper = filter.secondValue, !upper.trimmingCharacters(in: .whitespaces).isEmpty else { + throw QdrantFilterError.missingUpperBound(column: column) + } + let lower = lowerBound(of: filter.value, upperBound: upper) + return try range(column: column, type: type, op: op, bounds: ["gte": lower, "lte": upper]) + case "IN": + return ["key": column, "match": ["any": try list(filter.value, column: column, type: type)]] + case "NOT IN": + return ["key": column, "match": ["except": try list(filter.value, column: column, type: type)]] + case "CONTAINS": + return try textMatch(column: column, value: filter.value, type: type, op: op) + case "NOT CONTAINS": + return negated(try textMatch(column: column, value: filter.value, type: type, op: op)) + case "IS NULL": + return ["is_null": ["key": column]] + case "IS NOT NULL": + return negated(["is_null": ["key": column]]) + case "IS EMPTY": + return ["is_empty": ["key": column]] + case "IS NOT EMPTY": + return negated(["is_empty": ["key": column]]) + default: + throw QdrantFilterError.unsupportedOperator(op) + } + } + + private static func raw(_ text: String) throws -> [String: Any] { + guard let data = text.data(using: .utf8), + let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any] + else { throw QdrantFilterError.rawFilterNotAnObject } + return object + } + + private static func idCondition(op: String, filter: QdrantFilterSpec) throws -> [String: Any] { + func ids(_ text: String) throws -> [Any] { + let parts = text.split(separator: ",").map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty } + guard !parts.isEmpty else { throw QdrantFilterError.emptyList(column: QdrantSchema.idColumn) } + return try parts.map { part in + guard let id = QdrantPointID.json(from: part) else { throw QdrantFilterError.notAPointID(value: part) } + return id + } + } + switch op { + case "=", "IN": + return ["has_id": try ids(filter.value)] + case "!=", "<>", "NOT IN": + return negated(["has_id": try ids(filter.value)]) + default: + throw QdrantFilterError.unsupportedOperator(op) + } + } + + /// `match.value` takes a string, an integer or a boolean. A float has no exact match in Qdrant, + /// so equality on one is the closed range around it. + private static func equality(column: String, value: String, type: QdrantFieldType) throws -> [String: Any] { + switch type { + case .float, .datetime: + return try range(column: column, type: type, op: "=", bounds: ["gte": value, "lte": value]) + default: + return ["key": column, "match": ["value": try scalar(value, column: column, type: type)]] + } + } + + private static func range( + column: String, + type: QdrantFieldType, + op: String, + bounds: [String: String] + ) throws -> [String: Any] { + guard type.isOrdered || type == .json else { + throw QdrantFilterError.comparisonNeedsNumberOrDate(column: column, op: op) + } + var range: [String: Any] = [:] + for (key, text) in bounds { + let trimmed = text.trimmingCharacters(in: .whitespaces) + if type == .datetime { + range[key] = trimmed + } else if let number = Double(trimmed) { + range[key] = number + } else if type == .json { + range[key] = trimmed + } else { + throw QdrantFilterError.notANumber(column: column, value: text) + } + } + return ["key": column, "range": range] + } + + private static func textMatch(column: String, value: String, type: QdrantFieldType, op: String) throws -> [String: Any] { + guard type == .text else { + throw QdrantFilterError.textMatchNeedsTextIndex(column: column, op: op) + } + return ["key": column, "match": ["text": value]] + } + + private static func list(_ value: String, column: String, type: QdrantFieldType) throws -> [Any] { + let parts = value.split(separator: ",").map { $0.trimmingCharacters(in: .whitespaces) }.filter { !$0.isEmpty } + guard !parts.isEmpty else { throw QdrantFilterError.emptyList(column: column) } + return try parts.map { try scalar($0, column: column, type: type) } + } + + private static func scalar(_ value: String, column: String, type: QdrantFieldType) throws -> Any { + let trimmed = value.trimmingCharacters(in: .whitespaces) + switch type { + case .integer: + guard let number = Int64(trimmed) else { throw QdrantFilterError.notANumber(column: column, value: value) } + return NSNumber(value: number) + case .bool: + switch trimmed.lowercased() { + case "true", "1": return true + case "false", "0": return false + default: throw QdrantFilterError.notABoolean(column: column, value: value) + } + case .json: + if let number = Int64(trimmed) { return NSNumber(value: number) } + if trimmed == "true" { return true } + if trimmed == "false" { return false } + return value + default: + return value + } + } + + private static func rangeKey(_ op: String) -> String { + switch op { + case ">": return "gt" + case ">=": return "gte" + case "<": return "lt" + default: return "lte" + } + } + + private static func lowerBound(of joinedValue: String, upperBound: String) -> String { + let suffix = "," + upperBound + guard joinedValue.hasSuffix(suffix) else { return joinedValue } + return String(joinedValue.dropLast(suffix.count)) + } + + private static func negated(_ condition: [String: Any]) -> [String: Any] { + ["must_not": [condition]] + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantJSON.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantJSON.swift new file mode 100644 index 0000000000..bee609b383 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantJSON.swift @@ -0,0 +1,71 @@ +import Foundation + +public enum QdrantJSON { + public static func dictionary(_ value: Any?) -> [String: Any]? { + value as? [String: Any] + } + + public static func data(_ object: Any, pretty: Bool = false) throws -> Data { + guard JSONSerialization.isValidJSONObject(object) else { + throw QdrantError.malformedResponse(String(localized: "Request body is not valid JSON.")) + } + var options: JSONSerialization.WritingOptions = [.sortedKeys, .withoutEscapingSlashes] + if pretty { + options.insert(.prettyPrinted) + } + return try JSONSerialization.data(withJSONObject: object, options: options) + } + + public static func text(_ object: Any, pretty: Bool = false) throws -> String { + String(data: try data(object, pretty: pretty), encoding: .utf8) ?? "{}" + } + + public static func isBoolean(_ number: NSNumber) -> Bool { + CFGetTypeID(number) == CFBooleanGetTypeID() + } + + public static func displayText(_ value: Any?) -> String? { + switch value { + case nil, is NSNull: + return nil + case let text as String: + return text + case let number as NSNumber: + if isBoolean(number) { + return number.boolValue ? "true" : "false" + } + return number.stringValue + case let object as [String: Any]: + return try? text(object) + case let array as [Any]: + return try? text(array) + default: + return String(describing: value as Any) + } + } + + /// A payload is schemaless, so a cell is written back as the type its column holds. Parsing every + /// cell as JSON would turn a string field holding `42` into a number and break a `match` on it. + public static func parsedValue(_ text: String, fieldType: QdrantFieldType) -> Any { + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + switch fieldType { + case .keyword, .text, .string, .uuid, .datetime: + return text + case .integer: + return Int64(trimmed) ?? Double(trimmed).map { $0 as Any } ?? text + case .float: + return Double(trimmed) ?? text + case .bool: + switch trimmed.lowercased() { + case "true": return true + case "false": return false + default: return text + } + case .geo, .object, .array, .json: + guard let data = trimmed.data(using: .utf8), + let parsed = try? JSONSerialization.jsonObject(with: data, options: [.fragmentsAllowed]) + else { return text } + return parsed + } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantPoint.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantPoint.swift new file mode 100644 index 0000000000..c3846c1466 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantPoint.swift @@ -0,0 +1,118 @@ +import Foundation + +/// A point id is an unsigned integer or a UUID, and Qdrant refuses one in the other's form, so the +/// grid's text is turned back into the kind it came from. +public enum QdrantPointID { + public static func json(from text: String) -> Any? { + let trimmed = text.trimmingCharacters(in: .whitespacesAndNewlines) + guard !trimmed.isEmpty else { return nil } + if trimmed.allSatisfy(\.isASCII), trimmed.allSatisfy(\.isNumber), let number = UInt64(trimmed) { + return NSNumber(value: number) + } + guard UUID(uuidString: trimmed) != nil else { return nil } + return trimmed.lowercased() + } + + public static func text(_ value: Any?) -> String? { + QdrantJSON.displayText(value) + } +} + +public struct QdrantPoint: @unchecked Sendable { + public let id: Any? + public let payload: [String: Any] + /// `nil` when the request left vectors out; an array for an unnamed vector; a map by name otherwise. + public let vector: Any? + public let score: Double? + + public init(id: Any?, payload: [String: Any], vector: Any?, score: Double?) { + self.id = id + self.payload = payload + self.vector = vector + self.score = score + } + + public var idText: String? { QdrantPointID.text(id) } + + public var payloadKeys: [String] { payload.keys.sorted() } + + public static func parse(_ json: Any) -> QdrantPoint? { + guard let object = json as? [String: Any], object.keys.contains("id") else { return nil } + let vector = object["vector"] ?? object["vectors"] + return QdrantPoint( + id: object["id"], + payload: object["payload"] as? [String: Any] ?? [:], + vector: vector is NSNull ? nil : vector, + score: (object["score"] as? NSNumber)?.doubleValue + ) + } + + /// The point list of any read endpoint: `scroll` and `query` wrap it in `points`, retrieve and the + /// older `search` answer the array itself. `nil` when the result is not a list of points. + public static func list(fromResponse json: Any?) -> [QdrantPoint]? { + guard let result = (json as? [String: Any])?["result"] else { return nil } + let items: [Any] + if let wrapped = (result as? [String: Any])?["points"] as? [Any] { + items = wrapped + } else if let array = result as? [Any] { + items = array + } else { + return nil + } + let points = items.compactMap(parse) + return points.count == items.count ? points : nil + } + + public func vectorValue(for column: String) -> Any? { + guard let vector else { return nil } + if column == QdrantSchema.vectorColumn { + if vector is [Any] { return vector } + return (vector as? [String: Any])?[""] + } + let name = String(column.dropFirst(QdrantSchema.vectorColumn.count + 1)) + return (vector as? [String: Any])?[name] + } + + public func cell(for column: String) -> String? { + if column == QdrantSchema.idColumn { + return idText + } + if QdrantSchema.isVectorColumn(column) { + return QdrantJSON.displayText(vectorValue(for: column)) + } + if column == QdrantSchema.scoreColumn { + return score.map { String($0) } + } + return QdrantJSON.displayText(payload[QdrantSchema.payloadKey(forColumn: column)]) + } + + public func row(columns: [String]) -> [String?] { + columns.map(cell(for:)) + } + + /// The columns a console result shows: what came back, in the order the collection lists it. + public static func columns(of points: [QdrantPoint], collection: QdrantCollection?) -> [String] { + var columns = [QdrantSchema.idColumn] + if points.contains(where: { $0.score != nil }) { + columns.append(QdrantSchema.scoreColumn) + } + let returned = Set(points.flatMap(\.payloadKeys).map(QdrantSchema.column(forPayloadKey:))) + let declared = collection?.payloadFields.map { QdrantSchema.column(forPayloadKey: $0.name) } ?? [] + columns += declared.filter { returned.contains($0) && !columns.contains($0) } + columns += returned.subtracting(declared).subtracting(columns).sorted() + + var vectorColumns: [String] = [] + for point in points { + if point.vector is [Any] { + vectorColumns.append(QdrantSchema.vectorColumn) + } else if let named = point.vector as? [String: Any] { + vectorColumns += named.keys.map { "\(QdrantSchema.vectorColumn).\($0)" } + } + } + var seen = Set(columns) + for column in vectorColumns.sorted() where seen.insert(column).inserted { + columns.append(column) + } + return columns + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantQuery.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantQuery.swift new file mode 100644 index 0000000000..ad7f1d9c09 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantQuery.swift @@ -0,0 +1,294 @@ +import Foundation + +public struct QdrantSortSpec: Sendable, Equatable { + public let column: String + public let ascending: Bool + + public init(column: String, ascending: Bool) { + self.column = column + self.ascending = ascending + } +} + +public struct QdrantBrowseRequest: Sendable, Equatable { + public let collection: String + public let offset: Int + public let limit: Int + public let sort: QdrantSortSpec? + public let filters: [QdrantFilterSpec] + public let logicMode: String + + public init( + collection: String, + offset: Int, + limit: Int, + sort: QdrantSortSpec?, + filters: [QdrantFilterSpec], + logicMode: String + ) { + self.collection = collection + self.offset = max(offset, 0) + self.limit = max(limit, 1) + self.sort = sort + self.filters = filters + self.logicMode = logicMode + } +} + +/// A grid page travels from `buildFilteredQuery` to `execute` as tagged text, because the page is +/// built before the driver knows the collection's field types the filter JSON depends on. +public enum QdrantBrowseQuery { + public static let tag = "QDRANT_BROWSE:" + + public static func encode(_ request: QdrantBrowseRequest) -> String { + var payload: [String: Any] = [ + "collection": request.collection, + "offset": request.offset, + "limit": request.limit, + "logicMode": request.logicMode, + "filters": request.filters.map { filter -> [String: Any] in + var encoded: [String: Any] = ["column": filter.column, "op": filter.op, "value": filter.value] + if let secondValue = filter.secondValue { + encoded["secondValue"] = secondValue + } + return encoded + } + ] + if let sort = request.sort { + payload["sort"] = ["column": sort.column, "ascending": sort.ascending] + } + let body = (try? QdrantJSON.data(payload)) ?? Data() + return tag + body.base64EncodedString() + } + + public static func parse(_ query: String) -> QdrantBrowseRequest? { + guard query.hasPrefix(tag), + let data = Data(base64Encoded: String(query.dropFirst(tag.count))), + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let collection = json["collection"] as? String + else { return nil } + let sort = (json["sort"] as? [String: Any]).flatMap { item -> QdrantSortSpec? in + guard let column = item["column"] as? String else { return nil } + return QdrantSortSpec(column: column, ascending: item["ascending"] as? Bool ?? true) + } + let filters = (json["filters"] as? [[String: Any]] ?? []).compactMap { item -> QdrantFilterSpec? in + guard let column = item["column"] as? String, let op = item["op"] as? String else { return nil } + return QdrantFilterSpec( + column: column, + op: op, + value: item["value"] as? String ?? "", + secondValue: item["secondValue"] as? String + ) + } + return QdrantBrowseRequest( + collection: collection, + offset: json["offset"] as? Int ?? 0, + limit: json["limit"] as? Int ?? 100, + sort: sort, + filters: filters, + logicMode: json["logicMode"] as? String ?? "AND" + ) + } + + public static func isTagged(_ query: String) -> Bool { + query.hasPrefix(tag) + } + + /// The body of one page. `order_by` is only on the query endpoint, which takes a numeric offset; + /// an unsorted page goes through `scroll`, which pages by point id instead. + public static func queryBody( + for request: QdrantBrowseRequest, + filter: [String: Any]?, + sort: QdrantSortSpec + ) -> [String: Any] { + var body: [String: Any] = [ + "query": ["order_by": [ + "key": QdrantSchema.payloadKey(forColumn: sort.column), + "direction": sort.ascending ? "asc" : "desc" + ]], + "limit": request.limit, + "offset": request.offset, + "with_payload": true, + "with_vector": true + ] + if let filter { + body["filter"] = filter + } + return body + } + + public static func scrollBody( + filter: [String: Any]?, + limit: Int, + cursor: Any?, + withData: Bool, + withVector: Bool? = nil + ) -> [String: Any] { + var body: [String: Any] = [ + "limit": limit, + "with_payload": withData, + "with_vector": withVector ?? withData + ] + if let filter { + body["filter"] = filter + } + if let cursor { + body["offset"] = cursor + } + return body + } +} + +public struct QdrantWriteRequest: Sendable, Equatable { + public let method: String + public let path: String + public let body: String? + + public init(method: String, path: String, body: String?) { + self.method = method + self.path = path + self.body = body + } +} + +/// A grid save is shown in the review sheet and runs as the request line the console would send, +/// so what the user approves, what the read-only gate classifies and what runs are one string. +public enum QdrantWriteCodec { + public static func encode(_ request: QdrantWriteRequest) -> String { + guard let body = request.body else { return "\(request.method) \(request.path)" } + return "\(request.method) \(request.path)\n\(body)" + } +} + +public struct QdrantCellChange: Sendable, Equatable { + public let column: String + public let newText: String? + + public init(column: String, newText: String?) { + self.column = column + self.newText = newText + } +} + +public struct QdrantTrackedChange: Sendable, Equatable { + public enum Kind: String, Sendable, Equatable { + case insert + case update + case delete + } + + public let kind: Kind + public let id: String? + public let values: [String: String?] + public let cellChanges: [QdrantCellChange] + + public init(kind: Kind, id: String?, values: [String: String?] = [:], cellChanges: [QdrantCellChange] = []) { + self.kind = kind + self.id = id + self.values = values + self.cellChanges = cellChanges + } +} + +public enum QdrantSkipReason: String, Sendable, Equatable { + case missingID + case invalidID + case noEditableColumns + case payloadNotEncodable +} + +public struct QdrantSkippedChange: Sendable, Equatable { + public let kind: QdrantTrackedChange.Kind + public let reason: QdrantSkipReason +} + +public struct QdrantWriteBatch: Sendable, Equatable { + public let requests: [QdrantWriteRequest] + public let skipped: [QdrantSkippedChange] +} + +public enum QdrantStatementGenerator { + /// `uuid` supplies the id of an inserted row the user left blank. + public static func generate( + collection: QdrantCollection, + changes: [QdrantTrackedChange], + uuid: () -> String = { UUID().uuidString.lowercased() } + ) -> QdrantWriteBatch { + let base = "/collections/\(QdrantPathEncoding.segment(collection.name))/points" + var requests: [QdrantWriteRequest] = [] + var skipped: [QdrantSkippedChange] = [] + + for change in changes { + switch outcome(for: change, collection: collection, base: base, uuid: uuid) { + case .success(let request): + requests.append(request) + case .failure(let reason): + skipped.append(QdrantSkippedChange(kind: change.kind, reason: reason)) + } + } + return QdrantWriteBatch(requests: requests, skipped: skipped) + } + + private enum Outcome { + case success(QdrantWriteRequest) + case failure(QdrantSkipReason) + } + + private static func outcome( + for change: QdrantTrackedChange, + collection: QdrantCollection, + base: String, + uuid: () -> String + ) -> Outcome { + let immutable = Set(QdrantSchema.immutableColumns(for: collection)) + switch change.kind { + case .insert: + let idText = change.id.flatMap { $0.isEmpty ? nil : $0 } ?? uuid() + guard let id = QdrantPointID.json(from: idText) else { return .failure(.invalidID) } + let payload = payloadObject( + change.values.filter { !immutable.contains($0.key) }, + collection: collection + ) + // `insert_only` keeps an id that already exists from being overwritten, vector and all. + let body: [String: Any] = [ + "points": [["id": id, "vector": [String: Any](), "payload": payload]], + "update_mode": "insert_only" + ] + return request("PUT", "\(base)?wait=true", body) + case .update: + guard let idText = change.id, !idText.isEmpty else { return .failure(.missingID) } + guard let id = QdrantPointID.json(from: idText) else { return .failure(.invalidID) } + var patch: [String: String?] = [:] + for cell in change.cellChanges where !immutable.contains(cell.column) { + patch[cell.column] = cell.newText + } + guard !patch.isEmpty else { return .failure(.noEditableColumns) } + let body: [String: Any] = ["payload": payloadObject(patch, collection: collection), "points": [id]] + return request("POST", "\(base)/payload?wait=true", body) + case .delete: + guard let idText = change.id, !idText.isEmpty else { return .failure(.missingID) } + guard let id = QdrantPointID.json(from: idText) else { return .failure(.invalidID) } + return request("POST", "\(base)/delete?wait=true", ["points": [id]]) + } + } + + private static func request(_ method: String, _ path: String, _ body: [String: Any]) -> Outcome { + guard let text = try? QdrantJSON.text(body) else { return .failure(.payloadNotEncodable) } + return .success(QdrantWriteRequest(method: method, path: path, body: text)) + } + + private static func payloadObject(_ values: [String: String?], collection: QdrantCollection) -> [String: Any] { + var payload: [String: Any] = [:] + for (column, text) in values { + guard let text else { + payload[QdrantSchema.payloadKey(forColumn: column)] = NSNull() + continue + } + payload[QdrantSchema.payloadKey(forColumn: column)] = QdrantJSON.parsedValue( + text, + fieldType: collection.fieldType(of: column) ?? .string + ) + } + return payload + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift new file mode 100644 index 0000000000..e9d92af337 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/QdrantSchema.swift @@ -0,0 +1,239 @@ +import Foundation + +/// The type a payload column holds. The first seven are Qdrant's payload index schemas; the rest +/// are read off sampled values for a field nobody indexed. +public enum QdrantFieldType: String, Sendable, Equatable, CaseIterable { + case keyword + case integer + case float + case bool + case geo + case text + case datetime + case uuid + case string + case object + case array + case json + + public var isOrdered: Bool { + self == .integer || self == .float || self == .datetime + } + + public static func indexed(_ schema: Any?) -> QdrantFieldType? { + let name: String? + if let object = schema as? [String: Any] { + name = object["data_type"] as? String + ?? (object["params"] as? [String: Any])?["type"] as? String + } else { + name = schema as? String + } + return name.flatMap { QdrantFieldType(rawValue: $0.lowercased()) } + } + + public static func observed(_ value: Any) -> QdrantFieldType? { + switch value { + case is NSNull: + return nil + case is String: + return .string + case let number as NSNumber: + if QdrantJSON.isBoolean(number) { return .bool } + return CFNumberIsFloatType(number) ? .float : .integer + case is [String: Any]: + return .object + case is [Any]: + return .array + default: + return .json + } + } + + /// An integer column that also holds a fraction reads as a float; anything else that disagrees is + /// mixed JSON, written back as whatever the cell parses to. + func merged(with other: QdrantFieldType) -> QdrantFieldType { + if self == other { return self } + if Set([self, other]) == [.integer, .float] { return .float } + return .json + } +} + +public struct QdrantVectorSpec: Sendable, Equatable { + /// `nil` for the collection's single unnamed vector. + public let name: String? + public let isSparse: Bool + + public init(name: String?, isSparse: Bool = false) { + self.name = name + self.isSparse = isSparse + } + + public var column: String { + name.map { "\(QdrantSchema.vectorColumn).\($0)" } ?? QdrantSchema.vectorColumn + } +} + +public struct QdrantPayloadField: Sendable, Equatable { + public let name: String + public let type: QdrantFieldType + public let isIndexed: Bool + /// False while every value seen was null, so the first real value sets the type outright. + public let hasTypedValue: Bool + + public init(name: String, type: QdrantFieldType, isIndexed: Bool, hasTypedValue: Bool = true) { + self.name = name + self.type = type + self.isIndexed = isIndexed + self.hasTypedValue = hasTypedValue + } +} + +public struct QdrantCollection: Sendable, Equatable { + public let name: String + public let vectors: [QdrantVectorSpec] + public let payloadFields: [QdrantPayloadField] + public let pointsCount: Int? + public let info: String + + public init( + name: String, + vectors: [QdrantVectorSpec], + payloadFields: [QdrantPayloadField], + pointsCount: Int? = nil, + info: String = "{}" + ) { + self.name = name + self.vectors = vectors + self.payloadFields = payloadFields + self.pointsCount = pointsCount + self.info = info + } + + public func fieldType(of column: String) -> QdrantFieldType? { + let key = QdrantSchema.payloadKey(forColumn: column) + return payloadFields.first { $0.name == key }?.type + } + + public var vectorColumns: [String] { vectors.map(\.column) } + + /// The collection with every payload key `points` hold that it lacks, typed by their values. An + /// indexed field keeps the index's type. + public func including(_ points: [QdrantPoint]) -> QdrantCollection { + var fields = payloadFields + var position = Dictionary(uniqueKeysWithValues: fields.enumerated().map { ($1.name, $0) }) + for point in points { + for key in point.payloadKeys { + let observed = point.payload[key].flatMap(QdrantFieldType.observed) + guard let index = position[key] else { + position[key] = fields.count + fields.append(QdrantPayloadField( + name: key, type: observed ?? .json, isIndexed: false, hasTypedValue: observed != nil + )) + continue + } + let field = fields[index] + guard !field.isIndexed, let observed else { continue } + let type = field.hasTypedValue ? field.type.merged(with: observed) : observed + fields[index] = QdrantPayloadField(name: key, type: type, isIndexed: false) + } + } + return QdrantCollection(name: name, vectors: vectors, payloadFields: fields, pointsCount: pointsCount, info: info) + } + + /// `GET /collections/{name}` answers the config and the indexed payload fields; the fields nobody + /// indexed are only visible in the points themselves, so a sample of them fills the rest in. + public static func parse(name: String, info json: Any?, sample: [QdrantPoint]) -> QdrantCollection { + let result = (json as? [String: Any])?["result"] as? [String: Any] ?? [:] + let params = ((result["config"] as? [String: Any])?["params"] as? [String: Any]) ?? [:] + + var vectors: [QdrantVectorSpec] = [] + if let named = params["vectors"] as? [String: Any] { + if named["size"] != nil { + vectors.append(QdrantVectorSpec(name: nil)) + } else { + vectors += named.keys.sorted().map { QdrantVectorSpec(name: $0) } + } + } + if let sparse = params["sparse_vectors"] as? [String: Any] { + vectors += sparse.keys.sorted().map { QdrantVectorSpec(name: $0, isSparse: true) } + } + + let indexed = result["payload_schema"] as? [String: Any] ?? [:] + let fields = indexed.keys.sorted().compactMap { key in + QdrantFieldType.indexed(indexed[key]).map { QdrantPayloadField(name: key, type: $0, isIndexed: true) } + } + + return QdrantCollection( + name: name, + vectors: vectors, + payloadFields: fields, + pointsCount: result["points_count"] as? Int, + info: (try? QdrantJSON.text(result, pretty: true)) ?? "{}" + ).including(sample) + } +} + +public struct QdrantColumn: Sendable, Equatable { + public let name: String + public let type: String + public let isPrimaryKey: Bool +} + +public enum QdrantSchema { + public static let idColumn = "id" + public static let vectorColumn = "vector" + public static let scoreColumn = "score" + + public static func collectionNames(from json: Any?) -> [String] { + let result = (json as? [String: Any])?["result"] as? [String: Any] + let collections = result?["collections"] as? [[String: Any]] ?? [] + return collections.compactMap { $0["name"] as? String }.sorted() + } + + public static func isVectorColumn(_ column: String) -> Bool { + column == vectorColumn || column.hasPrefix("\(vectorColumn).") + } + + static let payloadPrefix = "payload." + + /// A payload key may be `id`, `score` or `vector`, which name the point's own columns, so such a + /// key shows as `payload.` and maps back to its key wherever a column is read or written. + public static func isReserved(_ key: String) -> Bool { + key == idColumn || key == scoreColumn || isVectorColumn(key) + } + + public static func column(forPayloadKey key: String) -> String { + isReserved(key) ? payloadPrefix + key : key + } + + public static func payloadKey(forColumn column: String) -> String { + guard column.hasPrefix(payloadPrefix) else { return column } + let key = String(column.dropFirst(payloadPrefix.count)) + return isReserved(key) ? key : column + } + + public static func immutableColumns(for collection: QdrantCollection) -> [String] { + [idColumn] + collection.vectorColumns + } + + public static func columns(for collection: QdrantCollection) -> [QdrantColumn] { + var columns = [QdrantColumn(name: idColumn, type: "id", isPrimaryKey: true)] + columns += collection.payloadFields.map { + QdrantColumn(name: column(forPayloadKey: $0.name), type: $0.type.rawValue, isPrimaryKey: false) + } + columns += collection.vectors.map { + QdrantColumn(name: $0.column, type: $0.isSparse ? "sparse_vector" : "vector", isPrimaryKey: false) + } + return columns + } + + public static func typeName(of column: String, in collection: QdrantCollection) -> String { + if column == idColumn { return "id" } + if column == scoreColumn { return "float" } + if let spec = collection.vectors.first(where: { $0.column == column }) { + return spec.isSparse ? "sparse_vector" : "vector" + } + if isVectorColumn(column) { return "vector" } + return collection.fieldType(of: column)?.rawValue ?? QdrantFieldType.json.rawValue + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantClient.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantClient.swift new file mode 100644 index 0000000000..75a2c7ed4f --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantClient.swift @@ -0,0 +1,167 @@ +import Foundation + +public struct QdrantScrollPage: @unchecked Sendable { + public let points: [QdrantPoint] + /// The id the next page starts from, `nil` once the collection is exhausted. + public let nextCursor: Any? +} + +public enum QdrantCursorLookup: @unchecked Sendable { + /// Scroll from this id; `nil` is the start of the collection. + case at(Any?) + /// The collection ends before the requested offset. + case pastEnd +} + +public final class QdrantClient: @unchecked Sendable { + public static let sampleSize = 100 + /// How many ids one request skips while walking to a page no cursor is held for. + public static let skipBatchSize = 1_000 + + public let settings: QdrantConnectionSettings + private let transport: QdrantTransport + private let timeout: @Sendable () -> TimeInterval + private let lock = NSLock() + private var version: String? + + public init( + settings: QdrantConnectionSettings, + transport: QdrantTransport, + timeout: @escaping @Sendable () -> TimeInterval + ) { + self.settings = settings + self.transport = transport + self.timeout = timeout + } + + public var serverVersion: String? { + lock.withLock { version } + } + + public func cancelAll() { + transport.cancelAll() + } + + public func connect(requestTimeout: @Sendable () -> TimeInterval) async throws { + let root = try await send(method: "GET", path: "/", timeoutInterval: requestTimeout()) + try throwIfFailed(root) + if let json = QdrantJSON.dictionary(root.json), let found = json["version"] as? String { + lock.withLock { version = found } + } + try throwIfFailed(try await send(method: "GET", path: "/collections", timeoutInterval: requestTimeout())) + } + + /// `/healthz` answers without the key, so a revoked key would leave the session looking live. + public func ping() async throws { + try throwIfFailed(try await send(method: "GET", path: "/collections")) + } + + public func collectionNames() async throws -> [String] { + let response = try await send(method: "GET", path: "/collections") + try throwIfFailed(response) + return QdrantSchema.collectionNames(from: response.json) + } + + public func collection(named name: String) async throws -> QdrantCollection { + let info = try await send(method: "GET", path: collectionPath(name)) + try throwIfFailed(info) + let sample = try await scroll( + collection: name, + body: ["limit": Self.sampleSize, "with_payload": true, "with_vector": false] + ) + return QdrantCollection.parse(name: name, info: info.json, sample: sample.points) + } + + public func scroll(collection: String, body: [String: Any]) async throws -> QdrantScrollPage { + let response = try await post("\(collectionPath(collection))/points/scroll", body) + let result = (response.json as? [String: Any])?["result"] as? [String: Any] + let points = (result?["points"] as? [Any] ?? []).compactMap(QdrantPoint.parse) + let next = result?["next_page_offset"] + return QdrantScrollPage(points: points, nextCursor: next is NSNull ? nil : next) + } + + public func query(collection: String, body: [String: Any]) async throws -> [QdrantPoint] { + let response = try await post("\(collectionPath(collection))/points/query", body) + return QdrantPoint.list(fromResponse: response.json) ?? [] + } + + public func count(collection: String, filter: [String: Any]?) async throws -> Int { + var body: [String: Any] = ["exact": true] + if let filter { + body["filter"] = filter + } + let response = try await post("\(collectionPath(collection))/points/count", body) + let result = (response.json as? [String: Any])?["result"] as? [String: Any] + guard let count = result?["count"] as? Int else { + throw QdrantError.malformedResponse(String(localized: "Qdrant answered a count without a number.")) + } + return count + } + + /// Walks to the cursor of `offset` from the nearest one already known, reading ids only. + public func cursor( + collection: String, + filter: [String: Any]?, + offset: Int, + from known: (offset: Int, cursor: Any?) + ) async throws -> QdrantCursorLookup { + var position = known.offset + var cursor = known.cursor + while position < offset { + try Task.checkCancellation() + let step = min(offset - position, Self.skipBatchSize) + let page = try await scroll( + collection: collection, + body: QdrantBrowseQuery.scrollBody(filter: filter, limit: step, cursor: cursor, withData: false) + ) + position += page.points.count + guard let next = page.nextCursor else { return .pastEnd } + cursor = next + } + return .at(cursor) + } + + public func execute(_ request: QdrantConsoleRequest) async throws -> QdrantHTTPResponse { + let response = try await send(method: request.method, path: request.path, body: request.body.map { Data($0.utf8) }) + try throwIfFailed(response) + return response + } + + public func send(method: String, path: String, body: Data? = nil) async throws -> QdrantHTTPResponse { + try await send(method: method, path: path, body: body, timeoutInterval: timeout()) + } + + private func post(_ path: String, _ body: [String: Any]) async throws -> QdrantHTTPResponse { + let response = try await send(method: "POST", path: path, body: try QdrantJSON.data(body)) + try throwIfFailed(response) + return response + } + + private func collectionPath(_ name: String) -> String { + "/collections/\(QdrantPathEncoding.segment(name))" + } + + private func send(method: String, path: String, body: Data? = nil, timeoutInterval: TimeInterval) async throws -> QdrantHTTPResponse { + guard let url = QdrantPathEncoding.resolve(path, against: try settings.baseURL()) else { + throw QdrantError.configuration(String(format: String(localized: "Invalid path: %@"), path)) + } + var headers = settings.headers + headers["Accept"] = "application/json" + if body != nil { + headers["Content-Type"] = "application/json" + } + return try await transport.send(QdrantHTTPRequest( + method: method, + url: url, + headers: headers, + body: body, + timeoutInterval: timeoutInterval + )) + } + + public func throwIfFailed(_ response: QdrantHTTPResponse) throws { + guard (200..<300).contains(response.statusCode) else { + throw QdrantError.from(status: response.statusCode, body: response.body) + } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift new file mode 100644 index 0000000000..e7e67dbbd0 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantScrollPager.swift @@ -0,0 +1,111 @@ +import Foundation + +/// The grid asks for a page by row offset and `scroll` answers with the id the next page starts +/// from. This remembers that id for every page already read, so paging forward costs one request +/// and a jump costs one id-only walk from the nearest page before it. +public final class QdrantCursorCache: @unchecked Sendable { + private static let maxQueries = 32 + + private let lock = NSLock() + private var cursors: [String: [Int: Any]] = [:] + private var order: [String] = [] + + public init() {} + + public func nearest(at offset: Int, key: String) -> (offset: Int, cursor: Any?) { + lock.withLock { + guard let known = cursors[key], + let best = known.keys.filter({ $0 <= offset }).max() + else { return (0, nil) } + return (best, known[best]) + } + } + + public func store(_ cursor: Any, at offset: Int, key: String) { + lock.withLock { + if cursors[key] == nil { + order.append(key) + if order.count > Self.maxQueries { + cursors[order.removeFirst()] = nil + } + } + cursors[key, default: [:]][offset] = cursor + } + } + + public func removeAll() { + lock.withLock { + cursors.removeAll() + order.removeAll() + } + } +} + +public extension QdrantClient { + func scrollPage( + collection: String, + filter: [String: Any]?, + offset: Int, + limit: Int, + cache: QdrantCursorCache + ) async throws -> [QdrantPoint] { + let filterKey = filter.flatMap { try? QdrantJSON.text($0) } ?? "" + let key = "\(collection)\u{0}\(filterKey)" + let start: Any? + switch try await cursor( + collection: collection, + filter: filter, + offset: offset, + from: cache.nearest(at: offset, key: key) + ) { + case .pastEnd: + return [] + case .at(let cursor): + start = cursor + if offset > 0, let cursor { + cache.store(cursor, at: offset, key: key) + } + } + let page = try await scroll( + collection: collection, + body: QdrantBrowseQuery.scrollBody(filter: filter, limit: limit, cursor: start, withData: true) + ) + if let next = page.nextCursor { + cache.store(next, at: offset + page.points.count, key: key) + } + return page.points + } + + /// Every point of a collection a page at a time, for export. Without vectors it reads payloads only, + /// which is how an export learns every payload key before it writes its header. + func pointPages( + collection: String, + pageSize: Int, + withVector: Bool = true + ) -> AsyncThrowingStream<[QdrantPoint], Error> { + AsyncThrowingStream { continuation in + let task = Task { + do { + var cursor: Any? + repeat { + try Task.checkCancellation() + let page = try await self.scroll( + collection: collection, + body: QdrantBrowseQuery.scrollBody( + filter: nil, limit: max(pageSize, 1), cursor: cursor, withData: true, withVector: withVector + ) + ) + if !page.points.isEmpty { + continuation.yield(page.points) + } + cursor = page.nextCursor + } while cursor != nil + continuation.finish() + } catch { + continuation.finish(throwing: error) + } + } + continuation.onTermination = { @Sendable _ in task.cancel() } + } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantTransport.swift b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantTransport.swift new file mode 100644 index 0000000000..b596180e49 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProQdrantCore/Wire/QdrantTransport.swift @@ -0,0 +1,149 @@ +import Foundation + +public struct QdrantHTTPRequest: Sendable, Equatable { + public let method: String + public let url: URL + public let headers: [String: String] + public let body: Data? + public let timeoutInterval: TimeInterval + + public init(method: String, url: URL, headers: [String: String], body: Data?, timeoutInterval: TimeInterval) { + self.method = method + self.url = url + self.headers = headers + self.body = body + self.timeoutInterval = timeoutInterval + } +} + +public struct QdrantHTTPResponse: Sendable, Equatable { + public let statusCode: Int + public let body: Data + + public init(statusCode: Int, body: Data) { + self.statusCode = statusCode + self.body = body + } + + public var json: Any? { + try? JSONSerialization.jsonObject(with: body, options: [.fragmentsAllowed]) + } + + public var text: String { + String(data: body, encoding: .utf8) ?? "" + } +} + +public protocol QdrantTransport: Sendable { + func send(_ request: QdrantHTTPRequest) async throws -> QdrantHTTPResponse + func cancelAll() +} + +public final class URLSessionQdrantTransport: QdrantTransport, @unchecked Sendable { + private let session: URLSession + private let lock = NSLock() + private var inFlight: [ObjectIdentifier: URLSessionTask] = [:] + + public init( + configuration: URLSessionConfiguration = .ephemeral, + resourceTimeout: TimeInterval, + skipTLSVerify: Bool = false + ) { + configuration.timeoutIntervalForResource = resourceTimeout + session = URLSession( + configuration: configuration, + delegate: skipTLSVerify ? TrustingTLSDelegate() : nil, + delegateQueue: nil + ) + } + + deinit { + session.invalidateAndCancel() + } + + public func cancelAll() { + let tasks = lock.withLock { Array(inFlight.values) } + tasks.forEach { $0.cancel() } + } + + public func send(_ request: QdrantHTTPRequest) async throws -> QdrantHTTPResponse { + var urlRequest = URLRequest(url: request.url) + urlRequest.httpMethod = request.method + urlRequest.httpBody = request.body + urlRequest.timeoutInterval = request.timeoutInterval + for (name, value) in request.headers { + urlRequest.setValue(value, forHTTPHeaderField: name) + } + + let tracker = TaskTracker(transport: self) + defer { tracker.finish() } + do { + let (data, response) = try await session.data(for: urlRequest, delegate: tracker) + guard let http = response as? HTTPURLResponse else { + throw QdrantError.transport(String(localized: "Qdrant answered with something other than HTTP.")) + } + return QdrantHTTPResponse(statusCode: http.statusCode, body: data) + } catch let error as URLError where error.code == .cancelled { + throw QdrantError.cancelled + } catch let error as QdrantError { + throw error + } catch let error as URLError { + throw QdrantError.transport(error.localizedDescription) + } + } + + fileprivate func register(_ task: URLSessionTask) { + lock.withLock { inFlight[ObjectIdentifier(task)] = task } + } + + fileprivate func unregister(_ task: URLSessionTask) { + lock.withLock { inFlight[ObjectIdentifier(task)] = nil } + } +} + +private final class TrustingTLSDelegate: NSObject, URLSessionDelegate { + func urlSession( + _ session: URLSession, + didReceive challenge: URLAuthenticationChallenge, + completionHandler: @escaping (URLSession.AuthChallengeDisposition, URLCredential?) -> Void + ) { + guard challenge.protectionSpace.authenticationMethod == NSURLAuthenticationMethodServerTrust, + let trust = challenge.protectionSpace.serverTrust + else { + completionHandler(.performDefaultHandling, nil) + return + } + completionHandler(.useCredential, URLCredential(trust: trust)) + } +} + +private final class TaskTracker: NSObject, URLSessionTaskDelegate, @unchecked Sendable { + private weak var transport: URLSessionQdrantTransport? + private let lock = NSLock() + private var task: URLSessionTask? + + init(transport: URLSessionQdrantTransport) { + self.transport = transport + } + + func urlSession(_ session: URLSession, didCreateTask task: URLSessionTask) { + lock.withLock { self.task = task } + transport?.register(task) + } + + // A redirect would carry the API key to wherever it points. + func urlSession( + _ session: URLSession, + task: URLSessionTask, + willPerformHTTPRedirection response: HTTPURLResponse, + newRequest request: URLRequest, + completionHandler: @escaping (URLRequest?) -> Void + ) { + completionHandler(nil) + } + + func finish() { + guard let task = lock.withLock({ task }) else { return } + transport?.unregister(task) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProQuery/FilterSQLGenerator.swift b/apple/Packages/TableProCore/Sources/TableProQuery/FilterSQLGenerator.swift index 93db218345..824aaf2522 100644 --- a/apple/Packages/TableProCore/Sources/TableProQuery/FilterSQLGenerator.swift +++ b/apple/Packages/TableProCore/Sources/TableProQuery/FilterSQLGenerator.swift @@ -87,13 +87,13 @@ public struct FilterSQLGenerator: Sendable { let escapedSecond = escapeValue(filter.secondValue, kind: kind) return "\(quotedColumn) BETWEEN \(escapedValue) AND \(escapedSecond)" case .contains: - let pattern = escapeLikePattern(filter.value) + let pattern = likePatternBody(filter.value) return likeCondition(operand, "'%\(pattern)%'", negated: false, folding: folding) case .startsWith: - let pattern = escapeLikePattern(filter.value) + let pattern = likePatternBody(filter.value) return likeCondition(operand, "'\(pattern)%'", negated: false, folding: folding) case .endsWith: - let pattern = escapeLikePattern(filter.value) + let pattern = likePatternBody(filter.value) return likeCondition(operand, "'%\(pattern)'", negated: false, folding: folding) } } @@ -117,7 +117,7 @@ public struct FilterSQLGenerator: Sendable { private func comparisonCondition( _ column: String, _ operand: String, _ operatorText: String, _ literal: String, folding: PluginSQLCaseFolding ) -> String { - guard folding.foldsComparisonOperands, literal.hasPrefix("'") else { + guard folding.foldsComparisonOperands, literal.unicodeScalars.first == "'" else { return "\(column) \(operatorText) \(literal)" } return "\(folding.fold(operand)) \(operatorText) \(folding.fold(literal))" @@ -159,7 +159,7 @@ public struct FilterSQLGenerator: Sendable { _ column: String, _ pattern: String, negated: Bool, folding: PluginSQLCaseFolding ) -> String { let keyword = negated ? folding.notLikeKeyword : folding.likeKeyword - return "\(folding.foldingLikeOperand(column)) \(keyword) \(folding.foldingLikeOperand(pattern))\(likeEscape)" + return "\(folding.foldingLikeOperand(column)) \(keyword) \(folding.foldingLikeOperand(pattern))\(likeEscapeClause)" } private func listCondition( @@ -172,13 +172,13 @@ public struct FilterSQLGenerator: Sendable { ) -> String { let keyword = negated ? "NOT IN" : "IN" let items = listItems(rawValue).map { escapeValue($0, kind: kind) } - let foldable = folding.foldsComparisonOperands && items.allSatisfy { $0.hasPrefix("'") } + let foldable = folding.foldsComparisonOperands && items.allSatisfy { $0.unicodeScalars.first == "'" } let rendered = foldable ? items.map { folding.fold($0) } : items let subject = foldable ? folding.fold(operand) : column return "\(subject) \(keyword) (\(rendered.joined(separator: ", ")))" } - private var likeEscape: String { + public var likeEscapeClause: String { switch dialect.likeEscapeStyle { case .explicit: return " ESCAPE '!'" @@ -187,11 +187,22 @@ public struct FilterSQLGenerator: Sendable { } } + // The LIKE layer first, with the escape `likeEscapeClause` names, then the literal rule over the whole pattern. + public func likePatternBody(_ value: String) -> String { + let escape = dialect.likeEscapeStyle == .implicit ? "\\" : "!" + // `[` opens a character class in a SQL Server LIKE pattern. + let metacharacters = [escape, "%", "_"] + (databaseType == .mssql ? ["["] : []) + let pattern = metacharacters.reduce(value) { pattern, metacharacter in + pattern.replacingOccurrences(of: metacharacter, with: escape + metacharacter, options: .literal) + } + return literalBody(pattern) + } + private func quoteIdentifier(_ name: String) -> String { if let identifierQuoter { return identifierQuoter(name) } let open = dialect.identifierQuote let close = open == "[" ? "]" : open - let escaped = name.replacingOccurrences(of: close, with: "\(close)\(close)") + let escaped = name.replacingOccurrences(of: close, with: "\(close)\(close)", options: .literal) return "\(open)\(escaped)\(close)" } @@ -199,32 +210,15 @@ public struct FilterSQLGenerator: Sendable { if FilterColumnKind.isBareNumber(value, kind: kind) { return value } - var escaped = value.replacingOccurrences(of: "\0", with: "") - if dialect.requiresBackslashEscaping { - escaped = escaped.replacingOccurrences(of: "\\", with: "\\\\") - } - escaped = escaped.replacingOccurrences(of: "'", with: "''") - return "'\(escaped)'" + return "'\(literalBody(value))'" } - private func escapeLikePattern(_ value: String) -> String { - var result = value - .replacingOccurrences(of: "'", with: "''") - .replacingOccurrences(of: "\0", with: "") - switch dialect.likeEscapeStyle { - case .explicit: - result = result - .replacingOccurrences(of: "!", with: "!!") - .replacingOccurrences(of: "%", with: "!%") - .replacingOccurrences(of: "_", with: "!_") - case .implicit: - if dialect.requiresBackslashEscaping { - result = result.replacingOccurrences(of: "\\", with: "\\\\") - } - result = result - .replacingOccurrences(of: "%", with: "\\%") - .replacingOccurrences(of: "_", with: "\\_") + // `.literal` compares code units: the default compares grapheme clusters and skips a quote a combining mark follows. + private func literalBody(_ value: String) -> String { + var body = value.replacingOccurrences(of: "\0", with: "", options: .literal) + if dialect.requiresBackslashEscaping { + body = body.replacingOccurrences(of: "\\", with: "\\\\", options: .literal) } - return result + return body.replacingOccurrences(of: "'", with: "''", options: .literal) } } diff --git a/apple/Packages/TableProCore/Sources/TableProQuery/SQLWriteClassifier.swift b/apple/Packages/TableProCore/Sources/TableProQuery/SQLWriteClassifier.swift index cc2d9b5894..ea62cfb7df 100644 --- a/apple/Packages/TableProCore/Sources/TableProQuery/SQLWriteClassifier.swift +++ b/apple/Packages/TableProCore/Sources/TableProQuery/SQLWriteClassifier.swift @@ -9,7 +9,9 @@ import TableProSQLGrammar /// The rule is fail-closed: a statement counts as a read only when its leading keyword is one of a /// short, closed set of read verbs. Everything else writes, including a keyword this classifier has /// never heard of. A write-keyword allowlist cannot be safe, because anything it has not been -/// taught, or anything hidden behind a leading comment, runs unguarded. +/// taught, or anything hidden behind a leading comment, runs unguarded. A query that passes the verb +/// rule is then read for what it calls, because `SELECT setval('seq', 1)` and +/// `SELECT pg_terminate_backend(pid)` open with a read verb and change state all the same. /// /// The batch is split into statements the way its engine lexes it, by the grammar the Mac reads /// too, and under every reading the engine could be using: the run splits by one reading, and a @@ -18,21 +20,37 @@ import TableProSQLGrammar /// engine that needs no `;`, every statement that begins inside one is read as well, because SQL /// Server runs `SELECT 1` followed by `DROP TABLE t` as two statements. public enum SQLWriteClassifier { - /// `EXPLAIN` and `PRAGMA` are deliberately absent: `EXPLAIN ANALYZE DELETE …` runs the delete on - /// PostgreSQL, and `PRAGMA journal_mode = WAL` writes on SQLite and DuckDB. - private static let readKeywords: Set = ["SHOW", "DESCRIBE", "DESC"] + // EXPLAIN and PRAGMA are absent: `EXPLAIN ANALYZE DELETE …` runs the delete on PostgreSQL, and + // `PRAGMA journal_mode = WAL` writes on SQLite and DuckDB. + private static let readKeywords: Set = ["SHOW"] - /// A `SELECT` reads unless it materialises a table, which `SELECT … INTO` does on SQL Server and - /// `SELECT … INTO OUTFILE` does on MySQL. Mirrors QueryClassifier.swift:288-291. + // MySQL and TiDB read `DESC ANALYZE DELETE …` as EXPLAIN ANALYZE, so only a DESCRIBE of a name reads. + private static let describeKeywords: Set = ["DESCRIBE", "DESC"] + + // `SELECT … INTO` materialises a table on SQL Server and a file on MySQL. private static let readUnlessIntoKeywords: Set = ["SELECT", "TABLE", "VALUES"] - /// Statement verbs only. `REPLACE`, `INTO` and `COPY` are left out because they collide with - /// ordinary function and column names, and a CTE that merely calls `replace()` is a read. - /// Matches QueryClassifier.swift:304-312. + // `REPLACE` and `COPY` are left out because they are also function and column names. private static let writeKeywordsInsideCTE: [String] = [ "INSERT", "UPDATE", "DELETE", "MERGE", "UPSERT", "DROP", "TRUNCATE", "ALTER", "CREATE", "INTO" ] + /// `NEXT VALUE FOR seq` (SQL Server, MariaDB, Db2) and `NEXTVAL FOR seq` (Db2) advance a sequence with + /// no call for the function scanner to see. Matches QueryClassifier.sequenceAdvanceRegex. + private static let sequenceAdvanceRegex = try? NSRegularExpression( + pattern: "\\bNEXT(?:\\s+VALUE|VAL)\\s+FOR\\b", + options: [] + ) + + /// `seq.NEXTVAL` does the same on the engines that have the pseudo-column. Elsewhere `t.nextval` is a column. + private static let sequencePseudoColumnRegex = try? NSRegularExpression( + pattern: "\\.\\s*NEXTVAL\\b", + options: [] + ) + + /// QueryClassifier.enginesWithSequencePseudoColumns without SAP HANA, which only the Mac connects to. + private static let enginesWithSequencePseudoColumns: Set = [.oracle, .snowflake, .dameng, .mariadb] + public static func isWriteQuery(_ sql: String, databaseType: DatabaseType) -> Bool { if databaseType == .redis { return RedisCommandSplitter.commands(in: sql).contains { redisWrites($0.text) } @@ -50,7 +68,7 @@ public enum SQLWriteClassifier { SQLStatementScanner.executableStatements(in: lexing.text, grammar: lexing.grammar).contains { statement in SQLUnterminatedStatements.runnable(in: statement.sql, grammar: lexing.grammar).contains { runnable in seen.insert(SQLLexing(grammar: lexing.grammar, text: runnable)).inserted - && statementWrites(runnable, grammar: lexing.grammar) + && statementWrites(runnable, grammar: lexing.grammar, databaseType: databaseType) } } } @@ -58,17 +76,58 @@ public enum SQLWriteClassifier { /// Reads the statement's code with every literal and comment blanked by its engine's own rules. A MySQL /// `/*! ... */` is kept as the code it is, so a keyword the server runs from one is never mistaken for a read. - private static func statementWrites(_ statement: String, grammar: SQLLexicalGrammar) -> Bool { + private static func statementWrites( + _ statement: String, + grammar: SQLLexicalGrammar, + databaseType: DatabaseType, + depth: Int = 0 + ) -> Bool { + // Each nested CTE copies and reads what is left of the text, so nesting past this counts as a write. + guard depth <= 32 else { return true } let code = SQLCodeProjection.code(of: statement, grammar: grammar, revealingExecutableComments: true) - let body = String(code.drop { $0.isWhitespace }) + let body = String(code.drop { $0.isWhitespace || $0 == "(" }) guard let keyword = leadingKeyword(of: body) else { return true } - if keyword == "WITH" { return commonTableExpressionWrites(code) } + if keyword == "WITH" { + return commonTableExpressionWrites( + statement, code: code, grammar: grammar, databaseType: databaseType, depth: depth + ) + || queryChangesState(statement, code: code, grammar: grammar, databaseType: databaseType) + } + if describeKeywords.contains(keyword) { + return SQLExplainStatement.read(statement, grammar: grammar)?.reading != .describesObject + } if readUnlessIntoKeywords.contains(keyword) { return containsWord("INTO", in: code.uppercased()) + || queryChangesState(statement, code: code, grammar: grammar, databaseType: databaseType) } return !readKeywords.contains(keyword) } + /// A query's verb says nothing about what it calls: a function in the select list can move a sequence, + /// end a session or drop a large object. A callee the scanner cannot name counts as one that changes + /// state. Mirrors QueryClassifier.stateChangingCallTier. + private static func queryChangesState( + _ statement: String, + code: String, + grammar: SQLLexicalGrammar, + databaseType: DatabaseType + ) -> Bool { + if advancesSequence(code.uppercased(), databaseType: databaseType) { return true } + let callsChangeState = SQLFunctionCallScanner.calls(in: statement as NSString, code: code as NSString) + .contains { call in + guard case .named(let name, _) = call.callee else { return true } + return SQLStateChangingFunctions.contains(name) + } + return callsChangeState || SQLStateChangingFunctions.namesStateChangingTable(in: statement, grammar: grammar) + } + + private static func advancesSequence(_ body: String, databaseType: DatabaseType) -> Bool { + let range = NSRange(body.startIndex..., in: body) + if sequenceAdvanceRegex?.firstMatch(in: body, options: [], range: range) != nil { return true } + guard enginesWithSequencePseudoColumns.contains(databaseType) else { return false } + return sequencePseudoColumnRegex?.firstMatch(in: body, options: [], range: range) != nil + } + // Read as the driver reads it, so a quoted verb is the command that runs. Unparseable text counts as a write. private static func redisWrites(_ command: String) -> Bool { guard let arguments = RedisArgumentCodec.split(command) else { return true } @@ -76,12 +135,23 @@ public enum SQLWriteClassifier { return !RedisReadCommands.onlyReads(words[...]) } - /// A CTE's leading keyword says nothing about what the statement finally does, so its code is - /// searched for a write verb. - private static func commonTableExpressionWrites(_ code: String) -> Bool { + // After the CTE list only a query reads, and each body is read as a statement of its own. + private static func commonTableExpressionWrites( + _ statement: String, + code: String, + grammar: SQLLexicalGrammar, + databaseType: DatabaseType, + depth: Int + ) -> Bool { let masked = code.uppercased() - return writeKeywordsInsideCTE.contains { keyword in - containsWord(keyword, in: masked) + if writeKeywordsInsideCTE.contains(where: { containsWord($0, in: masked) }) { return true } + guard let list = SQLCommonTableExpressions.read(statement, grammar: grammar) else { return true } + let text = statement as NSString + let main = text.substring(from: list.mainStatement) + let mainCode = SQLCodeProjection.code(of: main, grammar: grammar) + guard readUnlessIntoKeywords.contains(SQLLeadingComments.leadingCodeKeyword(mainCode)) else { return true } + return ([main] + list.bodies.map(text.substring(with:))).contains { part in + statementWrites(part, grammar: grammar, databaseType: databaseType, depth: depth + 1) } } diff --git a/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLIntrospectionSQL.swift b/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLIntrospectionSQL.swift index b8cb4bb289..cd67a7309f 100644 --- a/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLIntrospectionSQL.swift +++ b/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLIntrospectionSQL.swift @@ -33,7 +33,7 @@ public enum R2SQLIntrospectionSQL { } public static func quoteIdentifier(_ identifier: String) -> String { - "\"" + identifier.replacingOccurrences(of: "\"", with: "\"\"") + "\"" + "\"" + identifier.replacingOccurrences(of: "\"", with: "\"\"", options: .literal) + "\"" } static let namespaceColumns = ["namespace", "namespace_name", "database_name", "schema_name", "databaseName"] diff --git a/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLResponseDecoder.swift b/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLResponseDecoder.swift index a981a58ce2..0edd59042e 100644 --- a/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLResponseDecoder.swift +++ b/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLResponseDecoder.swift @@ -9,7 +9,8 @@ public enum R2SQLResponseDecoder { throw R2SQLError.malformedResponse(status: response.statusCode, detail: snippet(response.body)) } - guard envelope.success else { + // A refused redirect arrives as its 3xx, whatever the body says. + guard envelope.success, (200 ..< 300).contains(response.statusCode) else { switch response.statusCode { case 401, 403: throw R2SQLError.authentication(status: response.statusCode, errors: envelope.errors) diff --git a/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLURLSessionTransport.swift b/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLURLSessionTransport.swift index 8924638df3..2292e43911 100644 --- a/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLURLSessionTransport.swift +++ b/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLURLSessionTransport.swift @@ -75,6 +75,17 @@ private final class TaskTracker: NSObject, URLSessionTaskDelegate, @unchecked Se transport?.register(task) } + // A redirect would carry the API token to wherever it points. + func urlSession( + _ session: URLSession, + task: URLSessionTask, + willPerformHTTPRedirection response: HTTPURLResponse, + newRequest request: URLRequest, + completionHandler: @escaping (URLRequest?) -> Void + ) { + completionHandler(nil) + } + func finish() { guard let task = lock.withLock({ task }) else { return } transport?.unregister(task) diff --git a/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisQueryBuilder.swift b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisQueryBuilder.swift index b613314ba8..d2eb9468ba 100644 --- a/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisQueryBuilder.swift +++ b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisQueryBuilder.swift @@ -119,15 +119,15 @@ public struct RedisQueryBuilder: Sendable { } private func quoteForCommand(_ str: String) -> String { - var result = "" - for char in str { - switch char { - case "\\": result += "\\\\" - case "\"": result += "\\\"" - default: result.append(char) + var result = String.UnicodeScalarView() + for scalar in str.unicodeScalars { + switch scalar { + case "\\", "\"": result.append("\\") + default: break } + result.append(scalar) } - return result + return String(result) } /// Try to extract a SCAN-compatible glob pattern from key-column filters @@ -158,18 +158,14 @@ public struct RedisQueryBuilder: Sendable { /// Escape Redis glob special characters in user input. /// Redis SCAN MATCH uses glob-style patterns where *, ?, and [ are special. private func escapeGlobChars(_ str: String) -> String { - var result = "" - for char in str { - switch char { - case "*", "?", "[", "]": - result.append("\\") - result.append(char) - case "\\": - result.append("\\\\") - default: - result.append(char) + var result = String.UnicodeScalarView() + for scalar in str.unicodeScalars { + switch scalar { + case "*", "?", "[", "]", "\\": result.append("\\") + default: break } + result.append(scalar) } - return result + return String(result) } } diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/CQLBatchTracker.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/CQLBatchTracker.swift index 743f8cd2cb..fa0640b9c1 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/CQLBatchTracker.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/CQLBatchTracker.swift @@ -34,6 +34,10 @@ public struct CQLBatchTracker: SQLStatementBoundaryTracking { true } + public var holdsOpenBlock: Bool { + phase == .insideBatch || phase == .afterCloser + } + public mutating func observeWord(_ word: String) { switch phase { case .start: diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/JavaScript/MongoStatementClassifier.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/JavaScript/MongoStatementClassifier.swift index 0272332c25..2abb610a77 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/JavaScript/MongoStatementClassifier.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/JavaScript/MongoStatementClassifier.swift @@ -58,11 +58,15 @@ public enum MongoStatementClassifier { // Server-side JavaScript, plus the two stages that write to a collection the statement names. private static let codeExecutionKeys: Set = ["$where", "$function", "$accumulator", "$out", "$merge"] + // The server accepts its own operators from any client, and rewrites `mapReduce` into two of them that run a + // function: `$_internalJsEmit` and `$_internalJsReduce`. Nothing a person writes names one. + private static let internalOperatorPrefix = "$_internal" + private static let codeExecutionCalls: Set = ["mapreduce", "eval"] // For text the grammar rejects, where a false alarm only stops an external client sending code nobody could read. private static let codeExecutionMarkers: [String] = [ - "$where", "$function", "$accumulator", "mapreduce", ".eval(", "$out", "$merge" + "$where", "$function", "$accumulator", "$_internal", "mapreduce", ".eval(", "$out", "$merge" ] private static func statementVerdict( @@ -78,6 +82,7 @@ public enum MongoStatementClassifier { private static func verdict(of statement: MongoStatement) -> MongoStatementVerdict { let reachesCode = !statement.objectKeys.isDisjoint(with: codeExecutionKeys) + || statement.objectKeys.contains { $0.hasPrefix(internalOperatorPrefix) } || statement.calls.contains { call in codeExecutionCalls.contains(call.name.lowercased()) || call.commandName.map { codeExecutionCalls.contains($0.lowercased()) } == true diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/PLSQLUnitTracker.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/PLSQLUnitTracker.swift index f0c5161f62..74732eb67d 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/PLSQLUnitTracker.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/PLSQLUnitTracker.swift @@ -16,8 +16,11 @@ import Foundation /// Inside a unit the tracker counts the constructs that one bare `END` closes: a block, a subprogram or package body, /// a compound trigger and a `CASE`. A `BEGIN` continues the innermost construct when that construct is still in its /// declaration section, because `DECLARE ... BEGIN ... END` and `PROCEDURE p IS ... BEGIN ... END` each end once. -/// `END IF` and `END LOOP` close constructs that never opened one, and a word after `.` is a member name, never a -/// keyword. A `;` ends the unit only once every construct has closed. +/// `END IF` and `END LOOP` close constructs that never opened one, except between a loop's `FOR` or `WHILE` and its +/// `LOOP`: no statement ends there, so `CASE ... END LOOP` is a `CASE` expression and the `LOOP` that opens the loop. +/// A word after the `.` of `v.END` is a member name, never a keyword. The `..` of a range and the `.` that ends a +/// number name no member, so the word after `1..` or `2.` is a keyword. A `;` ends the unit only once every construct +/// has closed. /// /// The final `;` of a unit belongs to it, except after a trigger whose body is a `CALL`: measured on Oracle 23ai, /// `CREATE TRIGGER ... CALL p(:NEW.a);` is stored INVALID and the same text without the `;` compiles. @@ -50,6 +53,7 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { private static let lessThan = UInt16(UnicodeScalar("<").value) private static let greaterThan = UInt16(UnicodeScalar(">").value) private static let period = UInt16(UnicodeScalar(".").value) + private static let digits = UInt16(UnicodeScalar("0").value)...UInt16(UnicodeScalar("9").value) private static let createModifiers: Set = [ "OR", "REPLACE", "EDITIONABLE", "NONEDITIONABLE", "AND", "RESOLVE", "COMPILE", "NOFORCE", @@ -61,6 +65,10 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { /// One entry per construct a bare `END` will close, true while that construct still waits for its `BEGIN`. private var constructs: [Bool] = [] + + /// The open `CASE`s that have held no `;`, which only a `CASE` statement can. The next `END` closes one of them, + /// whatever word follows it. + private var openCaseExpressions = 0 private var parenDepth = 0 private var pendingEnd = false private var pendingBodyKeyword = false @@ -97,6 +105,18 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { } } + // An `END` still waiting for the word after it closes a construct when the text ends. + public var holdsOpenBlock: Bool { + switch kind { + case .anonymousBlock, .storedUnit: + return constructs.count > (pendingEnd ? 1 : 0) + case .inlineDeclarations: + return !inlineQueryStarted && constructs.count > (pendingEnd ? 1 : 0) + case .sql, .embeddedSourceHeader, .opaque: + return false + } + } + public mutating func observeWord(_ word: String) { previousSymbol = nil guard !inLabel else { return } @@ -153,6 +173,7 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { } public mutating func observeSymbol(_ symbol: UInt16) { + let precedingSymbol = previousSymbol if symbol == Self.lessThan, previousSymbol == Self.lessThan { inLabel = true previousSymbol = nil @@ -174,7 +195,7 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { default: break } - followsPeriod = symbol == Self.period + followsPeriod = symbol == Self.period && !Self.isPeriodOrDigit(precedingSymbol) } public mutating func observeOpaqueToken() { @@ -201,6 +222,7 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { public mutating func observeSemicolon() -> Bool { previousSymbol = nil followsPeriod = false + openCaseExpressions = 0 guard lead == .decided else { return true } switch kind { @@ -229,6 +251,12 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { // MARK: - Private + /// A name that ends in a digit arrives as a word, which leaves no preceding symbol, so `t1.END` is still a member. + private static func isPeriodOrDigit(_ symbol: UInt16?) -> Bool { + guard let symbol else { return false } + return symbol == period || digits.contains(symbol) + } + private var tracksConstructs: Bool { guard lead == .decided else { return false } switch kind { @@ -340,6 +368,7 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { constructs.append(true) case "CASE": constructs.append(false) + openCaseExpressions += 1 case "END": pendingEnd = true case "PROCEDURE", "FUNCTION": @@ -353,7 +382,7 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { /// Returns whether `word` was consumed by the `END` before it. private mutating func closeEnd(followedBy word: String) -> Bool { - switch SqlBlockStructure.endingFollowedBy(word) { + switch SqlBlockStructure.endingFollowedBy(word, closesCaseExpression: openCaseExpressions > 0) { case .closesControlFlow: return true case .closesCaseStatement: @@ -387,5 +416,6 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { private mutating func closeConstruct() { guard !constructs.isEmpty else { return } constructs.removeLast() + openCaseExpressions = max(0, openCaseExpressions - 1) } } diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCatalogEffect.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCatalogEffect.swift index a899b93894..2e44afde28 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCatalogEffect.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCatalogEffect.swift @@ -65,7 +65,7 @@ public enum SQLCatalogEffectClassifier { } private static func textEffect(of sql: String, grammar: SQLLexicalGrammar) -> SQLCatalogEffect { - let trimmed = StatementBlank.trimming(SQLLeadingComments.stripping(sql)) + let trimmed = StatementBlank.trimming(SQLLeadingComments.stripping(sql, grammar: grammar)) guard !trimmed.isEmpty else { return .none } if SQLProceduralBlock.runs(trimmed, grammar: grammar) { return .opaque @@ -215,20 +215,24 @@ public enum SQLLeadingComments { public static let lineCommentTerminators: Set = ["\n", "\r", "\r\n"] - public static func stripping(_ sql: String) -> String { - var remaining = sql[...] - while true { - let trimmed = StatementBlank.trimmingLeading(remaining) - if trimmed.hasPrefix("--") { - guard let lineBreak = trimmed.firstIndex(where: endsLineComment) else { return "" } - remaining = trimmed[trimmed.index(after: lineBreak)...] - } else if trimmed.hasPrefix("/*"), !startsConditionalComment(trimmed) { - guard let close = trimmed.range(of: "*/") else { return "" } - remaining = trimmed[close.upperBound...] - } else { - return String(trimmed) + /// `sql` from its first token, past the blanks and the comments `grammar` reads, so a comment ends exactly where + /// the engine ends it. An executable comment is code and stays. + public static func stripping(_ sql: String, grammar: SQLLexicalGrammar) -> String { + let text = sql as NSString + let length = text.length + var index = 0 + while index < length { + let blank = StatementBlank.blankLength(in: text, at: index) + if blank > 0 { + index += blank + continue + } + guard let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar), span.kind.isComment else { + break } + index = max(span.end, index + 1) } + return text.substring(from: min(index, length)) } public static func startsConditionalComment(_ text: Substring) -> Bool { @@ -261,20 +265,21 @@ public enum SQLProceduralBlock { } // A block may open after <