diff --git a/.claude/rules/data-grid.md b/.claude/rules/data-grid.md index 272876fd8b..e86d304b26 100644 --- a/.claude/rules/data-grid.md +++ b/.claude/rules/data-grid.md @@ -30,6 +30,7 @@ Three layers, knowledge pointing one way: an owner's adapter, then `DataGridMode - **Row height, row count, reloads, inserts and removes reach both panes in one call**, so their document heights never differ. - **Vertical scroll stays in step through bounds-change notifications**: each pane's clip moves the other with `clipView.scroll(to:)` and `reflectScrolledClipView`, only when its y differs. Anywhere else scroll with `NSView.scroll(_:)`: moving a clip's x that way leaves the header behind. - **The frozen pane forwards every wheel event to the main scroll view**, so one momentum engine moves both. +- **A `fitsRows` grid is as tall as its rows and scrolls only sideways.** It publishes `model.fittingHeight` for its host to size it, hands a vertical gesture to the enclosing scroll view (the axis is decided once per gesture), and reveals a row by scrolling that page. The Structure tab's sections use it. - **Frozen columns take at most two thirds of the grid** (`DataGridMetrics.maximumFrozenShare`). Columns past that scroll, and Freeze Up to This Column is refused in the menu. - **`DataGridTableView.scrollToVisible(_:)` widens the rect leftward by the frozen width**, so a revealed column never parks under the frozen pane. - **A column moves only within its side of the freeze line.** A freeze or unfreeze rebuilds both panes' column lists from the set, writing width and position explicitly: remove-then-add alone strands a column at the end and resets its width. @@ -65,4 +66,4 @@ Three layers, knowledge pointing one way: an owner's adapter, then `DataGridMode - **AppKit builds the AXCell tree from cell views only**, so while `DataGridAccessibility.isActive` the main table mounts a `DataGridAccessibilityCellView` (identifier `data-grid-cell`) for each on-screen cell. It draws nothing and takes no clicks. Keep that path working whenever row drawing changes. - **A label reads "Row N, column M: value"**: N is the row number on screen, page offset included, and M the display position. UI tests find cells by it (`.claude/rules/ui-tests.md`, The grid). - **The frozen pane is hidden from accessibility.** A frozen column's stand-in in the main table reports the frame the frozen pane shows it at. -- **Only the viewport mounts.** Mounting for every row starved the main thread behind a 1,000-row page (#2392); rows scrolled into view remount. +- **Only the viewport mounts.** Mounting for every row starved the main thread behind a 1,000-row page (#2392). A client's read prepares the rows it walks, off-screen ones without a stand-in, and AppKit never asks for a prepared row again. So each on-screen row still without one remounts after a scroll, a layout of the grid and a read of the table: a fitted grid grows to its rows, and a page moves a grid, with no scroll. diff --git a/.github/plugin-registry.json b/.github/plugin-registry.json index f6bf29a547..9e7edc2c93 100644 --- a/.github/plugin-registry.json +++ b/.github/plugin-registry.json @@ -40,6 +40,9 @@ "HanaDriver": { "name": "SAP HANA Driver" }, + "InfluxDBDriverPlugin": { + "name": "InfluxDB Driver" + }, "KafkaDriverPlugin": { "name": "Kafka Driver" }, @@ -55,6 +58,9 @@ "MongoDBDriver": { "name": "MongoDB Driver" }, + "Neo4jDriverPlugin": { + "name": "Neo4j Driver" + }, "OracleDriver": { "name": "Oracle Driver" }, @@ -67,6 +73,9 @@ "SpannerDriverPlugin": { "name": "Spanner Driver" }, + "StarRocksDriverPlugin": { + "name": "StarRocks Driver" + }, "SurrealDBDriverPlugin": { "name": "SurrealDB Driver" }, diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index f96745925a..77ded5c235 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -82,7 +82,7 @@ jobs: # install, then the meson toolchain, the -dev packages the # workspace links, the runtime pieces the test suites need (xvfb, # dbus, gnome-keyring for the Secret Service step, openssh-client - # for the SSH transport). + # for the SSH transport, glycin-loaders for a value's picture). run: | apt-get install -y --no-install-recommends \ build-essential \ @@ -100,9 +100,12 @@ jobs: libgtksourceview-5-dev \ libsqlite3-dev \ libkrb5-dev \ + liblcms2-dev \ + libseccomp-dev \ clang \ libclang-dev \ adwaita-icon-theme \ + glycin-loaders \ locales \ gnome-keyring \ openssh-client \ @@ -115,7 +118,7 @@ jobs: components: rustfmt, clippy # Meson points Cargo at _build/cargo-home and _build/cargo-target, # so those are what the cache keeps. - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: workspaces: linux -> _build/cargo-target cache-directories: linux/_build/cargo-home @@ -268,7 +271,7 @@ jobs: - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: toolchain: "1.98" - - uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29 + - uses: taiki-e/install-action@5488ac5a951645ce95062214902ac4b6bc1a1037 # v2.87.32 with: tool: cargo-deny@${{ env.CARGO_DENY_VERSION }},cargo-shear@${{ env.CARGO_SHEAR_VERSION }},cargo-about@${{ env.CARGO_ABOUT_VERSION }},zizmor@${{ env.ZIZMOR_VERSION }} - name: Advisories, bans, licenses and sources @@ -334,7 +337,7 @@ jobs: build-essential pkgconf meson ninja-build gettext \ desktop-file-utils appstream libglib2.0-bin libglib2.0-dev libxml2-utils \ libgtk-4-dev libadwaita-1-dev libgtksourceview-5-dev \ - libsqlite3-dev libkrb5-dev clang libclang-dev cargo rustc + libsqlite3-dev libkrb5-dev liblcms2-dev libseccomp-dev clang libclang-dev cargo rustc - name: POTFILES.in is current run: | git ls-files 'crates/app/src/*.rs' 'data/resources/*.ui' 'data/*.desktop.in.in' \ @@ -489,11 +492,11 @@ jobs: - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: toolchain: "1.98" - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: workspaces: linux key: ${{ matrix.package }} - - uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29 + - uses: taiki-e/install-action@5488ac5a951645ce95062214902ac4b6bc1a1037 # v2.87.32 with: tool: cargo-nextest@${{ env.CARGO_NEXTEST_VERSION }} - name: Verify docker is available @@ -564,11 +567,11 @@ jobs: - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: toolchain: "1.98" - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: workspaces: linux key: tablepro-session - - uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29 + - uses: taiki-e/install-action@5488ac5a951645ce95062214902ac4b6bc1a1037 # v2.87.32 with: tool: cargo-nextest@${{ env.CARGO_NEXTEST_VERSION }} - name: Verify docker is available @@ -626,6 +629,8 @@ jobs: libgtksourceview-5-dev \ libsqlite3-dev \ libkrb5-dev \ + liblcms2-dev \ + libseccomp-dev \ clang \ libclang-dev \ xvfb \ @@ -634,11 +639,11 @@ jobs: - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: toolchain: "1.98" - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: workspaces: linux key: tablepro-app - - uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29 + - uses: taiki-e/install-action@5488ac5a951645ce95062214902ac4b6bc1a1037 # v2.87.32 with: tool: cargo-nextest@${{ env.CARGO_NEXTEST_VERSION }} - name: Verify docker is available diff --git a/.github/workflows/release-linux.yml b/.github/workflows/release-linux.yml index 416e184d6f..83a6ed42d8 100644 --- a/.github/workflows/release-linux.yml +++ b/.github/workflows/release-linux.yml @@ -115,7 +115,7 @@ jobs: build-essential pkgconf meson ninja-build gettext \ desktop-file-utils appstream libglib2.0-bin libglib2.0-dev libxml2-utils \ libgtk-4-dev libadwaita-1-dev libgtksourceview-5-dev \ - libsqlite3-dev libkrb5-dev clang libclang-dev cargo rustc + libsqlite3-dev libkrb5-dev liblcms2-dev libseccomp-dev clang libclang-dev cargo rustc working-directory: / - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9b1b4d8bf2..bbd31b8db5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -37,7 +37,7 @@ jobs: - name: Set up Python 3.9 id: python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: "3.9" update-environment: false diff --git a/.github/workflows/repo-hygiene.yml b/.github/workflows/repo-hygiene.yml index 2ecf1f6edf..3a8f59100f 100644 --- a/.github/workflows/repo-hygiene.yml +++ b/.github/workflows/repo-hygiene.yml @@ -55,7 +55,7 @@ jobs: # runner's Python, which has PyYAML. - name: Set up Python 3.9 for the release tooling id: python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: "3.9" update-environment: false @@ -117,6 +117,10 @@ jobs: - name: Check every test target is in its test plan run: python3 apple/scripts/ci/check-test-plans.py + # Oracle reads a bare dictionary name from the current schema before the SYS synonym. + - name: Check Oracle SQL names dictionary objects through OracleDictionary + run: python3 apple/scripts/ci/check-oracle-dictionary-names.py + # The corpora run by hand against servers (scripts/probes/check-column-type-corpus.sh), so a type a driver # starts listing is caught here. - name: Check every listed column type has a corpus column diff --git a/.oss-scanner/Dockerfile b/.oss-scanner/Dockerfile index 88b83578d0..09a6864eef 100644 --- a/.oss-scanner/Dockerfile +++ b/.oss-scanner/Dockerfile @@ -12,7 +12,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates curl git \ build-essential pkgconf meson ninja-build gettext desktop-file-utils appstream libglib2.0-bin \ libxml2-utils libgtk-4-dev libadwaita-1-dev libgtksourceview-5-dev libsqlite3-dev libkrb5-dev \ - clang libclang-dev adwaita-icon-theme locales gnome-keyring openssh-client xvfb xauth dbus-daemon \ + liblcms2-dev libseccomp-dev clang libclang-dev adwaita-icon-theme glycin-loaders locales gnome-keyring \ + openssh-client xvfb xauth dbus-daemon \ && rm -rf /var/lib/apt/lists/* \ && localedef -i en_US -f UTF-8 en_US.UTF-8 diff --git a/.oss-scanner/swift/Package.swift b/.oss-scanner/swift/Package.swift index 9e02a473ad..49240392bd 100644 --- a/.oss-scanner/swift/Package.swift +++ b/.oss-scanner/swift/Package.swift @@ -14,7 +14,9 @@ let package = Package( .target(name: "TableProCoreTypes"), .target(name: "TableProDocumentPath"), .target(name: "TableProJavaScriptText"), + .target(name: "TableProDamengCore", dependencies: ["TableProSQLGrammar"]), .testTarget(name: "TableProSQLGrammarTests", dependencies: ["TableProSQLGrammar"]), - .testTarget(name: "TableProLogRedactionTests", dependencies: ["TableProLogRedaction"]) + .testTarget(name: "TableProLogRedactionTests", dependencies: ["TableProLogRedaction"]), + .testTarget(name: "TableProDamengCoreTests", dependencies: ["TableProDamengCore", "TableProSQLGrammar"]) ] ) diff --git a/.oss-scanner/threat_model.md b/.oss-scanner/threat_model.md index 9e67d73ebe..4fe2a19276 100644 --- a/.oss-scanner/threat_model.md +++ b/.oss-scanner/threat_model.md @@ -32,7 +32,7 @@ The user is trusted: the SQL they type, the connections they create, the plugins 7. Secrets: `apple/TablePro/Core/Storage/KeychainHelper.swift`, `apple/TablePro/Core/Sync/SyncRecordMapper.swift` (passwords must never reach CloudKit), `linux/crates/storage/src/secrets`, `TableProLogRedaction`, the MCP error redactor. 8. SQL the app builds from data: `apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift`, `apple/TablePro/Core/Database/FilterSQLGenerator.swift`, `SQLEscaping.swift`, `DialectQuoteHelper.swift`, each driver's `quoteIdentifier`; on Linux `linux/crates/core/src/dml`, `linux/crates/core/src/filter/validate_raw_predicate.rs` and each driver's `dialect.rs`. A crafted table or column name that escapes quoting runs SQL with the user's privileges. 9. SSH and TLS: `apple/TablePro/Core/SSH/` (libssh2, host key prompts), `apple/Packages/TableProMacCore/Sources/TableProSSHConfig`, `linux/crates/ssh` (drives the system `ssh`, so watch for option injection from connection fields), `linux/crates/net/src/tls`. -10. Rendering of cell values: the out-of-process image renderer (`apple/TablePro/CLI/ImageRenderMain.swift`), `apple/TablePro/Core/Services/Formatting/PhpSerializeParser.swift`, the MongoDB shell's JavaScriptCore runtime. +10. Rendering of cell values: the out-of-process image renderer (`apple/TablePro/CLI/ImageRenderMain.swift`), the Linux Row Details picture, which glycin decodes in a sandbox (`linux/crates/app/src/ui/data_grid/row_detail/picture.rs`), `apple/TablePro/Core/Services/Formatting/PhpSerializeParser.swift`, the MongoDB shell's JavaScriptCore runtime. Matter least: layout, theming and localization code, `linux/crates/test-fixtures`, `linux/crates/test-support`, `apple/TableProUITests`, `scripts/`, `.github/`, docs. diff --git a/README.ko.md b/README.ko.md index d066b55520..a976414762 100644 --- a/README.ko.md +++ b/README.ko.md @@ -171,4 +171,4 @@ TablePro에 기여해 주신 모든 분께 감사드립니다. ## 라이선스 -이 프로젝트는 [GNU Affero General Public License v3.0(AGPLv3)](LICENSE)에 따라 라이선스가 부여됩니다. +이 프로젝트는 [GNU Affero General Public License v3.0(AGPLv3)](LICENSE)에 따라 라이선스가 부여됩니다. TablePro 이름과 아이콘은 이 라이선스에 포함되지 않으며, 사용 방법은 [TRADEMARKS.md](TRADEMARKS.md)에 있습니다. diff --git a/README.md b/README.md index 3c07e7197e..c2ed042266 100644 --- a/README.md +++ b/README.md @@ -182,5 +182,5 @@ Want to join them? Read [CONTRIBUTING.md](CONTRIBUTING.md). ## License -This project is licensed under the [GNU Affero General Public License v3.0 (AGPLv3)](LICENSE). +This project is licensed under the [GNU Affero General Public License v3.0 (AGPLv3)](LICENSE). The TablePro name and icon are not part of that license: [TRADEMARKS.md](TRADEMARKS.md) says how you can use them. diff --git a/README.vi.md b/README.vi.md index 4ea88ccfab..3886e7a2c8 100644 --- a/README.vi.md +++ b/README.vi.md @@ -146,5 +146,5 @@ Muốn tham gia? Đọc [CONTRIBUTING.md](CONTRIBUTING.md). ## Bản quyền -Dự án này cấp phép theo [GNU Affero General Public License v3.0 (AGPLv3)](LICENSE). +Dự án này cấp phép theo [GNU Affero General Public License v3.0 (AGPLv3)](LICENSE). Tên và biểu tượng TablePro không thuộc giấy phép đó: cách dùng được ghi trong [TRADEMARKS.md](TRADEMARKS.md). diff --git a/README.zh.md b/README.zh.md index 7fb23a2c33..fd8d3c0fdd 100644 --- a/README.zh.md +++ b/README.zh.md @@ -146,5 +146,5 @@ TablePro 由用户和以下赞助者支持。[成为赞助者](https://tablepro. ## 许可证 -本项目采用 [GNU Affero General Public License v3.0 (AGPLv3)](LICENSE) 许可。 +本项目采用 [GNU Affero General Public License v3.0 (AGPLv3)](LICENSE) 许可。TablePro 名称和图标不在该许可范围内,使用方式见 [TRADEMARKS.md](TRADEMARKS.md)。 diff --git a/TRADEMARKS.md b/TRADEMARKS.md new file mode 100644 index 0000000000..35e46b4862 --- /dev/null +++ b/TRADEMARKS.md @@ -0,0 +1,24 @@ +# Trademark policy + +The code in this repository is licensed under the [AGPLv3](LICENSE). That license covers the code. It does not cover the name **TablePro** or the TablePro icon and logo, which identify the builds this project publishes. Section 7(e) of the AGPL lets a project keep those out of the license, and this file says what you can do with them. + +The aim is simple: anyone who downloads something called TablePro gets the app this project ships, not a build someone else changed. + +## What you can do + +- Build TablePro from this source for your own use, changed or not. The AGPL allows that, including a build with the license check removed. +- Redistribute an official release unchanged, for example through a package manager, under the name TablePro. +- Package the unchanged source for a Linux distribution or a package manager under the name TablePro, with only the changes packaging needs. +- Say that your work is based on TablePro, works with TablePro, or is a fork of TablePro, in plain words that do not suggest it is TablePro. +- Use the name and icon to write about TablePro, review it, teach it, or link to it. + +## What needs a different name and icon + +- A build with any other change, such as new or removed features, a removed license check, or other branding. Rename it and replace the icon before you share it. +- A product, company, domain, app store listing, or package name that people could take for TablePro or for something made by this project. +- Anything that suggests this project endorses, sponsors, or supports your work when it does not. +- Selling copies, or builds with the paid features unlocked, under the TablePro name. + +## Questions + +For anything this file does not cover, or to ask for permission, write to [hello@tablepro.app](mailto:hello@tablepro.app). diff --git a/apple/.swiftlint.yml b/apple/.swiftlint.yml index c31e1bc9cd..698a5f7bcd 100644 --- a/apple/.swiftlint.yml +++ b/apple/.swiftlint.yml @@ -214,7 +214,7 @@ custom_rules: severity: error appstorage_key_namespace: name: "AppStorage Key Namespace" - regex: '@AppStorage\(\s*"(?!com\.TablePro)(?!(?:hideExportSuccessDialog|skipSchemaPreview|structureCodeFontSize)")[^"\\]+"' + regex: '@AppStorage\(\s*"(?!com\.TablePro)(?!(?:hideExportSuccessDialog|structureCodeFontSize)")[^"\\]+"' excluded: ['.*/(Packages|Plugins)/'] message: "Route an @AppStorage key through the preferences layer, under com.TablePro." severity: error diff --git a/apple/CHANGELOG.md b/apple/CHANGELOG.md index f60114318e..7a25751fe7 100644 --- a/apple/CHANGELOG.md +++ b/apple/CHANGELOG.md @@ -9,6 +9,79 @@ Unreleased changes are files in `changelog.d/`. +## [0.83.1](https://github.com/TableProApp/TablePro/releases/tag/v0.83.1) - 2026-11-01 [CRITICAL] + +### Changed + +- With CanonicalizeHostname, a DNS alias is followed only where CanonicalizePermittedCNAMEs allows it. + +### Removed + +- DuckDB Restore no longer accepts a folder of Parquet files, only a .duckdb file. + +### Fixed + +- A table tab could reload under an open cell and drop what you were typing after a save in another window. ([#3848](https://github.com/TableProApp/TablePro/pull/3848)) +- Edits made to rows while a query or refresh was still loading new ones were discarded when it finished. ([#3848](https://github.com/TableProApp/TablePro/pull/3848)) +- Table tabs ran their query again when you switched back to them or returned to TablePro from another app. ([#3848](https://github.com/TableProApp/TablePro/pull/3848)) +- A table tab waiting for its connection could run the statement under the cursor in a query tab instead. ([#3848](https://github.com/TableProApp/TablePro/pull/3848)) +- On iOS, a search or filter with a backslash on MySQL or ClickHouse, or with `[` on SQL Server, matched the wrong rows. ([#3906](https://github.com/TableProApp/TablePro/pull/3906)) +- A DELETE with no WHERE running unconfirmed behind EXPLAIN ANALYZE, an unusual CTE, or a WHERE glued to a name. ([#3907](https://github.com/TableProApp/TablePro/pull/3907)) +- CREATE OR REPLACE TABLE, SCHEMA or DATABASE replacing data without the confirmation DROP gets. ([#3907](https://github.com/TableProApp/TablePro/pull/3907)) +- HTTP drivers such as Typesense following a server's redirect to another address with the connection's key. ([#3907](https://github.com/TableProApp/TablePro/pull/3907)) +- Spanner, BigQuery and Teradata saving a grid change they could not key to the default schema's table. ([#3907](https://github.com/TableProApp/TablePro/pull/3907)) +- Teradata showing each result one statement late after a statement hit the 20-second receive timeout. ([#3907](https://github.com/TableProApp/TablePro/pull/3907)) +- DuckDB's Open the File Read-Only being ignored for a database not named .duckdb or .ddb. ([#3907](https://github.com/TableProApp/TablePro/pull/3907)) +- T-SQL BULK INSERT and OPENROWSET, and MongoDB $_internalJs operators, passing the external file and code refusals. ([#3907](https://github.com/TableProApp/TablePro/pull/3907)) +- Bigtable saving a grid change to another row when the key starts with a combining mark. ([#3907](https://github.com/TableProApp/TablePro/pull/3907)) +- DuckDB Back Up to a .duckdb file, and DuckDB Restore, failed with "Cannot prepare multiple statements at once". +- Pull in a linked folder failing when the repository is a partial clone. + +### Security + +- A backslash in a Snowflake filter value could end its string literal and run the rest as SQL. ([#3906](https://github.com/TableProApp/TablePro/pull/3906)) +- A Firebase Database URL with an encoded slash sending your Google access token to another host. +- A PostgreSQL or SQL Server database name could send Back Up or Restore, with its password, to another server. +- A PostgreSQL type named geometry or geography that PostGIS did not create could run its owner's code as you. +- A SQL export carrying a statement of its own in a cell value, a type name or a table comment when restored. +- A Typesense path with dot segments, or a Firebase or Bigtable request behind a comment, passing the read-only check. +- A credential profile in a shared connection file reading ~/.pgpass and sending the password to a host the file named. +- A name with a quote before an accent mark could run its own SQL when you browsed, searched or exported the table. +- A remote SQLite file path with a quote and a combining accent could run commands on the SSH server. +- A write after a CTE list, a Snowflake ->> chain, or inside a SurrealDB or CockroachDB SELECT passing as a read. +- ANALYZE TABLE, EXPLAIN PRAGMA and SQLite pragma_optimize passing as reads for read-only external clients. +- An imported SSH host or user could run commands through a Match exec line in ~/.ssh/config. +- An imported SSH host with a line break could make TablePro trust a host key for another server. +- Browsing a PostgreSQL schema could run a function or operator its owner planted there, with your privileges. +- CanonicalizeFallbackLocal no was ignored, and an SSH tunnel dialed the short host name anyway. +- CanonicalizeHostname followed any DNS alias, so DNS chose the host key an SSH tunnel was checked against. +- Cassandra and ScyllaDB connections through a tunnel signing in to other cluster nodes outside the tunnel. +- Dameng connections set to Required or a Verify mode connecting without TLS. +- EXPLAIN or DESCRIBE with a parenthesised query or a quoted option passing as a read for read-only MCP clients. +- File and URL reads through DuckDB, ClickHouse, quoted function names or VACUUM INTO passing the external check. +- Git status for a linked folder running filter and fetch commands set up in that folder's own Git configuration. +- Import Commands showing the first 300 characters of Startup SQL while keeping all of it. +- MCP clients and the assistant running statements past Read Only on Dameng behind a comment the driver read differently. +- MCP clients running DROP or server API calls unasked through grid requests on BigQuery, Elasticsearch and Typesense. +- MCP clients running DROP unasked on Trino and Presto behind a statement the driver split differently from the gate. +- MCP clients running statements or changing consistency past Read Only on Cassandra and ScyllaDB with a forged comment. +- MCP clients with read access running writes through browse_table or get_table_statistics with a crafted table name. +- MCP clients, the AI assistant and AppleScript running a second statement the Read Only and destructive checks missed. +- Open Project Folder filling a project's ${VAR} references from TablePro's own environment. +- Opening a crafted table on iPhone or iPad running a second statement on PostgreSQL, Redshift or SQL Server. +- Oracle Explain reading a PLAN_TABLE or V$ views planted in the tab's schema instead of Oracle's own. +- Restoring a DuckDB folder of Parquet files ran its schema.sql with file access, so the folder could write any file. +- Restoring a MySQL or MariaDB dump could run shell commands written in the .sql file. +- Restoring a SQLite dump could run shell commands written in the .sql file. +- SQL Server EXEC of a string, sp_executesql, OPENQUERY and OPENROWSET passing as reads or without the code check. +- SQL Server Verify CA and Verify Identity on iPhone and iPad connecting without checking the certificate. +- Secrets in a pasted MongoDB connection string saved outside the Keychain and carried by exports, links and sync. +- Shared connections reusing your Entra ID token, Snowflake CLI profile, MongoDB AWS credentials or Run Ledger Plugins. +- Teradata with SSL through a tunnel sending the password to port 443 on this Mac. +- Trino and Presto sending the password or token to another host the server named in its reply. +- iPhone and iPad Safe Mode letting DESC ANALYZE of a write, OPENQUERY and state-changing calls run as reads. + + ## [0.83.0](https://github.com/TableProApp/TablePro/releases/tag/v0.83.0) - 2026-10-26 Search Database finds a value in every table, Compare Results matches two results row by row, and Generate Rows fills a table with test data. diff --git a/apple/Configs/Version-iOS.xcconfig b/apple/Configs/Version-iOS.xcconfig index 705cfb6c43..09f58d5e70 100644 --- a/apple/Configs/Version-iOS.xcconfig +++ b/apple/Configs/Version-iOS.xcconfig @@ -1,5 +1,5 @@ // Single source of truth for the iOS app version. // The app and its widget extension must ship the same values, so both read this file. -MARKETING_VERSION = 1.1 -CURRENT_PROJECT_VERSION = 23 +MARKETING_VERSION = 1.1.1 +CURRENT_PROJECT_VERSION = 24 diff --git a/apple/Libs/checksums.sha256 b/apple/Libs/checksums.sha256 index 58559a0baf..228b6a3f62 100644 --- a/apple/Libs/checksums.sha256 +++ b/apple/Libs/checksums.sha256 @@ -10,10 +10,10 @@ c47c8406cdb817e121a89ad5171f6931fd73c4bce6c73c3809e626257afd7374 Libs/libcrypto e274f77ed3feaee3d4f8f6ac297376d56a1369388f037599a90ea440390349b4 Libs/libcrypto_universal.a ec04bc233ea917de16a48b130d1467c68f5b95d469f423b7b9054d0e45bb4e16 Libs/libcrypto_x86_64.a e274f77ed3feaee3d4f8f6ac297376d56a1369388f037599a90ea440390349b4 Libs/libcrypto.a -f833d1edba062e03a5a7f7b5e40fbfe0f015c081158b8dd455d6129bf4515c12 Libs/libduckdb_arm64.a -c1aef28aa225d34bb4d8be853000f4d2640a77366ff4b2cfa61f53648290c357 Libs/libduckdb_universal.a -e971a4c84689626699482ba5e2f148345b50e53fc1aed57b6e37fbb90c85db9c Libs/libduckdb_x86_64.a -c1aef28aa225d34bb4d8be853000f4d2640a77366ff4b2cfa61f53648290c357 Libs/libduckdb.a +09b2deb312d03840dda422a6931a9918ad81c308941c774556adb1386d6ab589 Libs/libduckdb_arm64.a +9749f1be288f03cc4362b15fbc0137edd8aee75542f52f7d7ca8d563586a1e77 Libs/libduckdb_universal.a +3183445676921ea6e4ea4cb7f0689703fb3ceba3d1699b21e88a2e36f12e0e2e Libs/libduckdb_x86_64.a +9749f1be288f03cc4362b15fbc0137edd8aee75542f52f7d7ca8d563586a1e77 Libs/libduckdb.a 5017c3fecadf4955fad1db923a8358fc9943fa17e8a5d54288db6a28fc5a3aa8 Libs/libhiredis_arm64.a 58974eb81d5fefca9fb6732bd15921f1cf12a23f9a2402b1b099ec6c089f1e35 Libs/libhiredis_ssl_arm64.a 09c8cb4acb62a54c09c4f2605bbc464b93d9e2ea5b2ae89a1d2165f8adaa9b9c Libs/libhiredis_ssl_universal.a diff --git a/apple/Libs/ios/checksums.sha256 b/apple/Libs/ios/checksums.sha256 index bd457322f7..f6e64e4c7a 100644 --- a/apple/Libs/ios/checksums.sha256 +++ b/apple/Libs/ios/checksums.sha256 @@ -1,8 +1,10 @@ -840741c66e9ee257f97eb7cc16407c37960aa49faa4b110e56ccc0f8dc3a6db2 ./DuckDB.xcframework/Info.plist -c8dd688cf14559c744255d1bb3879330d6ae3523268e8904fa7b885e140030aa ./DuckDB.xcframework/ios-arm64/Headers/duckdb.h -f4073989393e739b0b5ad68c7f47f7750715d2d46e46141e81b1bc49cf782620 ./DuckDB.xcframework/ios-arm64/libduckdb-device.a -c8dd688cf14559c744255d1bb3879330d6ae3523268e8904fa7b885e140030aa ./DuckDB.xcframework/ios-arm64_x86_64-simulator/Headers/duckdb.h -75523877523153f63249a39eb4ec0d7f1bb05db3a4dbb5e6c35ee848b09afb71 ./DuckDB.xcframework/ios-arm64_x86_64-simulator/libduckdb-sim.a +fcfb2160ac0c32d40fd3241bb0ec0ec157238a5db6ef68a5d98e93a616956cf6 ./DuckDB.xcframework/Info.plist +8dbc9c71a10f0a75fcacc1a3f4f7c76093a84d232fc152222ee5b6e512ab4560 ./DuckDB.xcframework/ios-arm64-simulator/Headers/duckdb.h +a90f28cbe2662c69fc73f726832479ae7b5d043566b83eb1ac95a8ab35d3e37d ./DuckDB.xcframework/ios-arm64-simulator/Headers/duckdb_static_extensions.h +842bd80d7f54472c4161434474af155675e0d7b19c7751e1d1046d8e144a19b6 ./DuckDB.xcframework/ios-arm64-simulator/libduckdb-sim.a +8dbc9c71a10f0a75fcacc1a3f4f7c76093a84d232fc152222ee5b6e512ab4560 ./DuckDB.xcframework/ios-arm64/Headers/duckdb.h +a90f28cbe2662c69fc73f726832479ae7b5d043566b83eb1ac95a8ab35d3e37d ./DuckDB.xcframework/ios-arm64/Headers/duckdb_static_extensions.h +c667055fe35aa1e47ca8f2dd36b8328743429872022a0bb25701b593d9a7d1f0 ./DuckDB.xcframework/ios-arm64/libduckdb-device.a 0481f79db3949ba16425ffd8e15e640757a208ca05728d98cdbfb4bc617e06c3 ./FreeTDS.xcframework/Info.plist 734d8502be13513e974a621557b1a4776d9f605b5a48039e28a3597f27f704ce ./FreeTDS.xcframework/ios-arm64-simulator/Headers/sybdb.h eaa21732fa7cfb538b3bb9bcca7342b99b407d2e838c09e5f3d07398c21406a3 ./FreeTDS.xcframework/ios-arm64-simulator/Headers/sybfront.h diff --git a/apple/Libs/release b/apple/Libs/release index edf700ee23..f05581e49b 100644 --- a/apple/Libs/release +++ b/apple/Libs/release @@ -1 +1 @@ -libs-5 +libs-6 diff --git a/apple/Packages/TableProCore/Package.swift b/apple/Packages/TableProCore/Package.swift index 34aae47217..e79edac2cb 100644 --- a/apple/Packages/TableProCore/Package.swift +++ b/apple/Packages/TableProCore/Package.swift @@ -31,11 +31,14 @@ let package = Package( .library(name: "TableProKerberos", targets: ["TableProKerberos"]), .library(name: "TableProTeradataCore", targets: ["TableProTeradataCore"]), .library(name: "TableProTrinoCore", targets: ["TableProTrinoCore"]), + .library(name: "TableProDamengCore", targets: ["TableProDamengCore"]), .library(name: "TableProTLSClientIdentity", targets: ["TableProTLSClientIdentity"]), .library(name: "TableProTLSTestFixtures", targets: ["TableProTLSTestFixtures"]), .library(name: "TableProGoogleCloud", targets: ["TableProGoogleCloud"]), .library(name: "TableProSpannerCore", targets: ["TableProSpannerCore"]), .library(name: "TableProWeaviateCore", targets: ["TableProWeaviateCore"]), + .library(name: "TableProInfluxDBCore", targets: ["TableProInfluxDBCore"]), + .library(name: "TableProNeo4jCore", targets: ["TableProNeo4jCore"]), .library(name: "TableProFirebaseCore", targets: ["TableProFirebaseCore"]), .library(name: "TableProBigtableCore", targets: ["TableProBigtableCore"]), .library(name: "TableProNumberFormatting", targets: ["TableProNumberFormatting"]), @@ -105,7 +108,7 @@ let package = Package( ), .target( name: "TableProImport", - dependencies: ["TableProConnectionLibrary"], + dependencies: ["TableProConnectionLibrary", "TableProSSHTransport"], path: "Sources/TableProImport" ), .target( @@ -152,9 +155,15 @@ let package = Package( ), .target( name: "TableProTrinoCore", - dependencies: [], + dependencies: ["TableProSQLGrammar"], path: "Sources/TableProTrinoCore" ), + .target( + name: "TableProDamengCore", + dependencies: ["TableProSQLGrammar"], + path: "Sources/TableProDamengCore", + swiftSettings: pluginCoreSettings + ), .target( name: "TableProTLSClientIdentity", dependencies: [], @@ -185,6 +194,16 @@ let package = Package( dependencies: [], path: "Sources/TableProWeaviateCore" ), + .target( + name: "TableProInfluxDBCore", + dependencies: [], + path: "Sources/TableProInfluxDBCore" + ), + .target( + name: "TableProNeo4jCore", + dependencies: [], + path: "Sources/TableProNeo4jCore" + ), .target( name: "TableProBigtableCore", dependencies: ["TableProGoogleCloud"], @@ -273,16 +292,42 @@ let package = Package( dependencies: ["TableProBigtableCore", "TableProGoogleCloud"], path: "Tests/TableProBigtableCoreTests" ), + .testTarget( + name: "TableProNeo4jCoreTests", + dependencies: ["TableProNeo4jCore"], + path: "Tests/TableProNeo4jCoreTests" + ), + .testTarget( + name: "TableProFirebaseCoreTests", + dependencies: ["TableProFirebaseCore"], + path: "Tests/TableProFirebaseCoreTests" + ), .testTarget( name: "TableProSpannerCoreTests", dependencies: ["TableProSpannerCore"], path: "Tests/TableProSpannerCoreTests" ), + .testTarget( + name: "TableProTeradataCoreTests", + dependencies: ["TableProTeradataCore"], + path: "Tests/TableProTeradataCoreTests" + ), + .testTarget( + name: "TableProInfluxDBCoreTests", + dependencies: ["TableProInfluxDBCore"], + path: "Tests/TableProInfluxDBCoreTests" + ), .testTarget( name: "TableProTrinoCoreTests", - dependencies: ["TableProTrinoCore"], + dependencies: ["TableProTrinoCore", "TableProSQLGrammar"], path: "Tests/TableProTrinoCoreTests" ), + .testTarget( + name: "TableProDamengCoreTests", + dependencies: ["TableProDamengCore", "TableProSQLGrammar"], + path: "Tests/TableProDamengCoreTests", + swiftSettings: pluginCoreSettings + ), .testTarget( name: "TableProTabularIOTests", dependencies: ["TableProTabularIO", "TableProPluginKit"], diff --git a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionProvider.swift b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionProvider.swift index a190774b8c..ca5511a9f1 100644 --- a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionProvider.swift +++ b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionProvider.swift @@ -150,22 +150,25 @@ public final class SQLCompletionProvider { // A dot prefix is a table or alias first, then a schema (its tables) or a database (its schemas, or its // tables on an engine with none). The namespace fallback also covers an alias that resolves to a schema - // name parsed out of the FROM clause. + // name parsed out of the FROM clause. In a table slot it can only qualify the table being named, so a table + // that shares the name is never read for columns. if let dotPrefix = context.dotPrefix { guard let catalog else { return [] } - if let derived = context.tableReferences.first(where: { - $0.isDerived && $0.identifier.caseInsensitiveCompare(dotPrefix) == .orderedSame - }) { - let columns = await derivedColumns(of: derived) - if !columns.isEmpty { - return [columns.map { SQLCompletionItem.column($0, dataType: nil, tableName: derived.identifier) }] + if !context.expectsObjectName { + if let derived = context.tableReferences.first(where: { + $0.isDerived && $0.identifier.caseInsensitiveCompare(dotPrefix) == .orderedSame + }) { + let columns = await derivedColumns(of: derived) + if !columns.isEmpty { + return [columns.map { SQLCompletionItem.column($0, dataType: nil, tableName: derived.identifier) }] + } + } + if let tableName = await catalog.resolveAlias(dotPrefix, in: context.tableReferences) { + let schema = context.tableReferences.first { + $0.tableName.caseInsensitiveCompare(tableName) == .orderedSame + }?.schema + items = await catalog.columnCompletionItems(for: tableName, schema: schema) } - } - if let tableName = await catalog.resolveAlias(dotPrefix, in: context.tableReferences) { - let schema = context.tableReferences.first { - $0.tableName.caseInsensitiveCompare(tableName) == .orderedSame - }?.schema - items = await catalog.columnCompletionItems(for: tableName, schema: schema) } if items.isEmpty { if await catalog.isKnownSchema(dotPrefix) { @@ -493,7 +496,7 @@ public final class SQLCompletionProvider { var item = SQLCompletionItem( label: "'\(value)'", kind: .keyword, - insertText: "'\(value.replacingOccurrences(of: "'", with: "''"))'", + insertText: "'\(value.replacingOccurrences(of: "'", with: "''", options: .literal))'", detail: column, filterText: value.lowercased() ) @@ -598,13 +601,13 @@ public final class SQLCompletionProvider { // `AS 'Total'` names the column on MySQL and SQLite, but ORDER BY 'Total' sorts by a constant. private func identifierSpelling(of spelling: String, name: String) -> String { - guard spelling.hasPrefix("'") else { return spelling } + guard spelling.unicodeScalars.first == "'" else { return spelling } let isBare = name.unicodeScalars.first.map { CharacterSet.letters.contains($0) || $0 == "_" } == true && name.unicodeScalars.allSatisfy { CharacterSet.alphanumerics.contains($0) || $0 == "_" } guard !isBare || SQLKeywords.isKeyword(name) else { return name } // The analyzer reads a backslash-escaped `"` as a string, which is where a backtick quotes a name instead. let quote = grammar.isSuperset(of: [.backslashEscapesInDoubleQuotes, .backtickQuotes]) ? "`" : "\"" - return quote + name.replacingOccurrences(of: quote, with: quote + quote) + quote + return quote + name.replacingOccurrences(of: quote, with: quote + quote, options: .literal) + quote } // CTEs were bound to their own scope by the parser, so what is left to expand is a table the catalog knows. diff --git a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLContextAnalyzer.swift b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLContextAnalyzer.swift index aadeb59b74..841ac697c2 100644 --- a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLContextAnalyzer.swift +++ b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLContextAnalyzer.swift @@ -209,6 +209,7 @@ public final class SQLContextAnalyzer { private static let doubleQuote = UInt16(UnicodeScalar("\"").value) private static let backtick = UInt16(UnicodeScalar("`").value) private static let openBracket = UInt16(UnicodeScalar("[").value) + private static let closeBracket = UInt16(UnicodeScalar("]").value) private static let dollar = UInt16(UnicodeScalar("$").value) private static let hash = UInt16(UnicodeScalar("#").value) @@ -260,8 +261,9 @@ public final class SQLContextAnalyzer { // and hold any character. private static let quotedSegment = "[`\"'\\[]?\\w+[`\"'\\]]?" + // A trailing dot stays in the path, so `JOIN sales.` reads as unfinished instead of naming a table `sales`. private static let tableRefRegexes: [NSRegularExpression] = { - let path = "(\(quotedSegment)(?:\\.\(quotedSegment))*)" + let path = "(\(quotedSegment)(?:\\.\(quotedSegment))*\\.?)" let alias = "(?:\\s+(?:AS\\s+)?(\(quotedSegment)))?" let patterns = [ // MySQL's STRAIGHT_JOIN has no word boundary before JOIN. @@ -282,7 +284,7 @@ public final class SQLContextAnalyzer { ) private static let alterTableRegex = compileRegex( - "(?i)\\bALTER\\s+TABLE\\s+(\(quotedSegment)(?:\\.\(quotedSegment))*)" + "(?i)\\bALTER\\s+TABLE\\s+(\(quotedSegment)(?:\\.\(quotedSegment))*\\.?)" ) private static let derivedTableParser = DerivedTableParser() @@ -855,7 +857,13 @@ public final class SQLContextAnalyzer { append(upTo: index) start = index + 1 } - append(upTo: end) + // The name after a trailing dot is still being typed: an empty last segment, so no caller reads `sales.` as + // a table named `sales`. + if start == end, end > range.location { + segments.append(PathSegment(masked: "", name: "", isQuoted: false)) + } else { + append(upTo: end) + } return segments } @@ -1048,10 +1056,6 @@ public final class SQLContextAnalyzer { dotPrefix: String?, currentFunction: String? = nil ) -> ClauseResolution { - if dotPrefix != nil { - return ClauseResolution(clause: .select, expectsObjectName: false) - } - // Bounds the regex work on a long statement. SQLContextAnalyzerWindowingTests pins the size. let windowSize = 5_000 let nsText = textBeforeCursor as NSString @@ -1059,6 +1063,10 @@ public final class SQLContextAnalyzer { ? nsText.substring(from: nsText.length - windowSize) : textBeforeCursor + if dotPrefix != nil { + return ClauseResolution(clause: .select, expectsObjectName: qualifiesTableName(after: windowedText)) + } + // Structural contexts first: `CREATE TABLE test (id ` looks like a call to `test(` but is a column definition. let range = NSRange(location: 0, length: (windowedText as NSString).length) for (regex, clause) in Self.structuralClauseRegexes { @@ -1079,6 +1087,42 @@ public final class SQLContextAnalyzer { return ClauseResolution(clause: .unknown, expectsObjectName: false) } + // `FROM sales.` qualifies the table being named. A FROM after `IS DISTINCT` or inside a call such as + // `EXTRACT(YEAR FROM o.`, and any qualifier this cannot place, stays a table or alias. + private func qualifiesTableName(after textBeforePrefix: String) -> Bool { + let tokens = tokenize(Self.removingTrailingPath(textBeforePrefix)) + guard let keyword = tokens.last, keyword.kind == .word, + ["FROM", "JOIN", "INTO"].contains(keyword.upper) else { return false } + guard keyword.upper == "FROM" else { return true } + if tokens.count >= 2, tokens[tokens.count - 2].upper == "DISTINCT" { return false } + var open: [Int] = [] + for (index, token) in tokens.enumerated() { + if token.kind == .openParen { + open.append(index) + } else if token.kind == .closeParen, !open.isEmpty { + open.removeLast() + } + } + guard let paren = open.last, paren > 0, tokens[paren - 1].kind == .word else { return true } + return Self.groupingKeywords.contains(tokens[paren - 1].upper) + } + + // A parenthesis after one of these opens a subquery or a join group, so a FROM inside it names a table. + private static let groupingKeywords: Set = [ + "FROM", "JOIN", "LATERAL", "AS", "IN", "EXISTS", "ANY", "ALL", "SOME", "UNION", "INTERSECT", "EXCEPT" + ] + + private static func removingTrailingPath(_ text: String) -> String { + let ns = text as NSString + var start = ns.length + while start > 0 { + let ch = ns.character(at: start - 1) + guard SQLTokenBoundary.isTokenChar(ch) || ch == dot || ch == openBracket || ch == closeBracket else { break } + start -= 1 + } + return ns.substring(to: start) + } + // MARK: - Linear Clause Scan private enum ScanTokenKind { diff --git a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLIdentifierSpelling.swift b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLIdentifierSpelling.swift index 5531a4ec99..8eadaa7789 100644 --- a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLIdentifierSpelling.swift +++ b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLIdentifierSpelling.swift @@ -103,7 +103,7 @@ public struct SQLIdentifierSpelling: Sendable { public func quoted(_ name: String) -> String { let quote = String(rules.quote) - return quote + name.replacingOccurrences(of: quote, with: quote + quote) + quote + return quote + name.replacingOccurrences(of: quote, with: quote + quote, options: .literal) + quote } // A name written into a statement with nothing typed: as stored keeps the caller's own spelling. @@ -131,7 +131,7 @@ public struct SQLIdentifierSpelling: Sendable { if quoting != .none, identifier.parts.count == 1, case .catalog(let name) = identifier.parts[0] { // The user's own quote asks for the exact name: inside a closed pair it goes in raw. let quote = String(rules.quote) - let text = quoting == .enclosed ? name.replacingOccurrences(of: quote, with: quote + quote) : quoted(name) + let text = quoting == .enclosed ? name.replacingOccurrences(of: quote, with: quote + quote, options: .literal) : quoted(name) return item.recased(label: name, insertText: text) } // A qualified reference replaces the user's quotes, so it is spelled as if none were typed. diff --git a/apple/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift b/apple/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift index d96c5be4f0..bc2ce87ab9 100644 --- a/apple/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift +++ b/apple/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift @@ -15,14 +15,16 @@ public struct BigtableColumn: Hashable, Sendable, Comparable { family + ":" + BigtableBytes.display(qualifier) } + // By scalar: a qualifier that starts with a combining mark joins the colon into one Character. public init(name: String) throws { - guard let colon = name.firstIndex(of: ":"), colon != name.startIndex else { + guard let colon = name.unicodeScalars.firstIndex(of: ":"), colon != name.startIndex else { throw BigtableError.configuration(String( format: String(localized: "\"%@\" is not a column. Name it as family:qualifier."), name )) } - self.family = String(name[.. Bool { diff --git a/apple/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift b/apple/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift index 08b8966201..c05591c4f0 100644 --- a/apple/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift +++ b/apple/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift @@ -157,7 +157,7 @@ public struct BigtableRequest: Sendable, Equatable { /// Text that would not read back as one bare word goes in JSON quotes. static func token(_ text: String, reserved: String) -> String { - let plain = !text.isEmpty && !text.hasPrefix("\"") && text.unicodeScalars.allSatisfy { scalar in + let plain = !text.isEmpty && text.unicodeScalars.first != "\"" && text.unicodeScalars.allSatisfy { scalar in !CharacterSet.whitespacesAndNewlines.contains(scalar) && !reserved.unicodeScalars.contains(scalar) } if plain { return text } @@ -173,22 +173,23 @@ public struct BigtableRequest: Sendable, Equatable { static let missingTable = BigtableError.configuration(String(localized: "Name the table before the key, as in GET events/user#42.")) } -/// Reads a request line one token at a time. +// Read by scalar, as `token` writes: a key that starts with a combining mark joins the quote or slash before it +// into one Character. private struct RequestScanner { - private var text: Substring + private var text: Substring.UnicodeScalarView private var query: [String: String] = [:] init(_ text: String) { - self.text = Substring(text) + self.text = Substring(text).unicodeScalars } var isAtEnd: Bool { - text.allSatisfy(\.isWhitespace) + text.allSatisfy(\.properties.isWhitespace) } - mutating func word(stoppingAt stop: Character? = nil) -> String? { + mutating func word(stoppingAt stop: Unicode.Scalar? = nil) -> String? { skipSpace() - let word = text.prefix { !$0.isWhitespace && $0 != stop } + let word = text.prefix { !$0.properties.isWhitespace && $0 != stop } guard !word.isEmpty else { return nil } text = text.dropFirst(word.count) return String(word) @@ -196,7 +197,7 @@ private struct RequestScanner { mutating func rest() -> String { let remaining = String(text).trimmingCharacters(in: .whitespacesAndNewlines) - text = "" + text.removeAll() return remaining } @@ -204,7 +205,7 @@ private struct RequestScanner { /// no slash is a table alone, as in `SCAN events`. mutating func target(tableOnly: Bool) throws -> (table: String, key: Data?) { skipSpace() - let head = text.prefix { !$0.isWhitespace && $0 != "\"" && $0 != "?" } + let head = text.prefix { !$0.properties.isWhitespace && $0 != "\"" && $0 != "?" } let table: String if let slash = head.firstIndex(of: "/") { table = String(text[.. = [ + "tlscertificatekeyfilepassword", + "sslclientcertificatekeypassword", + "proxypassword", + ] + static let mongoKeyPassphraseOptions: Set = [ + "tlscertificatekeyfilepassword", + "sslclientcertificatekeypassword", + ] + static let mechanismPropertiesOption = "authmechanismproperties" + static let sessionTokenProperty = "AWS_SESSION_TOKEN" + + public static func isSecret(key: String, value: String) -> Bool { + if key.caseInsensitiveCompare(mongoSessionTokenFieldId) == .orderedSame { return true } + guard let option = mongoOption(in: key) else { return false } + if mongoSecretOptions.contains(option) { return true } + return option == mechanismPropertiesOption && splitSessionToken(from: value).sessionToken != nil + } + + // Mechanism properties keep everything but the token. + public static func removingSecrets(from fields: [String: String]) -> [String: String] { + var kept: [String: String] = [:] + for (key, value) in fields { + guard isSecret(key: key, value: value) else { + kept[key] = value + continue + } + if let option = mongoOption(in: key), isMongoMechanismProperties(option: option), + let remaining = splitSessionToken(from: value).remaining { + kept[key] = remaining + } + } + return kept + } + + // The lowercased option name of a `mongoParam_` field, nil for any other field. + public static func mongoOption(in key: String) -> String? { + guard key.count > mongoOptionPrefix.count, + key.lowercased().hasPrefix(mongoOptionPrefix.lowercased()) + else { return nil } + return String(key.dropFirst(mongoOptionPrefix.count)).lowercased() + } + + public static func isMongoKeyPassphrase(option: String) -> Bool { + mongoKeyPassphraseOptions.contains(option.lowercased()) + } + + public static func isMongoProxyPassword(option: String) -> Bool { + option.lowercased() == "proxypassword" + } + + public static func isMongoMechanismProperties(option: String) -> Bool { + option.lowercased() == mechanismPropertiesOption + } + + // `authMechanismProperties` is `KEY:value` pairs joined by commas; libmongoc reads keys without case. + public static func splitSessionToken(from properties: String) -> (remaining: String?, sessionToken: String?) { + var kept: [String] = [] + var token: String? + for pair in properties.split(separator: ",", omittingEmptySubsequences: true) { + let parts = pair.split(separator: ":", maxSplits: 1, omittingEmptySubsequences: false) + let name = parts[0].trimmingCharacters(in: .whitespaces) + if name.caseInsensitiveCompare(sessionTokenProperty) == .orderedSame { + let value = parts.count > 1 ? String(parts[1]) : "" + if !value.isEmpty { token = value } + } else { + kept.append(String(pair)) + } + } + return (kept.isEmpty ? nil : kept.joined(separator: ","), token) + } + + public static func joiningSessionToken(_ token: String, to properties: String?) -> String { + let remaining = properties.flatMap { splitSessionToken(from: $0).remaining } + let pair = "\(sessionTokenProperty):\(token)" + guard let remaining else { return pair } + return "\(remaining),\(pair)" + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift b/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift index d6962fdd63..09f552b188 100644 --- a/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift +++ b/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift @@ -46,6 +46,7 @@ public enum SQLDDLFallbackPolicy { "Firestore", "Firebase Realtime Database", "Bigtable", + "Neo4j", ] /// True when `DROP`/`TRUNCATE` built by the app is something this engine could run. diff --git a/apple/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift b/apple/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift index bb002a60ef..cfa5a5fb91 100644 --- a/apple/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift +++ b/apple/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift @@ -14,6 +14,7 @@ public struct DatabaseType: Hashable, Codable, Sendable, RawRepresentable { public static let tidb = DatabaseType(rawValue: "TiDB") public static let databend = DatabaseType(rawValue: "Databend") public static let oceanbase = DatabaseType(rawValue: "OceanBase") + public static let starrocks = DatabaseType(rawValue: "StarRocks") public static let postgresql = DatabaseType(rawValue: "PostgreSQL") public static let sqlite = DatabaseType(rawValue: "SQLite") public static let redis = DatabaseType(rawValue: "Redis") @@ -47,7 +48,7 @@ public struct DatabaseType: Hashable, Codable, Sendable, RawRepresentable { public static let bigtable = DatabaseType(rawValue: "Bigtable") public static let allKnownTypes: [DatabaseType] = [ - .mysql, .mariadb, .tidb, .databend, .oceanbase, .postgresql, .sqlite, .redis, .mongodb, + .mysql, .mariadb, .tidb, .databend, .oceanbase, .starrocks, .postgresql, .sqlite, .redis, .mongodb, .clickhouse, .mssql, .oracle, .dameng, .duckdb, .cassandra, .redshift, .etcd, .cloudflareD1, .dynamodb, .bigquery, .spanner, .snowflake, .libsql, .beancount, .surrealdb, .teradata, .trino, .kafka, .cloudflareR2SQL, .weaviate, .firestore, .firebaseRealtimeDatabase, .bigtable @@ -61,6 +62,7 @@ public struct DatabaseType: Hashable, Codable, Sendable, RawRepresentable { case .tidb: return "tidb-icon" case .databend: return "databend-icon" case .oceanbase: return "oceanbase-icon" + case .starrocks: return "starrocks-icon" case .postgresql: return "postgresql-icon" case .redshift: return "redshift-icon" case .sqlite: return "sqlite-icon" diff --git a/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengGrammar.swift b/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengGrammar.swift new file mode 100644 index 0000000000..490ce4e799 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengGrammar.swift @@ -0,0 +1,14 @@ +import Foundation +import TableProSQLGrammar + +enum DamengGrammar { + // The reading the app runs a Dameng session's text with: DM8's measured grammar plus the BACKSLASH_ESCAPE mode the + // driver detects at connect and reports as its session state. + static func execution(backslashEscapes: Bool) -> SQLLexicalGrammar { + let session = SQLSessionLexicalFacts( + determined: .backslashEscapesInSingleQuotes, + enabled: backslashEscapes ? .backslashEscapesInSingleQuotes : [] + ) + return SQLLexicalReadings.resolve(databaseTypeId: "Dameng", declared: nil, session: session).execution + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengPlaceholders.swift b/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengPlaceholders.swift new file mode 100644 index 0000000000..d4eda1777a --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengPlaceholders.swift @@ -0,0 +1,28 @@ +import Foundation +import TableProSQLGrammar + +public enum DamengPlaceholders { + private static let questionMark = UInt16(UnicodeScalar("?").value) + + // One more piece than there are placeholders. A `?` in a literal, a quoted name or a comment stays in its piece. + public static func pieces(of template: String, backslashEscapes: Bool) -> [String] { + let grammar = DamengGrammar.execution(backslashEscapes: backslashEscapes) + let text = template as NSString + var pieces: [String] = [] + var start = 0 + var index = 0 + while index < text.length { + if let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar) { + index = max(span.end, index + 1) + continue + } + if text.character(at: index) == questionMark { + pieces.append(text.substring(with: NSRange(location: start, length: index - start))) + start = index + 1 + } + index += 1 + } + pieces.append(text.substring(from: min(start, text.length))) + return pieces + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengScriptSplitter.swift b/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengScriptSplitter.swift new file mode 100644 index 0000000000..4911eacf04 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengScriptSplitter.swift @@ -0,0 +1,22 @@ +import Foundation +import TableProSQLGrammar + +// Cuts where the app's gates cut, so the driver never sends a piece they did not count. A PL/SQL block goes whole: +// the Dameng grammar does not delimit one, and a wrong cut sends the server a fragment. +public enum DamengScriptSplitter { + private static let blockKeywords: Set = [ + "BEGIN", "CASE", "DECLARE", "END", "IF", "LOOP" + ] + + public static func statements(in script: String, backslashEscapes: Bool) -> [String] { + let grammar = DamengGrammar.execution(backslashEscapes: backslashEscapes) + guard !holdsBlock(script, grammar: grammar) else { return [script] } + return SQLStatementScanner.allStatements(in: script, grammar: grammar) + } + + private static func holdsBlock(_ script: String, grammar: SQLLexicalGrammar) -> Bool { + SQLCodeProjection.code(of: script, grammar: grammar) + .split { !($0.isLetter || $0.isNumber || $0 == "_") } + .contains { blockKeywords.contains($0.uppercased()) } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengStatementClassifier.swift b/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengStatementClassifier.swift new file mode 100644 index 0000000000..c61ef9fed3 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengStatementClassifier.swift @@ -0,0 +1,51 @@ +import Foundation +import TableProSQLGrammar + +// A framing hint and the replay rule, never a verdict on what the user sees: the bridge converts whatever comes back. +public enum DamengStatementClassifier { + private static let resultKeywords: Set = [ + "DESC", "DESCRIBE", "EXPLAIN", "SELECT", "SHOW", "VALUES", "WITH" + ] + + // Shorter than resultKeywords: WITH and VALUES can head a write, and a replayed write may apply twice. + private static let readOnlyKeywords: Set = [ + "DESC", "DESCRIBE", "EXPLAIN", "SELECT", "SHOW" + ] + + private static let openParenthesis = UInt16(UnicodeScalar("(").value) + private static let semicolon = UInt16(UnicodeScalar(";").value) + + public static func likelyReturnsRows(_ query: String) -> Bool { + guard let keyword = firstKeyword(query) else { return false } + return resultKeywords.contains(keyword) + } + + public static func isReadOnly(_ query: String) -> Bool { + guard let keyword = firstKeyword(query) else { return false } + return readOnlyKeywords.contains(keyword) + } + + private static func firstKeyword(_ query: String) -> String? { + let text = query as NSString + let grammar = DamengGrammar.execution(backslashEscapes: false) + var index = 0 + while index < text.length { + if let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar) { + guard span.kind.isComment else { return nil } + index = max(span.end, index + 1) + continue + } + let blank = StatementBlank.blankLength(in: text, at: index) + if blank > 0 { + index += blank + continue + } + let unit = text.character(at: index) + guard unit == openParenthesis || unit == semicolon else { break } + index += 1 + } + guard index < text.length else { return nil } + let keyword = text.substring(from: index).prefix { $0.isLetter || $0 == "_" } + return keyword.isEmpty ? nil : keyword.uppercased() + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProDatabase/SQLEscaping.swift b/apple/Packages/TableProCore/Sources/TableProDatabase/SQLEscaping.swift index 4a31b07ddf..f992ca4a5e 100644 --- a/apple/Packages/TableProCore/Sources/TableProDatabase/SQLEscaping.swift +++ b/apple/Packages/TableProCore/Sources/TableProDatabase/SQLEscaping.swift @@ -3,17 +3,17 @@ import Foundation public enum SQLEscaping { public static func ansiStringLiteral(_ value: String) -> String { value - .replacingOccurrences(of: "\0", with: "") - .replacingOccurrences(of: "'", with: "''") + .replacingOccurrences(of: "\0", with: "", options: .literal) + .replacingOccurrences(of: "'", with: "''", options: .literal) } public static func backslashStringLiteral(_ value: String) -> String { value - .replacingOccurrences(of: "\\", with: "\\\\") - .replacingOccurrences(of: "\0", with: "\\0") - .replacingOccurrences(of: "\n", with: "\\n") - .replacingOccurrences(of: "\r", with: "\\r") - .replacingOccurrences(of: "\u{1a}", with: "\\Z") - .replacingOccurrences(of: "'", with: "''") + .replacingOccurrences(of: "\\", with: "\\\\", options: .literal) + .replacingOccurrences(of: "\0", with: "\\0", options: .literal) + .replacingOccurrences(of: "\n", with: "\\n", options: .literal) + .replacingOccurrences(of: "\r", with: "\\r", options: .literal) + .replacingOccurrences(of: "\u{1a}", with: "\\Z", options: .literal) + .replacingOccurrences(of: "'", with: "''", options: .literal) } } diff --git a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Config/FirebaseConnectionSettings.swift b/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Config/FirebaseConnectionSettings.swift index 84175aef2d..b76179a1c6 100644 --- a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Config/FirebaseConnectionSettings.swift +++ b/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Config/FirebaseConnectionSettings.swift @@ -113,8 +113,16 @@ public struct FirebaseConnectionSettings: Sendable, Equatable { return url } let trusted = host.hasSuffix(".firebaseio.com") || host.hasSuffix(".firebasedatabase.app") - guard scheme == "https", trusted else { throw invalidURL(text) } - return URL(string: "https://\(host)/") ?? url + guard scheme == "https", trusted, isDNSName(host), let instance = URL(string: "https://\(host)/") else { + throw invalidURL(text) + } + return instance + } + + // `URL.host` is percent-decoded, so an encoded `/`, `:`, `?`, `#` or `@` would pass the suffix check and then end + // the host early once the text is parsed again. + static func isDNSName(_ host: String) -> Bool { + host.range(of: #"^[a-z0-9-]+(\.[a-z0-9-]+)+$"#, options: .regularExpression) != nil } static func parseEmulatorHost(_ text: String?) throws -> String { diff --git a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreQuery.swift b/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreQuery.swift index 8760f9b3f7..9fc630c138 100644 --- a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreQuery.swift +++ b/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreQuery.swift @@ -268,7 +268,7 @@ public enum FirestoreQueryBuilder { if column == nameField { return column } let simple = column.range(of: "^[A-Za-z_][A-Za-z0-9_]*$", options: .regularExpression) != nil if simple { return column } - let escaped = column.replacingOccurrences(of: "\\", with: "\\\\").replacingOccurrences(of: "`", with: "\\`") + let escaped = column.replacingOccurrences(of: "\\", with: "\\\\", options: .literal).replacingOccurrences(of: "`", with: "\\`", options: .literal) return "`" + escaped + "`" } diff --git a/apple/Packages/TableProCore/Sources/TableProImport/Bundle/ConnectionBundle+Validation.swift b/apple/Packages/TableProCore/Sources/TableProImport/Bundle/ConnectionBundle+Validation.swift index c7780f3ed9..c45cbd0787 100644 --- a/apple/Packages/TableProCore/Sources/TableProImport/Bundle/ConnectionBundle+Validation.swift +++ b/apple/Packages/TableProCore/Sources/TableProImport/Bundle/ConnectionBundle+Validation.swift @@ -7,6 +7,7 @@ enum BundleViolation: Equatable, Sendable { case unresolvedCredentials(BundleRef) case cycle(BundleRef) case tooDeep(BundleRef) + case invalidSSHDestination(connection: String) /// Deeper than any folder or group the app creates, and shallow enough that a crafted file cannot /// make every query row carry a path of thousands of names. @@ -30,6 +31,8 @@ enum BundleViolation: Equatable, Sendable { String(format: String(localized: "the reference “%@” is inside itself"), ref.rawValue) case .tooDeep(let ref): String(format: String(localized: "the reference “%@” is nested too deeply"), ref.rawValue) + case .invalidSSHDestination(let connection): + String(format: String(localized: "the SSH host or user of “%@” contains characters ssh does not allow"), connection) } } @@ -51,6 +54,9 @@ enum BundleViolation: Equatable, Sendable { for refs in refLists { if let violation = identityViolation(refs) { return violation } } + if let connection = connections.first(where: { $0.settings.refusedSSHDestination != nil }) { + return .invalidSSHDestination(connection: connection.settings.name) + } let connectionRefs = Set(connections.map(\.ref)) let groupRefs = Set(groups.map(\.ref)) diff --git a/apple/Packages/TableProCore/Sources/TableProImport/Bundle/ExportableConnection.swift b/apple/Packages/TableProCore/Sources/TableProImport/Bundle/ExportableConnection.swift index 1ad029607f..178826dc3c 100644 --- a/apple/Packages/TableProCore/Sources/TableProImport/Bundle/ExportableConnection.swift +++ b/apple/Packages/TableProCore/Sources/TableProImport/Bundle/ExportableConnection.swift @@ -1,5 +1,6 @@ import Foundation import TableProConnectionLibrary +import TableProSSHTransport import UniformTypeIdentifiers public extension UTType { @@ -143,17 +144,24 @@ public extension ExportableConnection { private static let queryTimeoutSecondsRange = 0 ... Int(Int32.max) / 1_000 static let importBlockedAdditionalFieldKeys: Set = [ + "beancountrunledgerplugins", "postdisconnectscript", "preconnectscript", "pretunnelhost", "pretunnelport", "promptforpassword", + "snowflakeconnectionname", "spendpoint", "sslclientkeypassphrase", "usepgpass", ] - static let importBlockedAdditionalFieldPrefixes: Set = ["aws"] + static let importBlockedAdditionalFieldPrefixes: Set = ["aws", "entra"] + + // Values that pick a credential kept on this Mac, under keys whose other values are plain settings. + static let importBlockedAdditionalFieldValues: [String: Set] = [ + "mongoauthmechanism": ["mongodb-aws"], + ] static func isImportBlockedAdditionalFieldKey(_ key: String) -> Bool { let normalized = key.lowercased() @@ -161,12 +169,19 @@ public extension ExportableConnection { return importBlockedAdditionalFieldPrefixes.contains { normalized.hasPrefix($0) } } + static func isImportBlockedAdditionalField(key: String, value: String) -> Bool { + if isImportBlockedAdditionalFieldKey(key) { return true } + if ConnectionSecretOptions.isSecret(key: key, value: value) { return true } + let blockedValues = importBlockedAdditionalFieldValues[key.lowercased()] ?? [] + return blockedValues.contains(value.trimmingCharacters(in: .whitespaces).lowercased()) + } + static func shareableAdditionalFields( _ fields: [String: String], excluding excludedKeys: Set = [] ) -> [String: String]? { - let shareable = fields.filter { key, _ in - !excludedKeys.contains(key) && !isImportBlockedAdditionalFieldKey(key) + let shareable = fields.filter { key, value in + !excludedKeys.contains(key) && !isImportBlockedAdditionalField(key: key, value: value) } return shareable.isEmpty ? nil : shareable } @@ -191,7 +206,7 @@ public extension ExportableConnection { /// An explicit timeout that is out of range is dropped, not replaced by a legacy additional field. func sanitizedForImport() -> ExportableConnection { - var allowed = (additionalFields ?? [:]).filter { !Self.isImportBlockedAdditionalFieldKey($0.key) } + var allowed = (additionalFields ?? [:]).filter { !Self.isImportBlockedAdditionalField(key: $0.key, value: $0.value) } let legacyConnectTimeout = allowed.removeValue(forKey: Self.connectTimeoutSecondsKey).flatMap(Int.init) let legacyQueryTimeout = allowed.removeValue(forKey: Self.queryTimeoutSecondsKey).flatMap(Int.init) var copy = self @@ -243,6 +258,15 @@ public struct ExportableTunnelCommand: Codable, Sendable, Equatable { } } +public extension ExportableConnection { + // Match exec hands the host and user to /bin/sh, and the host key store is one entry per line. + var refusedSSHDestination: SSHDestinationName.Problem? { + guard let ssh = sshConfig, ssh.enabled else { return nil } + let hops = [(ssh.host, ssh.username)] + (ssh.jumpHosts ?? []).map { ($0.host, $0.username) } + return hops.lazy.compactMap { SSHDestinationName.problem(host: $0.0, user: $0.1) }.first + } +} + public struct ExportableSSHConfig: Codable, Sendable, Equatable { public var enabled: Bool public var host: String diff --git a/apple/Packages/TableProCore/Sources/TableProImport/Import/ImportPlanner.swift b/apple/Packages/TableProCore/Sources/TableProImport/Import/ImportPlanner.swift index 5cf8e769e7..aa939d542c 100644 --- a/apple/Packages/TableProCore/Sources/TableProImport/Import/ImportPlanner.swift +++ b/apple/Packages/TableProCore/Sources/TableProImport/Import/ImportPlanner.swift @@ -88,7 +88,8 @@ extension ImportPlanner { ref: profile.ref, name: profile.name, username: profile.username, - passwordMode: profile.passwordMode == .pgpass ? .pgpass : .prompt, + // A file never chooses how this Mac finds a password: pgpass would read ~/.pgpass for its host. + passwordMode: .prompt, secureFieldIds: profile.secureFieldIds )) } diff --git a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxCSV.swift b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxCSV.swift new file mode 100644 index 0000000000..938f6b2202 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxCSV.swift @@ -0,0 +1,79 @@ +import Foundation + +/// RFC 4180 records. A blank line comes back as an empty record because the Flux annotated CSV +/// uses it to separate one table from the next. +enum InfluxCSV { + static func records(_ text: String) -> [[String]] { + var records: [[String]] = [] + var record: [String] = [] + var field = "" + var inQuotes = false + var fieldStarted = false + var iterator = Array(text.unicodeScalars).makeIterator() + var pending: Unicode.Scalar? + + func next() -> Unicode.Scalar? { + if let scalar = pending { + pending = nil + return scalar + } + return iterator.next() + } + + func endRecord() { + if fieldStarted || !record.isEmpty { + record.append(field) + records.append(record) + } else { + records.append([]) + } + record = [] + field = "" + fieldStarted = false + } + + while let scalar = next() { + if inQuotes { + if scalar == "\"" { + if let following = next() { + if following == "\"" { + field.unicodeScalars.append("\"") + } else { + inQuotes = false + pending = following + } + } else { + inQuotes = false + } + } else { + field.unicodeScalars.append(scalar) + } + continue + } + switch scalar { + case "\"": + inQuotes = true + fieldStarted = true + case ",": + record.append(field) + field = "" + fieldStarted = true + case "\r": + if let following = next(), following != "\n" { + pending = following + } + endRecord() + case "\n": + endRecord() + default: + field.unicodeScalars.append(scalar) + fieldStarted = true + } + } + if fieldStarted || !record.isEmpty { + record.append(field) + records.append(record) + } + return records + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxError.swift b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxError.swift new file mode 100644 index 0000000000..2d3681778d --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxError.swift @@ -0,0 +1,43 @@ +import Foundation + +public enum InfluxError: Error, Equatable, LocalizedError { + case notConnected + case invalidURL(String) + case unauthorized(String) + case server(status: Int, message: String) + case query(String) + case malformedResponse(String) + case missingDatabase + case missingOrganization + case unsupportedLanguage(InfluxQueryLanguage, generation: InfluxGeneration) + + public var errorDescription: String? { + switch self { + case .notConnected: + return String(localized: "Not connected to InfluxDB.") + case .invalidURL(let text): + return String(format: String(localized: "Invalid InfluxDB address: %@"), text) + case .unauthorized(let message): + return message.isEmpty + ? String(localized: "InfluxDB refused the token.") + : String(format: String(localized: "InfluxDB refused the token: %@"), message) + case .server(let status, let message): + return message.isEmpty ? "HTTP \(status)" : "HTTP \(status): \(message)" + case .query(let message): + return message + case .malformedResponse(let detail): + return String(format: String(localized: "Unexpected InfluxDB response: %@"), detail) + case .missingDatabase: + return String(localized: "Choose a database or bucket first.") + case .missingOrganization: + return String(localized: "InfluxDB 2 needs an organization. Set it in the connection settings.") + case .unsupportedLanguage(let language, let generation): + let available = generation.languages.map(\.displayName).joined(separator: ", ") + return String( + format: String(localized: "%@ is not available on this server. It accepts %@."), + language.displayName, + available + ) + } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxQueryLanguage.swift b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxQueryLanguage.swift new file mode 100644 index 0000000000..069220c585 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxQueryLanguage.swift @@ -0,0 +1,94 @@ +import Foundation + +public enum InfluxQueryLanguage: String, Sendable, Equatable, CaseIterable { + case sql + case influxQL = "influxql" + case flux + + public var displayName: String { + switch self { + case .sql: return "SQL" + case .influxQL: return "InfluxQL" + case .flux: return "Flux" + } + } +} + +/// Picks the language a statement is sent in. The server version decides which languages exist; +/// within them the statement's own text decides, so the grid's `SELECT` and a typed Flux pipeline +/// both run without a setting to flip. A first-line comment naming a language (`-- influxql`, +/// `// flux`) overrides the guess. +public enum InfluxLanguageRouter { + public static func language( + for statement: String, + generation: InfluxGeneration + ) throws -> InfluxQueryLanguage { + let language = hint(in: statement) ?? guess(statement, generation: generation) + guard generation.languages.contains(language) else { + throw InfluxError.unsupportedLanguage(language, generation: generation) + } + return language + } + + static func hint(in statement: String) -> InfluxQueryLanguage? { + let firstLine = statement + .trimmingCharacters(in: .whitespacesAndNewlines) + .split(separator: "\n", maxSplits: 1, omittingEmptySubsequences: false) + .first + .map(String.init) ?? "" + let body: Substring + if firstLine.hasPrefix("--") { + body = firstLine.dropFirst(2) + } else if firstLine.hasPrefix("//") { + body = firstLine.dropFirst(2) + } else { + return nil + } + return InfluxQueryLanguage(rawValue: body.trimmingCharacters(in: .whitespaces).lowercased()) + } + + static func guess(_ statement: String, generation: InfluxGeneration) -> InfluxQueryLanguage { + let code = strippingLeadingComments(statement) + if looksLikeFlux(code) { + return .flux + } + switch generation { + case .v1, .v2: + return .influxQL + case .v3: + return looksLikeInfluxQLOnly(code) ? .influxQL : .sql + } + } + + static func looksLikeFlux(_ code: String) -> Bool { + if code.contains("|>") { return true } + let lowered = code.lowercased() + return ["from(", "import ", "buckets(", "option "].contains { lowered.hasPrefix($0) } + } + + private static let influxQLOnlyPrefixes = [ + "SHOW MEASUREMENT", "SHOW TAG", "SHOW FIELD", "SHOW SERIES", "SHOW RETENTION", "SHOW DATABASES", + ] + + /// Statements DataFusion SQL rejects but InfluxQL reads: its `SHOW` catalog commands, + /// `GROUP BY time(...)` buckets and duration literals such as `now() - 1h`. + static func looksLikeInfluxQLOnly(_ code: String) -> Bool { + let normalized = code.uppercased() + .components(separatedBy: .whitespacesAndNewlines) + .filter { !$0.isEmpty } + .joined(separator: " ") + if influxQLOnlyPrefixes.contains(where: normalized.hasPrefix) { return true } + if normalized.range(of: #"GROUP BY .*\bTIME\s*\("#, options: .regularExpression) != nil { return true } + return normalized.range(of: #"NOW\(\)\s*[-+]\s*\d+(NS|U|µ|MS|S|M|H|D|W)\b"#, options: .regularExpression) != nil + } + + static func strippingLeadingComments(_ statement: String) -> String { + var lines = statement.components(separatedBy: .newlines)[...] + while let first = lines.first { + let trimmed = first.trimmingCharacters(in: .whitespaces) + guard trimmed.isEmpty || trimmed.hasPrefix("--") || trimmed.hasPrefix("//") else { break } + lines = lines.dropFirst() + } + return lines.joined(separator: "\n").trimmingCharacters(in: .whitespacesAndNewlines) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxRequest.swift b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxRequest.swift new file mode 100644 index 0000000000..f807a0b24e --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxRequest.swift @@ -0,0 +1,219 @@ +import Foundation + +public struct InfluxRequest: Sendable, Equatable { + public let method: String + public let path: String + public let queryItems: [URLQueryItem] + public let headers: [String: String] + public let body: Data? + + public init( + method: String, + path: String, + queryItems: [URLQueryItem] = [], + headers: [String: String] = [:], + body: Data? = nil + ) { + self.method = method + self.path = path + self.queryItems = queryItems + self.headers = headers + self.body = body + } + + public func url(base: URL) -> URL? { + guard var components = URLComponents(url: base, resolvingAgainstBaseURL: false) else { return nil } + let basePath = components.path.hasSuffix("/") ? String(components.path.dropLast()) : components.path + components.path = basePath + path + components.queryItems = queryItems.isEmpty ? nil : queryItems + // `+` is legal in a query, but every InfluxDB line reads it as a space. + components.percentEncodedQuery = components.percentEncodedQuery? + .replacingOccurrences(of: "+", with: "%2B") + return components.url + } +} + +public enum InfluxCredentials: Sendable, Equatable { + case none + case token(String) + + public init(token: String?) { + let token = (token ?? "").trimmingCharacters(in: .whitespacesAndNewlines) + self = token.isEmpty ? .none : .token(token) + } + + /// 2.x reads `Token`, 3.x reads `Bearer`; both accept the other's word today, but the + /// documented one is what a proxy in front of them is most likely to pass through. + func authorization(for generation: InfluxGeneration?) -> String? { + switch self { + case .none: + return nil + case .token(let token): + return generation == .v3 ? "Bearer \(token)" : "Token \(token)" + } + } +} + +public enum InfluxRequests { + /// 3.x with auth on refuses an anonymous `/ping`; before the version is known the token goes + /// out with the `Token` word, which every line accepts. + public static func ping(credentials: InfluxCredentials) -> InfluxRequest { + InfluxRequest(method: "GET", path: "/ping", headers: authHeaders(credentials, generation: nil)) + } + + public static func listDatabases( + generation: InfluxGeneration, + organization: String, + credentials: InfluxCredentials, + offset: Int = 0 + ) -> InfluxRequest { + let headers = authHeaders(credentials, generation: generation) + switch generation { + case .v3: + return InfluxRequest( + method: "GET", + path: "/api/v3/configure/database", + queryItems: [URLQueryItem(name: "format", value: "json")], + headers: headers + ) + case .v2: + return InfluxRequest( + method: "GET", + path: "/api/v2/buckets", + queryItems: [ + URLQueryItem(name: "org", value: organization), + URLQueryItem(name: "limit", value: String(bucketPageSize)), + URLQueryItem(name: "offset", value: String(offset)), + ], + headers: headers + ) + case .v1: + return query("SHOW DATABASES", language: .influxQL, database: "", organization: organization, + generation: generation, credentials: credentials) + } + } + + public static let bucketPageSize = 100 + + public static func listDBRPs(organization: String, credentials: InfluxCredentials) -> InfluxRequest { + InfluxRequest( + method: "GET", + path: "/api/v2/dbrps", + queryItems: [URLQueryItem(name: "org", value: organization)], + headers: authHeaders(credentials, generation: .v2) + ) + } + + public static func listOrganizations(credentials: InfluxCredentials) -> InfluxRequest { + InfluxRequest(method: "GET", path: "/api/v2/orgs", headers: authHeaders(credentials, generation: .v2)) + } + + public static func query( + _ statement: String, + language: InfluxQueryLanguage, + database: String, + organization: String, + generation: InfluxGeneration, + credentials: InfluxCredentials + ) -> InfluxRequest { + var headers = authHeaders(credentials, generation: generation) + switch (generation, language) { + case (.v3, .sql), (.v3, .influxQL), (.v3, .flux): + headers["Content-Type"] = "application/json" + headers["Accept"] = "text/csv" + let payload: [String: String] = ["db": database, "q": statement, "format": "csv"] + return InfluxRequest( + method: "POST", + path: language == .sql ? "/api/v3/query_sql" : "/api/v3/query_influxql", + headers: headers, + body: try? JSONSerialization.data(withJSONObject: payload, options: [.sortedKeys]) + ) + case (_, .flux): + headers["Content-Type"] = "application/json" + headers["Accept"] = "application/csv" + let payload: [String: Any] = [ + "query": statement, + "type": "flux", + "dialect": ["annotations": ["datatype"], "header": true], + ] + return InfluxRequest( + method: "POST", + path: "/api/v2/query", + queryItems: [URLQueryItem(name: "org", value: organization)], + headers: headers, + body: try? JSONSerialization.data(withJSONObject: payload, options: [.sortedKeys]) + ) + case (_, .sql), (_, .influxQL): + headers["Content-Type"] = "application/x-www-form-urlencoded" + headers["Accept"] = "application/json" + var form = URLComponents() + form.queryItems = [URLQueryItem(name: "q", value: statement)] + if !database.isEmpty { + form.queryItems?.append(URLQueryItem(name: "db", value: database)) + } + let body = (form.percentEncodedQuery ?? "").replacingOccurrences(of: "+", with: "%2B") + return InfluxRequest(method: "POST", path: "/query", headers: headers, body: Data(body.utf8)) + } + } + + static func authHeaders(_ credentials: InfluxCredentials, generation: InfluxGeneration?) -> [String: String] { + guard let value = credentials.authorization(for: generation) else { return [:] } + return ["Authorization": value] + } +} + +public enum InfluxCatalogDecoder { + public static let systemDatabaseNames = ["_internal", "_monitoring", "_tasks"] + + /// `/api/v3/configure/database?format=json`: `[{"iox::database": "metrics"}]`. + public static func v3Databases(_ data: Data) throws -> [String] { + guard let rows = try? JSONSerialization.jsonObject(with: data) as? [[String: Any]] else { + throw InfluxError.malformedResponse(String(bytes: data.prefix(200), encoding: .utf8) ?? "") + } + return rows.compactMap { $0["iox::database"] as? String } + } + + /// `/api/v2/orgs`: `{"orgs": [{"name": "tp"}, ...]}`. + public static func v2Organizations(_ data: Data) throws -> [String] { + guard let root = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let orgs = root["orgs"] as? [[String: Any]] + else { + throw InfluxError.malformedResponse(String(bytes: data.prefix(200), encoding: .utf8) ?? "") + } + return orgs.compactMap { $0["name"] as? String } + } + + /// `/api/v2/buckets`: `{"buckets": [{"id": "1ef8...", "name": "metrics"}, ...]}`, as id and name. + public static func v2Buckets(_ data: Data) throws -> [(id: String, name: String)] { + guard let root = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let buckets = root["buckets"] as? [[String: Any]] + else { + throw InfluxError.malformedResponse(String(bytes: data.prefix(200), encoding: .utf8) ?? "") + } + return buckets.compactMap { bucket in + guard let name = bucket["name"] as? String else { return nil } + return (bucket["id"] as? String ?? "", name) + } + } + + /// `/api/v2/dbrps`: the InfluxQL database name each bucket answers to, keyed by bucket id. + /// InfluxQL resolves `db` through these mappings, not through bucket names. A mapping someone + /// made wins over the virtual one 2.x makes per bucket, and a default one over the rest. + public static func v2InfluxQLDatabases(_ data: Data) throws -> [String: String] { + guard let root = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let mappings = root["content"] as? [[String: Any]] + else { + throw InfluxError.malformedResponse(String(bytes: data.prefix(200), encoding: .utf8) ?? "") + } + func rank(_ mapping: [String: Any]) -> Int { + (mapping["default"] as? Bool == true ? 2 : 0) + (mapping["virtual"] as? Bool == true ? 0 : 1) + } + var best: [String: [String: Any]] = [:] + for mapping in mappings { + guard let bucket = mapping["bucketID"] as? String, mapping["database"] is String else { continue } + if let current = best[bucket], rank(current) >= rank(mapping) { continue } + best[bucket] = mapping + } + return best.compactMapValues { $0["database"] as? String } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxResponseDecoder.swift b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxResponseDecoder.swift new file mode 100644 index 0000000000..ece22ca03e --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxResponseDecoder.swift @@ -0,0 +1,217 @@ +import Foundation + +public enum InfluxResponseDecoder { + // MARK: - 3.x CSV + + /// `/api/v3/query_sql` and `/api/v3/query_influxql` with `format=csv`. CSV is the one 3.x format + /// that keeps the column order of the statement; JSON returns objects and drops null columns. + /// An empty cell reads as null, since the format has no other way to write one. + public static func decodeV3CSV(_ text: String) -> InfluxResultTable { + var records = InfluxCSV.records(text).filter { !$0.isEmpty } + guard !records.isEmpty else { return .empty } + let columns = records.removeFirst() + let rows: [[String?]] = records.map { record in + columns.indices.map { index in + guard index < record.count, !record[index].isEmpty else { return nil } + return record[index] + } + } + let typeNames = columns.enumerated().map { index, column in + InfluxTypeName.inferred(column: column, values: rows.map { $0[index] }) + } + let normalized = rows.map { row in + row.enumerated().map { index, value in + typeNames[index] == InfluxTypeName.timestamp + ? value.map(InfluxTypeName.normalizedTimestamp) + : value + } + } + return InfluxResultTable(columns: columns, typeNames: typeNames, rows: normalized) + } + + // MARK: - InfluxQL over /query (1.x and the 2.x compatibility API) + + /// Flattens every series of every statement into one grid. A statement that groups by tag + /// returns one series per tag set; the tags become leading columns, and when the series name + /// varies (`SHOW FIELD KEYS` over several measurements) a `name` column comes first. + public static func decodeV1JSON(_ data: Data) throws -> InfluxResultTable { + guard let root = try? JSONSerialization.jsonObject(with: data) as? [String: Any] else { + throw InfluxError.malformedResponse(String(bytes: data.prefix(200), encoding: .utf8) ?? "") + } + if let error = root["error"] as? String { + throw InfluxError.query(error) + } + let results = root["results"] as? [[String: Any]] ?? [] + if let error = results.compactMap({ $0["error"] as? String }).first { + throw InfluxError.query(error) + } + let series = results.flatMap { $0["series"] as? [[String: Any]] ?? [] } + guard !series.isEmpty else { return .empty } + + let names = Set(series.compactMap { $0["name"] as? String }) + let includesName = names.count > 1 + var tagKeys: [String] = [] + var valueColumns: [String] = [] + for item in series { + for key in (item["tags"] as? [String: Any] ?? [:]).keys.sorted() where !tagKeys.contains(key) { + tagKeys.append(key) + } + for column in item["columns"] as? [String] ?? [] where !valueColumns.contains(column) { + valueColumns.append(column) + } + } + valueColumns.removeAll { tagKeys.contains($0) } + let columns = (includesName ? ["name"] : []) + tagKeys + valueColumns + + var rows: [[String?]] = [] + var jsonTypes = [String?](repeating: nil, count: columns.count) + for item in series { + let seriesColumns = item["columns"] as? [String] ?? [] + let tags = item["tags"] as? [String: Any] ?? [:] + for values in item["values"] as? [[Any]] ?? [] { + var row = [String?](repeating: nil, count: columns.count) + var offset = 0 + if includesName { + row[0] = item["name"] as? String + offset = 1 + } + for (index, key) in tagKeys.enumerated() { + row[offset + index] = text(tags[key]) + } + for (index, column) in seriesColumns.enumerated() where index < values.count { + guard let target = columns.firstIndex(of: column) else { continue } + row[target] = text(values[index]) + if jsonTypes[target] == nil { + jsonTypes[target] = typeName(of: values[index]) + } + } + rows.append(row) + } + } + let typeNames = columns.enumerated().map { index, column -> String in + if column == InfluxTypeName.timeColumn { return InfluxTypeName.timestamp } + if tagKeys.contains(column) { return InfluxTypeName.tag } + return jsonTypes[index] ?? InfluxTypeName.string + } + return InfluxResultTable(columns: columns, typeNames: typeNames, rows: rows) + } + + // MARK: - Flux annotated CSV (2.x) + + /// Each Flux table arrives as its own block with its own header. The grid gets the union of + /// the headers in first-seen order, without the empty annotation column and `result`. + public static func decodeFluxCSV(_ text: String) throws -> InfluxResultTable { + var columns: [String] = [] + var typeNames: [String] = [] + var rows: [[String?]] = [] + + var datatypes: [String] = [] + var header: [String]? + for record in InfluxCSV.records(text) { + if record.isEmpty || record == [""] { + datatypes = [] + header = nil + continue + } + if let first = record.first, first.hasPrefix("#") { + if first == "#datatype" { datatypes = record } + continue + } + guard let currentHeader = header else { + header = record + if isFluxErrorHeader(record) { continue } + for (index, column) in record.enumerated() where keepsFluxColumn(column, index: index) { + if !columns.contains(column) { + columns.append(column) + typeNames.append(fluxTypeName(index < datatypes.count ? datatypes[index] : "")) + } + } + continue + } + if isFluxErrorHeader(currentHeader) { + throw InfluxError.query(record.count > 1 ? record[1] : "Flux error") + } + var row = [String?](repeating: nil, count: columns.count) + for (index, column) in currentHeader.enumerated() where keepsFluxColumn(column, index: index) { + guard index < record.count, let target = columns.firstIndex(of: column) else { continue } + row[target] = record[index].isEmpty ? nil : record[index] + } + rows.append(row) + } + for index in rows.indices where rows[index].count < columns.count { + rows[index] += [String?](repeating: nil, count: columns.count - rows[index].count) + } + return InfluxResultTable(columns: columns, typeNames: typeNames, rows: rows) + } + + private static func isFluxErrorHeader(_ header: [String]) -> Bool { + header.dropFirst().elementsEqual(["error", "reference"]) + } + + private static func keepsFluxColumn(_ column: String, index: Int) -> Bool { + !(index == 0 && column.isEmpty) && column != "result" + } + + static func fluxTypeName(_ annotation: String) -> String { + switch annotation { + case "dateTime:RFC3339", "dateTime:RFC3339Nano": return InfluxTypeName.timestamp + case "long": return InfluxTypeName.integer + case "unsignedLong": return InfluxTypeName.unsigned + case "double": return InfluxTypeName.float + case "boolean": return InfluxTypeName.boolean + case "duration": return "duration" + case "base64Binary": return "binary" + default: return InfluxTypeName.string + } + } + + // MARK: - Errors + + /// The message out of an error body: `{"message": ...}` on 2.x, `{"error": ...}` on 1.x and some + /// 3.x paths, plain text on others. + public static func errorMessage(from data: Data) -> String { + if let object = try? JSONSerialization.jsonObject(with: data) as? [String: Any] { + for key in ["message", "error", "data"] { + if let message = object[key] as? String, !message.isEmpty { return message } + } + } + return (String(bytes: data.prefix(2_000), encoding: .utf8) ?? "") + .trimmingCharacters(in: .whitespacesAndNewlines) + } + + // MARK: - Values + + static func text(_ value: Any?) -> String? { + switch value { + case nil, is NSNull: + return nil + case let string as String: + return string + case let number as NSNumber: + if CFGetTypeID(number) == CFBooleanGetTypeID() { + return number.boolValue ? "true" : "false" + } + return number.stringValue + case let other?: + if JSONSerialization.isValidJSONObject(other), + let data = try? JSONSerialization.data(withJSONObject: other, options: [.sortedKeys]) { + return String(bytes: data, encoding: .utf8) ?? "" + } + return "\(other)" + } + } + + static func typeName(of value: Any) -> String? { + switch value { + case is NSNull: + return nil + case is String: + return InfluxTypeName.string + case let number as NSNumber: + if CFGetTypeID(number) == CFBooleanGetTypeID() { return InfluxTypeName.boolean } + return CFNumberIsFloatType(number) ? InfluxTypeName.float : InfluxTypeName.integer + default: + return InfluxTypeName.string + } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxResultTable.swift b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxResultTable.swift new file mode 100644 index 0000000000..a7ce8d00d5 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxResultTable.swift @@ -0,0 +1,54 @@ +import Foundation + +public struct InfluxResultTable: Sendable, Equatable { + public var columns: [String] + public var typeNames: [String] + public var rows: [[String?]] + + public init(columns: [String], typeNames: [String], rows: [[String?]]) { + self.columns = columns + self.typeNames = typeNames + self.rows = rows + } + + public static let empty = InfluxResultTable(columns: [], typeNames: [], rows: []) + + public func column(named name: String) -> [String?] { + guard let index = columns.firstIndex(of: name) else { return [] } + return rows.map { index < $0.count ? $0[index] : nil } + } +} + +public enum InfluxTypeName { + public static let timestamp = "timestamp" + public static let tag = "tag" + public static let float = "float" + public static let integer = "integer" + public static let unsigned = "unsigned" + public static let string = "string" + public static let boolean = "boolean" + + public static let timeColumn = "time" + + /// Names the text a column holds when the response carries no types (the 3.x CSV format): + /// integer, float or boolean when every value reads as one, else string. + static func inferred(column: String, values: [String?]) -> String { + if column == timeColumn { return timestamp } + let present = values.compactMap { $0 } + guard !present.isEmpty else { return string } + if present.allSatisfy({ Int64($0) != nil }) { return integer } + if present.allSatisfy({ UInt64($0) != nil }) { return unsigned } + if present.allSatisfy({ Double($0) != nil }) { return float } + if present.allSatisfy({ $0 == "true" || $0 == "false" }) { return boolean } + return string + } + + /// 3.x returns `2024-10-27T03:33:20` for a UTC timestamp. Adding the zone makes it read the + /// same as the 2.x value and parse as RFC 3339 when pasted back into a filter. + static func normalizedTimestamp(_ value: String) -> String { + guard value.count >= 19, value.dropFirst(10).first == "T" else { return value } + let time = value.dropFirst(11) + if time.contains("Z") || time.contains("+") || time.contains("-") { return value } + return value + "Z" + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxSchema.swift b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxSchema.swift new file mode 100644 index 0000000000..e2eeedc8c3 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxSchema.swift @@ -0,0 +1,107 @@ +import Foundation + +public struct InfluxColumn: Sendable, Equatable { + public enum Role: Sendable, Equatable { + case time + case tag + case field + } + + public let name: String + public let typeName: String + public let role: Role + + public init(name: String, typeName: String, role: Role) { + self.name = name + self.typeName = typeName + self.role = role + } + + public var isText: Bool { + role == .tag || typeName == InfluxTypeName.string + } +} + +/// The catalog statements behind the sidebar, and the mapping of what they return into columns. +/// Columns come back with `time` first and the rest by name, the order InfluxQL's `SELECT *` uses. +public enum InfluxSchema { + public static func measurementsStatement(generation: InfluxGeneration) -> (String, InfluxQueryLanguage) { + switch generation { + case .v3: + return ( + "SELECT table_name FROM information_schema.tables WHERE table_schema = 'iox' ORDER BY table_name", + .sql + ) + case .v1, .v2: + return ("SHOW MEASUREMENTS", .influxQL) + } + } + + public static func measurements(from table: InfluxResultTable, generation: InfluxGeneration) -> [String] { + let column = generation == .v3 ? "table_name" : "name" + return table.column(named: column).compactMap { $0 } + } + + public static func v3ColumnsStatement(measurement: String) -> String { + let literal = InfluxSelectBuilder.stringLiteral(measurement, language: .sql) + return "SELECT column_name, data_type FROM information_schema.columns " + + "WHERE table_schema = 'iox' AND table_name = \(literal)" + } + + public static func v3Columns(from table: InfluxResultTable) -> [InfluxColumn] { + let names = table.column(named: "column_name") + let types = table.column(named: "data_type") + let columns = zip(names, types).compactMap { name, arrowType -> InfluxColumn? in + guard let name else { return nil } + return column(name: name, arrowType: arrowType ?? "") + } + return ordered(columns) + } + + public static func v1TagKeysStatement(measurement: String) -> String { + "SHOW TAG KEYS FROM \(InfluxSelectBuilder.quoteIdentifier(measurement, language: .influxQL))" + } + + public static func v1FieldKeysStatement(measurement: String) -> String { + "SHOW FIELD KEYS FROM \(InfluxSelectBuilder.quoteIdentifier(measurement, language: .influxQL))" + } + + public static func v1Columns(tagKeys: InfluxResultTable, fieldKeys: InfluxResultTable) -> [InfluxColumn] { + var columns = [InfluxColumn(name: InfluxTypeName.timeColumn, typeName: InfluxTypeName.timestamp, role: .time)] + for tag in tagKeys.column(named: "tagKey").compactMap({ $0 }) { + columns.append(InfluxColumn(name: tag, typeName: InfluxTypeName.tag, role: .tag)) + } + let names = fieldKeys.column(named: "fieldKey") + let types = fieldKeys.column(named: "fieldType") + for (name, type) in zip(names, types) { + guard let name, !columns.contains(where: { $0.name == name }) else { continue } + columns.append(InfluxColumn(name: name, typeName: type ?? InfluxTypeName.string, role: .field)) + } + return ordered(columns) + } + + static func column(name: String, arrowType: String) -> InfluxColumn { + if arrowType.hasPrefix("Timestamp") { + return InfluxColumn(name: name, typeName: InfluxTypeName.timestamp, role: .time) + } + if arrowType.hasPrefix("Dictionary") { + return InfluxColumn(name: name, typeName: InfluxTypeName.tag, role: .tag) + } + let typeName: String + switch arrowType { + case "Float64", "Float32": typeName = InfluxTypeName.float + case "Int64", "Int32": typeName = InfluxTypeName.integer + case "UInt64", "UInt32": typeName = InfluxTypeName.unsigned + case "Boolean": typeName = InfluxTypeName.boolean + default: typeName = InfluxTypeName.string + } + return InfluxColumn(name: name, typeName: typeName, role: .field) + } + + static func ordered(_ columns: [InfluxColumn]) -> [InfluxColumn] { + let time = columns.filter { $0.name == InfluxTypeName.timeColumn } + let rest = columns.filter { $0.name != InfluxTypeName.timeColumn } + .sorted { $0.name < $1.name } + return time + rest + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxSelectBuilder.swift b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxSelectBuilder.swift new file mode 100644 index 0000000000..3cc09b7336 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxSelectBuilder.swift @@ -0,0 +1,221 @@ +import Foundation + +public struct InfluxFilter: Sendable, Equatable { + public let column: String + public let op: String + public let value: String + public let secondValue: String? + public let isCaseSensitive: Bool + public let isText: Bool + + public init( + column: String, + op: String, + value: String, + secondValue: String? = nil, + isCaseSensitive: Bool = true, + isText: Bool = false + ) { + self.column = column + self.op = op + self.value = value + self.secondValue = secondValue + self.isCaseSensitive = isCaseSensitive + self.isText = isText + } +} + +public struct InfluxSort: Sendable, Equatable { + public let column: String + public let ascending: Bool + + public init(column: String, ascending: Bool) { + self.column = column + self.ascending = ascending + } +} + +/// Writes the grid's `SELECT` in SQL (3.x) or InfluxQL (1.x, 2.x). The two share the shape +/// `SELECT ... FROM "m" WHERE ... ORDER BY time DESC LIMIT n OFFSET m`, so a query built before +/// the server version is known runs on either. They differ in literals, regex syntax and in what +/// InfluxQL leaves out: no `IN`, no `IS NULL`, and `ORDER BY time` only. +public enum InfluxSelectBuilder { + public static func select( + measurement: String, + columns: [String], + filters: [InfluxFilter] = [], + logicMode: String = "and", + sorts: [InfluxSort] = [], + limit: Int, + offset: Int, + language: InfluxQueryLanguage + ) -> String { + let projection = language == .sql && !columns.isEmpty + ? columns.map { quoteIdentifier($0, language: language) }.joined(separator: ", ") + : "*" + var statement = "SELECT \(projection) FROM \(quoteIdentifier(measurement, language: language))" + statement += whereClause(filters, logicMode: logicMode, language: language) + statement += " ORDER BY " + orderBy(sorts, language: language) + statement += " LIMIT \(max(limit, 0)) OFFSET \(max(offset, 0))" + return statement + } + + /// InfluxQL answers `COUNT(*)` with one `count_` column per field, which is the point + /// count only for a measurement with one field; the caller decides whether to ask. + public static func count( + measurement: String, + filters: [InfluxFilter] = [], + logicMode: String = "and", + language: InfluxQueryLanguage + ) -> String { + "SELECT COUNT(*) FROM \(quoteIdentifier(measurement, language: language))" + + whereClause(filters, logicMode: logicMode, language: language) + } + + public static func rowCount(from table: InfluxResultTable) -> Int? { + guard let row = table.rows.first else { return 0 } + let counts = zip(table.columns, row).compactMap { column, value -> Int? in + guard column.lowercased().hasPrefix("count"), let value else { return nil } + return Int(value) + } + return counts.max() + } + + static func whereClause(_ filters: [InfluxFilter], logicMode: String, language: InfluxQueryLanguage) -> String { + let conditions = filters.compactMap { condition(for: $0, language: language) } + guard !conditions.isEmpty else { return "" } + let joiner = logicMode.lowercased() == "or" ? " OR " : " AND " + return " WHERE " + conditions.joined(separator: joiner) + } + + static func orderBy(_ sorts: [InfluxSort], language: InfluxQueryLanguage) -> String { + let timeFirst = "time DESC" + switch language { + case .sql: + guard !sorts.isEmpty else { return timeFirst } + return sorts + .map { "\(quoteIdentifier($0.column, language: language)) \($0.ascending ? "ASC" : "DESC")" } + .joined(separator: ", ") + case .influxQL, .flux: + guard let time = sorts.first(where: { $0.column == InfluxTypeName.timeColumn }) else { + return timeFirst + } + return time.ascending ? "time ASC" : timeFirst + } + } + + // MARK: - Conditions + + static func condition(for filter: InfluxFilter, language: InfluxQueryLanguage) -> String? { + let column = quoteIdentifier(filter.column, language: language) + let op = filter.op.uppercased() + switch op { + case "=", "!=", "<>", "<", ">", "<=", ">=": + let sqlOp = op == "<>" ? "!=" : op + return "\(column) \(sqlOp) \(literal(filter.value, filter: filter, language: language))" + case "CONTAINS": + return regex(column, escapedRegex(filter.value), filter, language) + case "NOT CONTAINS": + return regex(column, escapedRegex(filter.value), filter, language, negated: true) + case "STARTS WITH": + return regex(column, "^" + escapedRegex(filter.value), filter, language) + case "ENDS WITH": + return regex(column, escapedRegex(filter.value) + "$", filter, language) + case "REGEX": + return regex(column, filter.value, filter, language) + case "IN", "NOT IN": + let values = filter.value + .split(separator: ",") + .map { $0.trimmingCharacters(in: .whitespaces) } + .filter { !$0.isEmpty } + guard !values.isEmpty else { return nil } + let negated = op == "NOT IN" + let parts = values.map { + "\(column) \(negated ? "!=" : "=") \(literal($0, filter: filter, language: language))" + } + return "(" + parts.joined(separator: negated ? " AND " : " OR ") + ")" + case "BETWEEN": + guard let upper = filter.secondValue else { return nil } + let low = literal(filter.value, filter: filter, language: language) + let high = literal(upper, filter: filter, language: language) + return "(\(column) >= \(low) AND \(column) <= \(high))" + case "IS NULL": + return language == .sql ? "\(column) IS NULL" : "\(column) = ''" + case "IS NOT NULL": + return language == .sql ? "\(column) IS NOT NULL" : "\(column) != ''" + case "IS EMPTY": + return "\(column) = ''" + case "IS NOT EMPTY": + return "\(column) != ''" + default: + return nil + } + } + + /// Escapes the metacharacters RE2 and the Rust regex crate share. `NSRegularExpression`'s + /// escaper also escapes `/`, which InfluxQL's `/.../` literal would then escape twice. + static func escapedRegex(_ text: String) -> String { + var escaped = "" + for character in text { + if "\\^$.|?*+()[]{}".contains(character) { escaped.append("\\") } + escaped.append(character) + } + return escaped + } + + private static func regex( + _ column: String, + _ pattern: String, + _ filter: InfluxFilter, + _ language: InfluxQueryLanguage, + negated: Bool = false + ) -> String { + let flagged = filter.isCaseSensitive ? pattern : "(?i)" + pattern + switch language { + case .sql: + return "\(column) \(negated ? "!~" : "~") \(stringLiteral(flagged, language: language))" + case .influxQL, .flux: + let delimited = flagged.replacingOccurrences(of: "/", with: "\\/") + return "\(column) \(negated ? "!~" : "=~") /\(delimited)/" + } + } + + static func literal(_ value: String, filter: InfluxFilter, language: InfluxQueryLanguage) -> String { + if filter.column == InfluxTypeName.timeColumn || filter.isText { + return stringLiteral(value, language: language) + } + if Double(value) != nil, value.rangeOfCharacter(from: .letters) == nil { + return value + } + if value == "true" || value == "false" { + return value + } + return stringLiteral(value, language: language) + } + + // MARK: - Quoting + + public static func quoteIdentifier(_ name: String, language: InfluxQueryLanguage) -> String { + switch language { + case .sql: + return "\"" + name.replacingOccurrences(of: "\"", with: "\"\"") + "\"" + case .influxQL, .flux: + let escaped = name + .replacingOccurrences(of: "\\", with: "\\\\") + .replacingOccurrences(of: "\"", with: "\\\"") + return "\"" + escaped + "\"" + } + } + + public static func stringLiteral(_ value: String, language: InfluxQueryLanguage) -> String { + switch language { + case .sql: + return "'" + value.replacingOccurrences(of: "'", with: "''") + "'" + case .influxQL, .flux: + let escaped = value + .replacingOccurrences(of: "\\", with: "\\\\") + .replacingOccurrences(of: "'", with: "\\'") + return "'" + escaped + "'" + } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxServer.swift b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxServer.swift new file mode 100644 index 0000000000..9be3276b4f --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProInfluxDBCore/InfluxServer.swift @@ -0,0 +1,49 @@ +import Foundation + +public enum InfluxGeneration: Sendable, Equatable { + case v1 + case v2 + case v3 + + /// Reads the `X-Influxdb-Version` header `/ping` returns: `v2.9.1` on 2.x, `3.12.2` on 3.x, + /// `1.11.8` on 1.x. A server that sends none is taken as 3.x, the only line whose `/ping` + /// body also names the version. + public init(versionHeader: String?) { + let trimmed = (versionHeader ?? "").trimmingCharacters(in: .whitespaces) + let digits = trimmed.hasPrefix("v") ? String(trimmed.dropFirst()) : trimmed + switch digits.split(separator: ".").first.flatMap({ Int($0) }) { + case 1: self = .v1 + case 2: self = .v2 + default: self = .v3 + } + } + + public var languages: [InfluxQueryLanguage] { + switch self { + case .v1: return [.influxQL] + case .v2: return [.flux, .influxQL] + case .v3: return [.sql, .influxQL] + } + } + + /// The language the data grid's queries are written in: both read the same + /// `SELECT ... FROM "m" ORDER BY time DESC LIMIT n OFFSET m`. + public var gridLanguage: InfluxQueryLanguage { + self == .v3 ? .sql : .influxQL + } +} + +public struct InfluxServerInfo: Sendable, Equatable { + public let generation: InfluxGeneration + public let version: String + + public init(generation: InfluxGeneration, version: String) { + self.generation = generation + self.version = version + } + + public init(versionHeader: String?) { + let version = (versionHeader ?? "").trimmingCharacters(in: .whitespaces) + self.init(generation: InfluxGeneration(versionHeader: versionHeader), version: version) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLConnectionOptions.swift b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLConnectionOptions.swift index ac0cf7b97b..aedd33873c 100644 --- a/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLConnectionOptions.swift +++ b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLConnectionOptions.swift @@ -14,14 +14,11 @@ public struct MSSQLConnectionOptions: Sendable, Equatable { public var kerberosCachePath: String? public var kerberosServicePrincipal: String? - /// How far to check the server certificate. Set after init by the plugin from the - /// connection's SSL mode, so the existing initializer keeps its signature. - public var certificateVerification: MSSQLCertificateVerification = .none + // No default: a caller that forgets it would encrypt without checking who answers. + public var certificateVerification: MSSQLCertificateVerification public var caCertificatePath: String? - /// Microsoft Entra ID access token, sent in place of a user name and password. Set after - /// init for the same reason as `certificateVerification`. Only read when - /// `authMethod == .entra`. + // Sent in place of a user name and password, read only when `authMethod == .entra`. public var fedAuthToken: String? public static let defaultPort = 1_433 @@ -38,6 +35,7 @@ public struct MSSQLConnectionOptions: Sendable, Equatable { database: String, schema: String = MSSQLConnectionOptions.defaultSchema, encryptionLevel: MSSQLEncryptionLevel = MSSQLConnectionOptions.defaultEncryptionLevel, + certificateVerification: MSSQLCertificateVerification, applicationName: String = MSSQLConnectionOptions.defaultApplicationName, loginTimeoutSeconds: Int = MSSQLConnectionOptions.defaultLoginTimeoutSeconds, authMethod: MSSQLAuthMethod = .sqlServer, @@ -60,6 +58,7 @@ public struct MSSQLConnectionOptions: Sendable, Equatable { self.database = database self.schema = schema self.encryptionLevel = encryptionLevel + self.certificateVerification = certificateVerification self.applicationName = applicationName self.loginTimeoutSeconds = loginTimeoutSeconds } diff --git a/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLViewSources.swift b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLViewSources.swift new file mode 100644 index 0000000000..35ca9a4a2d --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLViewSources.swift @@ -0,0 +1,364 @@ +import Foundation + +// INFORMATION_SCHEMA says NO for every view; sys.dm_exec_describe_first_result_set compiles without running. +public enum MSSQLViewSources { + public enum Refusal: Equatable, Sendable { + case notUpdatable + case writesRunTriggers + case readsSeveralTables + case baseTableHidden + case couldNotCheck + } + + public enum Operation: String, CaseIterable, Sendable { + case update + case insert + case delete + } + + public struct BaseTable: Hashable, Sendable { + // Set for a table on a linked server. + public let server: String? + public let database: String? + public let schema: String + public let name: String + + public init(server: String? = nil, database: String?, schema: String, name: String) { + self.server = server + self.database = database + self.schema = schema + self.name = name + } + } + + public struct ViewColumn: Equatable, Sendable { + public let name: String + public let baseColumn: String? + public let acceptsWrites: Bool + + public init(name: String, baseColumn: String?, acceptsWrites: Bool) { + self.name = name + self.baseColumn = baseColumn + self.acceptsWrites = acceptsWrites + } + } + + public struct BaseColumn: Equatable, Sendable { + public let name: String + public let isNullable: Bool + // Nil also for a default whose text needs VIEW DEFINITION to read. + public let defaultValue: String? + public let hasDefault: Bool + public let isIdentity: Bool + public let isComputed: Bool + // A rowversion or a GENERATED ALWAYS period column. + public let isServerGenerated: Bool + + public init( + name: String, + isNullable: Bool, + defaultValue: String?, + hasDefault: Bool, + isIdentity: Bool, + isComputed: Bool, + isServerGenerated: Bool + ) { + self.name = name + self.isNullable = isNullable + self.defaultValue = defaultValue + self.hasDefault = hasDefault + self.isIdentity = isIdentity + self.isComputed = isComputed + self.isServerGenerated = isServerGenerated + } + + // A default the user cannot read still fills the column when an INSERT leaves it out. + public var assignsValue: Bool { + isIdentity || isServerGenerated || (hasDefault && defaultValue == nil) + } + + public var acceptsWrites: Bool { + !isComputed && !isIdentity && !isServerGenerated + } + } + + static let maxViewDepth = 8 + + // MARK: - The view itself + + // sys.sql_expression_dependencies needs VIEW DEFINITION; sys.sql_dependencies does not but drops recreated views. + // A schema-bound view has one sys.sql_expression_dependencies row per column; minor id 0 is the object. + public static func viewFactsQuery(schema: String, view: String) -> String { + let object = MSSQLStringLiteral.quoted(MSSQLSchemaQueries.bracketed(schema: schema, table: view)) + return """ + DECLARE @view int = OBJECT_ID(\(object)); + DECLARE @edges TABLE (referencing_id int NOT NULL, referenced_id int NOT NULL); + IF EXISTS (SELECT 1 FROM sys.views WHERE object_id = @view) + BEGIN + INSERT INTO @edges (referencing_id, referenced_id) + SELECT DISTINCT d.object_id, d.referenced_major_id + FROM sys.sql_dependencies d + JOIN sys.views r ON r.object_id = d.referenced_major_id; + IF HAS_PERMS_BY_NAME(N'sys.sql_expression_dependencies', 'OBJECT', 'SELECT') = 1 + INSERT INTO @edges (referencing_id, referenced_id) + SELECT d.referencing_id, d.referenced_id + FROM sys.sql_expression_dependencies d + JOIN sys.views r ON r.object_id = d.referenced_id + WHERE d.referenced_minor_id = 0 + AND NOT EXISTS ( + SELECT 1 + FROM @edges e + WHERE e.referencing_id = d.referencing_id AND e.referenced_id = d.referenced_id + ); + END; + WITH chain AS ( + SELECT v.object_id, 0 AS depth + FROM sys.views v + WHERE v.object_id = @view + UNION ALL + SELECT e.referenced_id, c.depth + 1 + FROM chain c + JOIN @edges e ON e.referencing_id = c.object_id + WHERE c.depth < \(maxViewDepth) + ) + SELECT + v.has_opaque_metadata, + CASE WHEN EXISTS ( + SELECT 1 + FROM chain c + JOIN sys.triggers tr ON tr.parent_id = c.object_id + WHERE tr.is_instead_of_trigger = 1 AND tr.is_disabled = 0 + ) THEN 1 ELSE 0 END AS runs_instead_of_trigger, + CASE WHEN EXISTS ( + SELECT 1 + FROM chain c + JOIN @edges e ON e.referencing_id = c.object_id + WHERE c.depth = \(maxViewDepth) + ) THEN 1 ELSE 0 END AS chain_continues + FROM sys.views v + WHERE v.object_id = @view + """ + } + + public struct ViewFacts: Equatable, Sendable { + public let hasOpaqueMetadata: Bool + public let runsInsteadOfTrigger: Bool + // Views nest deeper than the walk followed, so a trigger further down went unseen. + public let chainContinues: Bool + + public init(hasOpaqueMetadata: Bool, runsInsteadOfTrigger: Bool, chainContinues: Bool = false) { + self.hasOpaqueMetadata = hasOpaqueMetadata + self.runsInsteadOfTrigger = runsInsteadOfTrigger + self.chainContinues = chainContinues + } + + // Nil when no view of that name is visible to the user. + public init?(rows: [[String?]]) { + guard let row = rows.first else { return nil } + self.init( + hasOpaqueMetadata: row.cell(0) == "1", + runsInsteadOfTrigger: row.cell(1) == "1", + chainContinues: row.cell(2) == "1" + ) + } + + public var refusal: Refusal? { + // WITH VIEW_METADATA makes browse mode name the view in place of its base table. + if hasOpaqueMetadata { return .baseTableHidden } + if runsInsteadOfTrigger { return .writesRunTriggers } + if chainContinues { return .couldNotCheck } + return nil + } + } + + // MARK: - Browse mode + + // Mode 1 is FOR BROWSE: base table and column per column, plus hidden key and rowversion rows of every table. + public static func browseQuery(schema: String, view: String) -> String { + let select = "SELECT * FROM \(MSSQLSchemaQueries.bracketed(schema: schema, table: view))" + return """ + SELECT name, is_hidden, source_server, source_database, source_schema, source_table, source_column, + is_updateable + FROM sys.dm_exec_describe_first_result_set(\(MSSQLStringLiteral.quoted(select)), NULL, 1) + ORDER BY column_ordinal + """ + } + + public enum Source: Equatable, Sendable { + case table(BaseTable) + case refused(Refusal) + } + + public struct BrowseFacts: Equatable, Sendable { + public let columns: [ViewColumn] + public let baseTables: [BaseTable] + // View columns the probes write, one per base column, in view order. + public let writableColumns: [String] + + public init(rows: [[String?]]) { + var columns: [ViewColumn] = [] + var baseTables: [BaseTable] = [] + var writableColumns: [String] = [] + var probedBaseColumns: Set = [] + for row in rows { + // A view that no longer compiles yields error rows with no name. + guard let name = row.cell(0) else { continue } + let server = row.cell(2) + if let schema = row.cell(4), let table = row.cell(5) { + let base = BaseTable(server: server, database: row.cell(3), schema: schema, name: table) + if !baseTables.contains(base) { baseTables.append(base) } + } + guard row.cell(1) != "1" else { continue } + let baseColumn = server == nil ? row.cell(6) : nil + let acceptsWrites = row.cell(7) == "1" + columns.append(ViewColumn(name: name, baseColumn: baseColumn, acceptsWrites: acceptsWrites)) + if acceptsWrites, let baseColumn, probedBaseColumns.insert(baseColumn).inserted { + writableColumns.append(name) + } + } + self.columns = columns + self.baseTables = baseTables + self.writableColumns = writableColumns + } + + public var source: Source { + guard baseTables.count == 1, let table = baseTables.first else { + return .refused(baseTables.isEmpty ? .notUpdatable : .readsSeveralTables) + } + // Its key and columns live in another server's catalog. + return table.server == nil ? .table(table) : .refused(.baseTableHidden) + } + } + + // MARK: - Operations + + public static func probesQuery(schema: String, view: String, writableColumns: [String]) -> String { + let target = MSSQLSchemaQueries.bracketed(schema: schema, table: view) + var probes: [(Operation, String)] = [(.delete, "DELETE FROM \(target) WHERE 1 = 0")] + if let first = writableColumns.first.map(bracketedColumn) { + let list = writableColumns.map(bracketedColumn).joined(separator: ", ") + probes.append((.update, "UPDATE \(target) SET \(first) = \(first) WHERE 1 = 0")) + probes.append((.insert, "INSERT INTO \(target) (\(list)) SELECT \(list) FROM \(target) WHERE 1 = 0")) + } + let values = probes + .map { "('\($0.0.rawValue)', \(MSSQLStringLiteral.quoted($0.1)))" } + .joined(separator: ",\n ") + return """ + SELECT p.operation, r.error_number + FROM (VALUES + \(values) + ) AS p(operation, statement) + CROSS APPLY sys.dm_exec_describe_first_result_set(p.statement, NULL, 0) r + WHERE r.error_number IS NOT NULL + """ + } + + // Browse mode names a table once however often the view joins it, so a self-join shows only as 4405 on DELETE. + public static func refusal(probeRows: [[String?]]) -> Refusal? { + let joins = probeRows.contains { $0.cell(0) == Operation.delete.rawValue && $0.cell(1) == "4405" } + return joins ? .readsSeveralTables : nil + } + + public static func allowedOperations(writableColumns: [String], probeRows: [[String?]]) -> Set { + var allowed: Set = writableColumns.isEmpty ? [.delete] : Set(Operation.allCases) + for row in probeRows { + guard let operation = row.cell(0).flatMap(Operation.init(rawValue:)), + let number = row.cell(1).flatMap({ Int($0) }), + structuralErrors.contains(number) + else { continue } + allowed.remove(operation) + } + return allowed + } + + // 11501 marks any compile error. 229 and 230 come with 11529 and wait for the save, as on a table. + static let structuralErrors: Set = [ + 208, // invalid object name + 264, // a column assigned twice + 271, // computed column + 272, // timestamp column + 4_403, // aggregate, DISTINCT, GROUP BY, PIVOT + 4_405, // the statement touches several base tables + 4_406, // derived or constant column + 4_426, // UNION + 8_102, // identity column + 11_501, // the batch has compile errors + 13_537, // GENERATED ALWAYS column + ] + + // MARK: - Base table + + // Read in the base table's own database. sys.tables skips a view that browse mode stopped at. + public static func baseColumnsQuery(_ table: BaseTable) -> String { + let catalog = table.database.map { "[\(MSSQLSchemaQueries.escapeBracket($0))]." } ?? "" + let object = MSSQLStringLiteral.quoted( + catalog + MSSQLSchemaQueries.bracketed(schema: table.schema, table: table.name) + ) + return """ + SELECT + c.name, + c.is_nullable, + d.definition, + CASE WHEN c.default_object_id <> 0 THEN 1 ELSE 0 END AS has_default, + c.is_identity, + c.is_computed, + CASE + WHEN TYPE_NAME(c.system_type_id) = 'timestamp' THEN 1 + WHEN c.generated_always_type > 0 THEN 1 + ELSE 0 + END AS is_server_generated, + ic.key_ordinal + FROM \(catalog)sys.columns c + JOIN \(catalog)sys.tables t ON t.object_id = c.object_id + LEFT JOIN \(catalog)sys.default_constraints d ON d.object_id = c.default_object_id + LEFT JOIN \(catalog)sys.indexes i ON i.object_id = c.object_id AND i.is_primary_key = 1 + LEFT JOIN \(catalog)sys.index_columns ic + ON ic.object_id = i.object_id AND ic.index_id = i.index_id AND ic.column_id = c.column_id + WHERE c.object_id = OBJECT_ID(\(object)) + ORDER BY c.column_id + """ + } + + public struct BaseFacts: Equatable, Sendable { + public let keyColumns: [String] + public let columns: [BaseColumn] + + public init(keyColumns: [String], columns: [BaseColumn]) { + self.keyColumns = keyColumns + self.columns = columns + } + + // Nil when no row comes back: OBJECT_ID is NULL for a table the user has no permission on. + public init?(rows: [[String?]]) { + var columns: [BaseColumn] = [] + var keyed: [(ordinal: Int, name: String)] = [] + for row in rows { + guard let name = row.cell(0) else { continue } + columns.append(BaseColumn( + name: name, + isNullable: row.cell(1) == "1", + defaultValue: row.cell(2), + hasDefault: row.cell(3) == "1", + isIdentity: row.cell(4) == "1", + isComputed: row.cell(5) == "1", + isServerGenerated: row.cell(6) == "1" + )) + if let ordinal = row.cell(7).flatMap({ Int($0) }) { + keyed.append((ordinal, name)) + } + } + guard !columns.isEmpty else { return nil } + self.init(keyColumns: keyed.sorted { $0.ordinal < $1.ordinal }.map(\.name), columns: columns) + } + } + + private static func bracketedColumn(_ column: String) -> String { + "[\(MSSQLSchemaQueries.escapeBracket(column))]" + } +} + +private extension Array where Element == String? { + func cell(_ index: Int) -> String? { + indices.contains(index) ? self[index] : nil + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLLiteralSpelling.swift b/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLLiteralSpelling.swift index c9098218b2..3b0aaf5808 100644 --- a/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLLiteralSpelling.swift +++ b/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLLiteralSpelling.swift @@ -24,7 +24,7 @@ public enum MySQLLiteralSpelling: Equatable, Sendable { case .backslashEscapes: return mysqlEscapeStringLiteral(value) case .quoteDoubling: - return value.replacingOccurrences(of: "'", with: "''") + return value.replacingOccurrences(of: "'", with: "''", options: .literal) } } @@ -140,11 +140,12 @@ public enum MySQLLiteralSpelling: Equatable, Sendable { return scalars.count } + /// `--` opens a comment only when a space or a control character follows it, U+007F among them. private static func opensDashComment(_ scalars: [Unicode.Scalar], at index: Int) -> Bool { guard index + 1 < scalars.count, scalars[index + 1] == "-" else { return false } guard index + 2 < scalars.count else { return true } let next = scalars[index + 2] - return next == " " || next == "\t" || next == "\n" || next == "\r" + return next <= " " || next == "\u{7F}" } private static func endOfLine(in scalars: [Unicode.Scalar], from start: Int) -> Int { diff --git a/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLObjectQueries.swift b/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLObjectQueries.swift index cfb77225bf..76d48cf2c5 100644 --- a/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLObjectQueries.swift +++ b/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLObjectQueries.swift @@ -14,7 +14,7 @@ public enum MySQLObjectQueries { } public static func quoteIdentifier(_ value: String) -> String { - "`\(value.replacingOccurrences(of: "`", with: "``"))`" + "`\(value.replacingOccurrences(of: "`", with: "``", options: .literal))`" } /// The database a catalog read means. @@ -129,9 +129,9 @@ public enum MySQLObjectQueries { public static func likePattern(matching name: String) -> String { name - .replacingOccurrences(of: "\\", with: "\\\\") - .replacingOccurrences(of: "%", with: "\\%") - .replacingOccurrences(of: "_", with: "\\_") + .replacingOccurrences(of: "\\", with: "\\\\", options: .literal) + .replacingOccurrences(of: "%", with: "\\%", options: .literal) + .replacingOccurrences(of: "_", with: "\\_", options: .literal) } /// Whether `information_schema` describes this database at all, in one scalar. diff --git a/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLStatementKeyword.swift b/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLStatementKeyword.swift index 35bbf14782..1cb3bae3ff 100644 --- a/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLStatementKeyword.swift +++ b/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLStatementKeyword.swift @@ -20,7 +20,8 @@ nonisolated public func mysqlLeadingKeyword(_ query: String) -> String { while true { rest = rest.drop(while: { $0.isWhitespace }) if rest.hasPrefix("/*"), !rest.hasPrefix("/*!") { - guard let end = rest.range(of: "*/") else { return "" } + // Past the opener: its `*` closes nothing, so `/*/` is still open. + guard let end = rest.dropFirst(2).range(of: "*/") else { return "" } rest = rest[end.upperBound...] } else if rest.hasPrefix("#") || (rest.hasPrefix("--") && rest.dropFirst(2).first?.isWhitespace != false) { guard let end = rest.firstIndex(where: { $0.isNewline }) else { return "" } diff --git a/apple/Packages/TableProCore/Sources/TableProNeo4jCore/BoltMessage.swift b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/BoltMessage.swift new file mode 100644 index 0000000000..5b13e9e2e2 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/BoltMessage.swift @@ -0,0 +1,175 @@ +import Foundation + +public struct BoltVersion: Comparable, Hashable, Sendable, CustomStringConvertible { + public let major: UInt8 + public let minor: UInt8 + + public init(major: UInt8, minor: UInt8) { + self.major = major + self.minor = minor + } + + public var description: String { "\(major).\(minor)" } + + public static func < (lhs: BoltVersion, rhs: BoltVersion) -> Bool { + (lhs.major, lhs.minor) < (rhs.major, rhs.minor) + } + + public var separatesLogon: Bool { self >= BoltVersion(major: 5, minor: 1) } + public var sendsBoltAgent: Bool { self >= BoltVersion(major: 5, minor: 3) } + public var hasElementIds: Bool { major >= 5 } +} + +public enum BoltHandshake { + public static let magic: [UInt8] = [0x60, 0x60, 0xB0, 0x17] + + // Each proposal is [0, range, minor, major]: major.minor down to major.(minor - range). + public static let proposals: [UInt8] = [ + 0x00, 0x04, 0x04, 0x05, + 0x00, 0x02, 0x04, 0x04, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 + ] + + public static var request: Data { Data(magic + proposals) } + + public static func version(fromResponse bytes: [UInt8]) -> BoltVersion? { + guard bytes.count == 4, bytes != [0, 0, 0, 0] else { return nil } + return BoltVersion(major: bytes[3], minor: bytes[2]) + } +} + +public enum BoltRequest: Equatable, Sendable { + case hello(extra: [String: PackStreamValue]) + case logon(auth: [String: PackStreamValue]) + case run(query: String, parameters: [String: PackStreamValue], extra: [String: PackStreamValue]) + case pull(count: Int64) + case discard + case reset + case goodbye + + public var structure: PackStreamValue { + switch self { + case .hello(let extra): + return .structure(tag: 0x01, fields: [.map(extra)]) + case .logon(let auth): + return .structure(tag: 0x6A, fields: [.map(auth)]) + case .run(let query, let parameters, let extra): + return .structure(tag: 0x10, fields: [.string(query), .map(parameters), .map(extra)]) + case .pull(let count): + return .structure(tag: 0x3F, fields: [.map(["n": .int(count)])]) + case .discard: + return .structure(tag: 0x2F, fields: [.map(["n": .int(-1)])]) + case .reset: + return .structure(tag: 0x0F, fields: []) + case .goodbye: + return .structure(tag: 0x02, fields: []) + } + } + + public var expectsResponse: Bool { self != .goodbye } + + public var framed: Data { BoltChunker.frame(PackStreamEncoder.encode(structure)) } +} + +public enum BoltResponse: Equatable, Sendable { + case success([String: PackStreamValue]) + case record([PackStreamValue]) + case ignored + case failure(code: String, message: String) + + public static func decode(_ message: Data) throws -> BoltResponse { + guard case .structure(let tag, let fields) = try PackStreamDecoder.decode(message) else { + throw BoltProtocolError.unexpectedMessage("not a structure") + } + switch tag { + case 0x70: + return .success(fields.first?.mapValue ?? [:]) + case 0x71: + return .record(fields.first?.listValue ?? []) + case 0x7E: + return .ignored + case 0x7F: + let metadata = fields.first?.mapValue ?? [:] + let code = metadata["code"]?.stringValue ?? metadata["neo4j_code"]?.stringValue ?? "" + return .failure(code: code, message: metadata["message"]?.stringValue ?? "") + default: + throw BoltProtocolError.unexpectedMessage(String(format: "tag 0x%02X", tag)) + } + } +} + +public enum BoltProtocolError: Error, Equatable, Sendable, LocalizedError { + case unexpectedMessage(String) + case unsupportedServer + case chunkTooLarge(Int) + + public var errorDescription: String? { + switch self { + case .unexpectedMessage(let detail): + return String(format: String(localized: "The server sent a message this client does not understand (%@)."), detail) + case .unsupportedServer: + return String(localized: "The server does not speak Bolt 4.2 to 5.4. Neo4j 4.2 or later is required.") + case .chunkTooLarge(let size): + return String(format: String(localized: "The server sent a %d-byte message, which is over the limit."), size) + } + } +} + +public enum BoltChunker { + public static let maximumChunkSize = Int(UInt16.max) + + public static func frame(_ message: Data) -> Data { + var framed = Data() + framed.reserveCapacity(message.count + 4) + var offset = message.startIndex + while offset < message.endIndex { + let end = min(offset + maximumChunkSize, message.endIndex) + let size = UInt16(end - offset) + framed.append(UInt8(size >> 8)) + framed.append(UInt8(size & 0xFF)) + framed.append(message[offset ..< end]) + offset = end + } + framed.append(contentsOf: [0x00, 0x00]) + return framed + } +} + +// A zero-size chunk with nothing before it is a NOOP keep-alive, not an empty message. +public struct BoltDechunker: Sendable { + public static let maximumMessageSize = 512 * 1_024 * 1_024 + + private var buffer: [UInt8] = [] + private var cursor = 0 + private var message = Data() + + public init() {} + + public mutating func append(_ bytes: Bytes) where Bytes.Element == UInt8 { + if cursor > 0, cursor >= buffer.count / 2 { + buffer.removeFirst(cursor) + cursor = 0 + } + buffer.append(contentsOf: bytes) + } + + public mutating func nextMessage() throws -> Data? { + while buffer.count - cursor >= 2 { + let size = Int(buffer[cursor]) << 8 | Int(buffer[cursor + 1]) + if size == 0 { + cursor += 2 + guard !message.isEmpty else { continue } + defer { message = Data() } + return message + } + guard buffer.count - cursor >= 2 + size else { return nil } + message.append(contentsOf: buffer[(cursor + 2) ..< (cursor + 2 + size)]) + cursor += 2 + size + guard message.count <= Self.maximumMessageSize else { + throw BoltProtocolError.chunkTooLarge(message.count) + } + } + return nil + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jCypher.swift b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jCypher.swift new file mode 100644 index 0000000000..22d79a0760 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jCypher.swift @@ -0,0 +1,345 @@ +import Foundation + +public enum Neo4jEntityKind: String, Sendable, CaseIterable { + case node + case relationship + + public var schemaName: String { + switch self { + case .node: "Labels" + case .relationship: "Relationship Types" + } + } + + public var entityType: String { + switch self { + case .node: "NODE" + case .relationship: "RELATIONSHIP" + } + } + + public init?(schemaName: String?) { + guard let match = Self.allCases.first(where: { $0.schemaName == schemaName }) else { return nil } + self = match + } +} + +public struct Neo4jFilter: Sendable, Equatable { + public let column: String + public let op: String + public let value: String + public let secondValue: String? + public let isCaseSensitive: Bool + + public init(column: String, op: String, value: String, secondValue: String? = nil, isCaseSensitive: Bool = true) { + self.column = column + self.op = op + self.value = value + self.secondValue = secondValue + self.isCaseSensitive = isCaseSensitive + } +} + +public enum Neo4jCypherError: Error, Equatable, Sendable, LocalizedError { + case unsupportedOperator(String) + case unquotableName(String) + case incompleteRange + + public var errorDescription: String? { + switch self { + case .unsupportedOperator(let op): + return String(format: String(localized: "The %@ filter is not available for Neo4j."), op) + case .unquotableName(let name): + return String(format: String(localized: "Neo4j cannot query the name %@ because it contains \\u."), name) + case .incompleteRange: + return String(localized: "Enter both bounds to filter with BETWEEN.") + } + } +} + +public enum Neo4jCypher { + public static let idColumn = "" + public static let labelsColumn = "" + public static let startColumn = "" + public static let endColumn = "" + public static let rawFilterColumn = "__RAW__" + + // A comment keeps a browse query valid Cypher when pasted into the editor. + public static let browseMarker = "// TablePro browse: " + + // Neo4j decodes `\uXXXX` before parsing, inside backticks too, so no spelling of `\u` stays inside its quotes. + // `\uZZZZ` is an escape the server rejects; the query builders refuse such a name before it gets that far. + public static func quote(_ identifier: String) -> String { + let escaped = identifier + .replacingOccurrences(of: "\\u", with: "\\uZZZZ") + .replacingOccurrences(of: "`", with: "``") + return "`" + escaped + "`" + } + + public static func isQuotable(_ name: String) -> Bool { + !name.contains("\\u") + } + + static func requireQuotable(_ names: [String]) throws { + if let name = names.first(where: { !isQuotable($0) }) { throw Neo4jCypherError.unquotableName(name) } + } + + public static func literal(_ text: String) -> String { + let escaped = text + .replacingOccurrences(of: "\\", with: "\\\\") + .replacingOccurrences(of: "'", with: "\\'") + return "'" + escaped + "'" + } + + public static func browseKind(of query: String) -> Neo4jEntityKind? { + let trimmed = query.drop { $0.isWhitespace } + guard trimmed.hasPrefix(browseMarker) else { return nil } + let rest = trimmed.dropFirst(browseMarker.count).prefix { !$0.isNewline } + return Neo4jEntityKind(rawValue: rest.trimmingCharacters(in: .whitespaces)) + } + + public static func browsedName(in query: String) -> String? { + guard browseKind(of: query) != nil, let start = query.range(of: ":`") else { return nil } + var name = "" + var index = start.upperBound + while index < query.endIndex { + let character = query[index] + let next = query.index(after: index) + guard character == "`" else { + name.append(character) + index = next + continue + } + guard next < query.endIndex, query[next] == "`" else { return name } + name.append("`") + index = query.index(after: next) + } + return nil + } + + public static func fixedColumns(for kind: Neo4jEntityKind) -> [String] { + switch kind { + case .node: [idColumn, labelsColumn] + case .relationship: [idColumn, startColumn, endColumn] + } + } + + public static func pattern(for kind: Neo4jEntityKind, name: String) -> String { + switch kind { + case .node: "MATCH (n:\(quote(name)))" + case .relationship: "MATCH ()-[n:\(quote(name))]->()" + } + } + + public static func browseQuery( + kind: Neo4jEntityKind, + name: String, + filters: [Neo4jFilter] = [], + matchesAll: Bool = true, + sort: [(column: String, ascending: Bool)] = [], + limit: Int, + offset: Int, + usesElementId: Bool = true + ) throws -> String { + try requireQuotable([name] + sort.map(\.column)) + var lines = [browseMarker + kind.rawValue, pattern(for: kind, name: name)] + if let filter = try whereClause(filters, matchesAll: matchesAll, usesElementId: usesElementId) { + lines.append(filter) + } + lines.append("RETURN n") + if !sort.isEmpty { + let keys = sort.map { expression(for: $0.column, usesElementId: usesElementId) + ($0.ascending ? " ASC" : " DESC") } + lines.append("ORDER BY " + keys.joined(separator: ", ")) + } + lines.append("SKIP \(max(0, offset)) LIMIT \(max(0, limit))") + return lines.joined(separator: "\n") + } + + public static func exportQuery(kind: Neo4jEntityKind, name: String) throws -> String { + try requireQuotable([name]) + return [browseMarker + kind.rawValue, pattern(for: kind, name: name), "RETURN n"].joined(separator: "\n") + } + + public static func countQuery( + kind: Neo4jEntityKind, + name: String, + filters: [Neo4jFilter] = [], + matchesAll: Bool = true, + usesElementId: Bool = true + ) throws -> String { + try requireQuotable([name]) + var lines = [pattern(for: kind, name: name)] + if let filter = try whereClause(filters, matchesAll: matchesAll, usesElementId: usesElementId) { + lines.append(filter) + } + lines.append("RETURN count(n) AS count") + return lines.joined(separator: "\n") + } + + // Bounded so the Structure tab stays fast on a large graph, unlike `db.schema.nodeTypeProperties()`. + public static func propertySampleQuery(kind: Neo4jEntityKind, name: String, sampleSize: Int = 1_000) -> String { + pattern(for: kind, name: name) + + " WITH n LIMIT \(sampleSize) UNWIND keys(n) AS key" + + " RETURN key, head(collect(n[key])) AS sample ORDER BY key" + } + + public static func propertyKeysQuery(kind: Neo4jEntityKind, name: String) throws -> String { + try requireQuotable([name]) + return pattern(for: kind, name: name) + " UNWIND keys(n) AS key RETURN DISTINCT key ORDER BY key" + } + + public static func expression(for column: String, usesElementId: Bool) -> String { + let idFunction = usesElementId ? "elementId" : "id" + switch column { + case idColumn: return "\(idFunction)(n)" + case labelsColumn: return "labels(n)" + case startColumn: return "\(idFunction)(startNode(n))" + case endColumn: return "\(idFunction)(endNode(n))" + default: return "n." + quote(column) + } + } + + private static func whereClause(_ filters: [Neo4jFilter], matchesAll: Bool, usesElementId: Bool) throws -> String? { + let clauses = try filters.map { try condition(for: $0, usesElementId: usesElementId) } + guard !clauses.isEmpty else { return nil } + let joined = clauses.count == 1 + ? clauses[0] + : clauses.map { "(\($0))" }.joined(separator: matchesAll ? " AND " : " OR ") + return "WHERE " + joined + } + + private static let negations = ["!=": "=", "<>": "=", "NOT CONTAINS": "CONTAINS", "NOT IN": "IN"] + + static func condition(for filter: Neo4jFilter, usesElementId: Bool) throws -> String { + if filter.column == rawFilterColumn { return filter.value } + try requireQuotable([filter.column]) + let op = filter.op.uppercased() + if let presence = presence(op, column: filter.column, usesElementId: usesElementId) { return presence } + + let subject = Subject(filter: filter, usesElementId: usesElementId) + var predicate = try positive(negations[op] ?? op, filter: filter, subject: subject) + // `labels(n)` is a list, so a label filter matches when any one label does. + if filter.column == labelsColumn { predicate = "any(label IN labels(n) WHERE \(predicate))" } + return negations[op] == nil ? predicate : "NOT (\(predicate))" + } + + private struct Subject { + let expression: String + // An element id or a label is always a string, so it never compares as a number. + let isText: Bool + let foldsCase: Bool + + init(filter: Neo4jFilter, usesElementId: Bool) { + let isLabel = filter.column == labelsColumn + let isElementId = usesElementId && [idColumn, startColumn, endColumn].contains(filter.column) + expression = isLabel ? "label" : Neo4jCypher.expression(for: filter.column, usesElementId: usesElementId) + isText = isLabel || isElementId + foldsCase = !filter.isCaseSensitive && !isElementId + } + } + + private static func presence(_ op: String, column: String, usesElementId: Bool) -> String? { + let target = expression(for: column, usesElementId: usesElementId) + switch op { + case "IS NULL": return "\(target) IS NULL" + case "IS NOT NULL": return "\(target) IS NOT NULL" + case "IS EMPTY" where column == labelsColumn: return "size(labels(n)) = 0" + case "IS NOT EMPTY" where column == labelsColumn: return "size(labels(n)) > 0" + case "IS EMPTY": return "(\(target) IS NULL OR \(target) = '')" + case "IS NOT EMPTY": return "(\(target) IS NOT NULL AND \(target) <> '')" + default: return nil + } + } + + private static func positive(_ op: String, filter: Neo4jFilter, subject: Subject) throws -> String { + let target = subject.expression + switch op { + case "=": + return matchesAny(subject, [filter.value]) + case "IN": + return matchesAny(subject, filter.value.split(separator: ",").map { $0.trimmingCharacters(in: .whitespaces) }) + case ">", ">=", "<", "<=": + return "\(target) \(op) \(bound(filter.value, subject))" + case "BETWEEN": + let range = try bounds(of: filter) + return "\(target) >= \(bound(range.lower, subject)) AND \(target) <= \(bound(range.upper, subject))" + case "CONTAINS", "STARTS WITH", "ENDS WITH": + let needle = literal(subject.foldsCase ? filter.value.lowercased() : filter.value) + return "\(text(subject, folded: subject.foldsCase)) \(op) \(needle)" + case "REGEX": + let pattern = subject.foldsCase ? "(?i)" + filter.value : filter.value + return "\(text(subject, folded: false)) =~ \(literal(pattern))" + default: + throw Neo4jCypherError.unsupportedOperator(filter.op) + } + } + + // A property can hold any type, so text operators read it as text first: a number or a list is then a + // non-match rather than a type error that fails the whole page. + private static func text(_ subject: Subject, folded: Bool) -> String { + let text = subject.isText ? subject.expression : "toStringOrNull(\(subject.expression))" + return folded ? "toLower(\(text))" : text + } + + private static func bound(_ value: String, _ subject: Subject) -> String { + subject.isText ? literal(value) : typedLiteral(value) + } + + private static func matchesAny(_ subject: Subject, _ values: [String]) -> String { + let target = subject.expression + if subject.isText { + let items = values.map { literal(subject.foldsCase ? $0.lowercased() : $0) } + let lhs = subject.foldsCase ? "toLower(\(target))" : target + return items.count == 1 ? "\(lhs) = \(items[0])" : "\(lhs) IN [\(items.joined(separator: ", "))]" + } + if subject.foldsCase { + // Case folding applies to text only, so `30.0` still matches `= 30`. + let folded = "\(text(subject, folded: true)) IN [\(values.map { literal($0.lowercased()) }.joined(separator: ", "))]" + let typed = values.map(typedLiteral).filter { !$0.hasPrefix("'") } + guard !typed.isEmpty else { return folded } + return "(\(target) IN [\(typed.joined(separator: ", "))] OR \(folded))" + } + // A property has no declared type, so `123` also matches the text '123'. + let forms = values.flatMap(spellings) + if forms.count == 1 { return "\(target) = \(forms[0])" } + if values.count == 1 { return "(" + forms.map { "\(target) = \($0)" }.joined(separator: " OR ") + ")" } + return "\(target) IN [\(forms.joined(separator: ", "))]" + } + + private static func spellings(_ value: String) -> [String] { + let typed = typedLiteral(value) + return typed == literal(value) ? [typed] : [typed, literal(value)] + } + + // The host sends BETWEEN's value as "lower,upper" and the upper bound again on its own. + private static func bounds(of filter: Neo4jFilter) throws -> (lower: String, upper: String) { + let lower: String + let upper: String + if let second = filter.secondValue { + let suffix = "," + second + lower = filter.value.hasSuffix(suffix) ? String(filter.value.dropLast(suffix.count)) : filter.value + upper = second + } else if let comma = filter.value.firstIndex(of: ",") { + lower = String(filter.value[.. String { + if let integer = Int64(value), String(integer) == value { return value } + if value.range(of: #"^-?(0|[1-9][0-9]*)\.[0-9]+([eE][+-]?[0-9]+)?$"#, options: .regularExpression) != nil, + let number = Double(value), number.isFinite { + return value + } + switch value.lowercased() { + case "true", "false": return value.lowercased() + default: return literal(value) + } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jEntity.swift b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jEntity.swift new file mode 100644 index 0000000000..8d2a53ad0c --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jEntity.swift @@ -0,0 +1,64 @@ +import Foundation + +public enum Neo4jStructureTag { + public static let node: UInt8 = 0x4E + public static let relationship: UInt8 = 0x52 + public static let unboundRelationship: UInt8 = 0x72 + public static let path: UInt8 = 0x50 + public static let date: UInt8 = 0x44 + public static let time: UInt8 = 0x54 + public static let localTime: UInt8 = 0x74 + public static let dateTime: UInt8 = 0x49 + public static let dateTimeZoneId: UInt8 = 0x69 + public static let legacyDateTime: UInt8 = 0x46 + public static let legacyDateTimeZoneId: UInt8 = 0x66 + public static let localDateTime: UInt8 = 0x64 + public static let duration: UInt8 = 0x45 + public static let point2D: UInt8 = 0x58 + public static let point3D: UInt8 = 0x59 +} + +// Bolt 4 has no element id, so the numeric id stands in, written as text. +public struct Neo4jEntity: Equatable, Sendable { + public enum Kind: Equatable, Sendable { + case node(labels: [String]) + case relationship(type: String, startId: String, endId: String) + } + + public let elementId: String + public let kind: Kind + public let properties: [String: PackStreamValue] + + public init(elementId: String, kind: Kind, properties: [String: PackStreamValue]) { + self.elementId = elementId + self.kind = kind + self.properties = properties + } + + public init?(_ value: PackStreamValue) { + guard case .structure(let tag, let fields) = value else { return nil } + switch tag { + case Neo4jStructureTag.node where fields.count >= 3: + elementId = Self.identifier(fields, elementIdIndex: 3, idIndex: 0) + kind = .node(labels: fields[1].listValue?.compactMap(\.stringValue) ?? []) + properties = fields[2].mapValue ?? [:] + case Neo4jStructureTag.relationship where fields.count >= 5: + elementId = Self.identifier(fields, elementIdIndex: 5, idIndex: 0) + kind = .relationship( + type: fields[3].stringValue ?? "", + startId: Self.identifier(fields, elementIdIndex: 6, idIndex: 1), + endId: Self.identifier(fields, elementIdIndex: 7, idIndex: 2) + ) + properties = fields[4].mapValue ?? [:] + default: + return nil + } + } + + private static func identifier(_ fields: [PackStreamValue], elementIdIndex: Int, idIndex: Int) -> String { + if elementIdIndex < fields.count, let elementId = fields[elementIdIndex].stringValue { + return elementId + } + return fields[idIndex].intValue.map(String.init) ?? "" + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jResultTable.swift b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jResultTable.swift new file mode 100644 index 0000000000..cefdfeca6f --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jResultTable.swift @@ -0,0 +1,62 @@ +import Foundation + +public struct Neo4jResultTable: Equatable, Sendable { + public let columns: [String] + public let typeNames: [String] + public let rows: [[String?]] + + public init(columns: [String], typeNames: [String], rows: [[String?]]) { + self.columns = columns + self.typeNames = typeNames + self.rows = rows + } + + public static func records(keys: [String], records: [[PackStreamValue]]) -> Neo4jResultTable { + let typeNames = keys.indices.map { index -> String in + for record in records where index < record.count && record[index] != .null { + return Neo4jValueText.typeName(record[index]) + } + return "ANY" + } + let rows = records.map { record in + keys.indices.map { index in index < record.count ? Neo4jValueText.text(record[index]) : nil } + } + return Neo4jResultTable(columns: keys, typeNames: typeNames, rows: rows) + } + + // Known keys keep their order; keys first seen on this page follow, sorted. A stream's columns are fixed by its + // header, so it passes `addsUnseenProperties: false`. + public static func entities( + kind: Neo4jEntityKind, + records: [[PackStreamValue]], + knownProperties: [String], + addsUnseenProperties: Bool = true + ) -> Neo4jResultTable { + let entities = records.compactMap { $0.first.flatMap(Neo4jEntity.init) } + let known = Set(knownProperties) + let extra = addsUnseenProperties ? Set(entities.flatMap(\.properties.keys)).subtracting(known).sorted() : [] + let properties = knownProperties + extra + let fixed = Neo4jCypher.fixedColumns(for: kind) + + var typeNames = fixed.map { _ in "STRING" } + if kind == .node { typeNames[1] = "LIST" } + for key in properties { + let sample = entities.lazy.compactMap { $0.properties[key] }.first { $0 != .null } + typeNames.append(sample.map(Neo4jValueText.typeName) ?? "ANY") + } + + let rows = entities.map { entity -> [String?] in + var row: [String?] = [entity.elementId] + switch entity.kind { + case .node(let labels): + row.append(Neo4jValueText.text(.list(labels.map(PackStreamValue.string)))) + case .relationship(_, let startId, let endId): + row.append(startId) + row.append(endId) + } + row += properties.map { entity.properties[$0].flatMap(Neo4jValueText.text) } + return row + } + return Neo4jResultTable(columns: fixed + properties, typeNames: typeNames, rows: rows) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jUpdateCounts.swift b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jUpdateCounts.swift new file mode 100644 index 0000000000..619c215945 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jUpdateCounts.swift @@ -0,0 +1,41 @@ +import Foundation + +public struct Neo4jUpdateCounts: Equatable, Sendable { + public enum Counter: String, CaseIterable, Sendable { + case nodesCreated = "nodes-created" + case nodesDeleted = "nodes-deleted" + case relationshipsCreated = "relationships-created" + case relationshipsDeleted = "relationships-deleted" + case propertiesSet = "properties-set" + case labelsAdded = "labels-added" + case labelsRemoved = "labels-removed" + case indexesAdded = "indexes-added" + case indexesRemoved = "indexes-removed" + case constraintsAdded = "constraints-added" + case constraintsRemoved = "constraints-removed" + case systemUpdates = "system-updates" + } + + public let counters: [(counter: Counter, count: Int)] + + public init(summary: [String: PackStreamValue]) { + let stats = summary["stats"]?.mapValue ?? [:] + counters = Counter.allCases.compactMap { counter in + let count = Int(stats[counter.rawValue]?.intValue ?? 0) + return count > 0 ? (counter, count) : nil + } + } + + public func count(_ counter: Counter) -> Int { + counters.first { $0.counter == counter }?.count ?? 0 + } + + // A property or label counter counts values, not entities. + public var entitiesChanged: Int { + [.nodesCreated, .nodesDeleted, .relationshipsCreated, .relationshipsDeleted].reduce(0) { $0 + count($1) } + } + + public static func == (lhs: Self, rhs: Self) -> Bool { + lhs.counters.elementsEqual(rhs.counters) { $0.counter == $1.counter && $0.count == $1.count } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jValueText.swift b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jValueText.swift new file mode 100644 index 0000000000..4285727595 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jValueText.swift @@ -0,0 +1,305 @@ +import Foundation + +public enum Neo4jValueText { + public static func text(_ value: PackStreamValue) -> String? { + switch value { + case .null: + return nil + case .bool(let flag): + return flag ? "true" : "false" + case .int(let number): + return String(number) + case .float(let number): + return floatText(number) + case .string(let text): + return text + case .bytes(let bytes): + return "0x" + bytes.map { String(format: "%02x", $0) }.joined() + case .list, .map: + return json(jsonObject(value)) + case .structure(let tag, let fields): + if let scalar = structureText(tag: tag, fields: fields) { return scalar } + return json(jsonObject(value)) + } + } + + public static func typeName(_ value: PackStreamValue) -> String { + switch value { + case .null: return "NULL" + case .bool: return "BOOLEAN" + case .int: return "INTEGER" + case .float: return "FLOAT" + case .string: return "STRING" + case .bytes: return "BYTES" + case .list: return "LIST" + case .map: return "MAP" + case .structure(let tag, _): + switch tag { + case Neo4jStructureTag.node: return "NODE" + case Neo4jStructureTag.relationship, Neo4jStructureTag.unboundRelationship: return "RELATIONSHIP" + case Neo4jStructureTag.path: return "PATH" + case Neo4jStructureTag.date: return "DATE" + case Neo4jStructureTag.time: return "ZONED TIME" + case Neo4jStructureTag.localTime: return "LOCAL TIME" + case Neo4jStructureTag.dateTime, Neo4jStructureTag.dateTimeZoneId, + Neo4jStructureTag.legacyDateTime, Neo4jStructureTag.legacyDateTimeZoneId: + return "ZONED DATETIME" + case Neo4jStructureTag.localDateTime: return "LOCAL DATETIME" + case Neo4jStructureTag.duration: return "DURATION" + case Neo4jStructureTag.point2D, Neo4jStructureTag.point3D: return "POINT" + default: return "ANY" + } + } + } + + public static func jsonObject(_ value: PackStreamValue) -> Any { + switch value { + case .null: + return NSNull() + case .bool(let flag): + return flag + case .int(let number): + return number + case .float(let number): + return number.isFinite ? number as Any : floatText(number) + case .string(let text): + return text + case .bytes: + return text(value) ?? "" + case .list(let items): + return items.map(jsonObject) + case .map(let entries): + return entries.mapValues(jsonObject) + case .structure(let tag, let fields): + if let entity = Neo4jEntity(value) { + return entityObject(entity) + } + if tag == Neo4jStructureTag.unboundRelationship, fields.count >= 3 { + return [ + "elementId": fields.count > 3 ? fields[3].stringValue ?? "" : fields[0].intValue.map(String.init) ?? "", + "type": fields[1].stringValue ?? "", + "properties": (fields[2].mapValue ?? [:]).mapValues(jsonObject) + ] as [String: Any] + } + if tag == Neo4jStructureTag.path, fields.count == 3 { + return pathObject(fields) ?? [ + "nodes": (fields[0].listValue ?? []).map(jsonObject), + "relationships": (fields[1].listValue ?? []).map(jsonObject) + ] + } + return structureText(tag: tag, fields: fields) ?? NSNull() + } + } + + // Bolt sends a path's distinct nodes and relationships plus (relationship, node) index pairs; a positive + // 1-based relationship index walks forward, a negative one backward. + private static func pathObject(_ fields: [PackStreamValue]) -> [String: Any]? { + let nodes = (fields[0].listValue ?? []).compactMap(Neo4jEntity.init) + let relationships = fields[1].listValue ?? [] + let sequence = (fields[2].listValue ?? []).compactMap(\.intValue) + guard let start = nodes.first, sequence.count.isMultiple(of: 2) else { return nil } + + var orderedNodes = [entityObject(start)] + var orderedRelationships: [Any] = [] + var current = start + for step in stride(from: 0, to: sequence.count, by: 2) { + let relationshipIndex = Int(sequence[step].magnitude) - 1 + let nodeIndex = Int(sequence[step + 1]) + guard sequence[step] != 0, relationships.indices.contains(relationshipIndex), + nodes.indices.contains(nodeIndex), + case .structure(_, let relationshipFields) = relationships[relationshipIndex], + relationshipFields.count >= 3 + else { return nil } + let next = nodes[nodeIndex] + let forward = sequence[step] > 0 + orderedRelationships.append(entityObject(Neo4jEntity( + elementId: relationshipFields.count > 3 + ? relationshipFields[3].stringValue ?? "" + : relationshipFields[0].intValue.map(String.init) ?? "", + kind: .relationship( + type: relationshipFields[1].stringValue ?? "", + startId: forward ? current.elementId : next.elementId, + endId: forward ? next.elementId : current.elementId + ), + properties: relationshipFields[2].mapValue ?? [:] + ))) + orderedNodes.append(entityObject(next)) + current = next + } + return ["nodes": orderedNodes, "relationships": orderedRelationships] + } + + private static func entityObject(_ entity: Neo4jEntity) -> [String: Any] { + var object: [String: Any] = [ + "elementId": entity.elementId, + "properties": entity.properties.mapValues(jsonObject) + ] + switch entity.kind { + case .node(let labels): + object["labels"] = labels + case .relationship(let type, let startId, let endId): + object["type"] = type + object["startElementId"] = startId + object["endElementId"] = endId + } + return object + } + + private static func json(_ object: Any) -> String { + guard JSONSerialization.isValidJSONObject(object), + let data = try? JSONSerialization.data(withJSONObject: object, options: [.sortedKeys, .withoutEscapingSlashes]), + let text = String(data: data, encoding: .utf8) + else { return String(describing: object) } + return text + } + + static func floatText(_ number: Double) -> String { + if number.isNaN { return "NaN" } + if number.isInfinite { return number < 0 ? "-Infinity" : "Infinity" } + return String(number) + } + + // MARK: - Temporal and spatial + + static func structureText(tag: UInt8, fields: [PackStreamValue]) -> String? { + let ints = fields.map { $0.intValue } + switch tag { + case Neo4jStructureTag.date: + guard let days = ints.first ?? nil else { return nil } + return dateText(days: days) + case Neo4jStructureTag.localTime: + guard let nanos = ints.first ?? nil else { return nil } + return timeText(nanosOfDay: nanos) + case Neo4jStructureTag.time: + guard ints.count == 2, let nanos = ints[0], let offset = ints[1] else { return nil } + return timeText(nanosOfDay: nanos) + offsetText(offset) + case Neo4jStructureTag.localDateTime: + guard ints.count == 2, let seconds = ints[0], let nanos = ints[1] else { return nil } + return dateTimeText(localSeconds: seconds, nanos: nanos) + case Neo4jStructureTag.dateTime, Neo4jStructureTag.legacyDateTime: + guard ints.count == 3, let seconds = ints[0], let nanos = ints[1], let offset = ints[2] else { return nil } + let local = tag == Neo4jStructureTag.dateTime ? seconds + offset : seconds + return dateTimeText(localSeconds: local, nanos: nanos) + offsetText(offset) + case Neo4jStructureTag.dateTimeZoneId, Neo4jStructureTag.legacyDateTimeZoneId: + guard fields.count == 3, let seconds = ints[0], let nanos = ints[1], + let zoneId = fields[2].stringValue else { return nil } + let zone = TimeZone(identifier: zoneId) + let offset: Int64 + if tag == Neo4jStructureTag.dateTimeZoneId { + offset = Int64(zone?.secondsFromGMT(for: Date(timeIntervalSince1970: TimeInterval(seconds))) ?? 0) + } else { + let guess = Int64(zone?.secondsFromGMT(for: Date(timeIntervalSince1970: TimeInterval(seconds))) ?? 0) + offset = Int64(zone?.secondsFromGMT(for: Date(timeIntervalSince1970: TimeInterval(seconds - guess))) ?? 0) + } + let local = tag == Neo4jStructureTag.dateTimeZoneId ? seconds + offset : seconds + return dateTimeText(localSeconds: local, nanos: nanos) + offsetText(offset) + "[\(zoneId)]" + case Neo4jStructureTag.duration: + guard ints.count == 4, let months = ints[0], let days = ints[1], + let seconds = ints[2], let nanos = ints[3] else { return nil } + return durationText(months: months, days: days, seconds: seconds, nanos: nanos) + case Neo4jStructureTag.point2D, Neo4jStructureTag.point3D: + guard let srid = ints.first ?? nil else { return nil } + let names = ["x", "y", "z"] + let coordinates = fields.dropFirst().enumerated().map { index, field -> String in + let number: Double + switch field { + case .float(let value): number = value + case .int(let value): number = Double(value) + default: number = .nan + } + return "\(names[min(index, 2)]): \(floatText(number))" + } + return "point({srid: \(srid), " + coordinates.joined(separator: ", ") + "})" + default: + return nil + } + } + + static func dateText(days: Int64) -> String { + let (year, month, day) = civilDate(daysSinceEpoch: days) + let yearText: String + if year < 0 || year > 9_999 { + yearText = (year < 0 ? "-" : "+") + String(format: "%04lld", abs(year)) + } else { + yearText = String(format: "%04lld", year) + } + return yearText + String(format: "-%02lld-%02lld", month, day) + } + + static func timeText(nanosOfDay: Int64) -> String { + let seconds = nanosOfDay / 1_000_000_000 + let nanos = nanosOfDay % 1_000_000_000 + return String(format: "%02lld:%02lld:%02lld", seconds / 3_600, seconds / 60 % 60, seconds % 60) + + fractionText(nanos) + } + + static func dateTimeText(localSeconds: Int64, nanos: Int64) -> String { + let days = floorDivide(localSeconds, 86_400) + let secondsOfDay = localSeconds - days * 86_400 + return dateText(days: days) + "T" + timeText(nanosOfDay: secondsOfDay * 1_000_000_000 + nanos) + } + + static func offsetText(_ offsetSeconds: Int64) -> String { + guard offsetSeconds != 0 else { return "Z" } + let sign = offsetSeconds < 0 ? "-" : "+" + let magnitude = abs(offsetSeconds) + var text = sign + String(format: "%02lld:%02lld", magnitude / 3_600, magnitude / 60 % 60) + if magnitude % 60 != 0 { text += String(format: ":%02lld", magnitude % 60) } + return text + } + + static func durationText(months: Int64, days: Int64, seconds: Int64, nanos: Int64) -> String { + var text = "P" + if months / 12 != 0 { text += "\(months / 12)Y" } + if months % 12 != 0 { text += "\(months % 12)M" } + if days != 0 { text += "\(days)D" } + // Seconds and nanoseconds stay apart: their product overflows Int64 for durations Neo4j accepts. + let carry = floorDivide(nanos, 1_000_000_000) + let (total, overflow) = seconds.addingReportingOverflow(carry) + let signedSeconds = overflow ? seconds : total + var fraction = nanos - carry * 1_000_000_000 + if signedSeconds != 0 || fraction != 0 { + let negative = signedSeconds < 0 + var wholeSeconds = signedSeconds.magnitude + if negative, fraction != 0 { + wholeSeconds -= 1 + fraction = 1_000_000_000 - fraction + } + var time = "T" + if wholeSeconds / 3_600 != 0 { time += (negative ? "-" : "") + "\(wholeSeconds / 3_600)H" } + if wholeSeconds / 60 % 60 != 0 { time += (negative ? "-" : "") + "\(wholeSeconds / 60 % 60)M" } + let remainder = wholeSeconds % 60 + if remainder != 0 || fraction != 0 { + time += (negative ? "-" : "") + "\(remainder)" + fractionText(fraction) + "S" + } + text += time + } + return text == "P" ? "PT0S" : text + } + + private static func fractionText(_ nanos: Int64) -> String { + guard nanos != 0 else { return "" } + var digits = String(format: "%09lld", nanos) + while digits.hasSuffix("0") { digits.removeLast() } + return "." + digits + } + + private static func floorDivide(_ value: Int64, _ divisor: Int64) -> Int64 { + let quotient = value / divisor + return (value % divisor != 0 && (value < 0) != (divisor < 0)) ? quotient - 1 : quotient + } + + // Howard Hinnant's `civil_from_days`. + private static func civilDate(daysSinceEpoch: Int64) -> (Int64, Int64, Int64) { + let shifted = daysSinceEpoch + 719_468 + let era = floorDivide(shifted, 146_097) + let dayOfEra = shifted - era * 146_097 + let yearOfEra = (dayOfEra - dayOfEra / 1_460 + dayOfEra / 36_524 - dayOfEra / 146_096) / 365 + let dayOfYear = dayOfEra - (365 * yearOfEra + yearOfEra / 4 - yearOfEra / 100) + let monthIndex = (5 * dayOfYear + 2) / 153 + let day = dayOfYear - (153 * monthIndex + 2) / 5 + 1 + let month = monthIndex < 10 ? monthIndex + 3 : monthIndex - 9 + let year = yearOfEra + era * 400 + (month <= 2 ? 1 : 0) + return (year, month, day) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProNeo4jCore/PackStream.swift b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/PackStream.swift new file mode 100644 index 0000000000..c6874b6a1c --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/PackStream.swift @@ -0,0 +1,257 @@ +import Foundation + +public indirect enum PackStreamValue: Equatable, Sendable { + case null + case bool(Bool) + case int(Int64) + case float(Double) + case bytes(Data) + case string(String) + case list([PackStreamValue]) + case map([String: PackStreamValue]) + case structure(tag: UInt8, fields: [PackStreamValue]) + + public var stringValue: String? { + if case .string(let value) = self { return value } + return nil + } + + public var intValue: Int64? { + if case .int(let value) = self { return value } + return nil + } + + public var boolValue: Bool? { + if case .bool(let value) = self { return value } + return nil + } + + public var listValue: [PackStreamValue]? { + if case .list(let value) = self { return value } + return nil + } + + public var mapValue: [String: PackStreamValue]? { + if case .map(let value) = self { return value } + return nil + } +} + +public enum PackStreamError: Error, Equatable, Sendable { + case truncated + case unknownMarker(UInt8) + case invalidString + case nonStringKey + case nestedTooDeeply +} + +public enum PackStreamEncoder { + public static func encode(_ value: PackStreamValue) -> Data { + var data = Data() + write(value, into: &data) + return data + } + + public static func write(_ value: PackStreamValue, into data: inout Data) { + switch value { + case .null: + data.append(0xC0) + case .bool(let flag): + data.append(flag ? 0xC3 : 0xC2) + case .int(let number): + writeInt(number, into: &data) + case .float(let number): + data.append(0xC1) + appendBigEndian(number.bitPattern, into: &data) + case .bytes(let bytes): + writeHeader(count: bytes.count, tiny: nil, markers: (0xCC, 0xCD, 0xCE), into: &data) + data.append(bytes) + case .string(let text): + let utf8 = Data(text.utf8) + writeHeader(count: utf8.count, tiny: 0x80, markers: (0xD0, 0xD1, 0xD2), into: &data) + data.append(utf8) + case .list(let items): + writeHeader(count: items.count, tiny: 0x90, markers: (0xD4, 0xD5, 0xD6), into: &data) + for item in items { write(item, into: &data) } + case .map(let entries): + writeHeader(count: entries.count, tiny: 0xA0, markers: (0xD8, 0xD9, 0xDA), into: &data) + for key in entries.keys.sorted() { + write(.string(key), into: &data) + write(entries[key] ?? .null, into: &data) + } + case .structure(let tag, let fields): + precondition(fields.count < 16, "Bolt has no marker for a structure over 15 fields") + data.append(0xB0 | UInt8(fields.count)) + data.append(tag) + for field in fields { write(field, into: &data) } + } + } + + private static func writeInt(_ number: Int64, into data: inout Data) { + switch number { + case -16 ... 127: + data.append(UInt8(bitPattern: Int8(number))) + case Int64(Int8.min) ... Int64(Int8.max): + data.append(0xC8) + data.append(UInt8(bitPattern: Int8(number))) + case Int64(Int16.min) ... Int64(Int16.max): + data.append(0xC9) + appendBigEndian(UInt16(bitPattern: Int16(number)), into: &data) + case Int64(Int32.min) ... Int64(Int32.max): + data.append(0xCA) + appendBigEndian(UInt32(bitPattern: Int32(number)), into: &data) + default: + data.append(0xCB) + appendBigEndian(UInt64(bitPattern: number), into: &data) + } + } + + private static func writeHeader( + count: Int, + tiny: UInt8?, + markers: (UInt8, UInt8, UInt8), + into data: inout Data + ) { + if let tiny, count < 16 { + data.append(tiny | UInt8(count)) + } else if count <= Int(UInt8.max) { + data.append(markers.0) + data.append(UInt8(count)) + } else if count <= Int(UInt16.max) { + data.append(markers.1) + appendBigEndian(UInt16(count), into: &data) + } else { + data.append(markers.2) + appendBigEndian(UInt32(count), into: &data) + } + } + + private static func appendBigEndian(_ value: T, into data: inout Data) { + withUnsafeBytes(of: value.bigEndian) { data.append(contentsOf: $0) } + } +} + +public struct PackStreamDecoder { + // Each level is a stack frame, so a server cannot overflow the stack with a deeply nested list. + public static let maximumDepth = 64 + + private let bytes: [UInt8] + private var index = 0 + private var depth = 0 + + public init(_ data: Data) { + bytes = [UInt8](data) + } + + public var isAtEnd: Bool { index >= bytes.count } + + public static func decode(_ data: Data) throws -> PackStreamValue { + var decoder = PackStreamDecoder(data) + return try decoder.next() + } + + public mutating func next() throws -> PackStreamValue { + depth += 1 + defer { depth -= 1 } + guard depth <= Self.maximumDepth else { throw PackStreamError.nestedTooDeeply } + let marker = try byte() + switch marker { + case 0x00 ... 0x7F: + return .int(Int64(marker)) + case 0xF0 ... 0xFF: + return .int(Int64(Int8(bitPattern: marker))) + case 0x80 ... 0x8F: + return try string(count: Int(marker & 0x0F)) + case 0x90 ... 0x9F: + return try list(count: Int(marker & 0x0F)) + case 0xA0 ... 0xAF: + return try map(count: Int(marker & 0x0F)) + case 0xB0 ... 0xBF: + let tag = try byte() + var fields: [PackStreamValue] = [] + for _ in 0 ..< Int(marker & 0x0F) { fields.append(try next()) } + return .structure(tag: tag, fields: fields) + case 0xC0: + return .null + case 0xC1: + return .float(Double(bitPattern: try integer(UInt64.self))) + case 0xC2: + return .bool(false) + case 0xC3: + return .bool(true) + case 0xC8: + return .int(Int64(Int8(bitPattern: try byte()))) + case 0xC9: + return .int(Int64(Int16(bitPattern: try integer(UInt16.self)))) + case 0xCA: + return .int(Int64(Int32(bitPattern: try integer(UInt32.self)))) + case 0xCB: + return .int(Int64(bitPattern: try integer(UInt64.self))) + case 0xCC: + return .bytes(Data(try take(Int(try byte())))) + case 0xCD: + return .bytes(Data(try take(Int(try integer(UInt16.self))))) + case 0xCE: + return .bytes(Data(try take(Int(try integer(UInt32.self))))) + case 0xD0: + return try string(count: Int(try byte())) + case 0xD1: + return try string(count: Int(try integer(UInt16.self))) + case 0xD2: + return try string(count: Int(try integer(UInt32.self))) + case 0xD4: + return try list(count: Int(try byte())) + case 0xD5: + return try list(count: Int(try integer(UInt16.self))) + case 0xD6: + return try list(count: Int(try integer(UInt32.self))) + case 0xD8: + return try map(count: Int(try byte())) + case 0xD9: + return try map(count: Int(try integer(UInt16.self))) + case 0xDA: + return try map(count: Int(try integer(UInt32.self))) + default: + throw PackStreamError.unknownMarker(marker) + } + } + + private mutating func byte() throws -> UInt8 { + guard index < bytes.count else { throw PackStreamError.truncated } + defer { index += 1 } + return bytes[index] + } + + private mutating func take(_ count: Int) throws -> ArraySlice { + guard count >= 0, bytes.count - index >= count else { throw PackStreamError.truncated } + defer { index += count } + return bytes[index ..< index + count] + } + + private mutating func integer(_: T.Type) throws -> T { + try take(MemoryLayout.size).reduce(T.zero) { ($0 << 8) | T($1) } + } + + private mutating func string(count: Int) throws -> PackStreamValue { + guard let text = String(bytes: try take(count), encoding: .utf8) else { + throw PackStreamError.invalidString + } + return .string(text) + } + + private mutating func list(count: Int) throws -> PackStreamValue { + var items: [PackStreamValue] = [] + items.reserveCapacity(min(count, 1_024)) + for _ in 0 ..< count { items.append(try next()) } + return .list(items) + } + + private mutating func map(count: Int) throws -> PackStreamValue { + var entries: [String: PackStreamValue] = [:] + for _ in 0 ..< count { + guard case .string(let key) = try next() else { throw PackStreamError.nonStringKey } + entries[key] = try next() + } + return .map(entries) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/LibPQStringConformance.swift b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/LibPQStringConformance.swift index f22d445738..8310bc99c2 100644 --- a/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/LibPQStringConformance.swift +++ b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/LibPQStringConformance.swift @@ -22,9 +22,9 @@ public enum LibPQStringConformance { public static func escape(_ value: String, standardConformingStrings: Bool) -> String { let quoted = value - .replacingOccurrences(of: "\0", with: "") - .replacingOccurrences(of: "'", with: "''") + .replacingOccurrences(of: "\0", with: "", options: .literal) + .replacingOccurrences(of: "'", with: "''", options: .literal) guard !standardConformingStrings else { return quoted } - return quoted.replacingOccurrences(of: "\\", with: "\\\\") + return quoted.replacingOccurrences(of: "\\", with: "\\\\", options: .literal) } } diff --git a/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCatalogSearchPath.swift b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCatalogSearchPath.swift new file mode 100644 index 0000000000..cda64d28c8 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCatalogSearchPath.swift @@ -0,0 +1,28 @@ +import Foundation + +/// The `search_path` a driver-authored catalog statement runs under, set with `SET LOCAL` in the same query string so it +/// lasts only as long as the read. +/// +/// PostgreSQL resolves a function or operator over every schema on the path, and a candidate that takes the call's +/// argument types exactly beats a `pg_catalog` one reached through a cast, wherever the two sit on the path. A +/// `pg_total_relation_size(oid)` or an `<>(oid, integer)` in a browsed schema would otherwise run with the reader's +/// privileges, the substitution of CVE-2018-1058. `pg_dump` runs with the same narrowed path for the same reason. +/// `pg_temp` is listed last so a temporary relation never shadows a catalog; functions and operators are never looked +/// up there. +/// +/// `SET LOCAL` in the read's own query string makes the pair one implicit transaction, so the path is back as soon as +/// the read returns; inside a transaction the caller holds, the driver scopes it with a savepoint. `pg_catalog` rather +/// than an empty path: PostgreSQL 9.1 rejects `''` ("schema "" does not exist"), and `pg_catalog` deparses every user +/// relation with its schema all the same (measured identical output on 9.2, 9.3, 9.6 and 17). +nonisolated public enum PostgreSQLCatalogSearchPath { + public static let pinned = "SET LOCAL search_path = pg_catalog, pg_temp; " + + /// For a read whose output the server deparses: `format_type`, `pg_get_expr` and the `pg_get_*def` functions leave + /// out the schema of a name the path resolves, so this writes the schema's own names bare. The schema is on the + /// path, so a statement run under it calls only `pg_catalog`-qualified functions and compares only operands a + /// `pg_catalog` operator takes exactly. + public static func relative(to schema: String) -> String { + let quoted = "\"\(schema.replacingOccurrences(of: "\"", with: "\"\"", options: .literal))\"" + return "SET LOCAL search_path = pg_catalog, \(quoted), pg_temp; " + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCatalogTypeNames.swift b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCatalogTypeNames.swift index 218ec3889d..e5bf1f8b15 100644 --- a/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCatalogTypeNames.swift +++ b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCatalogTypeNames.swift @@ -105,8 +105,11 @@ public enum PostgreSQLCatalogTypeNames { /// because Redshift's catalog predates `typcategory` and the predicate agrees with it on /// every real type: `point` and `name` carry a `typelem` at a fixed length, and the one /// disagreement is the pseudo-type `_record`, which no column has. + /// + /// Sent on the user's own session under its search path, so each oid is typed: a bare integer compares oid with + /// integer, which a `=(oid, integer)` in a schema on that path takes exactly. public static func lookupQuery(oids: [UInt32]) -> String? { - let list = Set(oids).sorted().map(String.init).joined(separator: ", ") + let list = Set(oids).sorted().map { "'\($0)'::pg_catalog.oid" }.joined(separator: ", ") guard !list.isEmpty else { return nil } return """ SELECT t.oid::text, t.typname, t.typtype::text, diff --git a/apple/Packages/TableProCore/Sources/TableProQuery/FilterSQLGenerator.swift b/apple/Packages/TableProCore/Sources/TableProQuery/FilterSQLGenerator.swift index 93db218345..824aaf2522 100644 --- a/apple/Packages/TableProCore/Sources/TableProQuery/FilterSQLGenerator.swift +++ b/apple/Packages/TableProCore/Sources/TableProQuery/FilterSQLGenerator.swift @@ -87,13 +87,13 @@ public struct FilterSQLGenerator: Sendable { let escapedSecond = escapeValue(filter.secondValue, kind: kind) return "\(quotedColumn) BETWEEN \(escapedValue) AND \(escapedSecond)" case .contains: - let pattern = escapeLikePattern(filter.value) + let pattern = likePatternBody(filter.value) return likeCondition(operand, "'%\(pattern)%'", negated: false, folding: folding) case .startsWith: - let pattern = escapeLikePattern(filter.value) + let pattern = likePatternBody(filter.value) return likeCondition(operand, "'\(pattern)%'", negated: false, folding: folding) case .endsWith: - let pattern = escapeLikePattern(filter.value) + let pattern = likePatternBody(filter.value) return likeCondition(operand, "'%\(pattern)'", negated: false, folding: folding) } } @@ -117,7 +117,7 @@ public struct FilterSQLGenerator: Sendable { private func comparisonCondition( _ column: String, _ operand: String, _ operatorText: String, _ literal: String, folding: PluginSQLCaseFolding ) -> String { - guard folding.foldsComparisonOperands, literal.hasPrefix("'") else { + guard folding.foldsComparisonOperands, literal.unicodeScalars.first == "'" else { return "\(column) \(operatorText) \(literal)" } return "\(folding.fold(operand)) \(operatorText) \(folding.fold(literal))" @@ -159,7 +159,7 @@ public struct FilterSQLGenerator: Sendable { _ column: String, _ pattern: String, negated: Bool, folding: PluginSQLCaseFolding ) -> String { let keyword = negated ? folding.notLikeKeyword : folding.likeKeyword - return "\(folding.foldingLikeOperand(column)) \(keyword) \(folding.foldingLikeOperand(pattern))\(likeEscape)" + return "\(folding.foldingLikeOperand(column)) \(keyword) \(folding.foldingLikeOperand(pattern))\(likeEscapeClause)" } private func listCondition( @@ -172,13 +172,13 @@ public struct FilterSQLGenerator: Sendable { ) -> String { let keyword = negated ? "NOT IN" : "IN" let items = listItems(rawValue).map { escapeValue($0, kind: kind) } - let foldable = folding.foldsComparisonOperands && items.allSatisfy { $0.hasPrefix("'") } + let foldable = folding.foldsComparisonOperands && items.allSatisfy { $0.unicodeScalars.first == "'" } let rendered = foldable ? items.map { folding.fold($0) } : items let subject = foldable ? folding.fold(operand) : column return "\(subject) \(keyword) (\(rendered.joined(separator: ", ")))" } - private var likeEscape: String { + public var likeEscapeClause: String { switch dialect.likeEscapeStyle { case .explicit: return " ESCAPE '!'" @@ -187,11 +187,22 @@ public struct FilterSQLGenerator: Sendable { } } + // The LIKE layer first, with the escape `likeEscapeClause` names, then the literal rule over the whole pattern. + public func likePatternBody(_ value: String) -> String { + let escape = dialect.likeEscapeStyle == .implicit ? "\\" : "!" + // `[` opens a character class in a SQL Server LIKE pattern. + let metacharacters = [escape, "%", "_"] + (databaseType == .mssql ? ["["] : []) + let pattern = metacharacters.reduce(value) { pattern, metacharacter in + pattern.replacingOccurrences(of: metacharacter, with: escape + metacharacter, options: .literal) + } + return literalBody(pattern) + } + private func quoteIdentifier(_ name: String) -> String { if let identifierQuoter { return identifierQuoter(name) } let open = dialect.identifierQuote let close = open == "[" ? "]" : open - let escaped = name.replacingOccurrences(of: close, with: "\(close)\(close)") + let escaped = name.replacingOccurrences(of: close, with: "\(close)\(close)", options: .literal) return "\(open)\(escaped)\(close)" } @@ -199,32 +210,15 @@ public struct FilterSQLGenerator: Sendable { if FilterColumnKind.isBareNumber(value, kind: kind) { return value } - var escaped = value.replacingOccurrences(of: "\0", with: "") - if dialect.requiresBackslashEscaping { - escaped = escaped.replacingOccurrences(of: "\\", with: "\\\\") - } - escaped = escaped.replacingOccurrences(of: "'", with: "''") - return "'\(escaped)'" + return "'\(literalBody(value))'" } - private func escapeLikePattern(_ value: String) -> String { - var result = value - .replacingOccurrences(of: "'", with: "''") - .replacingOccurrences(of: "\0", with: "") - switch dialect.likeEscapeStyle { - case .explicit: - result = result - .replacingOccurrences(of: "!", with: "!!") - .replacingOccurrences(of: "%", with: "!%") - .replacingOccurrences(of: "_", with: "!_") - case .implicit: - if dialect.requiresBackslashEscaping { - result = result.replacingOccurrences(of: "\\", with: "\\\\") - } - result = result - .replacingOccurrences(of: "%", with: "\\%") - .replacingOccurrences(of: "_", with: "\\_") + // `.literal` compares code units: the default compares grapheme clusters and skips a quote a combining mark follows. + private func literalBody(_ value: String) -> String { + var body = value.replacingOccurrences(of: "\0", with: "", options: .literal) + if dialect.requiresBackslashEscaping { + body = body.replacingOccurrences(of: "\\", with: "\\\\", options: .literal) } - return result + return body.replacingOccurrences(of: "'", with: "''", options: .literal) } } diff --git a/apple/Packages/TableProCore/Sources/TableProQuery/SQLWriteClassifier.swift b/apple/Packages/TableProCore/Sources/TableProQuery/SQLWriteClassifier.swift index cc2d9b5894..ea62cfb7df 100644 --- a/apple/Packages/TableProCore/Sources/TableProQuery/SQLWriteClassifier.swift +++ b/apple/Packages/TableProCore/Sources/TableProQuery/SQLWriteClassifier.swift @@ -9,7 +9,9 @@ import TableProSQLGrammar /// The rule is fail-closed: a statement counts as a read only when its leading keyword is one of a /// short, closed set of read verbs. Everything else writes, including a keyword this classifier has /// never heard of. A write-keyword allowlist cannot be safe, because anything it has not been -/// taught, or anything hidden behind a leading comment, runs unguarded. +/// taught, or anything hidden behind a leading comment, runs unguarded. A query that passes the verb +/// rule is then read for what it calls, because `SELECT setval('seq', 1)` and +/// `SELECT pg_terminate_backend(pid)` open with a read verb and change state all the same. /// /// The batch is split into statements the way its engine lexes it, by the grammar the Mac reads /// too, and under every reading the engine could be using: the run splits by one reading, and a @@ -18,21 +20,37 @@ import TableProSQLGrammar /// engine that needs no `;`, every statement that begins inside one is read as well, because SQL /// Server runs `SELECT 1` followed by `DROP TABLE t` as two statements. public enum SQLWriteClassifier { - /// `EXPLAIN` and `PRAGMA` are deliberately absent: `EXPLAIN ANALYZE DELETE …` runs the delete on - /// PostgreSQL, and `PRAGMA journal_mode = WAL` writes on SQLite and DuckDB. - private static let readKeywords: Set = ["SHOW", "DESCRIBE", "DESC"] + // EXPLAIN and PRAGMA are absent: `EXPLAIN ANALYZE DELETE …` runs the delete on PostgreSQL, and + // `PRAGMA journal_mode = WAL` writes on SQLite and DuckDB. + private static let readKeywords: Set = ["SHOW"] - /// A `SELECT` reads unless it materialises a table, which `SELECT … INTO` does on SQL Server and - /// `SELECT … INTO OUTFILE` does on MySQL. Mirrors QueryClassifier.swift:288-291. + // MySQL and TiDB read `DESC ANALYZE DELETE …` as EXPLAIN ANALYZE, so only a DESCRIBE of a name reads. + private static let describeKeywords: Set = ["DESCRIBE", "DESC"] + + // `SELECT … INTO` materialises a table on SQL Server and a file on MySQL. private static let readUnlessIntoKeywords: Set = ["SELECT", "TABLE", "VALUES"] - /// Statement verbs only. `REPLACE`, `INTO` and `COPY` are left out because they collide with - /// ordinary function and column names, and a CTE that merely calls `replace()` is a read. - /// Matches QueryClassifier.swift:304-312. + // `REPLACE` and `COPY` are left out because they are also function and column names. private static let writeKeywordsInsideCTE: [String] = [ "INSERT", "UPDATE", "DELETE", "MERGE", "UPSERT", "DROP", "TRUNCATE", "ALTER", "CREATE", "INTO" ] + /// `NEXT VALUE FOR seq` (SQL Server, MariaDB, Db2) and `NEXTVAL FOR seq` (Db2) advance a sequence with + /// no call for the function scanner to see. Matches QueryClassifier.sequenceAdvanceRegex. + private static let sequenceAdvanceRegex = try? NSRegularExpression( + pattern: "\\bNEXT(?:\\s+VALUE|VAL)\\s+FOR\\b", + options: [] + ) + + /// `seq.NEXTVAL` does the same on the engines that have the pseudo-column. Elsewhere `t.nextval` is a column. + private static let sequencePseudoColumnRegex = try? NSRegularExpression( + pattern: "\\.\\s*NEXTVAL\\b", + options: [] + ) + + /// QueryClassifier.enginesWithSequencePseudoColumns without SAP HANA, which only the Mac connects to. + private static let enginesWithSequencePseudoColumns: Set = [.oracle, .snowflake, .dameng, .mariadb] + public static func isWriteQuery(_ sql: String, databaseType: DatabaseType) -> Bool { if databaseType == .redis { return RedisCommandSplitter.commands(in: sql).contains { redisWrites($0.text) } @@ -50,7 +68,7 @@ public enum SQLWriteClassifier { SQLStatementScanner.executableStatements(in: lexing.text, grammar: lexing.grammar).contains { statement in SQLUnterminatedStatements.runnable(in: statement.sql, grammar: lexing.grammar).contains { runnable in seen.insert(SQLLexing(grammar: lexing.grammar, text: runnable)).inserted - && statementWrites(runnable, grammar: lexing.grammar) + && statementWrites(runnable, grammar: lexing.grammar, databaseType: databaseType) } } } @@ -58,17 +76,58 @@ public enum SQLWriteClassifier { /// Reads the statement's code with every literal and comment blanked by its engine's own rules. A MySQL /// `/*! ... */` is kept as the code it is, so a keyword the server runs from one is never mistaken for a read. - private static func statementWrites(_ statement: String, grammar: SQLLexicalGrammar) -> Bool { + private static func statementWrites( + _ statement: String, + grammar: SQLLexicalGrammar, + databaseType: DatabaseType, + depth: Int = 0 + ) -> Bool { + // Each nested CTE copies and reads what is left of the text, so nesting past this counts as a write. + guard depth <= 32 else { return true } let code = SQLCodeProjection.code(of: statement, grammar: grammar, revealingExecutableComments: true) - let body = String(code.drop { $0.isWhitespace }) + let body = String(code.drop { $0.isWhitespace || $0 == "(" }) guard let keyword = leadingKeyword(of: body) else { return true } - if keyword == "WITH" { return commonTableExpressionWrites(code) } + if keyword == "WITH" { + return commonTableExpressionWrites( + statement, code: code, grammar: grammar, databaseType: databaseType, depth: depth + ) + || queryChangesState(statement, code: code, grammar: grammar, databaseType: databaseType) + } + if describeKeywords.contains(keyword) { + return SQLExplainStatement.read(statement, grammar: grammar)?.reading != .describesObject + } if readUnlessIntoKeywords.contains(keyword) { return containsWord("INTO", in: code.uppercased()) + || queryChangesState(statement, code: code, grammar: grammar, databaseType: databaseType) } return !readKeywords.contains(keyword) } + /// A query's verb says nothing about what it calls: a function in the select list can move a sequence, + /// end a session or drop a large object. A callee the scanner cannot name counts as one that changes + /// state. Mirrors QueryClassifier.stateChangingCallTier. + private static func queryChangesState( + _ statement: String, + code: String, + grammar: SQLLexicalGrammar, + databaseType: DatabaseType + ) -> Bool { + if advancesSequence(code.uppercased(), databaseType: databaseType) { return true } + let callsChangeState = SQLFunctionCallScanner.calls(in: statement as NSString, code: code as NSString) + .contains { call in + guard case .named(let name, _) = call.callee else { return true } + return SQLStateChangingFunctions.contains(name) + } + return callsChangeState || SQLStateChangingFunctions.namesStateChangingTable(in: statement, grammar: grammar) + } + + private static func advancesSequence(_ body: String, databaseType: DatabaseType) -> Bool { + let range = NSRange(body.startIndex..., in: body) + if sequenceAdvanceRegex?.firstMatch(in: body, options: [], range: range) != nil { return true } + guard enginesWithSequencePseudoColumns.contains(databaseType) else { return false } + return sequencePseudoColumnRegex?.firstMatch(in: body, options: [], range: range) != nil + } + // Read as the driver reads it, so a quoted verb is the command that runs. Unparseable text counts as a write. private static func redisWrites(_ command: String) -> Bool { guard let arguments = RedisArgumentCodec.split(command) else { return true } @@ -76,12 +135,23 @@ public enum SQLWriteClassifier { return !RedisReadCommands.onlyReads(words[...]) } - /// A CTE's leading keyword says nothing about what the statement finally does, so its code is - /// searched for a write verb. - private static func commonTableExpressionWrites(_ code: String) -> Bool { + // After the CTE list only a query reads, and each body is read as a statement of its own. + private static func commonTableExpressionWrites( + _ statement: String, + code: String, + grammar: SQLLexicalGrammar, + databaseType: DatabaseType, + depth: Int + ) -> Bool { let masked = code.uppercased() - return writeKeywordsInsideCTE.contains { keyword in - containsWord(keyword, in: masked) + if writeKeywordsInsideCTE.contains(where: { containsWord($0, in: masked) }) { return true } + guard let list = SQLCommonTableExpressions.read(statement, grammar: grammar) else { return true } + let text = statement as NSString + let main = text.substring(from: list.mainStatement) + let mainCode = SQLCodeProjection.code(of: main, grammar: grammar) + guard readUnlessIntoKeywords.contains(SQLLeadingComments.leadingCodeKeyword(mainCode)) else { return true } + return ([main] + list.bodies.map(text.substring(with:))).contains { part in + statementWrites(part, grammar: grammar, databaseType: databaseType, depth: depth + 1) } } diff --git a/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLIntrospectionSQL.swift b/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLIntrospectionSQL.swift index b8cb4bb289..cd67a7309f 100644 --- a/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLIntrospectionSQL.swift +++ b/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLIntrospectionSQL.swift @@ -33,7 +33,7 @@ public enum R2SQLIntrospectionSQL { } public static func quoteIdentifier(_ identifier: String) -> String { - "\"" + identifier.replacingOccurrences(of: "\"", with: "\"\"") + "\"" + "\"" + identifier.replacingOccurrences(of: "\"", with: "\"\"", options: .literal) + "\"" } static let namespaceColumns = ["namespace", "namespace_name", "database_name", "schema_name", "databaseName"] diff --git a/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLResponseDecoder.swift b/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLResponseDecoder.swift index a981a58ce2..0edd59042e 100644 --- a/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLResponseDecoder.swift +++ b/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLResponseDecoder.swift @@ -9,7 +9,8 @@ public enum R2SQLResponseDecoder { throw R2SQLError.malformedResponse(status: response.statusCode, detail: snippet(response.body)) } - guard envelope.success else { + // A refused redirect arrives as its 3xx, whatever the body says. + guard envelope.success, (200 ..< 300).contains(response.statusCode) else { switch response.statusCode { case 401, 403: throw R2SQLError.authentication(status: response.statusCode, errors: envelope.errors) diff --git a/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLURLSessionTransport.swift b/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLURLSessionTransport.swift index 8924638df3..2292e43911 100644 --- a/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLURLSessionTransport.swift +++ b/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLURLSessionTransport.swift @@ -75,6 +75,17 @@ private final class TaskTracker: NSObject, URLSessionTaskDelegate, @unchecked Se transport?.register(task) } + // A redirect would carry the API token to wherever it points. + func urlSession( + _ session: URLSession, + task: URLSessionTask, + willPerformHTTPRedirection response: HTTPURLResponse, + newRequest request: URLRequest, + completionHandler: @escaping (URLRequest?) -> Void + ) { + completionHandler(nil) + } + func finish() { guard let task = lock.withLock({ task }) else { return } transport?.unregister(task) diff --git a/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisQueryBuilder.swift b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisQueryBuilder.swift index b613314ba8..d2eb9468ba 100644 --- a/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisQueryBuilder.swift +++ b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisQueryBuilder.swift @@ -119,15 +119,15 @@ public struct RedisQueryBuilder: Sendable { } private func quoteForCommand(_ str: String) -> String { - var result = "" - for char in str { - switch char { - case "\\": result += "\\\\" - case "\"": result += "\\\"" - default: result.append(char) + var result = String.UnicodeScalarView() + for scalar in str.unicodeScalars { + switch scalar { + case "\\", "\"": result.append("\\") + default: break } + result.append(scalar) } - return result + return String(result) } /// Try to extract a SCAN-compatible glob pattern from key-column filters @@ -158,18 +158,14 @@ public struct RedisQueryBuilder: Sendable { /// Escape Redis glob special characters in user input. /// Redis SCAN MATCH uses glob-style patterns where *, ?, and [ are special. private func escapeGlobChars(_ str: String) -> String { - var result = "" - for char in str { - switch char { - case "*", "?", "[", "]": - result.append("\\") - result.append(char) - case "\\": - result.append("\\\\") - default: - result.append(char) + var result = String.UnicodeScalarView() + for scalar in str.unicodeScalars { + switch scalar { + case "*", "?", "[", "]", "\\": result.append("\\") + default: break } + result.append(scalar) } - return result + return String(result) } } diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/CQLBatchTracker.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/CQLBatchTracker.swift index 743f8cd2cb..fa0640b9c1 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/CQLBatchTracker.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/CQLBatchTracker.swift @@ -34,6 +34,10 @@ public struct CQLBatchTracker: SQLStatementBoundaryTracking { true } + public var holdsOpenBlock: Bool { + phase == .insideBatch || phase == .afterCloser + } + public mutating func observeWord(_ word: String) { switch phase { case .start: diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/JavaScript/MongoStatementClassifier.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/JavaScript/MongoStatementClassifier.swift index 0272332c25..2abb610a77 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/JavaScript/MongoStatementClassifier.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/JavaScript/MongoStatementClassifier.swift @@ -58,11 +58,15 @@ public enum MongoStatementClassifier { // Server-side JavaScript, plus the two stages that write to a collection the statement names. private static let codeExecutionKeys: Set = ["$where", "$function", "$accumulator", "$out", "$merge"] + // The server accepts its own operators from any client, and rewrites `mapReduce` into two of them that run a + // function: `$_internalJsEmit` and `$_internalJsReduce`. Nothing a person writes names one. + private static let internalOperatorPrefix = "$_internal" + private static let codeExecutionCalls: Set = ["mapreduce", "eval"] // For text the grammar rejects, where a false alarm only stops an external client sending code nobody could read. private static let codeExecutionMarkers: [String] = [ - "$where", "$function", "$accumulator", "mapreduce", ".eval(", "$out", "$merge" + "$where", "$function", "$accumulator", "$_internal", "mapreduce", ".eval(", "$out", "$merge" ] private static func statementVerdict( @@ -78,6 +82,7 @@ public enum MongoStatementClassifier { private static func verdict(of statement: MongoStatement) -> MongoStatementVerdict { let reachesCode = !statement.objectKeys.isDisjoint(with: codeExecutionKeys) + || statement.objectKeys.contains { $0.hasPrefix(internalOperatorPrefix) } || statement.calls.contains { call in codeExecutionCalls.contains(call.name.lowercased()) || call.commandName.map { codeExecutionCalls.contains($0.lowercased()) } == true diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/PLSQLUnitTracker.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/PLSQLUnitTracker.swift index f0c5161f62..74732eb67d 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/PLSQLUnitTracker.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/PLSQLUnitTracker.swift @@ -16,8 +16,11 @@ import Foundation /// Inside a unit the tracker counts the constructs that one bare `END` closes: a block, a subprogram or package body, /// a compound trigger and a `CASE`. A `BEGIN` continues the innermost construct when that construct is still in its /// declaration section, because `DECLARE ... BEGIN ... END` and `PROCEDURE p IS ... BEGIN ... END` each end once. -/// `END IF` and `END LOOP` close constructs that never opened one, and a word after `.` is a member name, never a -/// keyword. A `;` ends the unit only once every construct has closed. +/// `END IF` and `END LOOP` close constructs that never opened one, except between a loop's `FOR` or `WHILE` and its +/// `LOOP`: no statement ends there, so `CASE ... END LOOP` is a `CASE` expression and the `LOOP` that opens the loop. +/// A word after the `.` of `v.END` is a member name, never a keyword. The `..` of a range and the `.` that ends a +/// number name no member, so the word after `1..` or `2.` is a keyword. A `;` ends the unit only once every construct +/// has closed. /// /// The final `;` of a unit belongs to it, except after a trigger whose body is a `CALL`: measured on Oracle 23ai, /// `CREATE TRIGGER ... CALL p(:NEW.a);` is stored INVALID and the same text without the `;` compiles. @@ -50,6 +53,7 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { private static let lessThan = UInt16(UnicodeScalar("<").value) private static let greaterThan = UInt16(UnicodeScalar(">").value) private static let period = UInt16(UnicodeScalar(".").value) + private static let digits = UInt16(UnicodeScalar("0").value)...UInt16(UnicodeScalar("9").value) private static let createModifiers: Set = [ "OR", "REPLACE", "EDITIONABLE", "NONEDITIONABLE", "AND", "RESOLVE", "COMPILE", "NOFORCE", @@ -61,6 +65,10 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { /// One entry per construct a bare `END` will close, true while that construct still waits for its `BEGIN`. private var constructs: [Bool] = [] + + /// The open `CASE`s that have held no `;`, which only a `CASE` statement can. The next `END` closes one of them, + /// whatever word follows it. + private var openCaseExpressions = 0 private var parenDepth = 0 private var pendingEnd = false private var pendingBodyKeyword = false @@ -97,6 +105,18 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { } } + // An `END` still waiting for the word after it closes a construct when the text ends. + public var holdsOpenBlock: Bool { + switch kind { + case .anonymousBlock, .storedUnit: + return constructs.count > (pendingEnd ? 1 : 0) + case .inlineDeclarations: + return !inlineQueryStarted && constructs.count > (pendingEnd ? 1 : 0) + case .sql, .embeddedSourceHeader, .opaque: + return false + } + } + public mutating func observeWord(_ word: String) { previousSymbol = nil guard !inLabel else { return } @@ -153,6 +173,7 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { } public mutating func observeSymbol(_ symbol: UInt16) { + let precedingSymbol = previousSymbol if symbol == Self.lessThan, previousSymbol == Self.lessThan { inLabel = true previousSymbol = nil @@ -174,7 +195,7 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { default: break } - followsPeriod = symbol == Self.period + followsPeriod = symbol == Self.period && !Self.isPeriodOrDigit(precedingSymbol) } public mutating func observeOpaqueToken() { @@ -201,6 +222,7 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { public mutating func observeSemicolon() -> Bool { previousSymbol = nil followsPeriod = false + openCaseExpressions = 0 guard lead == .decided else { return true } switch kind { @@ -229,6 +251,12 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { // MARK: - Private + /// A name that ends in a digit arrives as a word, which leaves no preceding symbol, so `t1.END` is still a member. + private static func isPeriodOrDigit(_ symbol: UInt16?) -> Bool { + guard let symbol else { return false } + return symbol == period || digits.contains(symbol) + } + private var tracksConstructs: Bool { guard lead == .decided else { return false } switch kind { @@ -340,6 +368,7 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { constructs.append(true) case "CASE": constructs.append(false) + openCaseExpressions += 1 case "END": pendingEnd = true case "PROCEDURE", "FUNCTION": @@ -353,7 +382,7 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { /// Returns whether `word` was consumed by the `END` before it. private mutating func closeEnd(followedBy word: String) -> Bool { - switch SqlBlockStructure.endingFollowedBy(word) { + switch SqlBlockStructure.endingFollowedBy(word, closesCaseExpression: openCaseExpressions > 0) { case .closesControlFlow: return true case .closesCaseStatement: @@ -387,5 +416,6 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { private mutating func closeConstruct() { guard !constructs.isEmpty else { return } constructs.removeLast() + openCaseExpressions = max(0, openCaseExpressions - 1) } } diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCatalogEffect.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCatalogEffect.swift index a899b93894..2e44afde28 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCatalogEffect.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCatalogEffect.swift @@ -65,7 +65,7 @@ public enum SQLCatalogEffectClassifier { } private static func textEffect(of sql: String, grammar: SQLLexicalGrammar) -> SQLCatalogEffect { - let trimmed = StatementBlank.trimming(SQLLeadingComments.stripping(sql)) + let trimmed = StatementBlank.trimming(SQLLeadingComments.stripping(sql, grammar: grammar)) guard !trimmed.isEmpty else { return .none } if SQLProceduralBlock.runs(trimmed, grammar: grammar) { return .opaque @@ -215,20 +215,24 @@ public enum SQLLeadingComments { public static let lineCommentTerminators: Set = ["\n", "\r", "\r\n"] - public static func stripping(_ sql: String) -> String { - var remaining = sql[...] - while true { - let trimmed = StatementBlank.trimmingLeading(remaining) - if trimmed.hasPrefix("--") { - guard let lineBreak = trimmed.firstIndex(where: endsLineComment) else { return "" } - remaining = trimmed[trimmed.index(after: lineBreak)...] - } else if trimmed.hasPrefix("/*"), !startsConditionalComment(trimmed) { - guard let close = trimmed.range(of: "*/") else { return "" } - remaining = trimmed[close.upperBound...] - } else { - return String(trimmed) + /// `sql` from its first token, past the blanks and the comments `grammar` reads, so a comment ends exactly where + /// the engine ends it. An executable comment is code and stays. + public static func stripping(_ sql: String, grammar: SQLLexicalGrammar) -> String { + let text = sql as NSString + let length = text.length + var index = 0 + while index < length { + let blank = StatementBlank.blankLength(in: text, at: index) + if blank > 0 { + index += blank + continue + } + guard let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar), span.kind.isComment else { + break } + index = max(span.end, index + 1) } + return text.substring(from: min(index, length)) } public static func startsConditionalComment(_ text: Substring) -> Bool { @@ -261,20 +265,21 @@ public enum SQLProceduralBlock { } // A block may open after <