diff --git a/.claude/rules/data-grid.md b/.claude/rules/data-grid.md index 272876fd8b..e86d304b26 100644 --- a/.claude/rules/data-grid.md +++ b/.claude/rules/data-grid.md @@ -30,6 +30,7 @@ Three layers, knowledge pointing one way: an owner's adapter, then `DataGridMode - **Row height, row count, reloads, inserts and removes reach both panes in one call**, so their document heights never differ. - **Vertical scroll stays in step through bounds-change notifications**: each pane's clip moves the other with `clipView.scroll(to:)` and `reflectScrolledClipView`, only when its y differs. Anywhere else scroll with `NSView.scroll(_:)`: moving a clip's x that way leaves the header behind. - **The frozen pane forwards every wheel event to the main scroll view**, so one momentum engine moves both. +- **A `fitsRows` grid is as tall as its rows and scrolls only sideways.** It publishes `model.fittingHeight` for its host to size it, hands a vertical gesture to the enclosing scroll view (the axis is decided once per gesture), and reveals a row by scrolling that page. The Structure tab's sections use it. - **Frozen columns take at most two thirds of the grid** (`DataGridMetrics.maximumFrozenShare`). Columns past that scroll, and Freeze Up to This Column is refused in the menu. - **`DataGridTableView.scrollToVisible(_:)` widens the rect leftward by the frozen width**, so a revealed column never parks under the frozen pane. - **A column moves only within its side of the freeze line.** A freeze or unfreeze rebuilds both panes' column lists from the set, writing width and position explicitly: remove-then-add alone strands a column at the end and resets its width. @@ -65,4 +66,4 @@ Three layers, knowledge pointing one way: an owner's adapter, then `DataGridMode - **AppKit builds the AXCell tree from cell views only**, so while `DataGridAccessibility.isActive` the main table mounts a `DataGridAccessibilityCellView` (identifier `data-grid-cell`) for each on-screen cell. It draws nothing and takes no clicks. Keep that path working whenever row drawing changes. - **A label reads "Row N, column M: value"**: N is the row number on screen, page offset included, and M the display position. UI tests find cells by it (`.claude/rules/ui-tests.md`, The grid). - **The frozen pane is hidden from accessibility.** A frozen column's stand-in in the main table reports the frame the frozen pane shows it at. -- **Only the viewport mounts.** Mounting for every row starved the main thread behind a 1,000-row page (#2392); rows scrolled into view remount. +- **Only the viewport mounts.** Mounting for every row starved the main thread behind a 1,000-row page (#2392). A client's read prepares the rows it walks, off-screen ones without a stand-in, and AppKit never asks for a prepared row again. So each on-screen row still without one remounts after a scroll, a layout of the grid and a read of the table: a fitted grid grows to its rows, and a page moves a grid, with no scroll. diff --git a/.claude/rules/data-sync-storage.md b/.claude/rules/data-sync-storage.md index daafa2f767..042a2bf20a 100644 --- a/.claude/rules/data-sync-storage.md +++ b/.claude/rules/data-sync-storage.md @@ -35,6 +35,7 @@ This path holds user data. A silent wrong answer here looks exactly like an empt | Filter defaults and presets | UserDefaults | `FilterSettingsStorage`, `FilterPresetStorage` | | Per-table filters | JSON, one file per table | `FilterSettingsStorage` | | Generate Rows settings per table | UserDefaults, device-local | `RowGenerationSettingsStore` | +| Row inspector field heights, per column name | UserDefaults, device-local | `InspectorFieldHeightStore` | | Favorite tables (iCloud-synced) | UserDefaults | `FavoriteTablesStorage` | | Sidebar database and schema filters, recent tables | UserDefaults, device-local | `DatabaseTreeFilterStorage`, `RecentTablesStore` | | History drawer state | UserDefaults, device-local | `HistoryPanelPreferencesStorage` | diff --git a/.github/plugin-registry.json b/.github/plugin-registry.json index f6bf29a547..778efe9f7e 100644 --- a/.github/plugin-registry.json +++ b/.github/plugin-registry.json @@ -49,12 +49,18 @@ "LibSQLDriverPlugin": { "name": "libSQL / Turso Driver" }, + "MemcachedDriverPlugin": { + "name": "Memcached Driver" + }, "MSSQLDriver": { "name": "MSSQL Driver" }, "MongoDBDriver": { "name": "MongoDB Driver" }, + "Neo4jDriverPlugin": { + "name": "Neo4j Driver" + }, "OracleDriver": { "name": "Oracle Driver" }, @@ -67,6 +73,9 @@ "SpannerDriverPlugin": { "name": "Spanner Driver" }, + "StarRocksDriverPlugin": { + "name": "StarRocks Driver" + }, "SurrealDBDriverPlugin": { "name": "SurrealDB Driver" }, diff --git a/.github/workflows/build-linux.yml b/.github/workflows/build-linux.yml index f96745925a..77ded5c235 100644 --- a/.github/workflows/build-linux.yml +++ b/.github/workflows/build-linux.yml @@ -82,7 +82,7 @@ jobs: # install, then the meson toolchain, the -dev packages the # workspace links, the runtime pieces the test suites need (xvfb, # dbus, gnome-keyring for the Secret Service step, openssh-client - # for the SSH transport). + # for the SSH transport, glycin-loaders for a value's picture). run: | apt-get install -y --no-install-recommends \ build-essential \ @@ -100,9 +100,12 @@ jobs: libgtksourceview-5-dev \ libsqlite3-dev \ libkrb5-dev \ + liblcms2-dev \ + libseccomp-dev \ clang \ libclang-dev \ adwaita-icon-theme \ + glycin-loaders \ locales \ gnome-keyring \ openssh-client \ @@ -115,7 +118,7 @@ jobs: components: rustfmt, clippy # Meson points Cargo at _build/cargo-home and _build/cargo-target, # so those are what the cache keeps. - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: workspaces: linux -> _build/cargo-target cache-directories: linux/_build/cargo-home @@ -268,7 +271,7 @@ jobs: - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: toolchain: "1.98" - - uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29 + - uses: taiki-e/install-action@5488ac5a951645ce95062214902ac4b6bc1a1037 # v2.87.32 with: tool: cargo-deny@${{ env.CARGO_DENY_VERSION }},cargo-shear@${{ env.CARGO_SHEAR_VERSION }},cargo-about@${{ env.CARGO_ABOUT_VERSION }},zizmor@${{ env.ZIZMOR_VERSION }} - name: Advisories, bans, licenses and sources @@ -334,7 +337,7 @@ jobs: build-essential pkgconf meson ninja-build gettext \ desktop-file-utils appstream libglib2.0-bin libglib2.0-dev libxml2-utils \ libgtk-4-dev libadwaita-1-dev libgtksourceview-5-dev \ - libsqlite3-dev libkrb5-dev clang libclang-dev cargo rustc + libsqlite3-dev libkrb5-dev liblcms2-dev libseccomp-dev clang libclang-dev cargo rustc - name: POTFILES.in is current run: | git ls-files 'crates/app/src/*.rs' 'data/resources/*.ui' 'data/*.desktop.in.in' \ @@ -489,11 +492,11 @@ jobs: - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: toolchain: "1.98" - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: workspaces: linux key: ${{ matrix.package }} - - uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29 + - uses: taiki-e/install-action@5488ac5a951645ce95062214902ac4b6bc1a1037 # v2.87.32 with: tool: cargo-nextest@${{ env.CARGO_NEXTEST_VERSION }} - name: Verify docker is available @@ -564,11 +567,11 @@ jobs: - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: toolchain: "1.98" - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: workspaces: linux key: tablepro-session - - uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29 + - uses: taiki-e/install-action@5488ac5a951645ce95062214902ac4b6bc1a1037 # v2.87.32 with: tool: cargo-nextest@${{ env.CARGO_NEXTEST_VERSION }} - name: Verify docker is available @@ -626,6 +629,8 @@ jobs: libgtksourceview-5-dev \ libsqlite3-dev \ libkrb5-dev \ + liblcms2-dev \ + libseccomp-dev \ clang \ libclang-dev \ xvfb \ @@ -634,11 +639,11 @@ jobs: - uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # master with: toolchain: "1.98" - - uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 + - uses: Swatinem/rust-cache@d1527c8f67ef9af357290475d4af4b0fab8f12e0 # v2.9.3 with: workspaces: linux key: tablepro-app - - uses: taiki-e/install-action@8cca0918f8471c33515202b73f56ab27f2bed283 # v2.87.29 + - uses: taiki-e/install-action@5488ac5a951645ce95062214902ac4b6bc1a1037 # v2.87.32 with: tool: cargo-nextest@${{ env.CARGO_NEXTEST_VERSION }} - name: Verify docker is available diff --git a/.github/workflows/release-linux.yml b/.github/workflows/release-linux.yml index 416e184d6f..83a6ed42d8 100644 --- a/.github/workflows/release-linux.yml +++ b/.github/workflows/release-linux.yml @@ -115,7 +115,7 @@ jobs: build-essential pkgconf meson ninja-build gettext \ desktop-file-utils appstream libglib2.0-bin libglib2.0-dev libxml2-utils \ libgtk-4-dev libadwaita-1-dev libgtksourceview-5-dev \ - libsqlite3-dev libkrb5-dev clang libclang-dev cargo rustc + libsqlite3-dev libkrb5-dev liblcms2-dev libseccomp-dev clang libclang-dev cargo rustc working-directory: / - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9b1b4d8bf2..bbd31b8db5 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -37,7 +37,7 @@ jobs: - name: Set up Python 3.9 id: python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: "3.9" update-environment: false diff --git a/.github/workflows/repo-hygiene.yml b/.github/workflows/repo-hygiene.yml index 2ecf1f6edf..3a8f59100f 100644 --- a/.github/workflows/repo-hygiene.yml +++ b/.github/workflows/repo-hygiene.yml @@ -55,7 +55,7 @@ jobs: # runner's Python, which has PyYAML. - name: Set up Python 3.9 for the release tooling id: python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + uses: actions/setup-python@6d4fd22b20465ccc0f7354476588e8df7c9ca0e0 # v7.1.0 with: python-version: "3.9" update-environment: false @@ -117,6 +117,10 @@ jobs: - name: Check every test target is in its test plan run: python3 apple/scripts/ci/check-test-plans.py + # Oracle reads a bare dictionary name from the current schema before the SYS synonym. + - name: Check Oracle SQL names dictionary objects through OracleDictionary + run: python3 apple/scripts/ci/check-oracle-dictionary-names.py + # The corpora run by hand against servers (scripts/probes/check-column-type-corpus.sh), so a type a driver # starts listing is caught here. - name: Check every listed column type has a corpus column diff --git a/.oss-scanner/Dockerfile b/.oss-scanner/Dockerfile index 88b83578d0..09a6864eef 100644 --- a/.oss-scanner/Dockerfile +++ b/.oss-scanner/Dockerfile @@ -12,7 +12,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ca-certificates curl git \ build-essential pkgconf meson ninja-build gettext desktop-file-utils appstream libglib2.0-bin \ libxml2-utils libgtk-4-dev libadwaita-1-dev libgtksourceview-5-dev libsqlite3-dev libkrb5-dev \ - clang libclang-dev adwaita-icon-theme locales gnome-keyring openssh-client xvfb xauth dbus-daemon \ + liblcms2-dev libseccomp-dev clang libclang-dev adwaita-icon-theme glycin-loaders locales gnome-keyring \ + openssh-client xvfb xauth dbus-daemon \ && rm -rf /var/lib/apt/lists/* \ && localedef -i en_US -f UTF-8 en_US.UTF-8 diff --git a/.oss-scanner/swift/Package.swift b/.oss-scanner/swift/Package.swift index 9e02a473ad..49240392bd 100644 --- a/.oss-scanner/swift/Package.swift +++ b/.oss-scanner/swift/Package.swift @@ -14,7 +14,9 @@ let package = Package( .target(name: "TableProCoreTypes"), .target(name: "TableProDocumentPath"), .target(name: "TableProJavaScriptText"), + .target(name: "TableProDamengCore", dependencies: ["TableProSQLGrammar"]), .testTarget(name: "TableProSQLGrammarTests", dependencies: ["TableProSQLGrammar"]), - .testTarget(name: "TableProLogRedactionTests", dependencies: ["TableProLogRedaction"]) + .testTarget(name: "TableProLogRedactionTests", dependencies: ["TableProLogRedaction"]), + .testTarget(name: "TableProDamengCoreTests", dependencies: ["TableProDamengCore", "TableProSQLGrammar"]) ] ) diff --git a/.oss-scanner/threat_model.md b/.oss-scanner/threat_model.md index 9e67d73ebe..4fe2a19276 100644 --- a/.oss-scanner/threat_model.md +++ b/.oss-scanner/threat_model.md @@ -32,7 +32,7 @@ The user is trusted: the SQL they type, the connections they create, the plugins 7. Secrets: `apple/TablePro/Core/Storage/KeychainHelper.swift`, `apple/TablePro/Core/Sync/SyncRecordMapper.swift` (passwords must never reach CloudKit), `linux/crates/storage/src/secrets`, `TableProLogRedaction`, the MCP error redactor. 8. SQL the app builds from data: `apple/TablePro/Core/ChangeTracking/SQLStatementGenerator.swift`, `apple/TablePro/Core/Database/FilterSQLGenerator.swift`, `SQLEscaping.swift`, `DialectQuoteHelper.swift`, each driver's `quoteIdentifier`; on Linux `linux/crates/core/src/dml`, `linux/crates/core/src/filter/validate_raw_predicate.rs` and each driver's `dialect.rs`. A crafted table or column name that escapes quoting runs SQL with the user's privileges. 9. SSH and TLS: `apple/TablePro/Core/SSH/` (libssh2, host key prompts), `apple/Packages/TableProMacCore/Sources/TableProSSHConfig`, `linux/crates/ssh` (drives the system `ssh`, so watch for option injection from connection fields), `linux/crates/net/src/tls`. -10. Rendering of cell values: the out-of-process image renderer (`apple/TablePro/CLI/ImageRenderMain.swift`), `apple/TablePro/Core/Services/Formatting/PhpSerializeParser.swift`, the MongoDB shell's JavaScriptCore runtime. +10. Rendering of cell values: the out-of-process image renderer (`apple/TablePro/CLI/ImageRenderMain.swift`), the Linux Row Details picture, which glycin decodes in a sandbox (`linux/crates/app/src/ui/data_grid/row_detail/picture.rs`), `apple/TablePro/Core/Services/Formatting/PhpSerializeParser.swift`, the MongoDB shell's JavaScriptCore runtime. Matter least: layout, theming and localization code, `linux/crates/test-fixtures`, `linux/crates/test-support`, `apple/TableProUITests`, `scripts/`, `.github/`, docs. diff --git a/README.ko.md b/README.ko.md index d066b55520..a976414762 100644 --- a/README.ko.md +++ b/README.ko.md @@ -171,4 +171,4 @@ TablePro에 기여해 주신 모든 분께 감사드립니다. ## 라이선스 -이 프로젝트는 [GNU Affero General Public License v3.0(AGPLv3)](LICENSE)에 따라 라이선스가 부여됩니다. +이 프로젝트는 [GNU Affero General Public License v3.0(AGPLv3)](LICENSE)에 따라 라이선스가 부여됩니다. TablePro 이름과 아이콘은 이 라이선스에 포함되지 않으며, 사용 방법은 [TRADEMARKS.md](TRADEMARKS.md)에 있습니다. diff --git a/README.md b/README.md index 3c07e7197e..c2ed042266 100644 --- a/README.md +++ b/README.md @@ -182,5 +182,5 @@ Want to join them? Read [CONTRIBUTING.md](CONTRIBUTING.md). ## License -This project is licensed under the [GNU Affero General Public License v3.0 (AGPLv3)](LICENSE). +This project is licensed under the [GNU Affero General Public License v3.0 (AGPLv3)](LICENSE). The TablePro name and icon are not part of that license: [TRADEMARKS.md](TRADEMARKS.md) says how you can use them. diff --git a/README.vi.md b/README.vi.md index 4ea88ccfab..3886e7a2c8 100644 --- a/README.vi.md +++ b/README.vi.md @@ -146,5 +146,5 @@ Muốn tham gia? Đọc [CONTRIBUTING.md](CONTRIBUTING.md). ## Bản quyền -Dự án này cấp phép theo [GNU Affero General Public License v3.0 (AGPLv3)](LICENSE). +Dự án này cấp phép theo [GNU Affero General Public License v3.0 (AGPLv3)](LICENSE). Tên và biểu tượng TablePro không thuộc giấy phép đó: cách dùng được ghi trong [TRADEMARKS.md](TRADEMARKS.md). diff --git a/README.zh.md b/README.zh.md index 7fb23a2c33..fd8d3c0fdd 100644 --- a/README.zh.md +++ b/README.zh.md @@ -146,5 +146,5 @@ TablePro 由用户和以下赞助者支持。[成为赞助者](https://tablepro. ## 许可证 -本项目采用 [GNU Affero General Public License v3.0 (AGPLv3)](LICENSE) 许可。 +本项目采用 [GNU Affero General Public License v3.0 (AGPLv3)](LICENSE) 许可。TablePro 名称和图标不在该许可范围内,使用方式见 [TRADEMARKS.md](TRADEMARKS.md)。 diff --git a/TRADEMARKS.md b/TRADEMARKS.md new file mode 100644 index 0000000000..35e46b4862 --- /dev/null +++ b/TRADEMARKS.md @@ -0,0 +1,24 @@ +# Trademark policy + +The code in this repository is licensed under the [AGPLv3](LICENSE). That license covers the code. It does not cover the name **TablePro** or the TablePro icon and logo, which identify the builds this project publishes. Section 7(e) of the AGPL lets a project keep those out of the license, and this file says what you can do with them. + +The aim is simple: anyone who downloads something called TablePro gets the app this project ships, not a build someone else changed. + +## What you can do + +- Build TablePro from this source for your own use, changed or not. The AGPL allows that, including a build with the license check removed. +- Redistribute an official release unchanged, for example through a package manager, under the name TablePro. +- Package the unchanged source for a Linux distribution or a package manager under the name TablePro, with only the changes packaging needs. +- Say that your work is based on TablePro, works with TablePro, or is a fork of TablePro, in plain words that do not suggest it is TablePro. +- Use the name and icon to write about TablePro, review it, teach it, or link to it. + +## What needs a different name and icon + +- A build with any other change, such as new or removed features, a removed license check, or other branding. Rename it and replace the icon before you share it. +- A product, company, domain, app store listing, or package name that people could take for TablePro or for something made by this project. +- Anything that suggests this project endorses, sponsors, or supports your work when it does not. +- Selling copies, or builds with the paid features unlocked, under the TablePro name. + +## Questions + +For anything this file does not cover, or to ask for permission, write to [hello@tablepro.app](mailto:hello@tablepro.app). diff --git a/apple/.swiftlint.yml b/apple/.swiftlint.yml index c31e1bc9cd..698a5f7bcd 100644 --- a/apple/.swiftlint.yml +++ b/apple/.swiftlint.yml @@ -214,7 +214,7 @@ custom_rules: severity: error appstorage_key_namespace: name: "AppStorage Key Namespace" - regex: '@AppStorage\(\s*"(?!com\.TablePro)(?!(?:hideExportSuccessDialog|skipSchemaPreview|structureCodeFontSize)")[^"\\]+"' + regex: '@AppStorage\(\s*"(?!com\.TablePro)(?!(?:hideExportSuccessDialog|structureCodeFontSize)")[^"\\]+"' excluded: ['.*/(Packages|Plugins)/'] message: "Route an @AppStorage key through the preferences layer, under com.TablePro." severity: error diff --git a/apple/CHANGELOG.md b/apple/CHANGELOG.md index f60114318e..7a25751fe7 100644 --- a/apple/CHANGELOG.md +++ b/apple/CHANGELOG.md @@ -9,6 +9,79 @@ Unreleased changes are files in `changelog.d/`. +## [0.83.1](https://github.com/TableProApp/TablePro/releases/tag/v0.83.1) - 2026-11-01 [CRITICAL] + +### Changed + +- With CanonicalizeHostname, a DNS alias is followed only where CanonicalizePermittedCNAMEs allows it. + +### Removed + +- DuckDB Restore no longer accepts a folder of Parquet files, only a .duckdb file. + +### Fixed + +- A table tab could reload under an open cell and drop what you were typing after a save in another window. ([#3848](https://github.com/TableProApp/TablePro/pull/3848)) +- Edits made to rows while a query or refresh was still loading new ones were discarded when it finished. ([#3848](https://github.com/TableProApp/TablePro/pull/3848)) +- Table tabs ran their query again when you switched back to them or returned to TablePro from another app. ([#3848](https://github.com/TableProApp/TablePro/pull/3848)) +- A table tab waiting for its connection could run the statement under the cursor in a query tab instead. ([#3848](https://github.com/TableProApp/TablePro/pull/3848)) +- On iOS, a search or filter with a backslash on MySQL or ClickHouse, or with `[` on SQL Server, matched the wrong rows. ([#3906](https://github.com/TableProApp/TablePro/pull/3906)) +- A DELETE with no WHERE running unconfirmed behind EXPLAIN ANALYZE, an unusual CTE, or a WHERE glued to a name. ([#3907](https://github.com/TableProApp/TablePro/pull/3907)) +- CREATE OR REPLACE TABLE, SCHEMA or DATABASE replacing data without the confirmation DROP gets. ([#3907](https://github.com/TableProApp/TablePro/pull/3907)) +- HTTP drivers such as Typesense following a server's redirect to another address with the connection's key. ([#3907](https://github.com/TableProApp/TablePro/pull/3907)) +- Spanner, BigQuery and Teradata saving a grid change they could not key to the default schema's table. ([#3907](https://github.com/TableProApp/TablePro/pull/3907)) +- Teradata showing each result one statement late after a statement hit the 20-second receive timeout. ([#3907](https://github.com/TableProApp/TablePro/pull/3907)) +- DuckDB's Open the File Read-Only being ignored for a database not named .duckdb or .ddb. ([#3907](https://github.com/TableProApp/TablePro/pull/3907)) +- T-SQL BULK INSERT and OPENROWSET, and MongoDB $_internalJs operators, passing the external file and code refusals. ([#3907](https://github.com/TableProApp/TablePro/pull/3907)) +- Bigtable saving a grid change to another row when the key starts with a combining mark. ([#3907](https://github.com/TableProApp/TablePro/pull/3907)) +- DuckDB Back Up to a .duckdb file, and DuckDB Restore, failed with "Cannot prepare multiple statements at once". +- Pull in a linked folder failing when the repository is a partial clone. + +### Security + +- A backslash in a Snowflake filter value could end its string literal and run the rest as SQL. ([#3906](https://github.com/TableProApp/TablePro/pull/3906)) +- A Firebase Database URL with an encoded slash sending your Google access token to another host. +- A PostgreSQL or SQL Server database name could send Back Up or Restore, with its password, to another server. +- A PostgreSQL type named geometry or geography that PostGIS did not create could run its owner's code as you. +- A SQL export carrying a statement of its own in a cell value, a type name or a table comment when restored. +- A Typesense path with dot segments, or a Firebase or Bigtable request behind a comment, passing the read-only check. +- A credential profile in a shared connection file reading ~/.pgpass and sending the password to a host the file named. +- A name with a quote before an accent mark could run its own SQL when you browsed, searched or exported the table. +- A remote SQLite file path with a quote and a combining accent could run commands on the SSH server. +- A write after a CTE list, a Snowflake ->> chain, or inside a SurrealDB or CockroachDB SELECT passing as a read. +- ANALYZE TABLE, EXPLAIN PRAGMA and SQLite pragma_optimize passing as reads for read-only external clients. +- An imported SSH host or user could run commands through a Match exec line in ~/.ssh/config. +- An imported SSH host with a line break could make TablePro trust a host key for another server. +- Browsing a PostgreSQL schema could run a function or operator its owner planted there, with your privileges. +- CanonicalizeFallbackLocal no was ignored, and an SSH tunnel dialed the short host name anyway. +- CanonicalizeHostname followed any DNS alias, so DNS chose the host key an SSH tunnel was checked against. +- Cassandra and ScyllaDB connections through a tunnel signing in to other cluster nodes outside the tunnel. +- Dameng connections set to Required or a Verify mode connecting without TLS. +- EXPLAIN or DESCRIBE with a parenthesised query or a quoted option passing as a read for read-only MCP clients. +- File and URL reads through DuckDB, ClickHouse, quoted function names or VACUUM INTO passing the external check. +- Git status for a linked folder running filter and fetch commands set up in that folder's own Git configuration. +- Import Commands showing the first 300 characters of Startup SQL while keeping all of it. +- MCP clients and the assistant running statements past Read Only on Dameng behind a comment the driver read differently. +- MCP clients running DROP or server API calls unasked through grid requests on BigQuery, Elasticsearch and Typesense. +- MCP clients running DROP unasked on Trino and Presto behind a statement the driver split differently from the gate. +- MCP clients running statements or changing consistency past Read Only on Cassandra and ScyllaDB with a forged comment. +- MCP clients with read access running writes through browse_table or get_table_statistics with a crafted table name. +- MCP clients, the AI assistant and AppleScript running a second statement the Read Only and destructive checks missed. +- Open Project Folder filling a project's ${VAR} references from TablePro's own environment. +- Opening a crafted table on iPhone or iPad running a second statement on PostgreSQL, Redshift or SQL Server. +- Oracle Explain reading a PLAN_TABLE or V$ views planted in the tab's schema instead of Oracle's own. +- Restoring a DuckDB folder of Parquet files ran its schema.sql with file access, so the folder could write any file. +- Restoring a MySQL or MariaDB dump could run shell commands written in the .sql file. +- Restoring a SQLite dump could run shell commands written in the .sql file. +- SQL Server EXEC of a string, sp_executesql, OPENQUERY and OPENROWSET passing as reads or without the code check. +- SQL Server Verify CA and Verify Identity on iPhone and iPad connecting without checking the certificate. +- Secrets in a pasted MongoDB connection string saved outside the Keychain and carried by exports, links and sync. +- Shared connections reusing your Entra ID token, Snowflake CLI profile, MongoDB AWS credentials or Run Ledger Plugins. +- Teradata with SSL through a tunnel sending the password to port 443 on this Mac. +- Trino and Presto sending the password or token to another host the server named in its reply. +- iPhone and iPad Safe Mode letting DESC ANALYZE of a write, OPENQUERY and state-changing calls run as reads. + + ## [0.83.0](https://github.com/TableProApp/TablePro/releases/tag/v0.83.0) - 2026-10-26 Search Database finds a value in every table, Compare Results matches two results row by row, and Generate Rows fills a table with test data. diff --git a/apple/Configs/Version-iOS.xcconfig b/apple/Configs/Version-iOS.xcconfig index 705cfb6c43..09f58d5e70 100644 --- a/apple/Configs/Version-iOS.xcconfig +++ b/apple/Configs/Version-iOS.xcconfig @@ -1,5 +1,5 @@ // Single source of truth for the iOS app version. // The app and its widget extension must ship the same values, so both read this file. -MARKETING_VERSION = 1.1 -CURRENT_PROJECT_VERSION = 23 +MARKETING_VERSION = 1.1.1 +CURRENT_PROJECT_VERSION = 24 diff --git a/apple/Libs/checksums.sha256 b/apple/Libs/checksums.sha256 index 58559a0baf..228b6a3f62 100644 --- a/apple/Libs/checksums.sha256 +++ b/apple/Libs/checksums.sha256 @@ -10,10 +10,10 @@ c47c8406cdb817e121a89ad5171f6931fd73c4bce6c73c3809e626257afd7374 Libs/libcrypto e274f77ed3feaee3d4f8f6ac297376d56a1369388f037599a90ea440390349b4 Libs/libcrypto_universal.a ec04bc233ea917de16a48b130d1467c68f5b95d469f423b7b9054d0e45bb4e16 Libs/libcrypto_x86_64.a e274f77ed3feaee3d4f8f6ac297376d56a1369388f037599a90ea440390349b4 Libs/libcrypto.a -f833d1edba062e03a5a7f7b5e40fbfe0f015c081158b8dd455d6129bf4515c12 Libs/libduckdb_arm64.a -c1aef28aa225d34bb4d8be853000f4d2640a77366ff4b2cfa61f53648290c357 Libs/libduckdb_universal.a -e971a4c84689626699482ba5e2f148345b50e53fc1aed57b6e37fbb90c85db9c Libs/libduckdb_x86_64.a -c1aef28aa225d34bb4d8be853000f4d2640a77366ff4b2cfa61f53648290c357 Libs/libduckdb.a +09b2deb312d03840dda422a6931a9918ad81c308941c774556adb1386d6ab589 Libs/libduckdb_arm64.a +9749f1be288f03cc4362b15fbc0137edd8aee75542f52f7d7ca8d563586a1e77 Libs/libduckdb_universal.a +3183445676921ea6e4ea4cb7f0689703fb3ceba3d1699b21e88a2e36f12e0e2e Libs/libduckdb_x86_64.a +9749f1be288f03cc4362b15fbc0137edd8aee75542f52f7d7ca8d563586a1e77 Libs/libduckdb.a 5017c3fecadf4955fad1db923a8358fc9943fa17e8a5d54288db6a28fc5a3aa8 Libs/libhiredis_arm64.a 58974eb81d5fefca9fb6732bd15921f1cf12a23f9a2402b1b099ec6c089f1e35 Libs/libhiredis_ssl_arm64.a 09c8cb4acb62a54c09c4f2605bbc464b93d9e2ea5b2ae89a1d2165f8adaa9b9c Libs/libhiredis_ssl_universal.a diff --git a/apple/Libs/ios/checksums.sha256 b/apple/Libs/ios/checksums.sha256 index bd457322f7..f6e64e4c7a 100644 --- a/apple/Libs/ios/checksums.sha256 +++ b/apple/Libs/ios/checksums.sha256 @@ -1,8 +1,10 @@ -840741c66e9ee257f97eb7cc16407c37960aa49faa4b110e56ccc0f8dc3a6db2 ./DuckDB.xcframework/Info.plist -c8dd688cf14559c744255d1bb3879330d6ae3523268e8904fa7b885e140030aa ./DuckDB.xcframework/ios-arm64/Headers/duckdb.h -f4073989393e739b0b5ad68c7f47f7750715d2d46e46141e81b1bc49cf782620 ./DuckDB.xcframework/ios-arm64/libduckdb-device.a -c8dd688cf14559c744255d1bb3879330d6ae3523268e8904fa7b885e140030aa ./DuckDB.xcframework/ios-arm64_x86_64-simulator/Headers/duckdb.h -75523877523153f63249a39eb4ec0d7f1bb05db3a4dbb5e6c35ee848b09afb71 ./DuckDB.xcframework/ios-arm64_x86_64-simulator/libduckdb-sim.a +fcfb2160ac0c32d40fd3241bb0ec0ec157238a5db6ef68a5d98e93a616956cf6 ./DuckDB.xcframework/Info.plist +8dbc9c71a10f0a75fcacc1a3f4f7c76093a84d232fc152222ee5b6e512ab4560 ./DuckDB.xcframework/ios-arm64-simulator/Headers/duckdb.h +a90f28cbe2662c69fc73f726832479ae7b5d043566b83eb1ac95a8ab35d3e37d ./DuckDB.xcframework/ios-arm64-simulator/Headers/duckdb_static_extensions.h +842bd80d7f54472c4161434474af155675e0d7b19c7751e1d1046d8e144a19b6 ./DuckDB.xcframework/ios-arm64-simulator/libduckdb-sim.a +8dbc9c71a10f0a75fcacc1a3f4f7c76093a84d232fc152222ee5b6e512ab4560 ./DuckDB.xcframework/ios-arm64/Headers/duckdb.h +a90f28cbe2662c69fc73f726832479ae7b5d043566b83eb1ac95a8ab35d3e37d ./DuckDB.xcframework/ios-arm64/Headers/duckdb_static_extensions.h +c667055fe35aa1e47ca8f2dd36b8328743429872022a0bb25701b593d9a7d1f0 ./DuckDB.xcframework/ios-arm64/libduckdb-device.a 0481f79db3949ba16425ffd8e15e640757a208ca05728d98cdbfb4bc617e06c3 ./FreeTDS.xcframework/Info.plist 734d8502be13513e974a621557b1a4776d9f605b5a48039e28a3597f27f704ce ./FreeTDS.xcframework/ios-arm64-simulator/Headers/sybdb.h eaa21732fa7cfb538b3bb9bcca7342b99b407d2e838c09e5f3d07398c21406a3 ./FreeTDS.xcframework/ios-arm64-simulator/Headers/sybfront.h diff --git a/apple/Libs/release b/apple/Libs/release index edf700ee23..f05581e49b 100644 --- a/apple/Libs/release +++ b/apple/Libs/release @@ -1 +1 @@ -libs-5 +libs-6 diff --git a/apple/Packages/TableProCore/Package.swift b/apple/Packages/TableProCore/Package.swift index 34aae47217..e0b658ff61 100644 --- a/apple/Packages/TableProCore/Package.swift +++ b/apple/Packages/TableProCore/Package.swift @@ -31,11 +31,13 @@ let package = Package( .library(name: "TableProKerberos", targets: ["TableProKerberos"]), .library(name: "TableProTeradataCore", targets: ["TableProTeradataCore"]), .library(name: "TableProTrinoCore", targets: ["TableProTrinoCore"]), + .library(name: "TableProDamengCore", targets: ["TableProDamengCore"]), .library(name: "TableProTLSClientIdentity", targets: ["TableProTLSClientIdentity"]), .library(name: "TableProTLSTestFixtures", targets: ["TableProTLSTestFixtures"]), .library(name: "TableProGoogleCloud", targets: ["TableProGoogleCloud"]), .library(name: "TableProSpannerCore", targets: ["TableProSpannerCore"]), .library(name: "TableProWeaviateCore", targets: ["TableProWeaviateCore"]), + .library(name: "TableProNeo4jCore", targets: ["TableProNeo4jCore"]), .library(name: "TableProFirebaseCore", targets: ["TableProFirebaseCore"]), .library(name: "TableProBigtableCore", targets: ["TableProBigtableCore"]), .library(name: "TableProNumberFormatting", targets: ["TableProNumberFormatting"]), @@ -55,6 +57,7 @@ let package = Package( .library(name: "TableProMySQLCore", targets: ["TableProMySQLCore"]), .library(name: "TableProPostgreSQLCore", targets: ["TableProPostgreSQLCore"]), .library(name: "TableProRedisCore", targets: ["TableProRedisCore"]), + .library(name: "TableProMemcachedCore", targets: ["TableProMemcachedCore"]), .library(name: "TableProTestSupport", targets: ["TableProTestSupport"]) ], targets: [ @@ -105,7 +108,7 @@ let package = Package( ), .target( name: "TableProImport", - dependencies: ["TableProConnectionLibrary"], + dependencies: ["TableProConnectionLibrary", "TableProSSHTransport"], path: "Sources/TableProImport" ), .target( @@ -152,9 +155,15 @@ let package = Package( ), .target( name: "TableProTrinoCore", - dependencies: [], + dependencies: ["TableProSQLGrammar"], path: "Sources/TableProTrinoCore" ), + .target( + name: "TableProDamengCore", + dependencies: ["TableProSQLGrammar"], + path: "Sources/TableProDamengCore", + swiftSettings: pluginCoreSettings + ), .target( name: "TableProTLSClientIdentity", dependencies: [], @@ -185,6 +194,11 @@ let package = Package( dependencies: [], path: "Sources/TableProWeaviateCore" ), + .target( + name: "TableProNeo4jCore", + dependencies: [], + path: "Sources/TableProNeo4jCore" + ), .target( name: "TableProBigtableCore", dependencies: ["TableProGoogleCloud"], @@ -262,6 +276,12 @@ let package = Package( path: "Sources/TableProPostgreSQLCore", swiftSettings: pluginCoreSettings ), + .target( + name: "TableProMemcachedCore", + dependencies: [], + path: "Sources/TableProMemcachedCore", + swiftSettings: pluginCoreSettings + ), .target( name: "TableProRedisCore", dependencies: [], @@ -273,16 +293,37 @@ let package = Package( dependencies: ["TableProBigtableCore", "TableProGoogleCloud"], path: "Tests/TableProBigtableCoreTests" ), + .testTarget( + name: "TableProNeo4jCoreTests", + dependencies: ["TableProNeo4jCore"], + path: "Tests/TableProNeo4jCoreTests" + ), + .testTarget( + name: "TableProFirebaseCoreTests", + dependencies: ["TableProFirebaseCore"], + path: "Tests/TableProFirebaseCoreTests" + ), .testTarget( name: "TableProSpannerCoreTests", dependencies: ["TableProSpannerCore"], path: "Tests/TableProSpannerCoreTests" ), + .testTarget( + name: "TableProTeradataCoreTests", + dependencies: ["TableProTeradataCore"], + path: "Tests/TableProTeradataCoreTests" + ), .testTarget( name: "TableProTrinoCoreTests", - dependencies: ["TableProTrinoCore"], + dependencies: ["TableProTrinoCore", "TableProSQLGrammar"], path: "Tests/TableProTrinoCoreTests" ), + .testTarget( + name: "TableProDamengCoreTests", + dependencies: ["TableProDamengCore", "TableProSQLGrammar"], + path: "Tests/TableProDamengCoreTests", + swiftSettings: pluginCoreSettings + ), .testTarget( name: "TableProTabularIOTests", dependencies: ["TableProTabularIO", "TableProPluginKit"], @@ -417,6 +458,12 @@ let package = Package( path: "Tests/TableProPostgreSQLCoreTests", swiftSettings: pluginCoreSettings ), + .testTarget( + name: "TableProMemcachedCoreTests", + dependencies: ["TableProMemcachedCore"], + path: "Tests/TableProMemcachedCoreTests", + swiftSettings: pluginCoreSettings + ), .testTarget( name: "TableProRedisCoreTests", dependencies: ["TableProRedisCore"], diff --git a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionProvider.swift b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionProvider.swift index a190774b8c..ca5511a9f1 100644 --- a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionProvider.swift +++ b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLCompletionProvider.swift @@ -150,22 +150,25 @@ public final class SQLCompletionProvider { // A dot prefix is a table or alias first, then a schema (its tables) or a database (its schemas, or its // tables on an engine with none). The namespace fallback also covers an alias that resolves to a schema - // name parsed out of the FROM clause. + // name parsed out of the FROM clause. In a table slot it can only qualify the table being named, so a table + // that shares the name is never read for columns. if let dotPrefix = context.dotPrefix { guard let catalog else { return [] } - if let derived = context.tableReferences.first(where: { - $0.isDerived && $0.identifier.caseInsensitiveCompare(dotPrefix) == .orderedSame - }) { - let columns = await derivedColumns(of: derived) - if !columns.isEmpty { - return [columns.map { SQLCompletionItem.column($0, dataType: nil, tableName: derived.identifier) }] + if !context.expectsObjectName { + if let derived = context.tableReferences.first(where: { + $0.isDerived && $0.identifier.caseInsensitiveCompare(dotPrefix) == .orderedSame + }) { + let columns = await derivedColumns(of: derived) + if !columns.isEmpty { + return [columns.map { SQLCompletionItem.column($0, dataType: nil, tableName: derived.identifier) }] + } + } + if let tableName = await catalog.resolveAlias(dotPrefix, in: context.tableReferences) { + let schema = context.tableReferences.first { + $0.tableName.caseInsensitiveCompare(tableName) == .orderedSame + }?.schema + items = await catalog.columnCompletionItems(for: tableName, schema: schema) } - } - if let tableName = await catalog.resolveAlias(dotPrefix, in: context.tableReferences) { - let schema = context.tableReferences.first { - $0.tableName.caseInsensitiveCompare(tableName) == .orderedSame - }?.schema - items = await catalog.columnCompletionItems(for: tableName, schema: schema) } if items.isEmpty { if await catalog.isKnownSchema(dotPrefix) { @@ -493,7 +496,7 @@ public final class SQLCompletionProvider { var item = SQLCompletionItem( label: "'\(value)'", kind: .keyword, - insertText: "'\(value.replacingOccurrences(of: "'", with: "''"))'", + insertText: "'\(value.replacingOccurrences(of: "'", with: "''", options: .literal))'", detail: column, filterText: value.lowercased() ) @@ -598,13 +601,13 @@ public final class SQLCompletionProvider { // `AS 'Total'` names the column on MySQL and SQLite, but ORDER BY 'Total' sorts by a constant. private func identifierSpelling(of spelling: String, name: String) -> String { - guard spelling.hasPrefix("'") else { return spelling } + guard spelling.unicodeScalars.first == "'" else { return spelling } let isBare = name.unicodeScalars.first.map { CharacterSet.letters.contains($0) || $0 == "_" } == true && name.unicodeScalars.allSatisfy { CharacterSet.alphanumerics.contains($0) || $0 == "_" } guard !isBare || SQLKeywords.isKeyword(name) else { return name } // The analyzer reads a backslash-escaped `"` as a string, which is where a backtick quotes a name instead. let quote = grammar.isSuperset(of: [.backslashEscapesInDoubleQuotes, .backtickQuotes]) ? "`" : "\"" - return quote + name.replacingOccurrences(of: quote, with: quote + quote) + quote + return quote + name.replacingOccurrences(of: quote, with: quote + quote, options: .literal) + quote } // CTEs were bound to their own scope by the parser, so what is left to expand is a table the catalog knows. diff --git a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLContextAnalyzer.swift b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLContextAnalyzer.swift index aadeb59b74..841ac697c2 100644 --- a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLContextAnalyzer.swift +++ b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLContextAnalyzer.swift @@ -209,6 +209,7 @@ public final class SQLContextAnalyzer { private static let doubleQuote = UInt16(UnicodeScalar("\"").value) private static let backtick = UInt16(UnicodeScalar("`").value) private static let openBracket = UInt16(UnicodeScalar("[").value) + private static let closeBracket = UInt16(UnicodeScalar("]").value) private static let dollar = UInt16(UnicodeScalar("$").value) private static let hash = UInt16(UnicodeScalar("#").value) @@ -260,8 +261,9 @@ public final class SQLContextAnalyzer { // and hold any character. private static let quotedSegment = "[`\"'\\[]?\\w+[`\"'\\]]?" + // A trailing dot stays in the path, so `JOIN sales.` reads as unfinished instead of naming a table `sales`. private static let tableRefRegexes: [NSRegularExpression] = { - let path = "(\(quotedSegment)(?:\\.\(quotedSegment))*)" + let path = "(\(quotedSegment)(?:\\.\(quotedSegment))*\\.?)" let alias = "(?:\\s+(?:AS\\s+)?(\(quotedSegment)))?" let patterns = [ // MySQL's STRAIGHT_JOIN has no word boundary before JOIN. @@ -282,7 +284,7 @@ public final class SQLContextAnalyzer { ) private static let alterTableRegex = compileRegex( - "(?i)\\bALTER\\s+TABLE\\s+(\(quotedSegment)(?:\\.\(quotedSegment))*)" + "(?i)\\bALTER\\s+TABLE\\s+(\(quotedSegment)(?:\\.\(quotedSegment))*\\.?)" ) private static let derivedTableParser = DerivedTableParser() @@ -855,7 +857,13 @@ public final class SQLContextAnalyzer { append(upTo: index) start = index + 1 } - append(upTo: end) + // The name after a trailing dot is still being typed: an empty last segment, so no caller reads `sales.` as + // a table named `sales`. + if start == end, end > range.location { + segments.append(PathSegment(masked: "", name: "", isQuoted: false)) + } else { + append(upTo: end) + } return segments } @@ -1048,10 +1056,6 @@ public final class SQLContextAnalyzer { dotPrefix: String?, currentFunction: String? = nil ) -> ClauseResolution { - if dotPrefix != nil { - return ClauseResolution(clause: .select, expectsObjectName: false) - } - // Bounds the regex work on a long statement. SQLContextAnalyzerWindowingTests pins the size. let windowSize = 5_000 let nsText = textBeforeCursor as NSString @@ -1059,6 +1063,10 @@ public final class SQLContextAnalyzer { ? nsText.substring(from: nsText.length - windowSize) : textBeforeCursor + if dotPrefix != nil { + return ClauseResolution(clause: .select, expectsObjectName: qualifiesTableName(after: windowedText)) + } + // Structural contexts first: `CREATE TABLE test (id ` looks like a call to `test(` but is a column definition. let range = NSRange(location: 0, length: (windowedText as NSString).length) for (regex, clause) in Self.structuralClauseRegexes { @@ -1079,6 +1087,42 @@ public final class SQLContextAnalyzer { return ClauseResolution(clause: .unknown, expectsObjectName: false) } + // `FROM sales.` qualifies the table being named. A FROM after `IS DISTINCT` or inside a call such as + // `EXTRACT(YEAR FROM o.`, and any qualifier this cannot place, stays a table or alias. + private func qualifiesTableName(after textBeforePrefix: String) -> Bool { + let tokens = tokenize(Self.removingTrailingPath(textBeforePrefix)) + guard let keyword = tokens.last, keyword.kind == .word, + ["FROM", "JOIN", "INTO"].contains(keyword.upper) else { return false } + guard keyword.upper == "FROM" else { return true } + if tokens.count >= 2, tokens[tokens.count - 2].upper == "DISTINCT" { return false } + var open: [Int] = [] + for (index, token) in tokens.enumerated() { + if token.kind == .openParen { + open.append(index) + } else if token.kind == .closeParen, !open.isEmpty { + open.removeLast() + } + } + guard let paren = open.last, paren > 0, tokens[paren - 1].kind == .word else { return true } + return Self.groupingKeywords.contains(tokens[paren - 1].upper) + } + + // A parenthesis after one of these opens a subquery or a join group, so a FROM inside it names a table. + private static let groupingKeywords: Set = [ + "FROM", "JOIN", "LATERAL", "AS", "IN", "EXISTS", "ANY", "ALL", "SOME", "UNION", "INTERSECT", "EXCEPT" + ] + + private static func removingTrailingPath(_ text: String) -> String { + let ns = text as NSString + var start = ns.length + while start > 0 { + let ch = ns.character(at: start - 1) + guard SQLTokenBoundary.isTokenChar(ch) || ch == dot || ch == openBracket || ch == closeBracket else { break } + start -= 1 + } + return ns.substring(to: start) + } + // MARK: - Linear Clause Scan private enum ScanTokenKind { diff --git a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLIdentifierSpelling.swift b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLIdentifierSpelling.swift index 5531a4ec99..8eadaa7789 100644 --- a/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLIdentifierSpelling.swift +++ b/apple/Packages/TableProCore/Sources/TableProAutocomplete/SQLIdentifierSpelling.swift @@ -103,7 +103,7 @@ public struct SQLIdentifierSpelling: Sendable { public func quoted(_ name: String) -> String { let quote = String(rules.quote) - return quote + name.replacingOccurrences(of: quote, with: quote + quote) + quote + return quote + name.replacingOccurrences(of: quote, with: quote + quote, options: .literal) + quote } // A name written into a statement with nothing typed: as stored keeps the caller's own spelling. @@ -131,7 +131,7 @@ public struct SQLIdentifierSpelling: Sendable { if quoting != .none, identifier.parts.count == 1, case .catalog(let name) = identifier.parts[0] { // The user's own quote asks for the exact name: inside a closed pair it goes in raw. let quote = String(rules.quote) - let text = quoting == .enclosed ? name.replacingOccurrences(of: quote, with: quote + quote) : quoted(name) + let text = quoting == .enclosed ? name.replacingOccurrences(of: quote, with: quote + quote, options: .literal) : quoted(name) return item.recased(label: name, insertText: text) } // A qualified reference replaces the user's quotes, so it is spelled as if none were typed. diff --git a/apple/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift b/apple/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift index d96c5be4f0..bc2ce87ab9 100644 --- a/apple/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift +++ b/apple/Packages/TableProCore/Sources/TableProBigtableCore/Data/BigtableRow.swift @@ -15,14 +15,16 @@ public struct BigtableColumn: Hashable, Sendable, Comparable { family + ":" + BigtableBytes.display(qualifier) } + // By scalar: a qualifier that starts with a combining mark joins the colon into one Character. public init(name: String) throws { - guard let colon = name.firstIndex(of: ":"), colon != name.startIndex else { + guard let colon = name.unicodeScalars.firstIndex(of: ":"), colon != name.startIndex else { throw BigtableError.configuration(String( format: String(localized: "\"%@\" is not a column. Name it as family:qualifier."), name )) } - self.family = String(name[.. Bool { diff --git a/apple/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift b/apple/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift index 08b8966201..c05591c4f0 100644 --- a/apple/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift +++ b/apple/Packages/TableProCore/Sources/TableProBigtableCore/Wire/BigtableStatement.swift @@ -157,7 +157,7 @@ public struct BigtableRequest: Sendable, Equatable { /// Text that would not read back as one bare word goes in JSON quotes. static func token(_ text: String, reserved: String) -> String { - let plain = !text.isEmpty && !text.hasPrefix("\"") && text.unicodeScalars.allSatisfy { scalar in + let plain = !text.isEmpty && text.unicodeScalars.first != "\"" && text.unicodeScalars.allSatisfy { scalar in !CharacterSet.whitespacesAndNewlines.contains(scalar) && !reserved.unicodeScalars.contains(scalar) } if plain { return text } @@ -173,22 +173,23 @@ public struct BigtableRequest: Sendable, Equatable { static let missingTable = BigtableError.configuration(String(localized: "Name the table before the key, as in GET events/user#42.")) } -/// Reads a request line one token at a time. +// Read by scalar, as `token` writes: a key that starts with a combining mark joins the quote or slash before it +// into one Character. private struct RequestScanner { - private var text: Substring + private var text: Substring.UnicodeScalarView private var query: [String: String] = [:] init(_ text: String) { - self.text = Substring(text) + self.text = Substring(text).unicodeScalars } var isAtEnd: Bool { - text.allSatisfy(\.isWhitespace) + text.allSatisfy(\.properties.isWhitespace) } - mutating func word(stoppingAt stop: Character? = nil) -> String? { + mutating func word(stoppingAt stop: Unicode.Scalar? = nil) -> String? { skipSpace() - let word = text.prefix { !$0.isWhitespace && $0 != stop } + let word = text.prefix { !$0.properties.isWhitespace && $0 != stop } guard !word.isEmpty else { return nil } text = text.dropFirst(word.count) return String(word) @@ -196,7 +197,7 @@ private struct RequestScanner { mutating func rest() -> String { let remaining = String(text).trimmingCharacters(in: .whitespacesAndNewlines) - text = "" + text.removeAll() return remaining } @@ -204,7 +205,7 @@ private struct RequestScanner { /// no slash is a table alone, as in `SCAN events`. mutating func target(tableOnly: Bool) throws -> (table: String, key: Data?) { skipSpace() - let head = text.prefix { !$0.isWhitespace && $0 != "\"" && $0 != "?" } + let head = text.prefix { !$0.properties.isWhitespace && $0 != "\"" && $0 != "?" } let table: String if let slash = head.firstIndex(of: "/") { table = String(text[.. = [ + "tlscertificatekeyfilepassword", + "sslclientcertificatekeypassword", + "proxypassword", + ] + static let mongoKeyPassphraseOptions: Set = [ + "tlscertificatekeyfilepassword", + "sslclientcertificatekeypassword", + ] + static let mechanismPropertiesOption = "authmechanismproperties" + static let sessionTokenProperty = "AWS_SESSION_TOKEN" + + public static func isSecret(key: String, value: String) -> Bool { + if key.caseInsensitiveCompare(mongoSessionTokenFieldId) == .orderedSame { return true } + guard let option = mongoOption(in: key) else { return false } + if mongoSecretOptions.contains(option) { return true } + return option == mechanismPropertiesOption && splitSessionToken(from: value).sessionToken != nil + } + + // Mechanism properties keep everything but the token. + public static func removingSecrets(from fields: [String: String]) -> [String: String] { + var kept: [String: String] = [:] + for (key, value) in fields { + guard isSecret(key: key, value: value) else { + kept[key] = value + continue + } + if let option = mongoOption(in: key), isMongoMechanismProperties(option: option), + let remaining = splitSessionToken(from: value).remaining { + kept[key] = remaining + } + } + return kept + } + + // The lowercased option name of a `mongoParam_` field, nil for any other field. + public static func mongoOption(in key: String) -> String? { + guard key.count > mongoOptionPrefix.count, + key.lowercased().hasPrefix(mongoOptionPrefix.lowercased()) + else { return nil } + return String(key.dropFirst(mongoOptionPrefix.count)).lowercased() + } + + public static func isMongoKeyPassphrase(option: String) -> Bool { + mongoKeyPassphraseOptions.contains(option.lowercased()) + } + + public static func isMongoProxyPassword(option: String) -> Bool { + option.lowercased() == "proxypassword" + } + + public static func isMongoMechanismProperties(option: String) -> Bool { + option.lowercased() == mechanismPropertiesOption + } + + // `authMechanismProperties` is `KEY:value` pairs joined by commas; libmongoc reads keys without case. + public static func splitSessionToken(from properties: String) -> (remaining: String?, sessionToken: String?) { + var kept: [String] = [] + var token: String? + for pair in properties.split(separator: ",", omittingEmptySubsequences: true) { + let parts = pair.split(separator: ":", maxSplits: 1, omittingEmptySubsequences: false) + let name = parts[0].trimmingCharacters(in: .whitespaces) + if name.caseInsensitiveCompare(sessionTokenProperty) == .orderedSame { + let value = parts.count > 1 ? String(parts[1]) : "" + if !value.isEmpty { token = value } + } else { + kept.append(String(pair)) + } + } + return (kept.isEmpty ? nil : kept.joined(separator: ","), token) + } + + public static func joiningSessionToken(_ token: String, to properties: String?) -> String { + let remaining = properties.flatMap { splitSessionToken(from: $0).remaining } + let pair = "\(sessionTokenProperty):\(token)" + guard let remaining else { return pair } + return "\(remaining),\(pair)" + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift b/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift index d6962fdd63..d5bc57829a 100644 --- a/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift +++ b/apple/Packages/TableProCore/Sources/TableProConnectionLibrary/SQLDDLFallbackPolicy.swift @@ -36,6 +36,7 @@ public enum SQLDDLFallbackPolicy { "Elasticsearch", "Typesense", "Weaviate", + "Memcached", "Kafka", "etcd", "Redis", @@ -46,6 +47,7 @@ public enum SQLDDLFallbackPolicy { "Firestore", "Firebase Realtime Database", "Bigtable", + "Neo4j", ] /// True when `DROP`/`TRUNCATE` built by the app is something this engine could run. diff --git a/apple/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift b/apple/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift index bb002a60ef..b9b9da2621 100644 --- a/apple/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift +++ b/apple/Packages/TableProCore/Sources/TableProCoreTypes/DatabaseType.swift @@ -14,6 +14,7 @@ public struct DatabaseType: Hashable, Codable, Sendable, RawRepresentable { public static let tidb = DatabaseType(rawValue: "TiDB") public static let databend = DatabaseType(rawValue: "Databend") public static let oceanbase = DatabaseType(rawValue: "OceanBase") + public static let starrocks = DatabaseType(rawValue: "StarRocks") public static let postgresql = DatabaseType(rawValue: "PostgreSQL") public static let sqlite = DatabaseType(rawValue: "SQLite") public static let redis = DatabaseType(rawValue: "Redis") @@ -42,15 +43,16 @@ public struct DatabaseType: Hashable, Codable, Sendable, RawRepresentable { public static let kafka = DatabaseType(rawValue: "Kafka") public static let cloudflareR2SQL = DatabaseType(rawValue: "Cloudflare R2 SQL") public static let weaviate = DatabaseType(rawValue: "Weaviate") + public static let memcached = DatabaseType(rawValue: "Memcached") public static let firestore = DatabaseType(rawValue: "Firestore") public static let firebaseRealtimeDatabase = DatabaseType(rawValue: "Firebase Realtime Database") public static let bigtable = DatabaseType(rawValue: "Bigtable") public static let allKnownTypes: [DatabaseType] = [ - .mysql, .mariadb, .tidb, .databend, .oceanbase, .postgresql, .sqlite, .redis, .mongodb, + .mysql, .mariadb, .tidb, .databend, .oceanbase, .starrocks, .postgresql, .sqlite, .redis, .mongodb, .clickhouse, .mssql, .oracle, .dameng, .duckdb, .cassandra, .redshift, .etcd, .cloudflareD1, .dynamodb, .bigquery, .spanner, .snowflake, .libsql, .beancount, - .surrealdb, .teradata, .trino, .kafka, .cloudflareR2SQL, .weaviate, .firestore, .firebaseRealtimeDatabase, .bigtable + .surrealdb, .teradata, .trino, .kafka, .cloudflareR2SQL, .weaviate, .memcached, .firestore, .firebaseRealtimeDatabase, .bigtable ] /// Icon name for this database type — asset catalog name (e.g. "mysql-icon") or SF Symbol fallback @@ -61,6 +63,7 @@ public struct DatabaseType: Hashable, Codable, Sendable, RawRepresentable { case .tidb: return "tidb-icon" case .databend: return "databend-icon" case .oceanbase: return "oceanbase-icon" + case .starrocks: return "starrocks-icon" case .postgresql: return "postgresql-icon" case .redshift: return "redshift-icon" case .sqlite: return "sqlite-icon" @@ -87,6 +90,7 @@ public struct DatabaseType: Hashable, Codable, Sendable, RawRepresentable { case .kafka: return "kafka-icon" case .cloudflareR2SQL: return "cloudflare-r2-sql-icon" case .weaviate: return "weaviate-icon" + case .memcached: return "memorychip" case .firestore, .firebaseRealtimeDatabase: return "firebase-icon" case .bigtable: return "bigtable-icon" default: return "externaldrive" diff --git a/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengGrammar.swift b/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengGrammar.swift new file mode 100644 index 0000000000..490ce4e799 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengGrammar.swift @@ -0,0 +1,14 @@ +import Foundation +import TableProSQLGrammar + +enum DamengGrammar { + // The reading the app runs a Dameng session's text with: DM8's measured grammar plus the BACKSLASH_ESCAPE mode the + // driver detects at connect and reports as its session state. + static func execution(backslashEscapes: Bool) -> SQLLexicalGrammar { + let session = SQLSessionLexicalFacts( + determined: .backslashEscapesInSingleQuotes, + enabled: backslashEscapes ? .backslashEscapesInSingleQuotes : [] + ) + return SQLLexicalReadings.resolve(databaseTypeId: "Dameng", declared: nil, session: session).execution + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengPlaceholders.swift b/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengPlaceholders.swift new file mode 100644 index 0000000000..d4eda1777a --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengPlaceholders.swift @@ -0,0 +1,28 @@ +import Foundation +import TableProSQLGrammar + +public enum DamengPlaceholders { + private static let questionMark = UInt16(UnicodeScalar("?").value) + + // One more piece than there are placeholders. A `?` in a literal, a quoted name or a comment stays in its piece. + public static func pieces(of template: String, backslashEscapes: Bool) -> [String] { + let grammar = DamengGrammar.execution(backslashEscapes: backslashEscapes) + let text = template as NSString + var pieces: [String] = [] + var start = 0 + var index = 0 + while index < text.length { + if let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar) { + index = max(span.end, index + 1) + continue + } + if text.character(at: index) == questionMark { + pieces.append(text.substring(with: NSRange(location: start, length: index - start))) + start = index + 1 + } + index += 1 + } + pieces.append(text.substring(from: min(start, text.length))) + return pieces + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengScriptSplitter.swift b/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengScriptSplitter.swift new file mode 100644 index 0000000000..4911eacf04 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengScriptSplitter.swift @@ -0,0 +1,22 @@ +import Foundation +import TableProSQLGrammar + +// Cuts where the app's gates cut, so the driver never sends a piece they did not count. A PL/SQL block goes whole: +// the Dameng grammar does not delimit one, and a wrong cut sends the server a fragment. +public enum DamengScriptSplitter { + private static let blockKeywords: Set = [ + "BEGIN", "CASE", "DECLARE", "END", "IF", "LOOP" + ] + + public static func statements(in script: String, backslashEscapes: Bool) -> [String] { + let grammar = DamengGrammar.execution(backslashEscapes: backslashEscapes) + guard !holdsBlock(script, grammar: grammar) else { return [script] } + return SQLStatementScanner.allStatements(in: script, grammar: grammar) + } + + private static func holdsBlock(_ script: String, grammar: SQLLexicalGrammar) -> Bool { + SQLCodeProjection.code(of: script, grammar: grammar) + .split { !($0.isLetter || $0.isNumber || $0 == "_") } + .contains { blockKeywords.contains($0.uppercased()) } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengStatementClassifier.swift b/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengStatementClassifier.swift new file mode 100644 index 0000000000..c61ef9fed3 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProDamengCore/DamengStatementClassifier.swift @@ -0,0 +1,51 @@ +import Foundation +import TableProSQLGrammar + +// A framing hint and the replay rule, never a verdict on what the user sees: the bridge converts whatever comes back. +public enum DamengStatementClassifier { + private static let resultKeywords: Set = [ + "DESC", "DESCRIBE", "EXPLAIN", "SELECT", "SHOW", "VALUES", "WITH" + ] + + // Shorter than resultKeywords: WITH and VALUES can head a write, and a replayed write may apply twice. + private static let readOnlyKeywords: Set = [ + "DESC", "DESCRIBE", "EXPLAIN", "SELECT", "SHOW" + ] + + private static let openParenthesis = UInt16(UnicodeScalar("(").value) + private static let semicolon = UInt16(UnicodeScalar(";").value) + + public static func likelyReturnsRows(_ query: String) -> Bool { + guard let keyword = firstKeyword(query) else { return false } + return resultKeywords.contains(keyword) + } + + public static func isReadOnly(_ query: String) -> Bool { + guard let keyword = firstKeyword(query) else { return false } + return readOnlyKeywords.contains(keyword) + } + + private static func firstKeyword(_ query: String) -> String? { + let text = query as NSString + let grammar = DamengGrammar.execution(backslashEscapes: false) + var index = 0 + while index < text.length { + if let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar) { + guard span.kind.isComment else { return nil } + index = max(span.end, index + 1) + continue + } + let blank = StatementBlank.blankLength(in: text, at: index) + if blank > 0 { + index += blank + continue + } + let unit = text.character(at: index) + guard unit == openParenthesis || unit == semicolon else { break } + index += 1 + } + guard index < text.length else { return nil } + let keyword = text.substring(from: index).prefix { $0.isLetter || $0 == "_" } + return keyword.isEmpty ? nil : keyword.uppercased() + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProDatabase/SQLEscaping.swift b/apple/Packages/TableProCore/Sources/TableProDatabase/SQLEscaping.swift index 4a31b07ddf..f992ca4a5e 100644 --- a/apple/Packages/TableProCore/Sources/TableProDatabase/SQLEscaping.swift +++ b/apple/Packages/TableProCore/Sources/TableProDatabase/SQLEscaping.swift @@ -3,17 +3,17 @@ import Foundation public enum SQLEscaping { public static func ansiStringLiteral(_ value: String) -> String { value - .replacingOccurrences(of: "\0", with: "") - .replacingOccurrences(of: "'", with: "''") + .replacingOccurrences(of: "\0", with: "", options: .literal) + .replacingOccurrences(of: "'", with: "''", options: .literal) } public static func backslashStringLiteral(_ value: String) -> String { value - .replacingOccurrences(of: "\\", with: "\\\\") - .replacingOccurrences(of: "\0", with: "\\0") - .replacingOccurrences(of: "\n", with: "\\n") - .replacingOccurrences(of: "\r", with: "\\r") - .replacingOccurrences(of: "\u{1a}", with: "\\Z") - .replacingOccurrences(of: "'", with: "''") + .replacingOccurrences(of: "\\", with: "\\\\", options: .literal) + .replacingOccurrences(of: "\0", with: "\\0", options: .literal) + .replacingOccurrences(of: "\n", with: "\\n", options: .literal) + .replacingOccurrences(of: "\r", with: "\\r", options: .literal) + .replacingOccurrences(of: "\u{1a}", with: "\\Z", options: .literal) + .replacingOccurrences(of: "'", with: "''", options: .literal) } } diff --git a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Config/FirebaseConnectionSettings.swift b/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Config/FirebaseConnectionSettings.swift index 84175aef2d..b76179a1c6 100644 --- a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Config/FirebaseConnectionSettings.swift +++ b/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Config/FirebaseConnectionSettings.swift @@ -113,8 +113,16 @@ public struct FirebaseConnectionSettings: Sendable, Equatable { return url } let trusted = host.hasSuffix(".firebaseio.com") || host.hasSuffix(".firebasedatabase.app") - guard scheme == "https", trusted else { throw invalidURL(text) } - return URL(string: "https://\(host)/") ?? url + guard scheme == "https", trusted, isDNSName(host), let instance = URL(string: "https://\(host)/") else { + throw invalidURL(text) + } + return instance + } + + // `URL.host` is percent-decoded, so an encoded `/`, `:`, `?`, `#` or `@` would pass the suffix check and then end + // the host early once the text is parsed again. + static func isDNSName(_ host: String) -> Bool { + host.range(of: #"^[a-z0-9-]+(\.[a-z0-9-]+)+$"#, options: .regularExpression) != nil } static func parseEmulatorHost(_ text: String?) throws -> String { diff --git a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreQuery.swift b/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreQuery.swift index 8760f9b3f7..9fc630c138 100644 --- a/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreQuery.swift +++ b/apple/Packages/TableProCore/Sources/TableProFirebaseCore/Firestore/FirestoreQuery.swift @@ -268,7 +268,7 @@ public enum FirestoreQueryBuilder { if column == nameField { return column } let simple = column.range(of: "^[A-Za-z_][A-Za-z0-9_]*$", options: .regularExpression) != nil if simple { return column } - let escaped = column.replacingOccurrences(of: "\\", with: "\\\\").replacingOccurrences(of: "`", with: "\\`") + let escaped = column.replacingOccurrences(of: "\\", with: "\\\\", options: .literal).replacingOccurrences(of: "`", with: "\\`", options: .literal) return "`" + escaped + "`" } diff --git a/apple/Packages/TableProCore/Sources/TableProImport/Bundle/ConnectionBundle+Validation.swift b/apple/Packages/TableProCore/Sources/TableProImport/Bundle/ConnectionBundle+Validation.swift index c7780f3ed9..c45cbd0787 100644 --- a/apple/Packages/TableProCore/Sources/TableProImport/Bundle/ConnectionBundle+Validation.swift +++ b/apple/Packages/TableProCore/Sources/TableProImport/Bundle/ConnectionBundle+Validation.swift @@ -7,6 +7,7 @@ enum BundleViolation: Equatable, Sendable { case unresolvedCredentials(BundleRef) case cycle(BundleRef) case tooDeep(BundleRef) + case invalidSSHDestination(connection: String) /// Deeper than any folder or group the app creates, and shallow enough that a crafted file cannot /// make every query row carry a path of thousands of names. @@ -30,6 +31,8 @@ enum BundleViolation: Equatable, Sendable { String(format: String(localized: "the reference “%@” is inside itself"), ref.rawValue) case .tooDeep(let ref): String(format: String(localized: "the reference “%@” is nested too deeply"), ref.rawValue) + case .invalidSSHDestination(let connection): + String(format: String(localized: "the SSH host or user of “%@” contains characters ssh does not allow"), connection) } } @@ -51,6 +54,9 @@ enum BundleViolation: Equatable, Sendable { for refs in refLists { if let violation = identityViolation(refs) { return violation } } + if let connection = connections.first(where: { $0.settings.refusedSSHDestination != nil }) { + return .invalidSSHDestination(connection: connection.settings.name) + } let connectionRefs = Set(connections.map(\.ref)) let groupRefs = Set(groups.map(\.ref)) diff --git a/apple/Packages/TableProCore/Sources/TableProImport/Bundle/ExportableConnection.swift b/apple/Packages/TableProCore/Sources/TableProImport/Bundle/ExportableConnection.swift index 1ad029607f..178826dc3c 100644 --- a/apple/Packages/TableProCore/Sources/TableProImport/Bundle/ExportableConnection.swift +++ b/apple/Packages/TableProCore/Sources/TableProImport/Bundle/ExportableConnection.swift @@ -1,5 +1,6 @@ import Foundation import TableProConnectionLibrary +import TableProSSHTransport import UniformTypeIdentifiers public extension UTType { @@ -143,17 +144,24 @@ public extension ExportableConnection { private static let queryTimeoutSecondsRange = 0 ... Int(Int32.max) / 1_000 static let importBlockedAdditionalFieldKeys: Set = [ + "beancountrunledgerplugins", "postdisconnectscript", "preconnectscript", "pretunnelhost", "pretunnelport", "promptforpassword", + "snowflakeconnectionname", "spendpoint", "sslclientkeypassphrase", "usepgpass", ] - static let importBlockedAdditionalFieldPrefixes: Set = ["aws"] + static let importBlockedAdditionalFieldPrefixes: Set = ["aws", "entra"] + + // Values that pick a credential kept on this Mac, under keys whose other values are plain settings. + static let importBlockedAdditionalFieldValues: [String: Set] = [ + "mongoauthmechanism": ["mongodb-aws"], + ] static func isImportBlockedAdditionalFieldKey(_ key: String) -> Bool { let normalized = key.lowercased() @@ -161,12 +169,19 @@ public extension ExportableConnection { return importBlockedAdditionalFieldPrefixes.contains { normalized.hasPrefix($0) } } + static func isImportBlockedAdditionalField(key: String, value: String) -> Bool { + if isImportBlockedAdditionalFieldKey(key) { return true } + if ConnectionSecretOptions.isSecret(key: key, value: value) { return true } + let blockedValues = importBlockedAdditionalFieldValues[key.lowercased()] ?? [] + return blockedValues.contains(value.trimmingCharacters(in: .whitespaces).lowercased()) + } + static func shareableAdditionalFields( _ fields: [String: String], excluding excludedKeys: Set = [] ) -> [String: String]? { - let shareable = fields.filter { key, _ in - !excludedKeys.contains(key) && !isImportBlockedAdditionalFieldKey(key) + let shareable = fields.filter { key, value in + !excludedKeys.contains(key) && !isImportBlockedAdditionalField(key: key, value: value) } return shareable.isEmpty ? nil : shareable } @@ -191,7 +206,7 @@ public extension ExportableConnection { /// An explicit timeout that is out of range is dropped, not replaced by a legacy additional field. func sanitizedForImport() -> ExportableConnection { - var allowed = (additionalFields ?? [:]).filter { !Self.isImportBlockedAdditionalFieldKey($0.key) } + var allowed = (additionalFields ?? [:]).filter { !Self.isImportBlockedAdditionalField(key: $0.key, value: $0.value) } let legacyConnectTimeout = allowed.removeValue(forKey: Self.connectTimeoutSecondsKey).flatMap(Int.init) let legacyQueryTimeout = allowed.removeValue(forKey: Self.queryTimeoutSecondsKey).flatMap(Int.init) var copy = self @@ -243,6 +258,15 @@ public struct ExportableTunnelCommand: Codable, Sendable, Equatable { } } +public extension ExportableConnection { + // Match exec hands the host and user to /bin/sh, and the host key store is one entry per line. + var refusedSSHDestination: SSHDestinationName.Problem? { + guard let ssh = sshConfig, ssh.enabled else { return nil } + let hops = [(ssh.host, ssh.username)] + (ssh.jumpHosts ?? []).map { ($0.host, $0.username) } + return hops.lazy.compactMap { SSHDestinationName.problem(host: $0.0, user: $0.1) }.first + } +} + public struct ExportableSSHConfig: Codable, Sendable, Equatable { public var enabled: Bool public var host: String diff --git a/apple/Packages/TableProCore/Sources/TableProImport/Import/ImportPlanner.swift b/apple/Packages/TableProCore/Sources/TableProImport/Import/ImportPlanner.swift index 5cf8e769e7..aa939d542c 100644 --- a/apple/Packages/TableProCore/Sources/TableProImport/Import/ImportPlanner.swift +++ b/apple/Packages/TableProCore/Sources/TableProImport/Import/ImportPlanner.swift @@ -88,7 +88,8 @@ extension ImportPlanner { ref: profile.ref, name: profile.name, username: profile.username, - passwordMode: profile.passwordMode == .pgpass ? .pgpass : .prompt, + // A file never chooses how this Mac finds a password: pgpass would read ~/.pgpass for its host. + passwordMode: .prompt, secureFieldIds: profile.secureFieldIds )) } diff --git a/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLConnectionOptions.swift b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLConnectionOptions.swift index ac0cf7b97b..aedd33873c 100644 --- a/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLConnectionOptions.swift +++ b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLConnectionOptions.swift @@ -14,14 +14,11 @@ public struct MSSQLConnectionOptions: Sendable, Equatable { public var kerberosCachePath: String? public var kerberosServicePrincipal: String? - /// How far to check the server certificate. Set after init by the plugin from the - /// connection's SSL mode, so the existing initializer keeps its signature. - public var certificateVerification: MSSQLCertificateVerification = .none + // No default: a caller that forgets it would encrypt without checking who answers. + public var certificateVerification: MSSQLCertificateVerification public var caCertificatePath: String? - /// Microsoft Entra ID access token, sent in place of a user name and password. Set after - /// init for the same reason as `certificateVerification`. Only read when - /// `authMethod == .entra`. + // Sent in place of a user name and password, read only when `authMethod == .entra`. public var fedAuthToken: String? public static let defaultPort = 1_433 @@ -38,6 +35,7 @@ public struct MSSQLConnectionOptions: Sendable, Equatable { database: String, schema: String = MSSQLConnectionOptions.defaultSchema, encryptionLevel: MSSQLEncryptionLevel = MSSQLConnectionOptions.defaultEncryptionLevel, + certificateVerification: MSSQLCertificateVerification, applicationName: String = MSSQLConnectionOptions.defaultApplicationName, loginTimeoutSeconds: Int = MSSQLConnectionOptions.defaultLoginTimeoutSeconds, authMethod: MSSQLAuthMethod = .sqlServer, @@ -60,6 +58,7 @@ public struct MSSQLConnectionOptions: Sendable, Equatable { self.database = database self.schema = schema self.encryptionLevel = encryptionLevel + self.certificateVerification = certificateVerification self.applicationName = applicationName self.loginTimeoutSeconds = loginTimeoutSeconds } diff --git a/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLViewSources.swift b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLViewSources.swift new file mode 100644 index 0000000000..35ca9a4a2d --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProMSSQLCore/MSSQLViewSources.swift @@ -0,0 +1,364 @@ +import Foundation + +// INFORMATION_SCHEMA says NO for every view; sys.dm_exec_describe_first_result_set compiles without running. +public enum MSSQLViewSources { + public enum Refusal: Equatable, Sendable { + case notUpdatable + case writesRunTriggers + case readsSeveralTables + case baseTableHidden + case couldNotCheck + } + + public enum Operation: String, CaseIterable, Sendable { + case update + case insert + case delete + } + + public struct BaseTable: Hashable, Sendable { + // Set for a table on a linked server. + public let server: String? + public let database: String? + public let schema: String + public let name: String + + public init(server: String? = nil, database: String?, schema: String, name: String) { + self.server = server + self.database = database + self.schema = schema + self.name = name + } + } + + public struct ViewColumn: Equatable, Sendable { + public let name: String + public let baseColumn: String? + public let acceptsWrites: Bool + + public init(name: String, baseColumn: String?, acceptsWrites: Bool) { + self.name = name + self.baseColumn = baseColumn + self.acceptsWrites = acceptsWrites + } + } + + public struct BaseColumn: Equatable, Sendable { + public let name: String + public let isNullable: Bool + // Nil also for a default whose text needs VIEW DEFINITION to read. + public let defaultValue: String? + public let hasDefault: Bool + public let isIdentity: Bool + public let isComputed: Bool + // A rowversion or a GENERATED ALWAYS period column. + public let isServerGenerated: Bool + + public init( + name: String, + isNullable: Bool, + defaultValue: String?, + hasDefault: Bool, + isIdentity: Bool, + isComputed: Bool, + isServerGenerated: Bool + ) { + self.name = name + self.isNullable = isNullable + self.defaultValue = defaultValue + self.hasDefault = hasDefault + self.isIdentity = isIdentity + self.isComputed = isComputed + self.isServerGenerated = isServerGenerated + } + + // A default the user cannot read still fills the column when an INSERT leaves it out. + public var assignsValue: Bool { + isIdentity || isServerGenerated || (hasDefault && defaultValue == nil) + } + + public var acceptsWrites: Bool { + !isComputed && !isIdentity && !isServerGenerated + } + } + + static let maxViewDepth = 8 + + // MARK: - The view itself + + // sys.sql_expression_dependencies needs VIEW DEFINITION; sys.sql_dependencies does not but drops recreated views. + // A schema-bound view has one sys.sql_expression_dependencies row per column; minor id 0 is the object. + public static func viewFactsQuery(schema: String, view: String) -> String { + let object = MSSQLStringLiteral.quoted(MSSQLSchemaQueries.bracketed(schema: schema, table: view)) + return """ + DECLARE @view int = OBJECT_ID(\(object)); + DECLARE @edges TABLE (referencing_id int NOT NULL, referenced_id int NOT NULL); + IF EXISTS (SELECT 1 FROM sys.views WHERE object_id = @view) + BEGIN + INSERT INTO @edges (referencing_id, referenced_id) + SELECT DISTINCT d.object_id, d.referenced_major_id + FROM sys.sql_dependencies d + JOIN sys.views r ON r.object_id = d.referenced_major_id; + IF HAS_PERMS_BY_NAME(N'sys.sql_expression_dependencies', 'OBJECT', 'SELECT') = 1 + INSERT INTO @edges (referencing_id, referenced_id) + SELECT d.referencing_id, d.referenced_id + FROM sys.sql_expression_dependencies d + JOIN sys.views r ON r.object_id = d.referenced_id + WHERE d.referenced_minor_id = 0 + AND NOT EXISTS ( + SELECT 1 + FROM @edges e + WHERE e.referencing_id = d.referencing_id AND e.referenced_id = d.referenced_id + ); + END; + WITH chain AS ( + SELECT v.object_id, 0 AS depth + FROM sys.views v + WHERE v.object_id = @view + UNION ALL + SELECT e.referenced_id, c.depth + 1 + FROM chain c + JOIN @edges e ON e.referencing_id = c.object_id + WHERE c.depth < \(maxViewDepth) + ) + SELECT + v.has_opaque_metadata, + CASE WHEN EXISTS ( + SELECT 1 + FROM chain c + JOIN sys.triggers tr ON tr.parent_id = c.object_id + WHERE tr.is_instead_of_trigger = 1 AND tr.is_disabled = 0 + ) THEN 1 ELSE 0 END AS runs_instead_of_trigger, + CASE WHEN EXISTS ( + SELECT 1 + FROM chain c + JOIN @edges e ON e.referencing_id = c.object_id + WHERE c.depth = \(maxViewDepth) + ) THEN 1 ELSE 0 END AS chain_continues + FROM sys.views v + WHERE v.object_id = @view + """ + } + + public struct ViewFacts: Equatable, Sendable { + public let hasOpaqueMetadata: Bool + public let runsInsteadOfTrigger: Bool + // Views nest deeper than the walk followed, so a trigger further down went unseen. + public let chainContinues: Bool + + public init(hasOpaqueMetadata: Bool, runsInsteadOfTrigger: Bool, chainContinues: Bool = false) { + self.hasOpaqueMetadata = hasOpaqueMetadata + self.runsInsteadOfTrigger = runsInsteadOfTrigger + self.chainContinues = chainContinues + } + + // Nil when no view of that name is visible to the user. + public init?(rows: [[String?]]) { + guard let row = rows.first else { return nil } + self.init( + hasOpaqueMetadata: row.cell(0) == "1", + runsInsteadOfTrigger: row.cell(1) == "1", + chainContinues: row.cell(2) == "1" + ) + } + + public var refusal: Refusal? { + // WITH VIEW_METADATA makes browse mode name the view in place of its base table. + if hasOpaqueMetadata { return .baseTableHidden } + if runsInsteadOfTrigger { return .writesRunTriggers } + if chainContinues { return .couldNotCheck } + return nil + } + } + + // MARK: - Browse mode + + // Mode 1 is FOR BROWSE: base table and column per column, plus hidden key and rowversion rows of every table. + public static func browseQuery(schema: String, view: String) -> String { + let select = "SELECT * FROM \(MSSQLSchemaQueries.bracketed(schema: schema, table: view))" + return """ + SELECT name, is_hidden, source_server, source_database, source_schema, source_table, source_column, + is_updateable + FROM sys.dm_exec_describe_first_result_set(\(MSSQLStringLiteral.quoted(select)), NULL, 1) + ORDER BY column_ordinal + """ + } + + public enum Source: Equatable, Sendable { + case table(BaseTable) + case refused(Refusal) + } + + public struct BrowseFacts: Equatable, Sendable { + public let columns: [ViewColumn] + public let baseTables: [BaseTable] + // View columns the probes write, one per base column, in view order. + public let writableColumns: [String] + + public init(rows: [[String?]]) { + var columns: [ViewColumn] = [] + var baseTables: [BaseTable] = [] + var writableColumns: [String] = [] + var probedBaseColumns: Set = [] + for row in rows { + // A view that no longer compiles yields error rows with no name. + guard let name = row.cell(0) else { continue } + let server = row.cell(2) + if let schema = row.cell(4), let table = row.cell(5) { + let base = BaseTable(server: server, database: row.cell(3), schema: schema, name: table) + if !baseTables.contains(base) { baseTables.append(base) } + } + guard row.cell(1) != "1" else { continue } + let baseColumn = server == nil ? row.cell(6) : nil + let acceptsWrites = row.cell(7) == "1" + columns.append(ViewColumn(name: name, baseColumn: baseColumn, acceptsWrites: acceptsWrites)) + if acceptsWrites, let baseColumn, probedBaseColumns.insert(baseColumn).inserted { + writableColumns.append(name) + } + } + self.columns = columns + self.baseTables = baseTables + self.writableColumns = writableColumns + } + + public var source: Source { + guard baseTables.count == 1, let table = baseTables.first else { + return .refused(baseTables.isEmpty ? .notUpdatable : .readsSeveralTables) + } + // Its key and columns live in another server's catalog. + return table.server == nil ? .table(table) : .refused(.baseTableHidden) + } + } + + // MARK: - Operations + + public static func probesQuery(schema: String, view: String, writableColumns: [String]) -> String { + let target = MSSQLSchemaQueries.bracketed(schema: schema, table: view) + var probes: [(Operation, String)] = [(.delete, "DELETE FROM \(target) WHERE 1 = 0")] + if let first = writableColumns.first.map(bracketedColumn) { + let list = writableColumns.map(bracketedColumn).joined(separator: ", ") + probes.append((.update, "UPDATE \(target) SET \(first) = \(first) WHERE 1 = 0")) + probes.append((.insert, "INSERT INTO \(target) (\(list)) SELECT \(list) FROM \(target) WHERE 1 = 0")) + } + let values = probes + .map { "('\($0.0.rawValue)', \(MSSQLStringLiteral.quoted($0.1)))" } + .joined(separator: ",\n ") + return """ + SELECT p.operation, r.error_number + FROM (VALUES + \(values) + ) AS p(operation, statement) + CROSS APPLY sys.dm_exec_describe_first_result_set(p.statement, NULL, 0) r + WHERE r.error_number IS NOT NULL + """ + } + + // Browse mode names a table once however often the view joins it, so a self-join shows only as 4405 on DELETE. + public static func refusal(probeRows: [[String?]]) -> Refusal? { + let joins = probeRows.contains { $0.cell(0) == Operation.delete.rawValue && $0.cell(1) == "4405" } + return joins ? .readsSeveralTables : nil + } + + public static func allowedOperations(writableColumns: [String], probeRows: [[String?]]) -> Set { + var allowed: Set = writableColumns.isEmpty ? [.delete] : Set(Operation.allCases) + for row in probeRows { + guard let operation = row.cell(0).flatMap(Operation.init(rawValue:)), + let number = row.cell(1).flatMap({ Int($0) }), + structuralErrors.contains(number) + else { continue } + allowed.remove(operation) + } + return allowed + } + + // 11501 marks any compile error. 229 and 230 come with 11529 and wait for the save, as on a table. + static let structuralErrors: Set = [ + 208, // invalid object name + 264, // a column assigned twice + 271, // computed column + 272, // timestamp column + 4_403, // aggregate, DISTINCT, GROUP BY, PIVOT + 4_405, // the statement touches several base tables + 4_406, // derived or constant column + 4_426, // UNION + 8_102, // identity column + 11_501, // the batch has compile errors + 13_537, // GENERATED ALWAYS column + ] + + // MARK: - Base table + + // Read in the base table's own database. sys.tables skips a view that browse mode stopped at. + public static func baseColumnsQuery(_ table: BaseTable) -> String { + let catalog = table.database.map { "[\(MSSQLSchemaQueries.escapeBracket($0))]." } ?? "" + let object = MSSQLStringLiteral.quoted( + catalog + MSSQLSchemaQueries.bracketed(schema: table.schema, table: table.name) + ) + return """ + SELECT + c.name, + c.is_nullable, + d.definition, + CASE WHEN c.default_object_id <> 0 THEN 1 ELSE 0 END AS has_default, + c.is_identity, + c.is_computed, + CASE + WHEN TYPE_NAME(c.system_type_id) = 'timestamp' THEN 1 + WHEN c.generated_always_type > 0 THEN 1 + ELSE 0 + END AS is_server_generated, + ic.key_ordinal + FROM \(catalog)sys.columns c + JOIN \(catalog)sys.tables t ON t.object_id = c.object_id + LEFT JOIN \(catalog)sys.default_constraints d ON d.object_id = c.default_object_id + LEFT JOIN \(catalog)sys.indexes i ON i.object_id = c.object_id AND i.is_primary_key = 1 + LEFT JOIN \(catalog)sys.index_columns ic + ON ic.object_id = i.object_id AND ic.index_id = i.index_id AND ic.column_id = c.column_id + WHERE c.object_id = OBJECT_ID(\(object)) + ORDER BY c.column_id + """ + } + + public struct BaseFacts: Equatable, Sendable { + public let keyColumns: [String] + public let columns: [BaseColumn] + + public init(keyColumns: [String], columns: [BaseColumn]) { + self.keyColumns = keyColumns + self.columns = columns + } + + // Nil when no row comes back: OBJECT_ID is NULL for a table the user has no permission on. + public init?(rows: [[String?]]) { + var columns: [BaseColumn] = [] + var keyed: [(ordinal: Int, name: String)] = [] + for row in rows { + guard let name = row.cell(0) else { continue } + columns.append(BaseColumn( + name: name, + isNullable: row.cell(1) == "1", + defaultValue: row.cell(2), + hasDefault: row.cell(3) == "1", + isIdentity: row.cell(4) == "1", + isComputed: row.cell(5) == "1", + isServerGenerated: row.cell(6) == "1" + )) + if let ordinal = row.cell(7).flatMap({ Int($0) }) { + keyed.append((ordinal, name)) + } + } + guard !columns.isEmpty else { return nil } + self.init(keyColumns: keyed.sorted { $0.ordinal < $1.ordinal }.map(\.name), columns: columns) + } + } + + private static func bracketedColumn(_ column: String) -> String { + "[\(MSSQLSchemaQueries.escapeBracket(column))]" + } +} + +private extension Array where Element == String? { + func cell(_ index: Int) -> String? { + indices.contains(index) ? self[index] : nil + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedBrowseSpec.swift b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedBrowseSpec.swift new file mode 100644 index 0000000000..bb65220078 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedBrowseSpec.swift @@ -0,0 +1,165 @@ +import Foundation + +public enum MemcachedColumn: String, CaseIterable, Sendable { + case key = "Key" + case value = "Value" + case ttl = "TTL" + case flags = "Flags" + + public var typeName: String { + switch self { + case .key, .value: return "String" + case .ttl, .flags: return "Int64" + } + } +} + +public enum MemcachedKeyComparison: String, Codable, CaseIterable, Sendable { + case equal = "=" + case notEqual = "!=" + case contains = "CONTAINS" + case notContains = "NOT CONTAINS" + case startsWith = "STARTS WITH" + case endsWith = "ENDS WITH" + case inList = "IN" + case notInList = "NOT IN" + case regex = "REGEX" +} + +public enum MemcachedFilterRefusal: Error, Equatable, Sendable { + case notKeyColumn(String) + case unsupportedOperator(String) + case emptyList(String) + case invalidPattern(String) +} + +/// A page of the key grid. The grid hands the driver this spec as a tagged statement, so a page, its sort and its +/// filter reach the driver the way every other statement does. +public struct MemcachedBrowseSpec: Codable, Equatable, Sendable { + public struct KeyCondition: Codable, Equatable, Sendable { + public let comparison: MemcachedKeyComparison + public let value: String + public let isCaseSensitive: Bool + + public init(comparison: MemcachedKeyComparison, value: String, isCaseSensitive: Bool) { + self.comparison = comparison + self.value = value + self.isCaseSensitive = isCaseSensitive + } + } + + public static let tag = "MEMCACHED_BROWSE:" + public static let refusalTag = "MEMCACHED_REFUSAL:" + + public let limit: Int + public let offset: Int + public let ascending: Bool + public let conditions: [KeyCondition] + public let matchesAll: Bool + + public init(limit: Int, offset: Int, ascending: Bool, conditions: [KeyCondition] = [], matchesAll: Bool = true) { + self.limit = limit + self.offset = offset + self.ascending = ascending + self.conditions = conditions + self.matchesAll = matchesAll + } + + public var statement: String { + let encoder = JSONEncoder() + encoder.outputFormatting = .sortedKeys + return Self.tag + ((try? encoder.encode(self)) ?? Data()).base64EncodedString() + } + + public static func parse(_ statement: String) -> MemcachedBrowseSpec? { + guard statement.hasPrefix(tag), let data = Data(base64Encoded: String(statement.dropFirst(tag.count))) else { + return nil + } + guard let spec = try? JSONDecoder().decode(MemcachedBrowseSpec.self, from: data), + spec.limit >= 0, spec.offset >= 0 else { return nil } + return spec + } + + public static func refusalStatement(_ message: String) -> String { + refusalTag + Data(message.utf8).base64EncodedString() + } + + public static func refusalMessage(_ statement: String) -> String? { + guard statement.hasPrefix(refusalTag), + let data = Data(base64Encoded: String(statement.dropFirst(refusalTag.count))) else { return nil } + return String(bytes: data, encoding: .utf8) + } + + public static func condition( + column: String, op: String, value: String, isCaseSensitive: Bool + ) throws(MemcachedFilterRefusal) -> KeyCondition { + guard column == MemcachedColumn.key.rawValue else { throw .notKeyColumn(column) } + let name = op.trimmingCharacters(in: .whitespaces).uppercased() + guard let comparison = MemcachedKeyComparison(rawValue: name) else { throw .unsupportedOperator(op) } + let condition = KeyCondition(comparison: comparison, value: value, isCaseSensitive: isCaseSensitive) + _ = try KeyTest(condition) + return condition + } + + /// The keys this page shows, out of every key listed: filtered, sorted by key, then cut to the page. + public func page(of keys: [MemcachedKeyEntry]) throws(MemcachedFilterRefusal) -> [MemcachedKeyEntry] { + let matching = try filter(keys) + let sorted = matching.sorted { ascending ? $0.key < $1.key : $0.key > $1.key } + guard offset < sorted.count else { return [] } + return Array(sorted[offset ..< offset + min(limit, sorted.count - offset)]) + } + + public func filter(_ keys: [MemcachedKeyEntry]) throws(MemcachedFilterRefusal) -> [MemcachedKeyEntry] { + guard !conditions.isEmpty else { return keys } + var tests: [KeyTest] = [] + for condition in conditions { tests.append(try KeyTest(condition)) } + return keys.filter { entry in + matchesAll ? tests.allSatisfy { $0.matches(entry.key) } : tests.contains { $0.matches(entry.key) } + } + } +} + +private struct KeyTest { + let condition: MemcachedBrowseSpec.KeyCondition + let items: [String] + let regex: NSRegularExpression? + + init(_ condition: MemcachedBrowseSpec.KeyCondition) throws(MemcachedFilterRefusal) { + self.condition = condition + switch condition.comparison { + case .inList, .notInList: + items = condition.value.split(separator: ",").map { $0.trimmingCharacters(in: .whitespaces) } + .filter { !$0.isEmpty } + guard !items.isEmpty else { throw .emptyList(condition.comparison.rawValue) } + regex = nil + case .regex: + items = [] + let options: NSRegularExpression.Options = condition.isCaseSensitive ? [] : [.caseInsensitive] + guard let compiled = try? NSRegularExpression(pattern: condition.value, options: options) else { + throw .invalidPattern(condition.value) + } + regex = compiled + default: + items = [] + regex = nil + } + } + + func matches(_ key: String) -> Bool { + let options: String.CompareOptions = condition.isCaseSensitive ? [.literal] : [.caseInsensitive] + let value = condition.value + switch condition.comparison { + case .equal: return key.compare(value, options: options) == .orderedSame + case .notEqual: return key.compare(value, options: options) != .orderedSame + case .contains: return key.range(of: value, options: options) != nil + case .notContains: return key.range(of: value, options: options) == nil + case .startsWith: return key.range(of: value, options: options.union(.anchored)) != nil + case .endsWith: return key.range(of: value, options: options.union([.anchored, .backwards])) != nil + case .inList: return items.contains { key.compare($0, options: options) == .orderedSame } + case .notInList: return !items.contains { key.compare($0, options: options) == .orderedSame } + case .regex: + let range = NSRange(key.startIndex..., in: key) + return regex?.firstMatch(in: key, range: range) != nil + } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedCommand.swift b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedCommand.swift new file mode 100644 index 0000000000..78e4a26736 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedCommand.swift @@ -0,0 +1,286 @@ +import Foundation + +public enum MemcachedStorageVerb: String, CaseIterable, Sendable { + case set, add, replace, append, prepend +} + +/// One statement of the console, in the order the text protocol takes its arguments. A value is the last argument +/// of the line instead of a data block, so a statement stays one line the review sheet can show. +public enum MemcachedCommand: Equatable, Sendable { + case get(keys: [String], withCAS: Bool) + case getAndTouch(exptime: Int64, keys: [String], withCAS: Bool) + case store(MemcachedStorageVerb, key: String, flags: UInt32, exptime: Int64, value: Data) + case compareAndSwap(key: String, flags: UInt32, exptime: Int64, unique: UInt64, value: Data) + case touch(key: String, exptime: Int64) + case delete(key: String) + case increment(key: String, delta: UInt64) + case decrement(key: String, delta: UInt64) + case flushAll(delay: Int64?) + case stats(arguments: [String]) + case version + case metadump(selector: String) + + public var replyShape: MemcachedReplyShape { + switch self { + case .get, .getAndTouch: return .values + case .stats(let arguments): return Self.statsChangingServer(arguments) ? .status : .lines + case .metadump: return .lines + default: return .status + } + } + + /// `stats reset`, `stats detail on|off` and `stats sizes_enable|sizes_disable` change the server and answer one + /// status line with no `END`. Every other `stats` form reads. + public static func statsChangingServer(_ arguments: [String]) -> Bool { + let words = arguments.map { $0.lowercased() } + switch words.first { + case "reset", "sizes_enable", "sizes_disable": return true + case "detail": return words.count > 1 && words[1] != "dump" + default: return false + } + } + + /// The bytes the text protocol takes for this command, data block included. + public var wireBytes: [UInt8] { + func line(_ words: [String]) -> [UInt8] { Array((words.joined(separator: " ") + "\r\n").utf8) } + func block(_ head: [String], _ value: Data) -> [UInt8] { + line(head + [String(value.count)]) + Array(value) + Array("\r\n".utf8) + } + switch self { + case .get(let keys, let withCAS): + return line([withCAS ? "gets" : "get"] + keys) + case .getAndTouch(let exptime, let keys, let withCAS): + return line([withCAS ? "gats" : "gat", String(exptime)] + keys) + case .store(let verb, let key, let flags, let exptime, let value): + return block([verb.rawValue, key, String(flags), String(exptime)], value) + case .compareAndSwap(let key, let flags, let exptime, let unique, let value): + return line(["cas", key, String(flags), String(exptime), String(value.count), String(unique)]) + + Array(value) + Array("\r\n".utf8) + case .touch(let key, let exptime): + return line(["touch", key, String(exptime)]) + case .delete(let key): + return line(["delete", key]) + case .increment(let key, let delta): + return line(["incr", key, String(delta)]) + case .decrement(let key, let delta): + return line(["decr", key, String(delta)]) + case .flushAll(let delay): + return line(["flush_all"] + (delay.map { [String($0)] } ?? [])) + case .stats(let arguments): + return line(["stats"] + arguments) + case .version: + return line(["version"]) + case .metadump(let selector): + return line(["lru_crawler", "metadump", selector]) + } + } +} + +public enum MemcachedCommandError: Error, Equatable, Sendable { + case empty + case unknownCommand(String) + case wrongArguments(command: String, usage: String) + case invalidKey(String) + case invalidNumber(String) + case unterminatedQuote +} + +public enum MemcachedCommandParser { + /// The longest key memcached accepts, in bytes. + public static let maximumKeyLength = 250 + + public static func parse(_ statement: String) throws(MemcachedCommandError) -> MemcachedCommand { + let words = try tokenize(statement) + guard let first = words.first else { throw .empty } + let name = Self.text(first).lowercased() + let arguments = Array(words.dropFirst()) + func text(_ index: Int) -> String { Self.text(arguments[index]) } + func usage(_ form: String) -> MemcachedCommandError { .wrongArguments(command: name, usage: form) } + + switch name { + case "get", "gets": + guard !arguments.isEmpty else { throw usage("\(name) [...]") } + return .get(keys: try keys(arguments[...]), withCAS: name == "gets") + case "gat", "gats": + guard arguments.count >= 2 else { throw usage("\(name) [...]") } + return .getAndTouch( + exptime: try integer(text(0)), keys: try keys(arguments.dropFirst()), withCAS: name == "gats" + ) + case "set", "add", "replace", "append", "prepend": + guard arguments.count == 4, let verb = MemcachedStorageVerb(rawValue: name) else { + throw usage("\(name) ") + } + return .store( + verb, key: try key(arguments[0]), flags: try flags(text(1)), exptime: try integer(text(2)), + value: Data(arguments[3]) + ) + case "cas": + guard arguments.count == 5 else { throw usage("cas ") } + return .compareAndSwap( + key: try key(arguments[0]), flags: try flags(text(1)), exptime: try integer(text(2)), + unique: try unsigned(text(3)), value: Data(arguments[4]) + ) + case "touch": + guard arguments.count == 2 else { throw usage("touch ") } + return .touch(key: try key(arguments[0]), exptime: try integer(text(1))) + case "delete": + guard arguments.count == 1 else { throw usage("delete ") } + return .delete(key: try key(arguments[0])) + case "incr", "decr": + guard arguments.count == 2 else { throw usage("\(name) ") } + let key = try key(arguments[0]) + let delta = try unsigned(text(1)) + return name == "incr" ? .increment(key: key, delta: delta) : .decrement(key: key, delta: delta) + case "flush_all": + guard arguments.count <= 1 else { throw usage("flush_all []") } + return .flushAll(delay: try arguments.isEmpty ? nil : integer(text(0))) + case "stats": + let words = arguments.indices.map(text) + guard words.allSatisfy(isPlainWord) else { throw usage("stats [ []]") } + return .stats(arguments: words) + case "version": + guard arguments.isEmpty else { throw usage("version") } + return .version + case "lru_crawler": + guard arguments.count == 2, text(0).lowercased() == "metadump", isPlainWord(text(1)) else { + throw usage("lru_crawler metadump ") + } + return .metadump(selector: text(1)) + default: + throw .unknownCommand(name) + } + } + + /// Writes a value so `parse` reads it back byte for byte. + public static func quoted(_ value: String) -> String { + escaped(Array(value.utf8), keepingUTF8: true) + } + + /// A key as one argument: as written unless it holds a quote or backslash, which the tokenizer would read. + public static func argument(_ key: String) -> String { + key.contains(where: { $0 == "\"" || $0 == "'" || $0 == "\\" }) ? quoted(key) : key + } + + /// Writes bytes that may not be UTF-8, every byte past ASCII as `\xHH`. + public static func quoted(bytes: Data) -> String { + escaped(Array(bytes), keepingUTF8: false) + } + + private static func escaped(_ bytes: [UInt8], keepingUTF8: Bool) -> String { + var out: [UInt8] = [0x22] + for byte in bytes { + switch byte { + case 0x22: out += Array(#"\""#.utf8) + case 0x5C: out += Array(#"\\"#.utf8) + case 0x0A: out += Array(#"\n"#.utf8) + case 0x0D: out += Array(#"\r"#.utf8) + case 0x09: out += Array(#"\t"#.utf8) + case 0x20 ..< 0x7F: out.append(byte) + case 0x80...: out += keepingUTF8 ? [byte] : Array(String(format: #"\x%02X"#, byte).utf8) + default: out += Array(String(format: #"\x%02X"#, byte).utf8) + } + } + out.append(0x22) + return text(out) + } + + /// Splits a line on whitespace. Double quotes take `\n`, `\r`, `\t`, `\xHH`, `\"` and `\\`; single quotes + /// take everything as written. + static func tokenize(_ text: String) throws(MemcachedCommandError) -> [[UInt8]] { + var words: [[UInt8]] = [] + var current: [UInt8] = [] + var inWord = false + var scalars = Array(text.unicodeScalars)[...] + while let scalar = scalars.popFirst() { + switch scalar { + case " ", "\t", "\n", "\r": + if inWord { words.append(current); current = []; inWord = false } + case "'": + inWord = true + guard let end = scalars.firstIndex(of: "'") else { throw .unterminatedQuote } + current += Array(String(String.UnicodeScalarView(scalars[.., + into current: inout [UInt8] + ) throws(MemcachedCommandError) { + while let scalar = scalars.popFirst() { + switch scalar { + case "\"": + return + case "\\": + guard let escaped = scalars.popFirst() else { throw .unterminatedQuote } + switch escaped { + case "n": current.append(0x0A) + case "r": current.append(0x0D) + case "t": current.append(0x09) + case "x": + let digits = String(String.UnicodeScalarView(scalars.prefix(2))) + guard digits.count == 2, let byte = UInt8(digits, radix: 16) else { + current += Array("\\x".utf8) + continue + } + current.append(byte) + scalars = scalars.dropFirst(2) + default: current += Array(String(escaped).utf8) + } + default: + current += Array(String(scalar).utf8) + } + } + throw .unterminatedQuote + } + + /// A key is 1 to 250 bytes with no space or control character, the limits memcached's text protocol sets. + public static func isValidKey(_ bytes: [UInt8]) -> Bool { + !bytes.isEmpty && bytes.count <= maximumKeyLength && !bytes.contains { $0 <= 0x20 || $0 == 0x7F } + } + + private static func key(_ bytes: [UInt8]) throws(MemcachedCommandError) -> String { + let text = Self.text(bytes) + guard isValidKey(bytes) else { throw .invalidKey(text) } + return text + } + + private static func keys(_ words: ArraySlice<[UInt8]>) throws(MemcachedCommandError) -> [String] { + var keys: [String] = [] + for word in words { keys.append(try key(word)) } + return keys + } + + /// Bytes as text for a message or a number, never for a value: a byte that is not UTF-8 reads as Latin-1. + static func text(_ bytes: [UInt8]) -> String { + String(bytes: bytes, encoding: .utf8) ?? String(bytes: bytes, encoding: .isoLatin1) ?? "" + } + + private static func isPlainWord(_ word: String) -> Bool { + isValidKey(Array(word.utf8)) + } + + private static func integer(_ text: String) throws(MemcachedCommandError) -> Int64 { + guard let value = Int64(text) else { throw .invalidNumber(text) } + return value + } + + private static func unsigned(_ text: String) throws(MemcachedCommandError) -> UInt64 { + guard let value = UInt64(text) else { throw .invalidNumber(text) } + return value + } + + private static func flags(_ text: String) throws(MemcachedCommandError) -> UInt32 { + guard let value = UInt32(text) else { throw .invalidNumber(text) } + return value + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedKeyListing.swift b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedKeyListing.swift new file mode 100644 index 0000000000..cd44dc0707 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedKeyListing.swift @@ -0,0 +1,95 @@ +import Foundation + +public struct MemcachedKeyEntry: Equatable, Sendable { + public let key: String + /// Unix time the item expires at, or nil for an item that never expires. + public let expiresAt: Int64? + public let flags: UInt32? + + public init(key: String, expiresAt: Int64?, flags: UInt32?) { + self.key = key + self.expiresAt = expiresAt + self.flags = flags + } + + /// Seconds left at `serverTime`, nil for no expiry. An item past its time reads as gone. + public func secondsLeft(at serverTime: Int64) -> Int64? { + expiresAt.map { $0 - serverTime } + } + + public func isExpired(at serverTime: Int64) -> Bool { + (secondsLeft(at: serverTime) ?? 1) <= 0 + } +} + +/// How the key list was read. A cachedump is a per-slab sample capped by the server, never the whole cache. +public enum MemcachedKeySource: String, Sendable, Equatable { + case metadump + case cachedump +} + +public enum MemcachedKeyListing { + /// One line of `lru_crawler metadump`: `key=a%3A1 exp=-1 la=1793499867 cas=2 fetch=yes cls=1 size=71 flags=5`. + /// The key is URL-encoded, and `exp` is a Unix time or -1 for never. + public static func metadumpEntry(_ line: String) -> MemcachedKeyEntry? { + var fields: [Substring: Substring] = [:] + for part in line.split(separator: " ") { + guard let equals = part.firstIndex(of: "=") else { continue } + fields[part[.. 0 ? $0 : nil }, + flags: fields["flags"].flatMap { UInt32($0) } + ) + } + + /// One line of `stats cachedump`: `ITEM user:42 [5 b; 1793500167 s]`. An expiry of 0 means never. + public static func cachedumpEntry(_ line: String) -> MemcachedKeyEntry? { + guard line.hasPrefix("ITEM "), let open = line.lastIndex(of: "[") else { return nil } + let key = line[line.index(line.startIndex, offsetBy: 5) ..< open].trimmingCharacters(in: .whitespaces) + guard !key.isEmpty else { return nil } + let inside = line[line.index(after: open)...].split(whereSeparator: { $0 == ";" || $0 == " " || $0 == "]" }) + let expiry = inside.count >= 3 ? Int64(inside[2]) : nil + return MemcachedKeyEntry(key: key, expiresAt: expiry.flatMap { $0 > 0 ? $0 : nil }, flags: nil) + } + + /// The slab classes holding items, read from `stats items` lines such as `STAT items:1:number 3`. + public static func slabClasses(fromItemStats lines: [String]) -> [Int] { + var classes: [Int] = [] + for (name, value) in statPairs(lines) where name.hasSuffix(":number") && (Int(value) ?? 0) > 0 { + let parts = name.split(separator: ":") + if parts.count == 3, parts[0] == "items", let id = Int(parts[1]), !classes.contains(id) { + classes.append(id) + } + } + return classes.sorted() + } + + /// `STAT ` lines as pairs, in the server's order. + public static func statPairs(_ lines: [String]) -> [(name: String, value: String)] { + lines.compactMap { line in + let parts = line.split(separator: " ", maxSplits: 2, omittingEmptySubsequences: true) + guard parts.count >= 2, parts[0] == "STAT" else { return nil } + return (String(parts[1]), parts.count == 3 ? String(parts[2]) : "") + } + } + + private static func percentDecoded(_ text: Substring) -> String? { + var bytes: [UInt8] = [] + var iterator = Array(text.utf8)[...] + while let byte = iterator.popFirst() { + guard byte == UInt8(ascii: "%") else { + bytes.append(byte) + continue + } + guard iterator.count >= 2, + let decoded = UInt8(String(bytes: iterator.prefix(2), encoding: .ascii) ?? "", radix: 16) else { return nil } + bytes.append(decoded) + iterator = iterator.dropFirst(2) + } + return String(bytes: bytes, encoding: .utf8) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedReplyReader.swift b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedReplyReader.swift new file mode 100644 index 0000000000..e90c657a49 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedReplyReader.swift @@ -0,0 +1,152 @@ +import Foundation + +public enum MemcachedReplyShape: Sendable, Equatable { + /// `VALUE` blocks closed by `END`, the reply to `get` and its kin. + case values + /// Lines closed by `END`, the reply to `stats` and `lru_crawler metadump`. + case lines + /// One status line such as `STORED`, `DELETED`, `VERSION 1.6.45` or a counter's new value. + case status +} + +public struct MemcachedValue: Equatable, Sendable { + public let key: String + public let flags: UInt32 + public let data: Data + public let unique: UInt64? + + public init(key: String, flags: UInt32, data: Data, unique: UInt64? = nil) { + self.key = key + self.flags = flags + self.data = data + self.unique = unique + } +} + +public enum MemcachedReply: Equatable, Sendable { + case values([MemcachedValue]) + /// `truncated` means the reader stopped at its line limit before `END`; the rest is still on the wire. + case lines([String], truncated: Bool) + case status(String) +} + +public struct MemcachedServerError: Error, Equatable, Sendable { + public enum Kind: String, Sendable { + case error = "ERROR" + case clientError = "CLIENT_ERROR" + case serverError = "SERVER_ERROR" + case busy = "BUSY" + } + + public let kind: Kind + public let message: String + + public init(kind: Kind, message: String) { + self.kind = kind + self.message = message + } + + /// `CLIENT_ERROR unauthenticated` is what a server started with an auth file says to any command before login. + public var needsAuthentication: Bool { + kind == .clientError && message.lowercased().contains("unauthenticated") + } +} + +public enum MemcachedProtocolError: Error, Equatable, Sendable { + case malformedLine(String) + case lineTooLong +} + +/// Reads one reply from a byte stream fed in pieces. A reply can be large, so lines already read are kept rather +/// than scanned again when more bytes arrive. +public struct MemcachedReplyReader: Sendable { + public let shape: MemcachedReplyShape + private let lineLimit: Int + private var buffer: [UInt8] = [] + private var cursor = 0 + private var values: [MemcachedValue] = [] + private var lines: [String] = [] + private var pendingValue: (key: String, flags: UInt32, length: Int, unique: UInt64?)? + + private static let maximumLineLength = 64 * 1_024 + /// Memcached's largest item size setting, `-I 1024m`. A longer data block is not a Memcached reply. + public static let maximumValueLength = 1 << 30 + + public init(shape: MemcachedReplyShape, lineLimit: Int = .max) { + self.shape = shape + self.lineLimit = lineLimit + } + + public mutating func append(_ bytes: Bytes) where Bytes.Element == UInt8 { + if cursor > 1 << 16 { + buffer.removeFirst(cursor) + cursor = 0 + } + buffer.append(contentsOf: bytes) + } + + /// The reply once it is whole, or nil while more bytes are needed. + public mutating func reply() throws -> MemcachedReply? { + while true { + if let pending = pendingValue { + guard buffer.count - cursor >= pending.length + 2 else { return nil } + let data = Data(buffer[cursor ..< cursor + pending.length]) + guard buffer[cursor + pending.length] == 0x0D, buffer[cursor + pending.length + 1] == 0x0A else { + throw MemcachedProtocolError.malformedLine("VALUE \(pending.key)") + } + cursor += pending.length + 2 + values.append(MemcachedValue(key: pending.key, flags: pending.flags, data: data, unique: pending.unique)) + pendingValue = nil + continue + } + guard let line = try nextLine() else { return nil } + if let error = Self.serverError(line) { throw error } + switch shape { + case .status: + return .status(line) + case .lines: + if line == "END" { return .lines(lines, truncated: false) } + if line == "OK", lines.isEmpty { continue } + lines.append(line) + if lines.count >= lineLimit { return .lines(lines, truncated: true) } + case .values: + if line == "END" { return .values(values) } + pendingValue = try Self.valueHeader(line) + } + } + } + + /// `lru_crawler metadump` ends its key lines with a bare LF and only `END` with CRLF, so a line ends at LF and + /// a CR before it is dropped. Data blocks are read by length, never as lines. + private mutating func nextLine() throws -> String? { + guard let newline = buffer[cursor...].firstIndex(of: 0x0A) else { + if buffer.count - cursor > Self.maximumLineLength { throw MemcachedProtocolError.lineTooLong } + return nil + } + let end = newline > cursor && buffer[newline - 1] == 0x0D ? newline - 1 : newline + let line = MemcachedCommandParser.text(Array(buffer[cursor ..< end])) + cursor = newline + 1 + return line + } + + static func serverError(_ line: String) -> MemcachedServerError? { + for kind in [MemcachedServerError.Kind.clientError, .serverError, .busy, .error] { + let name = kind.rawValue + if line == name { return MemcachedServerError(kind: kind, message: "") } + if line.hasPrefix(name + " ") { + return MemcachedServerError(kind: kind, message: String(line.dropFirst(name.count + 1))) + } + } + return nil + } + + private static func valueHeader(_ line: String) throws -> (key: String, flags: UInt32, length: Int, unique: UInt64?) { + let parts = line.split(separator: " ", omittingEmptySubsequences: true) + guard parts.count == 4 || parts.count == 5, parts[0] == "VALUE", + let flags = UInt32(parts[2]), let length = Int(parts[3]), (0 ... maximumValueLength).contains(length) else { + throw MemcachedProtocolError.malformedLine(line) + } + let unique = parts.count == 5 ? UInt64(parts[4]) : nil + return (String(parts[1]), flags, length, unique) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedStatements.swift b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedStatements.swift new file mode 100644 index 0000000000..dd91bb1722 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProMemcachedCore/MemcachedStatements.swift @@ -0,0 +1,35 @@ +import Foundation + +/// The console lines a grid edit becomes. Each one parses back into the command it stands for. +public enum MemcachedStatements { + /// Memcached reads an exptime above 30 days as a Unix time rather than seconds from now. + public static let relativeExptimeLimit: Int64 = 60 * 60 * 24 * 30 + + /// The text protocol reads an exptime as a 32-bit number. + public static let maximumExptime = Int64(Int32.max) + + /// No TTL, or 0, keeps the item until it is evicted. A TTL past 30 days is sent as the time it ends. Nil when + /// the end falls past what the protocol can carry. + public static func exptime(forTTL ttl: Int64?, now: Int64) -> Int64? { + guard let ttl, ttl != 0 else { return 0 } + guard ttl > relativeExptimeLimit else { return ttl >= -maximumExptime ? ttl : nil } + let (end, overflow) = now.addingReportingOverflow(ttl) + return !overflow && end <= maximumExptime ? end : nil + } + + public static func set(key: String, flags: UInt32, exptime: Int64, value: String) -> String { + "set \(MemcachedCommandParser.argument(key)) \(flags) \(exptime) \(MemcachedCommandParser.quoted(value))" + } + + public static func set(key: String, flags: UInt32, exptime: Int64, bytes: Data) -> String { + "set \(MemcachedCommandParser.argument(key)) \(flags) \(exptime) \(MemcachedCommandParser.quoted(bytes: bytes))" + } + + public static func touch(key: String, exptime: Int64) -> String { + "touch \(MemcachedCommandParser.argument(key)) \(exptime)" + } + + public static func delete(key: String) -> String { + "delete \(MemcachedCommandParser.argument(key))" + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLLiteralSpelling.swift b/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLLiteralSpelling.swift index c9098218b2..3b0aaf5808 100644 --- a/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLLiteralSpelling.swift +++ b/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLLiteralSpelling.swift @@ -24,7 +24,7 @@ public enum MySQLLiteralSpelling: Equatable, Sendable { case .backslashEscapes: return mysqlEscapeStringLiteral(value) case .quoteDoubling: - return value.replacingOccurrences(of: "'", with: "''") + return value.replacingOccurrences(of: "'", with: "''", options: .literal) } } @@ -140,11 +140,12 @@ public enum MySQLLiteralSpelling: Equatable, Sendable { return scalars.count } + /// `--` opens a comment only when a space or a control character follows it, U+007F among them. private static func opensDashComment(_ scalars: [Unicode.Scalar], at index: Int) -> Bool { guard index + 1 < scalars.count, scalars[index + 1] == "-" else { return false } guard index + 2 < scalars.count else { return true } let next = scalars[index + 2] - return next == " " || next == "\t" || next == "\n" || next == "\r" + return next <= " " || next == "\u{7F}" } private static func endOfLine(in scalars: [Unicode.Scalar], from start: Int) -> Int { diff --git a/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLObjectQueries.swift b/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLObjectQueries.swift index cfb77225bf..76d48cf2c5 100644 --- a/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLObjectQueries.swift +++ b/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLObjectQueries.swift @@ -14,7 +14,7 @@ public enum MySQLObjectQueries { } public static func quoteIdentifier(_ value: String) -> String { - "`\(value.replacingOccurrences(of: "`", with: "``"))`" + "`\(value.replacingOccurrences(of: "`", with: "``", options: .literal))`" } /// The database a catalog read means. @@ -129,9 +129,9 @@ public enum MySQLObjectQueries { public static func likePattern(matching name: String) -> String { name - .replacingOccurrences(of: "\\", with: "\\\\") - .replacingOccurrences(of: "%", with: "\\%") - .replacingOccurrences(of: "_", with: "\\_") + .replacingOccurrences(of: "\\", with: "\\\\", options: .literal) + .replacingOccurrences(of: "%", with: "\\%", options: .literal) + .replacingOccurrences(of: "_", with: "\\_", options: .literal) } /// Whether `information_schema` describes this database at all, in one scalar. diff --git a/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLStatementKeyword.swift b/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLStatementKeyword.swift index 35bbf14782..1cb3bae3ff 100644 --- a/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLStatementKeyword.swift +++ b/apple/Packages/TableProCore/Sources/TableProMySQLCore/MySQLStatementKeyword.swift @@ -20,7 +20,8 @@ nonisolated public func mysqlLeadingKeyword(_ query: String) -> String { while true { rest = rest.drop(while: { $0.isWhitespace }) if rest.hasPrefix("/*"), !rest.hasPrefix("/*!") { - guard let end = rest.range(of: "*/") else { return "" } + // Past the opener: its `*` closes nothing, so `/*/` is still open. + guard let end = rest.dropFirst(2).range(of: "*/") else { return "" } rest = rest[end.upperBound...] } else if rest.hasPrefix("#") || (rest.hasPrefix("--") && rest.dropFirst(2).first?.isWhitespace != false) { guard let end = rest.firstIndex(where: { $0.isNewline }) else { return "" } diff --git a/apple/Packages/TableProCore/Sources/TableProNeo4jCore/BoltMessage.swift b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/BoltMessage.swift new file mode 100644 index 0000000000..5b13e9e2e2 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/BoltMessage.swift @@ -0,0 +1,175 @@ +import Foundation + +public struct BoltVersion: Comparable, Hashable, Sendable, CustomStringConvertible { + public let major: UInt8 + public let minor: UInt8 + + public init(major: UInt8, minor: UInt8) { + self.major = major + self.minor = minor + } + + public var description: String { "\(major).\(minor)" } + + public static func < (lhs: BoltVersion, rhs: BoltVersion) -> Bool { + (lhs.major, lhs.minor) < (rhs.major, rhs.minor) + } + + public var separatesLogon: Bool { self >= BoltVersion(major: 5, minor: 1) } + public var sendsBoltAgent: Bool { self >= BoltVersion(major: 5, minor: 3) } + public var hasElementIds: Bool { major >= 5 } +} + +public enum BoltHandshake { + public static let magic: [UInt8] = [0x60, 0x60, 0xB0, 0x17] + + // Each proposal is [0, range, minor, major]: major.minor down to major.(minor - range). + public static let proposals: [UInt8] = [ + 0x00, 0x04, 0x04, 0x05, + 0x00, 0x02, 0x04, 0x04, + 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00 + ] + + public static var request: Data { Data(magic + proposals) } + + public static func version(fromResponse bytes: [UInt8]) -> BoltVersion? { + guard bytes.count == 4, bytes != [0, 0, 0, 0] else { return nil } + return BoltVersion(major: bytes[3], minor: bytes[2]) + } +} + +public enum BoltRequest: Equatable, Sendable { + case hello(extra: [String: PackStreamValue]) + case logon(auth: [String: PackStreamValue]) + case run(query: String, parameters: [String: PackStreamValue], extra: [String: PackStreamValue]) + case pull(count: Int64) + case discard + case reset + case goodbye + + public var structure: PackStreamValue { + switch self { + case .hello(let extra): + return .structure(tag: 0x01, fields: [.map(extra)]) + case .logon(let auth): + return .structure(tag: 0x6A, fields: [.map(auth)]) + case .run(let query, let parameters, let extra): + return .structure(tag: 0x10, fields: [.string(query), .map(parameters), .map(extra)]) + case .pull(let count): + return .structure(tag: 0x3F, fields: [.map(["n": .int(count)])]) + case .discard: + return .structure(tag: 0x2F, fields: [.map(["n": .int(-1)])]) + case .reset: + return .structure(tag: 0x0F, fields: []) + case .goodbye: + return .structure(tag: 0x02, fields: []) + } + } + + public var expectsResponse: Bool { self != .goodbye } + + public var framed: Data { BoltChunker.frame(PackStreamEncoder.encode(structure)) } +} + +public enum BoltResponse: Equatable, Sendable { + case success([String: PackStreamValue]) + case record([PackStreamValue]) + case ignored + case failure(code: String, message: String) + + public static func decode(_ message: Data) throws -> BoltResponse { + guard case .structure(let tag, let fields) = try PackStreamDecoder.decode(message) else { + throw BoltProtocolError.unexpectedMessage("not a structure") + } + switch tag { + case 0x70: + return .success(fields.first?.mapValue ?? [:]) + case 0x71: + return .record(fields.first?.listValue ?? []) + case 0x7E: + return .ignored + case 0x7F: + let metadata = fields.first?.mapValue ?? [:] + let code = metadata["code"]?.stringValue ?? metadata["neo4j_code"]?.stringValue ?? "" + return .failure(code: code, message: metadata["message"]?.stringValue ?? "") + default: + throw BoltProtocolError.unexpectedMessage(String(format: "tag 0x%02X", tag)) + } + } +} + +public enum BoltProtocolError: Error, Equatable, Sendable, LocalizedError { + case unexpectedMessage(String) + case unsupportedServer + case chunkTooLarge(Int) + + public var errorDescription: String? { + switch self { + case .unexpectedMessage(let detail): + return String(format: String(localized: "The server sent a message this client does not understand (%@)."), detail) + case .unsupportedServer: + return String(localized: "The server does not speak Bolt 4.2 to 5.4. Neo4j 4.2 or later is required.") + case .chunkTooLarge(let size): + return String(format: String(localized: "The server sent a %d-byte message, which is over the limit."), size) + } + } +} + +public enum BoltChunker { + public static let maximumChunkSize = Int(UInt16.max) + + public static func frame(_ message: Data) -> Data { + var framed = Data() + framed.reserveCapacity(message.count + 4) + var offset = message.startIndex + while offset < message.endIndex { + let end = min(offset + maximumChunkSize, message.endIndex) + let size = UInt16(end - offset) + framed.append(UInt8(size >> 8)) + framed.append(UInt8(size & 0xFF)) + framed.append(message[offset ..< end]) + offset = end + } + framed.append(contentsOf: [0x00, 0x00]) + return framed + } +} + +// A zero-size chunk with nothing before it is a NOOP keep-alive, not an empty message. +public struct BoltDechunker: Sendable { + public static let maximumMessageSize = 512 * 1_024 * 1_024 + + private var buffer: [UInt8] = [] + private var cursor = 0 + private var message = Data() + + public init() {} + + public mutating func append(_ bytes: Bytes) where Bytes.Element == UInt8 { + if cursor > 0, cursor >= buffer.count / 2 { + buffer.removeFirst(cursor) + cursor = 0 + } + buffer.append(contentsOf: bytes) + } + + public mutating func nextMessage() throws -> Data? { + while buffer.count - cursor >= 2 { + let size = Int(buffer[cursor]) << 8 | Int(buffer[cursor + 1]) + if size == 0 { + cursor += 2 + guard !message.isEmpty else { continue } + defer { message = Data() } + return message + } + guard buffer.count - cursor >= 2 + size else { return nil } + message.append(contentsOf: buffer[(cursor + 2) ..< (cursor + 2 + size)]) + cursor += 2 + size + guard message.count <= Self.maximumMessageSize else { + throw BoltProtocolError.chunkTooLarge(message.count) + } + } + return nil + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jCypher.swift b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jCypher.swift new file mode 100644 index 0000000000..22d79a0760 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jCypher.swift @@ -0,0 +1,345 @@ +import Foundation + +public enum Neo4jEntityKind: String, Sendable, CaseIterable { + case node + case relationship + + public var schemaName: String { + switch self { + case .node: "Labels" + case .relationship: "Relationship Types" + } + } + + public var entityType: String { + switch self { + case .node: "NODE" + case .relationship: "RELATIONSHIP" + } + } + + public init?(schemaName: String?) { + guard let match = Self.allCases.first(where: { $0.schemaName == schemaName }) else { return nil } + self = match + } +} + +public struct Neo4jFilter: Sendable, Equatable { + public let column: String + public let op: String + public let value: String + public let secondValue: String? + public let isCaseSensitive: Bool + + public init(column: String, op: String, value: String, secondValue: String? = nil, isCaseSensitive: Bool = true) { + self.column = column + self.op = op + self.value = value + self.secondValue = secondValue + self.isCaseSensitive = isCaseSensitive + } +} + +public enum Neo4jCypherError: Error, Equatable, Sendable, LocalizedError { + case unsupportedOperator(String) + case unquotableName(String) + case incompleteRange + + public var errorDescription: String? { + switch self { + case .unsupportedOperator(let op): + return String(format: String(localized: "The %@ filter is not available for Neo4j."), op) + case .unquotableName(let name): + return String(format: String(localized: "Neo4j cannot query the name %@ because it contains \\u."), name) + case .incompleteRange: + return String(localized: "Enter both bounds to filter with BETWEEN.") + } + } +} + +public enum Neo4jCypher { + public static let idColumn = "" + public static let labelsColumn = "" + public static let startColumn = "" + public static let endColumn = "" + public static let rawFilterColumn = "__RAW__" + + // A comment keeps a browse query valid Cypher when pasted into the editor. + public static let browseMarker = "// TablePro browse: " + + // Neo4j decodes `\uXXXX` before parsing, inside backticks too, so no spelling of `\u` stays inside its quotes. + // `\uZZZZ` is an escape the server rejects; the query builders refuse such a name before it gets that far. + public static func quote(_ identifier: String) -> String { + let escaped = identifier + .replacingOccurrences(of: "\\u", with: "\\uZZZZ") + .replacingOccurrences(of: "`", with: "``") + return "`" + escaped + "`" + } + + public static func isQuotable(_ name: String) -> Bool { + !name.contains("\\u") + } + + static func requireQuotable(_ names: [String]) throws { + if let name = names.first(where: { !isQuotable($0) }) { throw Neo4jCypherError.unquotableName(name) } + } + + public static func literal(_ text: String) -> String { + let escaped = text + .replacingOccurrences(of: "\\", with: "\\\\") + .replacingOccurrences(of: "'", with: "\\'") + return "'" + escaped + "'" + } + + public static func browseKind(of query: String) -> Neo4jEntityKind? { + let trimmed = query.drop { $0.isWhitespace } + guard trimmed.hasPrefix(browseMarker) else { return nil } + let rest = trimmed.dropFirst(browseMarker.count).prefix { !$0.isNewline } + return Neo4jEntityKind(rawValue: rest.trimmingCharacters(in: .whitespaces)) + } + + public static func browsedName(in query: String) -> String? { + guard browseKind(of: query) != nil, let start = query.range(of: ":`") else { return nil } + var name = "" + var index = start.upperBound + while index < query.endIndex { + let character = query[index] + let next = query.index(after: index) + guard character == "`" else { + name.append(character) + index = next + continue + } + guard next < query.endIndex, query[next] == "`" else { return name } + name.append("`") + index = query.index(after: next) + } + return nil + } + + public static func fixedColumns(for kind: Neo4jEntityKind) -> [String] { + switch kind { + case .node: [idColumn, labelsColumn] + case .relationship: [idColumn, startColumn, endColumn] + } + } + + public static func pattern(for kind: Neo4jEntityKind, name: String) -> String { + switch kind { + case .node: "MATCH (n:\(quote(name)))" + case .relationship: "MATCH ()-[n:\(quote(name))]->()" + } + } + + public static func browseQuery( + kind: Neo4jEntityKind, + name: String, + filters: [Neo4jFilter] = [], + matchesAll: Bool = true, + sort: [(column: String, ascending: Bool)] = [], + limit: Int, + offset: Int, + usesElementId: Bool = true + ) throws -> String { + try requireQuotable([name] + sort.map(\.column)) + var lines = [browseMarker + kind.rawValue, pattern(for: kind, name: name)] + if let filter = try whereClause(filters, matchesAll: matchesAll, usesElementId: usesElementId) { + lines.append(filter) + } + lines.append("RETURN n") + if !sort.isEmpty { + let keys = sort.map { expression(for: $0.column, usesElementId: usesElementId) + ($0.ascending ? " ASC" : " DESC") } + lines.append("ORDER BY " + keys.joined(separator: ", ")) + } + lines.append("SKIP \(max(0, offset)) LIMIT \(max(0, limit))") + return lines.joined(separator: "\n") + } + + public static func exportQuery(kind: Neo4jEntityKind, name: String) throws -> String { + try requireQuotable([name]) + return [browseMarker + kind.rawValue, pattern(for: kind, name: name), "RETURN n"].joined(separator: "\n") + } + + public static func countQuery( + kind: Neo4jEntityKind, + name: String, + filters: [Neo4jFilter] = [], + matchesAll: Bool = true, + usesElementId: Bool = true + ) throws -> String { + try requireQuotable([name]) + var lines = [pattern(for: kind, name: name)] + if let filter = try whereClause(filters, matchesAll: matchesAll, usesElementId: usesElementId) { + lines.append(filter) + } + lines.append("RETURN count(n) AS count") + return lines.joined(separator: "\n") + } + + // Bounded so the Structure tab stays fast on a large graph, unlike `db.schema.nodeTypeProperties()`. + public static func propertySampleQuery(kind: Neo4jEntityKind, name: String, sampleSize: Int = 1_000) -> String { + pattern(for: kind, name: name) + + " WITH n LIMIT \(sampleSize) UNWIND keys(n) AS key" + + " RETURN key, head(collect(n[key])) AS sample ORDER BY key" + } + + public static func propertyKeysQuery(kind: Neo4jEntityKind, name: String) throws -> String { + try requireQuotable([name]) + return pattern(for: kind, name: name) + " UNWIND keys(n) AS key RETURN DISTINCT key ORDER BY key" + } + + public static func expression(for column: String, usesElementId: Bool) -> String { + let idFunction = usesElementId ? "elementId" : "id" + switch column { + case idColumn: return "\(idFunction)(n)" + case labelsColumn: return "labels(n)" + case startColumn: return "\(idFunction)(startNode(n))" + case endColumn: return "\(idFunction)(endNode(n))" + default: return "n." + quote(column) + } + } + + private static func whereClause(_ filters: [Neo4jFilter], matchesAll: Bool, usesElementId: Bool) throws -> String? { + let clauses = try filters.map { try condition(for: $0, usesElementId: usesElementId) } + guard !clauses.isEmpty else { return nil } + let joined = clauses.count == 1 + ? clauses[0] + : clauses.map { "(\($0))" }.joined(separator: matchesAll ? " AND " : " OR ") + return "WHERE " + joined + } + + private static let negations = ["!=": "=", "<>": "=", "NOT CONTAINS": "CONTAINS", "NOT IN": "IN"] + + static func condition(for filter: Neo4jFilter, usesElementId: Bool) throws -> String { + if filter.column == rawFilterColumn { return filter.value } + try requireQuotable([filter.column]) + let op = filter.op.uppercased() + if let presence = presence(op, column: filter.column, usesElementId: usesElementId) { return presence } + + let subject = Subject(filter: filter, usesElementId: usesElementId) + var predicate = try positive(negations[op] ?? op, filter: filter, subject: subject) + // `labels(n)` is a list, so a label filter matches when any one label does. + if filter.column == labelsColumn { predicate = "any(label IN labels(n) WHERE \(predicate))" } + return negations[op] == nil ? predicate : "NOT (\(predicate))" + } + + private struct Subject { + let expression: String + // An element id or a label is always a string, so it never compares as a number. + let isText: Bool + let foldsCase: Bool + + init(filter: Neo4jFilter, usesElementId: Bool) { + let isLabel = filter.column == labelsColumn + let isElementId = usesElementId && [idColumn, startColumn, endColumn].contains(filter.column) + expression = isLabel ? "label" : Neo4jCypher.expression(for: filter.column, usesElementId: usesElementId) + isText = isLabel || isElementId + foldsCase = !filter.isCaseSensitive && !isElementId + } + } + + private static func presence(_ op: String, column: String, usesElementId: Bool) -> String? { + let target = expression(for: column, usesElementId: usesElementId) + switch op { + case "IS NULL": return "\(target) IS NULL" + case "IS NOT NULL": return "\(target) IS NOT NULL" + case "IS EMPTY" where column == labelsColumn: return "size(labels(n)) = 0" + case "IS NOT EMPTY" where column == labelsColumn: return "size(labels(n)) > 0" + case "IS EMPTY": return "(\(target) IS NULL OR \(target) = '')" + case "IS NOT EMPTY": return "(\(target) IS NOT NULL AND \(target) <> '')" + default: return nil + } + } + + private static func positive(_ op: String, filter: Neo4jFilter, subject: Subject) throws -> String { + let target = subject.expression + switch op { + case "=": + return matchesAny(subject, [filter.value]) + case "IN": + return matchesAny(subject, filter.value.split(separator: ",").map { $0.trimmingCharacters(in: .whitespaces) }) + case ">", ">=", "<", "<=": + return "\(target) \(op) \(bound(filter.value, subject))" + case "BETWEEN": + let range = try bounds(of: filter) + return "\(target) >= \(bound(range.lower, subject)) AND \(target) <= \(bound(range.upper, subject))" + case "CONTAINS", "STARTS WITH", "ENDS WITH": + let needle = literal(subject.foldsCase ? filter.value.lowercased() : filter.value) + return "\(text(subject, folded: subject.foldsCase)) \(op) \(needle)" + case "REGEX": + let pattern = subject.foldsCase ? "(?i)" + filter.value : filter.value + return "\(text(subject, folded: false)) =~ \(literal(pattern))" + default: + throw Neo4jCypherError.unsupportedOperator(filter.op) + } + } + + // A property can hold any type, so text operators read it as text first: a number or a list is then a + // non-match rather than a type error that fails the whole page. + private static func text(_ subject: Subject, folded: Bool) -> String { + let text = subject.isText ? subject.expression : "toStringOrNull(\(subject.expression))" + return folded ? "toLower(\(text))" : text + } + + private static func bound(_ value: String, _ subject: Subject) -> String { + subject.isText ? literal(value) : typedLiteral(value) + } + + private static func matchesAny(_ subject: Subject, _ values: [String]) -> String { + let target = subject.expression + if subject.isText { + let items = values.map { literal(subject.foldsCase ? $0.lowercased() : $0) } + let lhs = subject.foldsCase ? "toLower(\(target))" : target + return items.count == 1 ? "\(lhs) = \(items[0])" : "\(lhs) IN [\(items.joined(separator: ", "))]" + } + if subject.foldsCase { + // Case folding applies to text only, so `30.0` still matches `= 30`. + let folded = "\(text(subject, folded: true)) IN [\(values.map { literal($0.lowercased()) }.joined(separator: ", "))]" + let typed = values.map(typedLiteral).filter { !$0.hasPrefix("'") } + guard !typed.isEmpty else { return folded } + return "(\(target) IN [\(typed.joined(separator: ", "))] OR \(folded))" + } + // A property has no declared type, so `123` also matches the text '123'. + let forms = values.flatMap(spellings) + if forms.count == 1 { return "\(target) = \(forms[0])" } + if values.count == 1 { return "(" + forms.map { "\(target) = \($0)" }.joined(separator: " OR ") + ")" } + return "\(target) IN [\(forms.joined(separator: ", "))]" + } + + private static func spellings(_ value: String) -> [String] { + let typed = typedLiteral(value) + return typed == literal(value) ? [typed] : [typed, literal(value)] + } + + // The host sends BETWEEN's value as "lower,upper" and the upper bound again on its own. + private static func bounds(of filter: Neo4jFilter) throws -> (lower: String, upper: String) { + let lower: String + let upper: String + if let second = filter.secondValue { + let suffix = "," + second + lower = filter.value.hasSuffix(suffix) ? String(filter.value.dropLast(suffix.count)) : filter.value + upper = second + } else if let comma = filter.value.firstIndex(of: ",") { + lower = String(filter.value[.. String { + if let integer = Int64(value), String(integer) == value { return value } + if value.range(of: #"^-?(0|[1-9][0-9]*)\.[0-9]+([eE][+-]?[0-9]+)?$"#, options: .regularExpression) != nil, + let number = Double(value), number.isFinite { + return value + } + switch value.lowercased() { + case "true", "false": return value.lowercased() + default: return literal(value) + } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jEntity.swift b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jEntity.swift new file mode 100644 index 0000000000..8d2a53ad0c --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jEntity.swift @@ -0,0 +1,64 @@ +import Foundation + +public enum Neo4jStructureTag { + public static let node: UInt8 = 0x4E + public static let relationship: UInt8 = 0x52 + public static let unboundRelationship: UInt8 = 0x72 + public static let path: UInt8 = 0x50 + public static let date: UInt8 = 0x44 + public static let time: UInt8 = 0x54 + public static let localTime: UInt8 = 0x74 + public static let dateTime: UInt8 = 0x49 + public static let dateTimeZoneId: UInt8 = 0x69 + public static let legacyDateTime: UInt8 = 0x46 + public static let legacyDateTimeZoneId: UInt8 = 0x66 + public static let localDateTime: UInt8 = 0x64 + public static let duration: UInt8 = 0x45 + public static let point2D: UInt8 = 0x58 + public static let point3D: UInt8 = 0x59 +} + +// Bolt 4 has no element id, so the numeric id stands in, written as text. +public struct Neo4jEntity: Equatable, Sendable { + public enum Kind: Equatable, Sendable { + case node(labels: [String]) + case relationship(type: String, startId: String, endId: String) + } + + public let elementId: String + public let kind: Kind + public let properties: [String: PackStreamValue] + + public init(elementId: String, kind: Kind, properties: [String: PackStreamValue]) { + self.elementId = elementId + self.kind = kind + self.properties = properties + } + + public init?(_ value: PackStreamValue) { + guard case .structure(let tag, let fields) = value else { return nil } + switch tag { + case Neo4jStructureTag.node where fields.count >= 3: + elementId = Self.identifier(fields, elementIdIndex: 3, idIndex: 0) + kind = .node(labels: fields[1].listValue?.compactMap(\.stringValue) ?? []) + properties = fields[2].mapValue ?? [:] + case Neo4jStructureTag.relationship where fields.count >= 5: + elementId = Self.identifier(fields, elementIdIndex: 5, idIndex: 0) + kind = .relationship( + type: fields[3].stringValue ?? "", + startId: Self.identifier(fields, elementIdIndex: 6, idIndex: 1), + endId: Self.identifier(fields, elementIdIndex: 7, idIndex: 2) + ) + properties = fields[4].mapValue ?? [:] + default: + return nil + } + } + + private static func identifier(_ fields: [PackStreamValue], elementIdIndex: Int, idIndex: Int) -> String { + if elementIdIndex < fields.count, let elementId = fields[elementIdIndex].stringValue { + return elementId + } + return fields[idIndex].intValue.map(String.init) ?? "" + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jResultTable.swift b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jResultTable.swift new file mode 100644 index 0000000000..cefdfeca6f --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jResultTable.swift @@ -0,0 +1,62 @@ +import Foundation + +public struct Neo4jResultTable: Equatable, Sendable { + public let columns: [String] + public let typeNames: [String] + public let rows: [[String?]] + + public init(columns: [String], typeNames: [String], rows: [[String?]]) { + self.columns = columns + self.typeNames = typeNames + self.rows = rows + } + + public static func records(keys: [String], records: [[PackStreamValue]]) -> Neo4jResultTable { + let typeNames = keys.indices.map { index -> String in + for record in records where index < record.count && record[index] != .null { + return Neo4jValueText.typeName(record[index]) + } + return "ANY" + } + let rows = records.map { record in + keys.indices.map { index in index < record.count ? Neo4jValueText.text(record[index]) : nil } + } + return Neo4jResultTable(columns: keys, typeNames: typeNames, rows: rows) + } + + // Known keys keep their order; keys first seen on this page follow, sorted. A stream's columns are fixed by its + // header, so it passes `addsUnseenProperties: false`. + public static func entities( + kind: Neo4jEntityKind, + records: [[PackStreamValue]], + knownProperties: [String], + addsUnseenProperties: Bool = true + ) -> Neo4jResultTable { + let entities = records.compactMap { $0.first.flatMap(Neo4jEntity.init) } + let known = Set(knownProperties) + let extra = addsUnseenProperties ? Set(entities.flatMap(\.properties.keys)).subtracting(known).sorted() : [] + let properties = knownProperties + extra + let fixed = Neo4jCypher.fixedColumns(for: kind) + + var typeNames = fixed.map { _ in "STRING" } + if kind == .node { typeNames[1] = "LIST" } + for key in properties { + let sample = entities.lazy.compactMap { $0.properties[key] }.first { $0 != .null } + typeNames.append(sample.map(Neo4jValueText.typeName) ?? "ANY") + } + + let rows = entities.map { entity -> [String?] in + var row: [String?] = [entity.elementId] + switch entity.kind { + case .node(let labels): + row.append(Neo4jValueText.text(.list(labels.map(PackStreamValue.string)))) + case .relationship(_, let startId, let endId): + row.append(startId) + row.append(endId) + } + row += properties.map { entity.properties[$0].flatMap(Neo4jValueText.text) } + return row + } + return Neo4jResultTable(columns: fixed + properties, typeNames: typeNames, rows: rows) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jUpdateCounts.swift b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jUpdateCounts.swift new file mode 100644 index 0000000000..619c215945 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jUpdateCounts.swift @@ -0,0 +1,41 @@ +import Foundation + +public struct Neo4jUpdateCounts: Equatable, Sendable { + public enum Counter: String, CaseIterable, Sendable { + case nodesCreated = "nodes-created" + case nodesDeleted = "nodes-deleted" + case relationshipsCreated = "relationships-created" + case relationshipsDeleted = "relationships-deleted" + case propertiesSet = "properties-set" + case labelsAdded = "labels-added" + case labelsRemoved = "labels-removed" + case indexesAdded = "indexes-added" + case indexesRemoved = "indexes-removed" + case constraintsAdded = "constraints-added" + case constraintsRemoved = "constraints-removed" + case systemUpdates = "system-updates" + } + + public let counters: [(counter: Counter, count: Int)] + + public init(summary: [String: PackStreamValue]) { + let stats = summary["stats"]?.mapValue ?? [:] + counters = Counter.allCases.compactMap { counter in + let count = Int(stats[counter.rawValue]?.intValue ?? 0) + return count > 0 ? (counter, count) : nil + } + } + + public func count(_ counter: Counter) -> Int { + counters.first { $0.counter == counter }?.count ?? 0 + } + + // A property or label counter counts values, not entities. + public var entitiesChanged: Int { + [.nodesCreated, .nodesDeleted, .relationshipsCreated, .relationshipsDeleted].reduce(0) { $0 + count($1) } + } + + public static func == (lhs: Self, rhs: Self) -> Bool { + lhs.counters.elementsEqual(rhs.counters) { $0.counter == $1.counter && $0.count == $1.count } + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jValueText.swift b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jValueText.swift new file mode 100644 index 0000000000..4285727595 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/Neo4jValueText.swift @@ -0,0 +1,305 @@ +import Foundation + +public enum Neo4jValueText { + public static func text(_ value: PackStreamValue) -> String? { + switch value { + case .null: + return nil + case .bool(let flag): + return flag ? "true" : "false" + case .int(let number): + return String(number) + case .float(let number): + return floatText(number) + case .string(let text): + return text + case .bytes(let bytes): + return "0x" + bytes.map { String(format: "%02x", $0) }.joined() + case .list, .map: + return json(jsonObject(value)) + case .structure(let tag, let fields): + if let scalar = structureText(tag: tag, fields: fields) { return scalar } + return json(jsonObject(value)) + } + } + + public static func typeName(_ value: PackStreamValue) -> String { + switch value { + case .null: return "NULL" + case .bool: return "BOOLEAN" + case .int: return "INTEGER" + case .float: return "FLOAT" + case .string: return "STRING" + case .bytes: return "BYTES" + case .list: return "LIST" + case .map: return "MAP" + case .structure(let tag, _): + switch tag { + case Neo4jStructureTag.node: return "NODE" + case Neo4jStructureTag.relationship, Neo4jStructureTag.unboundRelationship: return "RELATIONSHIP" + case Neo4jStructureTag.path: return "PATH" + case Neo4jStructureTag.date: return "DATE" + case Neo4jStructureTag.time: return "ZONED TIME" + case Neo4jStructureTag.localTime: return "LOCAL TIME" + case Neo4jStructureTag.dateTime, Neo4jStructureTag.dateTimeZoneId, + Neo4jStructureTag.legacyDateTime, Neo4jStructureTag.legacyDateTimeZoneId: + return "ZONED DATETIME" + case Neo4jStructureTag.localDateTime: return "LOCAL DATETIME" + case Neo4jStructureTag.duration: return "DURATION" + case Neo4jStructureTag.point2D, Neo4jStructureTag.point3D: return "POINT" + default: return "ANY" + } + } + } + + public static func jsonObject(_ value: PackStreamValue) -> Any { + switch value { + case .null: + return NSNull() + case .bool(let flag): + return flag + case .int(let number): + return number + case .float(let number): + return number.isFinite ? number as Any : floatText(number) + case .string(let text): + return text + case .bytes: + return text(value) ?? "" + case .list(let items): + return items.map(jsonObject) + case .map(let entries): + return entries.mapValues(jsonObject) + case .structure(let tag, let fields): + if let entity = Neo4jEntity(value) { + return entityObject(entity) + } + if tag == Neo4jStructureTag.unboundRelationship, fields.count >= 3 { + return [ + "elementId": fields.count > 3 ? fields[3].stringValue ?? "" : fields[0].intValue.map(String.init) ?? "", + "type": fields[1].stringValue ?? "", + "properties": (fields[2].mapValue ?? [:]).mapValues(jsonObject) + ] as [String: Any] + } + if tag == Neo4jStructureTag.path, fields.count == 3 { + return pathObject(fields) ?? [ + "nodes": (fields[0].listValue ?? []).map(jsonObject), + "relationships": (fields[1].listValue ?? []).map(jsonObject) + ] + } + return structureText(tag: tag, fields: fields) ?? NSNull() + } + } + + // Bolt sends a path's distinct nodes and relationships plus (relationship, node) index pairs; a positive + // 1-based relationship index walks forward, a negative one backward. + private static func pathObject(_ fields: [PackStreamValue]) -> [String: Any]? { + let nodes = (fields[0].listValue ?? []).compactMap(Neo4jEntity.init) + let relationships = fields[1].listValue ?? [] + let sequence = (fields[2].listValue ?? []).compactMap(\.intValue) + guard let start = nodes.first, sequence.count.isMultiple(of: 2) else { return nil } + + var orderedNodes = [entityObject(start)] + var orderedRelationships: [Any] = [] + var current = start + for step in stride(from: 0, to: sequence.count, by: 2) { + let relationshipIndex = Int(sequence[step].magnitude) - 1 + let nodeIndex = Int(sequence[step + 1]) + guard sequence[step] != 0, relationships.indices.contains(relationshipIndex), + nodes.indices.contains(nodeIndex), + case .structure(_, let relationshipFields) = relationships[relationshipIndex], + relationshipFields.count >= 3 + else { return nil } + let next = nodes[nodeIndex] + let forward = sequence[step] > 0 + orderedRelationships.append(entityObject(Neo4jEntity( + elementId: relationshipFields.count > 3 + ? relationshipFields[3].stringValue ?? "" + : relationshipFields[0].intValue.map(String.init) ?? "", + kind: .relationship( + type: relationshipFields[1].stringValue ?? "", + startId: forward ? current.elementId : next.elementId, + endId: forward ? next.elementId : current.elementId + ), + properties: relationshipFields[2].mapValue ?? [:] + ))) + orderedNodes.append(entityObject(next)) + current = next + } + return ["nodes": orderedNodes, "relationships": orderedRelationships] + } + + private static func entityObject(_ entity: Neo4jEntity) -> [String: Any] { + var object: [String: Any] = [ + "elementId": entity.elementId, + "properties": entity.properties.mapValues(jsonObject) + ] + switch entity.kind { + case .node(let labels): + object["labels"] = labels + case .relationship(let type, let startId, let endId): + object["type"] = type + object["startElementId"] = startId + object["endElementId"] = endId + } + return object + } + + private static func json(_ object: Any) -> String { + guard JSONSerialization.isValidJSONObject(object), + let data = try? JSONSerialization.data(withJSONObject: object, options: [.sortedKeys, .withoutEscapingSlashes]), + let text = String(data: data, encoding: .utf8) + else { return String(describing: object) } + return text + } + + static func floatText(_ number: Double) -> String { + if number.isNaN { return "NaN" } + if number.isInfinite { return number < 0 ? "-Infinity" : "Infinity" } + return String(number) + } + + // MARK: - Temporal and spatial + + static func structureText(tag: UInt8, fields: [PackStreamValue]) -> String? { + let ints = fields.map { $0.intValue } + switch tag { + case Neo4jStructureTag.date: + guard let days = ints.first ?? nil else { return nil } + return dateText(days: days) + case Neo4jStructureTag.localTime: + guard let nanos = ints.first ?? nil else { return nil } + return timeText(nanosOfDay: nanos) + case Neo4jStructureTag.time: + guard ints.count == 2, let nanos = ints[0], let offset = ints[1] else { return nil } + return timeText(nanosOfDay: nanos) + offsetText(offset) + case Neo4jStructureTag.localDateTime: + guard ints.count == 2, let seconds = ints[0], let nanos = ints[1] else { return nil } + return dateTimeText(localSeconds: seconds, nanos: nanos) + case Neo4jStructureTag.dateTime, Neo4jStructureTag.legacyDateTime: + guard ints.count == 3, let seconds = ints[0], let nanos = ints[1], let offset = ints[2] else { return nil } + let local = tag == Neo4jStructureTag.dateTime ? seconds + offset : seconds + return dateTimeText(localSeconds: local, nanos: nanos) + offsetText(offset) + case Neo4jStructureTag.dateTimeZoneId, Neo4jStructureTag.legacyDateTimeZoneId: + guard fields.count == 3, let seconds = ints[0], let nanos = ints[1], + let zoneId = fields[2].stringValue else { return nil } + let zone = TimeZone(identifier: zoneId) + let offset: Int64 + if tag == Neo4jStructureTag.dateTimeZoneId { + offset = Int64(zone?.secondsFromGMT(for: Date(timeIntervalSince1970: TimeInterval(seconds))) ?? 0) + } else { + let guess = Int64(zone?.secondsFromGMT(for: Date(timeIntervalSince1970: TimeInterval(seconds))) ?? 0) + offset = Int64(zone?.secondsFromGMT(for: Date(timeIntervalSince1970: TimeInterval(seconds - guess))) ?? 0) + } + let local = tag == Neo4jStructureTag.dateTimeZoneId ? seconds + offset : seconds + return dateTimeText(localSeconds: local, nanos: nanos) + offsetText(offset) + "[\(zoneId)]" + case Neo4jStructureTag.duration: + guard ints.count == 4, let months = ints[0], let days = ints[1], + let seconds = ints[2], let nanos = ints[3] else { return nil } + return durationText(months: months, days: days, seconds: seconds, nanos: nanos) + case Neo4jStructureTag.point2D, Neo4jStructureTag.point3D: + guard let srid = ints.first ?? nil else { return nil } + let names = ["x", "y", "z"] + let coordinates = fields.dropFirst().enumerated().map { index, field -> String in + let number: Double + switch field { + case .float(let value): number = value + case .int(let value): number = Double(value) + default: number = .nan + } + return "\(names[min(index, 2)]): \(floatText(number))" + } + return "point({srid: \(srid), " + coordinates.joined(separator: ", ") + "})" + default: + return nil + } + } + + static func dateText(days: Int64) -> String { + let (year, month, day) = civilDate(daysSinceEpoch: days) + let yearText: String + if year < 0 || year > 9_999 { + yearText = (year < 0 ? "-" : "+") + String(format: "%04lld", abs(year)) + } else { + yearText = String(format: "%04lld", year) + } + return yearText + String(format: "-%02lld-%02lld", month, day) + } + + static func timeText(nanosOfDay: Int64) -> String { + let seconds = nanosOfDay / 1_000_000_000 + let nanos = nanosOfDay % 1_000_000_000 + return String(format: "%02lld:%02lld:%02lld", seconds / 3_600, seconds / 60 % 60, seconds % 60) + + fractionText(nanos) + } + + static func dateTimeText(localSeconds: Int64, nanos: Int64) -> String { + let days = floorDivide(localSeconds, 86_400) + let secondsOfDay = localSeconds - days * 86_400 + return dateText(days: days) + "T" + timeText(nanosOfDay: secondsOfDay * 1_000_000_000 + nanos) + } + + static func offsetText(_ offsetSeconds: Int64) -> String { + guard offsetSeconds != 0 else { return "Z" } + let sign = offsetSeconds < 0 ? "-" : "+" + let magnitude = abs(offsetSeconds) + var text = sign + String(format: "%02lld:%02lld", magnitude / 3_600, magnitude / 60 % 60) + if magnitude % 60 != 0 { text += String(format: ":%02lld", magnitude % 60) } + return text + } + + static func durationText(months: Int64, days: Int64, seconds: Int64, nanos: Int64) -> String { + var text = "P" + if months / 12 != 0 { text += "\(months / 12)Y" } + if months % 12 != 0 { text += "\(months % 12)M" } + if days != 0 { text += "\(days)D" } + // Seconds and nanoseconds stay apart: their product overflows Int64 for durations Neo4j accepts. + let carry = floorDivide(nanos, 1_000_000_000) + let (total, overflow) = seconds.addingReportingOverflow(carry) + let signedSeconds = overflow ? seconds : total + var fraction = nanos - carry * 1_000_000_000 + if signedSeconds != 0 || fraction != 0 { + let negative = signedSeconds < 0 + var wholeSeconds = signedSeconds.magnitude + if negative, fraction != 0 { + wholeSeconds -= 1 + fraction = 1_000_000_000 - fraction + } + var time = "T" + if wholeSeconds / 3_600 != 0 { time += (negative ? "-" : "") + "\(wholeSeconds / 3_600)H" } + if wholeSeconds / 60 % 60 != 0 { time += (negative ? "-" : "") + "\(wholeSeconds / 60 % 60)M" } + let remainder = wholeSeconds % 60 + if remainder != 0 || fraction != 0 { + time += (negative ? "-" : "") + "\(remainder)" + fractionText(fraction) + "S" + } + text += time + } + return text == "P" ? "PT0S" : text + } + + private static func fractionText(_ nanos: Int64) -> String { + guard nanos != 0 else { return "" } + var digits = String(format: "%09lld", nanos) + while digits.hasSuffix("0") { digits.removeLast() } + return "." + digits + } + + private static func floorDivide(_ value: Int64, _ divisor: Int64) -> Int64 { + let quotient = value / divisor + return (value % divisor != 0 && (value < 0) != (divisor < 0)) ? quotient - 1 : quotient + } + + // Howard Hinnant's `civil_from_days`. + private static func civilDate(daysSinceEpoch: Int64) -> (Int64, Int64, Int64) { + let shifted = daysSinceEpoch + 719_468 + let era = floorDivide(shifted, 146_097) + let dayOfEra = shifted - era * 146_097 + let yearOfEra = (dayOfEra - dayOfEra / 1_460 + dayOfEra / 36_524 - dayOfEra / 146_096) / 365 + let dayOfYear = dayOfEra - (365 * yearOfEra + yearOfEra / 4 - yearOfEra / 100) + let monthIndex = (5 * dayOfYear + 2) / 153 + let day = dayOfYear - (153 * monthIndex + 2) / 5 + 1 + let month = monthIndex < 10 ? monthIndex + 3 : monthIndex - 9 + let year = yearOfEra + era * 400 + (month <= 2 ? 1 : 0) + return (year, month, day) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProNeo4jCore/PackStream.swift b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/PackStream.swift new file mode 100644 index 0000000000..c6874b6a1c --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProNeo4jCore/PackStream.swift @@ -0,0 +1,257 @@ +import Foundation + +public indirect enum PackStreamValue: Equatable, Sendable { + case null + case bool(Bool) + case int(Int64) + case float(Double) + case bytes(Data) + case string(String) + case list([PackStreamValue]) + case map([String: PackStreamValue]) + case structure(tag: UInt8, fields: [PackStreamValue]) + + public var stringValue: String? { + if case .string(let value) = self { return value } + return nil + } + + public var intValue: Int64? { + if case .int(let value) = self { return value } + return nil + } + + public var boolValue: Bool? { + if case .bool(let value) = self { return value } + return nil + } + + public var listValue: [PackStreamValue]? { + if case .list(let value) = self { return value } + return nil + } + + public var mapValue: [String: PackStreamValue]? { + if case .map(let value) = self { return value } + return nil + } +} + +public enum PackStreamError: Error, Equatable, Sendable { + case truncated + case unknownMarker(UInt8) + case invalidString + case nonStringKey + case nestedTooDeeply +} + +public enum PackStreamEncoder { + public static func encode(_ value: PackStreamValue) -> Data { + var data = Data() + write(value, into: &data) + return data + } + + public static func write(_ value: PackStreamValue, into data: inout Data) { + switch value { + case .null: + data.append(0xC0) + case .bool(let flag): + data.append(flag ? 0xC3 : 0xC2) + case .int(let number): + writeInt(number, into: &data) + case .float(let number): + data.append(0xC1) + appendBigEndian(number.bitPattern, into: &data) + case .bytes(let bytes): + writeHeader(count: bytes.count, tiny: nil, markers: (0xCC, 0xCD, 0xCE), into: &data) + data.append(bytes) + case .string(let text): + let utf8 = Data(text.utf8) + writeHeader(count: utf8.count, tiny: 0x80, markers: (0xD0, 0xD1, 0xD2), into: &data) + data.append(utf8) + case .list(let items): + writeHeader(count: items.count, tiny: 0x90, markers: (0xD4, 0xD5, 0xD6), into: &data) + for item in items { write(item, into: &data) } + case .map(let entries): + writeHeader(count: entries.count, tiny: 0xA0, markers: (0xD8, 0xD9, 0xDA), into: &data) + for key in entries.keys.sorted() { + write(.string(key), into: &data) + write(entries[key] ?? .null, into: &data) + } + case .structure(let tag, let fields): + precondition(fields.count < 16, "Bolt has no marker for a structure over 15 fields") + data.append(0xB0 | UInt8(fields.count)) + data.append(tag) + for field in fields { write(field, into: &data) } + } + } + + private static func writeInt(_ number: Int64, into data: inout Data) { + switch number { + case -16 ... 127: + data.append(UInt8(bitPattern: Int8(number))) + case Int64(Int8.min) ... Int64(Int8.max): + data.append(0xC8) + data.append(UInt8(bitPattern: Int8(number))) + case Int64(Int16.min) ... Int64(Int16.max): + data.append(0xC9) + appendBigEndian(UInt16(bitPattern: Int16(number)), into: &data) + case Int64(Int32.min) ... Int64(Int32.max): + data.append(0xCA) + appendBigEndian(UInt32(bitPattern: Int32(number)), into: &data) + default: + data.append(0xCB) + appendBigEndian(UInt64(bitPattern: number), into: &data) + } + } + + private static func writeHeader( + count: Int, + tiny: UInt8?, + markers: (UInt8, UInt8, UInt8), + into data: inout Data + ) { + if let tiny, count < 16 { + data.append(tiny | UInt8(count)) + } else if count <= Int(UInt8.max) { + data.append(markers.0) + data.append(UInt8(count)) + } else if count <= Int(UInt16.max) { + data.append(markers.1) + appendBigEndian(UInt16(count), into: &data) + } else { + data.append(markers.2) + appendBigEndian(UInt32(count), into: &data) + } + } + + private static func appendBigEndian(_ value: T, into data: inout Data) { + withUnsafeBytes(of: value.bigEndian) { data.append(contentsOf: $0) } + } +} + +public struct PackStreamDecoder { + // Each level is a stack frame, so a server cannot overflow the stack with a deeply nested list. + public static let maximumDepth = 64 + + private let bytes: [UInt8] + private var index = 0 + private var depth = 0 + + public init(_ data: Data) { + bytes = [UInt8](data) + } + + public var isAtEnd: Bool { index >= bytes.count } + + public static func decode(_ data: Data) throws -> PackStreamValue { + var decoder = PackStreamDecoder(data) + return try decoder.next() + } + + public mutating func next() throws -> PackStreamValue { + depth += 1 + defer { depth -= 1 } + guard depth <= Self.maximumDepth else { throw PackStreamError.nestedTooDeeply } + let marker = try byte() + switch marker { + case 0x00 ... 0x7F: + return .int(Int64(marker)) + case 0xF0 ... 0xFF: + return .int(Int64(Int8(bitPattern: marker))) + case 0x80 ... 0x8F: + return try string(count: Int(marker & 0x0F)) + case 0x90 ... 0x9F: + return try list(count: Int(marker & 0x0F)) + case 0xA0 ... 0xAF: + return try map(count: Int(marker & 0x0F)) + case 0xB0 ... 0xBF: + let tag = try byte() + var fields: [PackStreamValue] = [] + for _ in 0 ..< Int(marker & 0x0F) { fields.append(try next()) } + return .structure(tag: tag, fields: fields) + case 0xC0: + return .null + case 0xC1: + return .float(Double(bitPattern: try integer(UInt64.self))) + case 0xC2: + return .bool(false) + case 0xC3: + return .bool(true) + case 0xC8: + return .int(Int64(Int8(bitPattern: try byte()))) + case 0xC9: + return .int(Int64(Int16(bitPattern: try integer(UInt16.self)))) + case 0xCA: + return .int(Int64(Int32(bitPattern: try integer(UInt32.self)))) + case 0xCB: + return .int(Int64(bitPattern: try integer(UInt64.self))) + case 0xCC: + return .bytes(Data(try take(Int(try byte())))) + case 0xCD: + return .bytes(Data(try take(Int(try integer(UInt16.self))))) + case 0xCE: + return .bytes(Data(try take(Int(try integer(UInt32.self))))) + case 0xD0: + return try string(count: Int(try byte())) + case 0xD1: + return try string(count: Int(try integer(UInt16.self))) + case 0xD2: + return try string(count: Int(try integer(UInt32.self))) + case 0xD4: + return try list(count: Int(try byte())) + case 0xD5: + return try list(count: Int(try integer(UInt16.self))) + case 0xD6: + return try list(count: Int(try integer(UInt32.self))) + case 0xD8: + return try map(count: Int(try byte())) + case 0xD9: + return try map(count: Int(try integer(UInt16.self))) + case 0xDA: + return try map(count: Int(try integer(UInt32.self))) + default: + throw PackStreamError.unknownMarker(marker) + } + } + + private mutating func byte() throws -> UInt8 { + guard index < bytes.count else { throw PackStreamError.truncated } + defer { index += 1 } + return bytes[index] + } + + private mutating func take(_ count: Int) throws -> ArraySlice { + guard count >= 0, bytes.count - index >= count else { throw PackStreamError.truncated } + defer { index += count } + return bytes[index ..< index + count] + } + + private mutating func integer(_: T.Type) throws -> T { + try take(MemoryLayout.size).reduce(T.zero) { ($0 << 8) | T($1) } + } + + private mutating func string(count: Int) throws -> PackStreamValue { + guard let text = String(bytes: try take(count), encoding: .utf8) else { + throw PackStreamError.invalidString + } + return .string(text) + } + + private mutating func list(count: Int) throws -> PackStreamValue { + var items: [PackStreamValue] = [] + items.reserveCapacity(min(count, 1_024)) + for _ in 0 ..< count { items.append(try next()) } + return .list(items) + } + + private mutating func map(count: Int) throws -> PackStreamValue { + var entries: [String: PackStreamValue] = [:] + for _ in 0 ..< count { + guard case .string(let key) = try next() else { throw PackStreamError.nonStringKey } + entries[key] = try next() + } + return .map(entries) + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/LibPQStringConformance.swift b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/LibPQStringConformance.swift index f22d445738..8310bc99c2 100644 --- a/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/LibPQStringConformance.swift +++ b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/LibPQStringConformance.swift @@ -22,9 +22,9 @@ public enum LibPQStringConformance { public static func escape(_ value: String, standardConformingStrings: Bool) -> String { let quoted = value - .replacingOccurrences(of: "\0", with: "") - .replacingOccurrences(of: "'", with: "''") + .replacingOccurrences(of: "\0", with: "", options: .literal) + .replacingOccurrences(of: "'", with: "''", options: .literal) guard !standardConformingStrings else { return quoted } - return quoted.replacingOccurrences(of: "\\", with: "\\\\") + return quoted.replacingOccurrences(of: "\\", with: "\\\\", options: .literal) } } diff --git a/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCatalogSearchPath.swift b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCatalogSearchPath.swift new file mode 100644 index 0000000000..cda64d28c8 --- /dev/null +++ b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCatalogSearchPath.swift @@ -0,0 +1,28 @@ +import Foundation + +/// The `search_path` a driver-authored catalog statement runs under, set with `SET LOCAL` in the same query string so it +/// lasts only as long as the read. +/// +/// PostgreSQL resolves a function or operator over every schema on the path, and a candidate that takes the call's +/// argument types exactly beats a `pg_catalog` one reached through a cast, wherever the two sit on the path. A +/// `pg_total_relation_size(oid)` or an `<>(oid, integer)` in a browsed schema would otherwise run with the reader's +/// privileges, the substitution of CVE-2018-1058. `pg_dump` runs with the same narrowed path for the same reason. +/// `pg_temp` is listed last so a temporary relation never shadows a catalog; functions and operators are never looked +/// up there. +/// +/// `SET LOCAL` in the read's own query string makes the pair one implicit transaction, so the path is back as soon as +/// the read returns; inside a transaction the caller holds, the driver scopes it with a savepoint. `pg_catalog` rather +/// than an empty path: PostgreSQL 9.1 rejects `''` ("schema "" does not exist"), and `pg_catalog` deparses every user +/// relation with its schema all the same (measured identical output on 9.2, 9.3, 9.6 and 17). +nonisolated public enum PostgreSQLCatalogSearchPath { + public static let pinned = "SET LOCAL search_path = pg_catalog, pg_temp; " + + /// For a read whose output the server deparses: `format_type`, `pg_get_expr` and the `pg_get_*def` functions leave + /// out the schema of a name the path resolves, so this writes the schema's own names bare. The schema is on the + /// path, so a statement run under it calls only `pg_catalog`-qualified functions and compares only operands a + /// `pg_catalog` operator takes exactly. + public static func relative(to schema: String) -> String { + let quoted = "\"\(schema.replacingOccurrences(of: "\"", with: "\"\"", options: .literal))\"" + return "SET LOCAL search_path = pg_catalog, \(quoted), pg_temp; " + } +} diff --git a/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCatalogTypeNames.swift b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCatalogTypeNames.swift index 218ec3889d..e5bf1f8b15 100644 --- a/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCatalogTypeNames.swift +++ b/apple/Packages/TableProCore/Sources/TableProPostgreSQLCore/PostgreSQLCatalogTypeNames.swift @@ -105,8 +105,11 @@ public enum PostgreSQLCatalogTypeNames { /// because Redshift's catalog predates `typcategory` and the predicate agrees with it on /// every real type: `point` and `name` carry a `typelem` at a fixed length, and the one /// disagreement is the pseudo-type `_record`, which no column has. + /// + /// Sent on the user's own session under its search path, so each oid is typed: a bare integer compares oid with + /// integer, which a `=(oid, integer)` in a schema on that path takes exactly. public static func lookupQuery(oids: [UInt32]) -> String? { - let list = Set(oids).sorted().map(String.init).joined(separator: ", ") + let list = Set(oids).sorted().map { "'\($0)'::pg_catalog.oid" }.joined(separator: ", ") guard !list.isEmpty else { return nil } return """ SELECT t.oid::text, t.typname, t.typtype::text, diff --git a/apple/Packages/TableProCore/Sources/TableProQuery/FilterSQLGenerator.swift b/apple/Packages/TableProCore/Sources/TableProQuery/FilterSQLGenerator.swift index 93db218345..824aaf2522 100644 --- a/apple/Packages/TableProCore/Sources/TableProQuery/FilterSQLGenerator.swift +++ b/apple/Packages/TableProCore/Sources/TableProQuery/FilterSQLGenerator.swift @@ -87,13 +87,13 @@ public struct FilterSQLGenerator: Sendable { let escapedSecond = escapeValue(filter.secondValue, kind: kind) return "\(quotedColumn) BETWEEN \(escapedValue) AND \(escapedSecond)" case .contains: - let pattern = escapeLikePattern(filter.value) + let pattern = likePatternBody(filter.value) return likeCondition(operand, "'%\(pattern)%'", negated: false, folding: folding) case .startsWith: - let pattern = escapeLikePattern(filter.value) + let pattern = likePatternBody(filter.value) return likeCondition(operand, "'\(pattern)%'", negated: false, folding: folding) case .endsWith: - let pattern = escapeLikePattern(filter.value) + let pattern = likePatternBody(filter.value) return likeCondition(operand, "'%\(pattern)'", negated: false, folding: folding) } } @@ -117,7 +117,7 @@ public struct FilterSQLGenerator: Sendable { private func comparisonCondition( _ column: String, _ operand: String, _ operatorText: String, _ literal: String, folding: PluginSQLCaseFolding ) -> String { - guard folding.foldsComparisonOperands, literal.hasPrefix("'") else { + guard folding.foldsComparisonOperands, literal.unicodeScalars.first == "'" else { return "\(column) \(operatorText) \(literal)" } return "\(folding.fold(operand)) \(operatorText) \(folding.fold(literal))" @@ -159,7 +159,7 @@ public struct FilterSQLGenerator: Sendable { _ column: String, _ pattern: String, negated: Bool, folding: PluginSQLCaseFolding ) -> String { let keyword = negated ? folding.notLikeKeyword : folding.likeKeyword - return "\(folding.foldingLikeOperand(column)) \(keyword) \(folding.foldingLikeOperand(pattern))\(likeEscape)" + return "\(folding.foldingLikeOperand(column)) \(keyword) \(folding.foldingLikeOperand(pattern))\(likeEscapeClause)" } private func listCondition( @@ -172,13 +172,13 @@ public struct FilterSQLGenerator: Sendable { ) -> String { let keyword = negated ? "NOT IN" : "IN" let items = listItems(rawValue).map { escapeValue($0, kind: kind) } - let foldable = folding.foldsComparisonOperands && items.allSatisfy { $0.hasPrefix("'") } + let foldable = folding.foldsComparisonOperands && items.allSatisfy { $0.unicodeScalars.first == "'" } let rendered = foldable ? items.map { folding.fold($0) } : items let subject = foldable ? folding.fold(operand) : column return "\(subject) \(keyword) (\(rendered.joined(separator: ", ")))" } - private var likeEscape: String { + public var likeEscapeClause: String { switch dialect.likeEscapeStyle { case .explicit: return " ESCAPE '!'" @@ -187,11 +187,22 @@ public struct FilterSQLGenerator: Sendable { } } + // The LIKE layer first, with the escape `likeEscapeClause` names, then the literal rule over the whole pattern. + public func likePatternBody(_ value: String) -> String { + let escape = dialect.likeEscapeStyle == .implicit ? "\\" : "!" + // `[` opens a character class in a SQL Server LIKE pattern. + let metacharacters = [escape, "%", "_"] + (databaseType == .mssql ? ["["] : []) + let pattern = metacharacters.reduce(value) { pattern, metacharacter in + pattern.replacingOccurrences(of: metacharacter, with: escape + metacharacter, options: .literal) + } + return literalBody(pattern) + } + private func quoteIdentifier(_ name: String) -> String { if let identifierQuoter { return identifierQuoter(name) } let open = dialect.identifierQuote let close = open == "[" ? "]" : open - let escaped = name.replacingOccurrences(of: close, with: "\(close)\(close)") + let escaped = name.replacingOccurrences(of: close, with: "\(close)\(close)", options: .literal) return "\(open)\(escaped)\(close)" } @@ -199,32 +210,15 @@ public struct FilterSQLGenerator: Sendable { if FilterColumnKind.isBareNumber(value, kind: kind) { return value } - var escaped = value.replacingOccurrences(of: "\0", with: "") - if dialect.requiresBackslashEscaping { - escaped = escaped.replacingOccurrences(of: "\\", with: "\\\\") - } - escaped = escaped.replacingOccurrences(of: "'", with: "''") - return "'\(escaped)'" + return "'\(literalBody(value))'" } - private func escapeLikePattern(_ value: String) -> String { - var result = value - .replacingOccurrences(of: "'", with: "''") - .replacingOccurrences(of: "\0", with: "") - switch dialect.likeEscapeStyle { - case .explicit: - result = result - .replacingOccurrences(of: "!", with: "!!") - .replacingOccurrences(of: "%", with: "!%") - .replacingOccurrences(of: "_", with: "!_") - case .implicit: - if dialect.requiresBackslashEscaping { - result = result.replacingOccurrences(of: "\\", with: "\\\\") - } - result = result - .replacingOccurrences(of: "%", with: "\\%") - .replacingOccurrences(of: "_", with: "\\_") + // `.literal` compares code units: the default compares grapheme clusters and skips a quote a combining mark follows. + private func literalBody(_ value: String) -> String { + var body = value.replacingOccurrences(of: "\0", with: "", options: .literal) + if dialect.requiresBackslashEscaping { + body = body.replacingOccurrences(of: "\\", with: "\\\\", options: .literal) } - return result + return body.replacingOccurrences(of: "'", with: "''", options: .literal) } } diff --git a/apple/Packages/TableProCore/Sources/TableProQuery/SQLWriteClassifier.swift b/apple/Packages/TableProCore/Sources/TableProQuery/SQLWriteClassifier.swift index cc2d9b5894..ea62cfb7df 100644 --- a/apple/Packages/TableProCore/Sources/TableProQuery/SQLWriteClassifier.swift +++ b/apple/Packages/TableProCore/Sources/TableProQuery/SQLWriteClassifier.swift @@ -9,7 +9,9 @@ import TableProSQLGrammar /// The rule is fail-closed: a statement counts as a read only when its leading keyword is one of a /// short, closed set of read verbs. Everything else writes, including a keyword this classifier has /// never heard of. A write-keyword allowlist cannot be safe, because anything it has not been -/// taught, or anything hidden behind a leading comment, runs unguarded. +/// taught, or anything hidden behind a leading comment, runs unguarded. A query that passes the verb +/// rule is then read for what it calls, because `SELECT setval('seq', 1)` and +/// `SELECT pg_terminate_backend(pid)` open with a read verb and change state all the same. /// /// The batch is split into statements the way its engine lexes it, by the grammar the Mac reads /// too, and under every reading the engine could be using: the run splits by one reading, and a @@ -18,21 +20,37 @@ import TableProSQLGrammar /// engine that needs no `;`, every statement that begins inside one is read as well, because SQL /// Server runs `SELECT 1` followed by `DROP TABLE t` as two statements. public enum SQLWriteClassifier { - /// `EXPLAIN` and `PRAGMA` are deliberately absent: `EXPLAIN ANALYZE DELETE …` runs the delete on - /// PostgreSQL, and `PRAGMA journal_mode = WAL` writes on SQLite and DuckDB. - private static let readKeywords: Set = ["SHOW", "DESCRIBE", "DESC"] + // EXPLAIN and PRAGMA are absent: `EXPLAIN ANALYZE DELETE …` runs the delete on PostgreSQL, and + // `PRAGMA journal_mode = WAL` writes on SQLite and DuckDB. + private static let readKeywords: Set = ["SHOW"] - /// A `SELECT` reads unless it materialises a table, which `SELECT … INTO` does on SQL Server and - /// `SELECT … INTO OUTFILE` does on MySQL. Mirrors QueryClassifier.swift:288-291. + // MySQL and TiDB read `DESC ANALYZE DELETE …` as EXPLAIN ANALYZE, so only a DESCRIBE of a name reads. + private static let describeKeywords: Set = ["DESCRIBE", "DESC"] + + // `SELECT … INTO` materialises a table on SQL Server and a file on MySQL. private static let readUnlessIntoKeywords: Set = ["SELECT", "TABLE", "VALUES"] - /// Statement verbs only. `REPLACE`, `INTO` and `COPY` are left out because they collide with - /// ordinary function and column names, and a CTE that merely calls `replace()` is a read. - /// Matches QueryClassifier.swift:304-312. + // `REPLACE` and `COPY` are left out because they are also function and column names. private static let writeKeywordsInsideCTE: [String] = [ "INSERT", "UPDATE", "DELETE", "MERGE", "UPSERT", "DROP", "TRUNCATE", "ALTER", "CREATE", "INTO" ] + /// `NEXT VALUE FOR seq` (SQL Server, MariaDB, Db2) and `NEXTVAL FOR seq` (Db2) advance a sequence with + /// no call for the function scanner to see. Matches QueryClassifier.sequenceAdvanceRegex. + private static let sequenceAdvanceRegex = try? NSRegularExpression( + pattern: "\\bNEXT(?:\\s+VALUE|VAL)\\s+FOR\\b", + options: [] + ) + + /// `seq.NEXTVAL` does the same on the engines that have the pseudo-column. Elsewhere `t.nextval` is a column. + private static let sequencePseudoColumnRegex = try? NSRegularExpression( + pattern: "\\.\\s*NEXTVAL\\b", + options: [] + ) + + /// QueryClassifier.enginesWithSequencePseudoColumns without SAP HANA, which only the Mac connects to. + private static let enginesWithSequencePseudoColumns: Set = [.oracle, .snowflake, .dameng, .mariadb] + public static func isWriteQuery(_ sql: String, databaseType: DatabaseType) -> Bool { if databaseType == .redis { return RedisCommandSplitter.commands(in: sql).contains { redisWrites($0.text) } @@ -50,7 +68,7 @@ public enum SQLWriteClassifier { SQLStatementScanner.executableStatements(in: lexing.text, grammar: lexing.grammar).contains { statement in SQLUnterminatedStatements.runnable(in: statement.sql, grammar: lexing.grammar).contains { runnable in seen.insert(SQLLexing(grammar: lexing.grammar, text: runnable)).inserted - && statementWrites(runnable, grammar: lexing.grammar) + && statementWrites(runnable, grammar: lexing.grammar, databaseType: databaseType) } } } @@ -58,17 +76,58 @@ public enum SQLWriteClassifier { /// Reads the statement's code with every literal and comment blanked by its engine's own rules. A MySQL /// `/*! ... */` is kept as the code it is, so a keyword the server runs from one is never mistaken for a read. - private static func statementWrites(_ statement: String, grammar: SQLLexicalGrammar) -> Bool { + private static func statementWrites( + _ statement: String, + grammar: SQLLexicalGrammar, + databaseType: DatabaseType, + depth: Int = 0 + ) -> Bool { + // Each nested CTE copies and reads what is left of the text, so nesting past this counts as a write. + guard depth <= 32 else { return true } let code = SQLCodeProjection.code(of: statement, grammar: grammar, revealingExecutableComments: true) - let body = String(code.drop { $0.isWhitespace }) + let body = String(code.drop { $0.isWhitespace || $0 == "(" }) guard let keyword = leadingKeyword(of: body) else { return true } - if keyword == "WITH" { return commonTableExpressionWrites(code) } + if keyword == "WITH" { + return commonTableExpressionWrites( + statement, code: code, grammar: grammar, databaseType: databaseType, depth: depth + ) + || queryChangesState(statement, code: code, grammar: grammar, databaseType: databaseType) + } + if describeKeywords.contains(keyword) { + return SQLExplainStatement.read(statement, grammar: grammar)?.reading != .describesObject + } if readUnlessIntoKeywords.contains(keyword) { return containsWord("INTO", in: code.uppercased()) + || queryChangesState(statement, code: code, grammar: grammar, databaseType: databaseType) } return !readKeywords.contains(keyword) } + /// A query's verb says nothing about what it calls: a function in the select list can move a sequence, + /// end a session or drop a large object. A callee the scanner cannot name counts as one that changes + /// state. Mirrors QueryClassifier.stateChangingCallTier. + private static func queryChangesState( + _ statement: String, + code: String, + grammar: SQLLexicalGrammar, + databaseType: DatabaseType + ) -> Bool { + if advancesSequence(code.uppercased(), databaseType: databaseType) { return true } + let callsChangeState = SQLFunctionCallScanner.calls(in: statement as NSString, code: code as NSString) + .contains { call in + guard case .named(let name, _) = call.callee else { return true } + return SQLStateChangingFunctions.contains(name) + } + return callsChangeState || SQLStateChangingFunctions.namesStateChangingTable(in: statement, grammar: grammar) + } + + private static func advancesSequence(_ body: String, databaseType: DatabaseType) -> Bool { + let range = NSRange(body.startIndex..., in: body) + if sequenceAdvanceRegex?.firstMatch(in: body, options: [], range: range) != nil { return true } + guard enginesWithSequencePseudoColumns.contains(databaseType) else { return false } + return sequencePseudoColumnRegex?.firstMatch(in: body, options: [], range: range) != nil + } + // Read as the driver reads it, so a quoted verb is the command that runs. Unparseable text counts as a write. private static func redisWrites(_ command: String) -> Bool { guard let arguments = RedisArgumentCodec.split(command) else { return true } @@ -76,12 +135,23 @@ public enum SQLWriteClassifier { return !RedisReadCommands.onlyReads(words[...]) } - /// A CTE's leading keyword says nothing about what the statement finally does, so its code is - /// searched for a write verb. - private static func commonTableExpressionWrites(_ code: String) -> Bool { + // After the CTE list only a query reads, and each body is read as a statement of its own. + private static func commonTableExpressionWrites( + _ statement: String, + code: String, + grammar: SQLLexicalGrammar, + databaseType: DatabaseType, + depth: Int + ) -> Bool { let masked = code.uppercased() - return writeKeywordsInsideCTE.contains { keyword in - containsWord(keyword, in: masked) + if writeKeywordsInsideCTE.contains(where: { containsWord($0, in: masked) }) { return true } + guard let list = SQLCommonTableExpressions.read(statement, grammar: grammar) else { return true } + let text = statement as NSString + let main = text.substring(from: list.mainStatement) + let mainCode = SQLCodeProjection.code(of: main, grammar: grammar) + guard readUnlessIntoKeywords.contains(SQLLeadingComments.leadingCodeKeyword(mainCode)) else { return true } + return ([main] + list.bodies.map(text.substring(with:))).contains { part in + statementWrites(part, grammar: grammar, databaseType: databaseType, depth: depth + 1) } } diff --git a/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLIntrospectionSQL.swift b/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLIntrospectionSQL.swift index b8cb4bb289..cd67a7309f 100644 --- a/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLIntrospectionSQL.swift +++ b/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLIntrospectionSQL.swift @@ -33,7 +33,7 @@ public enum R2SQLIntrospectionSQL { } public static func quoteIdentifier(_ identifier: String) -> String { - "\"" + identifier.replacingOccurrences(of: "\"", with: "\"\"") + "\"" + "\"" + identifier.replacingOccurrences(of: "\"", with: "\"\"", options: .literal) + "\"" } static let namespaceColumns = ["namespace", "namespace_name", "database_name", "schema_name", "databaseName"] diff --git a/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLResponseDecoder.swift b/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLResponseDecoder.swift index a981a58ce2..0edd59042e 100644 --- a/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLResponseDecoder.swift +++ b/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLResponseDecoder.swift @@ -9,7 +9,8 @@ public enum R2SQLResponseDecoder { throw R2SQLError.malformedResponse(status: response.statusCode, detail: snippet(response.body)) } - guard envelope.success else { + // A refused redirect arrives as its 3xx, whatever the body says. + guard envelope.success, (200 ..< 300).contains(response.statusCode) else { switch response.statusCode { case 401, 403: throw R2SQLError.authentication(status: response.statusCode, errors: envelope.errors) diff --git a/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLURLSessionTransport.swift b/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLURLSessionTransport.swift index 8924638df3..2292e43911 100644 --- a/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLURLSessionTransport.swift +++ b/apple/Packages/TableProCore/Sources/TableProR2SQLCore/R2SQLURLSessionTransport.swift @@ -75,6 +75,17 @@ private final class TaskTracker: NSObject, URLSessionTaskDelegate, @unchecked Se transport?.register(task) } + // A redirect would carry the API token to wherever it points. + func urlSession( + _ session: URLSession, + task: URLSessionTask, + willPerformHTTPRedirection response: HTTPURLResponse, + newRequest request: URLRequest, + completionHandler: @escaping (URLRequest?) -> Void + ) { + completionHandler(nil) + } + func finish() { guard let task = lock.withLock({ task }) else { return } transport?.unregister(task) diff --git a/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisQueryBuilder.swift b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisQueryBuilder.swift index b613314ba8..d2eb9468ba 100644 --- a/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisQueryBuilder.swift +++ b/apple/Packages/TableProCore/Sources/TableProRedisCore/RedisQueryBuilder.swift @@ -119,15 +119,15 @@ public struct RedisQueryBuilder: Sendable { } private func quoteForCommand(_ str: String) -> String { - var result = "" - for char in str { - switch char { - case "\\": result += "\\\\" - case "\"": result += "\\\"" - default: result.append(char) + var result = String.UnicodeScalarView() + for scalar in str.unicodeScalars { + switch scalar { + case "\\", "\"": result.append("\\") + default: break } + result.append(scalar) } - return result + return String(result) } /// Try to extract a SCAN-compatible glob pattern from key-column filters @@ -158,18 +158,14 @@ public struct RedisQueryBuilder: Sendable { /// Escape Redis glob special characters in user input. /// Redis SCAN MATCH uses glob-style patterns where *, ?, and [ are special. private func escapeGlobChars(_ str: String) -> String { - var result = "" - for char in str { - switch char { - case "*", "?", "[", "]": - result.append("\\") - result.append(char) - case "\\": - result.append("\\\\") - default: - result.append(char) + var result = String.UnicodeScalarView() + for scalar in str.unicodeScalars { + switch scalar { + case "*", "?", "[", "]", "\\": result.append("\\") + default: break } + result.append(scalar) } - return result + return String(result) } } diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/CQLBatchTracker.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/CQLBatchTracker.swift index 743f8cd2cb..fa0640b9c1 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/CQLBatchTracker.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/CQLBatchTracker.swift @@ -34,6 +34,10 @@ public struct CQLBatchTracker: SQLStatementBoundaryTracking { true } + public var holdsOpenBlock: Bool { + phase == .insideBatch || phase == .afterCloser + } + public mutating func observeWord(_ word: String) { switch phase { case .start: diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/JavaScript/MongoStatementClassifier.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/JavaScript/MongoStatementClassifier.swift index 0272332c25..2abb610a77 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/JavaScript/MongoStatementClassifier.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/JavaScript/MongoStatementClassifier.swift @@ -58,11 +58,15 @@ public enum MongoStatementClassifier { // Server-side JavaScript, plus the two stages that write to a collection the statement names. private static let codeExecutionKeys: Set = ["$where", "$function", "$accumulator", "$out", "$merge"] + // The server accepts its own operators from any client, and rewrites `mapReduce` into two of them that run a + // function: `$_internalJsEmit` and `$_internalJsReduce`. Nothing a person writes names one. + private static let internalOperatorPrefix = "$_internal" + private static let codeExecutionCalls: Set = ["mapreduce", "eval"] // For text the grammar rejects, where a false alarm only stops an external client sending code nobody could read. private static let codeExecutionMarkers: [String] = [ - "$where", "$function", "$accumulator", "mapreduce", ".eval(", "$out", "$merge" + "$where", "$function", "$accumulator", "$_internal", "mapreduce", ".eval(", "$out", "$merge" ] private static func statementVerdict( @@ -78,6 +82,7 @@ public enum MongoStatementClassifier { private static func verdict(of statement: MongoStatement) -> MongoStatementVerdict { let reachesCode = !statement.objectKeys.isDisjoint(with: codeExecutionKeys) + || statement.objectKeys.contains { $0.hasPrefix(internalOperatorPrefix) } || statement.calls.contains { call in codeExecutionCalls.contains(call.name.lowercased()) || call.commandName.map { codeExecutionCalls.contains($0.lowercased()) } == true diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/PLSQLUnitTracker.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/PLSQLUnitTracker.swift index f0c5161f62..74732eb67d 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/PLSQLUnitTracker.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/PLSQLUnitTracker.swift @@ -16,8 +16,11 @@ import Foundation /// Inside a unit the tracker counts the constructs that one bare `END` closes: a block, a subprogram or package body, /// a compound trigger and a `CASE`. A `BEGIN` continues the innermost construct when that construct is still in its /// declaration section, because `DECLARE ... BEGIN ... END` and `PROCEDURE p IS ... BEGIN ... END` each end once. -/// `END IF` and `END LOOP` close constructs that never opened one, and a word after `.` is a member name, never a -/// keyword. A `;` ends the unit only once every construct has closed. +/// `END IF` and `END LOOP` close constructs that never opened one, except between a loop's `FOR` or `WHILE` and its +/// `LOOP`: no statement ends there, so `CASE ... END LOOP` is a `CASE` expression and the `LOOP` that opens the loop. +/// A word after the `.` of `v.END` is a member name, never a keyword. The `..` of a range and the `.` that ends a +/// number name no member, so the word after `1..` or `2.` is a keyword. A `;` ends the unit only once every construct +/// has closed. /// /// The final `;` of a unit belongs to it, except after a trigger whose body is a `CALL`: measured on Oracle 23ai, /// `CREATE TRIGGER ... CALL p(:NEW.a);` is stored INVALID and the same text without the `;` compiles. @@ -50,6 +53,7 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { private static let lessThan = UInt16(UnicodeScalar("<").value) private static let greaterThan = UInt16(UnicodeScalar(">").value) private static let period = UInt16(UnicodeScalar(".").value) + private static let digits = UInt16(UnicodeScalar("0").value)...UInt16(UnicodeScalar("9").value) private static let createModifiers: Set = [ "OR", "REPLACE", "EDITIONABLE", "NONEDITIONABLE", "AND", "RESOLVE", "COMPILE", "NOFORCE", @@ -61,6 +65,10 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { /// One entry per construct a bare `END` will close, true while that construct still waits for its `BEGIN`. private var constructs: [Bool] = [] + + /// The open `CASE`s that have held no `;`, which only a `CASE` statement can. The next `END` closes one of them, + /// whatever word follows it. + private var openCaseExpressions = 0 private var parenDepth = 0 private var pendingEnd = false private var pendingBodyKeyword = false @@ -97,6 +105,18 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { } } + // An `END` still waiting for the word after it closes a construct when the text ends. + public var holdsOpenBlock: Bool { + switch kind { + case .anonymousBlock, .storedUnit: + return constructs.count > (pendingEnd ? 1 : 0) + case .inlineDeclarations: + return !inlineQueryStarted && constructs.count > (pendingEnd ? 1 : 0) + case .sql, .embeddedSourceHeader, .opaque: + return false + } + } + public mutating func observeWord(_ word: String) { previousSymbol = nil guard !inLabel else { return } @@ -153,6 +173,7 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { } public mutating func observeSymbol(_ symbol: UInt16) { + let precedingSymbol = previousSymbol if symbol == Self.lessThan, previousSymbol == Self.lessThan { inLabel = true previousSymbol = nil @@ -174,7 +195,7 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { default: break } - followsPeriod = symbol == Self.period + followsPeriod = symbol == Self.period && !Self.isPeriodOrDigit(precedingSymbol) } public mutating func observeOpaqueToken() { @@ -201,6 +222,7 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { public mutating func observeSemicolon() -> Bool { previousSymbol = nil followsPeriod = false + openCaseExpressions = 0 guard lead == .decided else { return true } switch kind { @@ -229,6 +251,12 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { // MARK: - Private + /// A name that ends in a digit arrives as a word, which leaves no preceding symbol, so `t1.END` is still a member. + private static func isPeriodOrDigit(_ symbol: UInt16?) -> Bool { + guard let symbol else { return false } + return symbol == period || digits.contains(symbol) + } + private var tracksConstructs: Bool { guard lead == .decided else { return false } switch kind { @@ -340,6 +368,7 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { constructs.append(true) case "CASE": constructs.append(false) + openCaseExpressions += 1 case "END": pendingEnd = true case "PROCEDURE", "FUNCTION": @@ -353,7 +382,7 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { /// Returns whether `word` was consumed by the `END` before it. private mutating func closeEnd(followedBy word: String) -> Bool { - switch SqlBlockStructure.endingFollowedBy(word) { + switch SqlBlockStructure.endingFollowedBy(word, closesCaseExpression: openCaseExpressions > 0) { case .closesControlFlow: return true case .closesCaseStatement: @@ -387,5 +416,6 @@ public struct PLSQLUnitTracker: SQLStatementBoundaryTracking { private mutating func closeConstruct() { guard !constructs.isEmpty else { return } constructs.removeLast() + openCaseExpressions = max(0, openCaseExpressions - 1) } } diff --git a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCatalogEffect.swift b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCatalogEffect.swift index a899b93894..2e44afde28 100644 --- a/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCatalogEffect.swift +++ b/apple/Packages/TableProCore/Sources/TableProSQLGrammar/SQLCatalogEffect.swift @@ -65,7 +65,7 @@ public enum SQLCatalogEffectClassifier { } private static func textEffect(of sql: String, grammar: SQLLexicalGrammar) -> SQLCatalogEffect { - let trimmed = StatementBlank.trimming(SQLLeadingComments.stripping(sql)) + let trimmed = StatementBlank.trimming(SQLLeadingComments.stripping(sql, grammar: grammar)) guard !trimmed.isEmpty else { return .none } if SQLProceduralBlock.runs(trimmed, grammar: grammar) { return .opaque @@ -215,20 +215,24 @@ public enum SQLLeadingComments { public static let lineCommentTerminators: Set = ["\n", "\r", "\r\n"] - public static func stripping(_ sql: String) -> String { - var remaining = sql[...] - while true { - let trimmed = StatementBlank.trimmingLeading(remaining) - if trimmed.hasPrefix("--") { - guard let lineBreak = trimmed.firstIndex(where: endsLineComment) else { return "" } - remaining = trimmed[trimmed.index(after: lineBreak)...] - } else if trimmed.hasPrefix("/*"), !startsConditionalComment(trimmed) { - guard let close = trimmed.range(of: "*/") else { return "" } - remaining = trimmed[close.upperBound...] - } else { - return String(trimmed) + /// `sql` from its first token, past the blanks and the comments `grammar` reads, so a comment ends exactly where + /// the engine ends it. An executable comment is code and stays. + public static func stripping(_ sql: String, grammar: SQLLexicalGrammar) -> String { + let text = sql as NSString + let length = text.length + var index = 0 + while index < length { + let blank = StatementBlank.blankLength(in: text, at: index) + if blank > 0 { + index += blank + continue + } + guard let span = SQLNonCodeSpan.span(at: index, in: text, grammar: grammar), span.kind.isComment else { + break } + index = max(span.end, index + 1) } + return text.substring(from: min(index, length)) } public static func startsConditionalComment(_ text: Substring) -> Bool { @@ -261,20 +265,21 @@ public enum SQLProceduralBlock { } // A block may open after <