From 4c8e365dbdfcf498fc8c80fab7e10ce9d81cae3e Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Fri, 13 Nov 2026 04:02:08 +0700 Subject: [PATCH 1/3] fix(macos): compress the remote SQLite agent so the exec request fits libssh2's packet limit --- .../RemoteSQLiteAgentSource.swift | 49 ++++++++--- .../RemoteSQLiteAdmissionTests.swift | 11 +-- .../RemoteSQLiteAgentPipeTests.swift | 7 +- .../scripts/probes/remote-sqlite-launcher.py | 87 +++++++++++++++++++ 4 files changed, 134 insertions(+), 20 deletions(-) create mode 100755 apple/scripts/probes/remote-sqlite-launcher.py diff --git a/apple/TablePro/Core/RemoteSQLite/RemoteSQLiteAgentSource.swift b/apple/TablePro/Core/RemoteSQLite/RemoteSQLiteAgentSource.swift index 7c4ca881b7..ce2bb6cc0f 100644 --- a/apple/TablePro/Core/RemoteSQLite/RemoteSQLiteAgentSource.swift +++ b/apple/TablePro/Core/RemoteSQLite/RemoteSQLiteAgentSource.swift @@ -3,6 +3,7 @@ // TablePro // +import Compression import Foundation /// The Python program TablePro runs on the SSH server to open a SQLite database with the server's @@ -19,27 +20,49 @@ enum RemoteSQLiteAgentSource { /// `platform-python` under `/usr/libexec` with no `python3` on `PATH`. private static let interpreters = ["python3", "/usr/bin/python3", "/usr/libexec/platform-python"] + /// The bootstrap each interpreter runs: decode and inflate the program from its first argument, + /// then run it. `COMPRESSION_ZLIB` emits raw DEFLATE, so `zlib.decompress` takes wbits `-15`. + static let interpreterBootstrap = "import base64,zlib,sys;exec(zlib.decompress(base64.b64decode(sys.argv[1]),-15))" + + /// The program, raw-DEFLATE compressed then base64, as the launcher ships it. Compressed because + /// the exec request is one SSH packet, and libssh2 rejects an uncompressed payload at or above + /// `MAX_SSH_PACKET_LEN - 0x100` (34744) with `LIBSSH2_ERROR_INVAL`; the raw 24 KB base64, carried + /// once per interpreter, ran past that. 4.8 KB compressed leaves room as the program grows. + static let encodedProgram: String = { + let source = Data(python.utf8) + let capacity = source.count + 1_024 + var deflated = Data(count: capacity) + let written = deflated.withUnsafeMutableBytes { destination -> Int in + source.withUnsafeBytes { input -> Int in + guard let destinationBase = destination.bindMemory(to: UInt8.self).baseAddress, + let inputBase = input.bindMemory(to: UInt8.self).baseAddress else { return 0 } + return compression_encode_buffer(destinationBase, capacity, inputBase, source.count, nil, COMPRESSION_ZLIB) + } + } + deflated.removeSubrange(written.. String { - let encoded = Data(python.utf8).base64EncodedString() - let loop = interpreters.map { interpreter -> String in + let branches = interpreters.map { interpreter -> String in let present = #"command -v "\#(interpreter)" >/dev/null 2>&1"# - let run = #"exec "\#(interpreter)" -I -S -c "import base64,sys;exec(base64.b64decode(sys.argv[1]))" \#(encoded)"# + let run = #"exec "\#(interpreter)" -I -S -c "\#(interpreterBootstrap)" "$b""# return "if \(present); then \(run); fi" } .joined(separator: "; ") - return "sh -c '" + loop + "; echo " + RemoteSQLiteWire.noPythonNotice + "; exit 0'" + return "sh -c 'b=" + encodedProgram + "; " + branches + "; echo " + RemoteSQLiteWire.noPythonNotice + "; exit 0'" } static let python = #""" diff --git a/apple/TableProTests/RemoteSQLite/RemoteSQLiteAdmissionTests.swift b/apple/TableProTests/RemoteSQLite/RemoteSQLiteAdmissionTests.swift index d33f1b313b..2f0c2f666f 100644 --- a/apple/TableProTests/RemoteSQLite/RemoteSQLiteAdmissionTests.swift +++ b/apple/TableProTests/RemoteSQLite/RemoteSQLiteAdmissionTests.swift @@ -42,12 +42,13 @@ struct RemoteSQLiteAdmissionTests { #expect(token.allSatisfy { $0.isHexDigit }) } - @Test func launcherCommandCarriesNoUserDataAndFitsTheArgumentLimit() { + @Test func launcherCommandCarriesNoUserDataAndFitsThePacketLimit() { let command = RemoteSQLiteAgentSource.launcherCommand() - #expect(command.hasPrefix("sh -c '")) + #expect(command.hasPrefix("sh -c 'b=")) #expect(command.contains(RemoteSQLiteWire.noPythonNotice)) - #expect(command.contains("base64.b64decode")) - // Well under the Linux MAX_ARG_STRLEN of 128 KiB. - #expect(command.utf8.count < 100_000) + #expect(command.contains("zlib.decompress")) + // The exec request is one SSH packet; libssh2 rejects an uncompressed payload at or above + // MAX_SSH_PACKET_LEN - 0x100 with -34. Stay under it with room to spare. + #expect(command.utf8.count < 34_744) } } diff --git a/apple/TableProTests/RemoteSQLite/RemoteSQLiteAgentPipeTests.swift b/apple/TableProTests/RemoteSQLite/RemoteSQLiteAgentPipeTests.swift index 2a1b34205a..ad448c1466 100644 --- a/apple/TableProTests/RemoteSQLite/RemoteSQLiteAgentPipeTests.swift +++ b/apple/TableProTests/RemoteSQLite/RemoteSQLiteAgentPipeTests.swift @@ -174,9 +174,12 @@ struct RemoteSQLiteAgentPipeTests { private var buffer: [UInt8] = [] init(pythonPath: String) throws { - let encoded = Data(RemoteSQLiteAgentSource.python.utf8).base64EncodedString() + // Launch the program the way the launcher ships it: the compressed, base64 blob through + // the same bootstrap, so this suite also covers the decompression round-trip. process.executableURL = URL(fileURLWithPath: pythonPath) - process.arguments = ["-I", "-S", "-c", "import base64,sys;exec(base64.b64decode(sys.argv[1]))", encoded] + process.arguments = [ + "-I", "-S", "-c", RemoteSQLiteAgentSource.interpreterBootstrap, RemoteSQLiteAgentSource.encodedProgram, + ] process.standardInput = toAgent process.standardOutput = fromAgent process.standardError = FileHandle.nullDevice diff --git a/apple/scripts/probes/remote-sqlite-launcher.py b/apple/scripts/probes/remote-sqlite-launcher.py new file mode 100755 index 0000000000..308e2a4831 --- /dev/null +++ b/apple/scripts/probes/remote-sqlite-launcher.py @@ -0,0 +1,87 @@ +#!/usr/bin/env python3 +"""Probe the remote SQLite launcher: exec-request size and a HELLO -> READY round-trip. + +Why this exists: the agent program is delivered inline on the SSH exec request, and libssh2 +rejects an uncompressed packet payload at or above MAX_SSH_PACKET_LEN - 0x100 = 34744 with +LIBSSH2_ERROR_INVAL (-34). RemoteSQLiteAgentSource compresses the program (Apple +COMPRESSION_ZLIB, raw DEFLATE) and carries it once. This mirrors that on the CLI: raw DEFLATE +built with wbits -15, decompressed with the production bootstrap, run against the local python3. + +Run from the repo root: python3 apple/scripts/probes/remote-sqlite-launcher.py +""" +import base64 +import re +import struct +import subprocess +import sys +import zlib +from pathlib import Path + +LIMIT = 35000 - 0x100 # MAX_SSH_PACKET_LEN - 0x100, libssh2 src/transport.c +INTERPRETERS = ["python3", "/usr/bin/python3", "/usr/libexec/platform-python"] +BOOTSTRAP = "import base64,zlib,sys;exec(zlib.decompress(base64.b64decode(sys.argv[1]),-15))" +NOTICE = "TPRSQL:NO_PYTHON" +SOURCE = Path(__file__).resolve().parents[2] / "TablePro/Core/RemoteSQLite/RemoteSQLiteAgentSource.swift" + + +def agent_program() -> bytes: + text = SOURCE.read_text(encoding="utf-8") + match = re.search(r'static let python = #"""\n(.*)\n"""#', text, re.S) + if not match: + sys.exit("could not extract the agent program from " + str(SOURCE)) + return (match.group(1) + "\n").encode("utf-8") + + +def raw_deflate(data: bytes) -> bytes: + comp = zlib.compressobj(9, zlib.DEFLATED, -zlib.MAX_WBITS) + return comp.compress(data) + comp.flush() + + +def launcher_command(program: bytes) -> str: + blob = base64.b64encode(raw_deflate(program)).decode() + branches = "; ".join( + f'if command -v "{i}" >/dev/null 2>&1; then exec "{i}" -I -S -c "{BOOTSTRAP}" "$b"; fi' + for i in INTERPRETERS + ) + return "sh -c 'b=" + blob + "; " + branches + "; echo " + NOTICE + "; exit 0'" + + +def frame(op: int, payload: bytes) -> bytes: + return struct.pack(">I", len(payload) + 1) + bytes([op]) + payload + + +def read_str(body: bytes, off: int): + n = struct.unpack(">I", body[off:off + 4])[0] + return body[off + 4:off + 4 + n].decode(), off + 4 + n + + +def main() -> int: + program = agent_program() + command = launcher_command(program) + size = len(command.encode("utf-8")) + print(f"program {len(program)} B -> exec request {size} B (limit {LIMIT}, headroom {LIMIT - size})") + if size >= LIMIT: + print("FAIL: exec request would be rejected by libssh2") + return 1 + + path = b":memory:" + hello = frame(0x01, struct.pack(">I", 1) + struct.pack(">I", len(path)) + path + struct.pack(">I", 0)) + result = subprocess.run(command, shell=True, input=hello, capture_output=True, timeout=30) + out = result.stdout + if len(out) < 5: + print("FAIL: no reply. stderr:", result.stderr.decode(errors="replace")[:400]) + return 1 + op = out[4] + body = out[5:4 + struct.unpack(">I", out[:4])[0]] + if op != 0x81: # OP_READY + print(f"FAIL: expected OP_READY, got 0x{op:02x} {body[:80]!r}") + return 1 + version = struct.unpack(">I", body[:4])[0] + sqlite_version, off = read_str(body, 4) + python_version, _ = read_str(body, off) + print(f"PASS: OP_READY proto={version} sqlite={sqlite_version} python={python_version}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 4c0b0a7551db058a6b6cadc951087bfb7a5906ac Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Fri, 13 Nov 2026 04:03:31 +0700 Subject: [PATCH 2/3] docs(changelog): remote SQLite on-server connection fix --- apple/changelog.d/4249.fixed.md | 1 + 1 file changed, 1 insertion(+) create mode 100644 apple/changelog.d/4249.fixed.md diff --git a/apple/changelog.d/4249.fixed.md b/apple/changelog.d/4249.fixed.md new file mode 100644 index 0000000000..d5c62eb041 --- /dev/null +++ b/apple/changelog.d/4249.fixed.md @@ -0,0 +1 @@ +A SQLite over SSH connection set to open the database on the server never connected. From 36e10262652bdb8e600b99a7ee0f47877cabfab6 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Fri, 13 Nov 2026 04:11:02 +0700 Subject: [PATCH 3/3] fix(macos): guard agent compression and tighten the packet-limit test and probe --- .../Core/RemoteSQLite/RemoteSQLiteAgentSource.swift | 2 ++ .../RemoteSQLite/RemoteSQLiteAdmissionTests.swift | 7 ++++--- apple/scripts/probes/remote-sqlite-launcher.py | 8 +++++--- 3 files changed, 11 insertions(+), 6 deletions(-) diff --git a/apple/TablePro/Core/RemoteSQLite/RemoteSQLiteAgentSource.swift b/apple/TablePro/Core/RemoteSQLite/RemoteSQLiteAgentSource.swift index ce2bb6cc0f..bb4a4b47ff 100644 --- a/apple/TablePro/Core/RemoteSQLite/RemoteSQLiteAgentSource.swift +++ b/apple/TablePro/Core/RemoteSQLite/RemoteSQLiteAgentSource.swift @@ -39,6 +39,8 @@ enum RemoteSQLiteAgentSource { return compression_encode_buffer(destinationBase, capacity, inputBase, source.count, nil, COMPRESSION_ZLIB) } } + // 0 means the output did not fit `capacity`; raw DEFLATE of this text never grows past it. + precondition(written > 0, "RemoteSQLiteAgentSource could not compress the agent program") deflated.removeSubrange(written.. bytes: match = re.search(r'static let python = #"""\n(.*)\n"""#', text, re.S) if not match: sys.exit("could not extract the agent program from " + str(SOURCE)) - return (match.group(1) + "\n").encode("utf-8") + # Matches Data(python.utf8): the multiline literal has no trailing newline. + return match.group(1).encode("utf-8") def raw_deflate(data: bytes) -> bytes: