diff --git a/.github/workflows/relational-package.yml b/.github/workflows/relational-package.yml index d770940..6970246 100644 --- a/.github/workflows/relational-package.yml +++ b/.github/workflows/relational-package.yml @@ -63,21 +63,95 @@ jobs: PYTHONPATH=examples/verl python -m pytest examples/verl/test_e2b_episode.py -q wheel: runs-on: ubuntu-22.04 + services: + postgres: + image: postgres:16-alpine + env: + POSTGRES_PASSWORD: postgres + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 5s + --health-timeout 5s + --health-retries 10 strategy: matrix: - python: ['3.10', '3.12'] + include: + - python: '3.10' + duckdb: 'OFF' + filesystem: 'OFF' + - python: '3.12' + duckdb: 'OFF' + filesystem: 'ON' + - python: '3.12' + duckdb: 'ON' + filesystem: 'OFF' steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - - run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev libpq-dev libboost-dev + - run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev libpq-dev libboost-dev libfuse3-dev - run: python -m pip install build pytest pytest-asyncio pytest-timeout - name: Build from source distribution and install the wheel run: | - python -m build packages/chronos-core --outdir /tmp/chronos-wheel -Cbuild-dir=/tmp/chronos-build -Cbuild.tool-args=-j2 + python -m build packages/chronos-core --outdir /tmp/chronos-wheel -Cbuild-dir=/tmp/chronos-build -Cbuild.tool-args=-j2 -Ccmake.define.CHRONOS_WITH_DUCKDB=${{ matrix.duckdb }} -Ccmake.define.CHRONOS_WITH_FILESYSTEM=${{ matrix.filesystem }} python -m pip install /tmp/chronos-wheel/*.whl - name: Test the installed relational artifact run: | python -I -m pytest tests/test_build_features.py tests/test_native_interval.py tests/test_interval_allocator.py tests/test_checkpoint_forks.py tests/test_reused_session_transactions.py -k 'not postgres' -q --timeout=60 python -I examples/single_store_branching.py + - name: Test the installed CLI against the enabled relational stores + env: + CHRONOS_CLI_TEST_POSTGRES_ADMIN_URL: postgresql://postgres:postgres@127.0.0.1:5432/postgres + CHRONOS_CLI_REQUIRE_DUCKDB: ${{ matrix.duckdb }} + run: | + python -I -m pytest tests/test_cli.py::TestTerminalBehavior \ + tests/test_cli.py::TestWorkspaceConfiguration \ + tests/test_cli.py::TestWorkspaceCommands \ + tests/test_cli.py::TestWorkspaceFilesystemCommands \ + tests/test_cli_integration.py::TestWorkspaceSQLWorkflows \ + tests/test_cli_integration.py::TestWorkspaceResources::test_database_paths_preserve_reserved_characters \ + tests/test_cli_integration.py::TestWorkspaceResources::test_hash_directories_keep_workspace_state_separate \ + tests/test_cli_integration.py::TestWorkspaceResources::test_sqlite_workspace_persists_after_close \ + tests/test_cli_integration.py::TestWorkspaceResources::test_sqlite_workspace_without_qdrant_dependency \ + -q --timeout=60 + - name: Test workspace filesystem on Linux + if: matrix.python == '3.12' && matrix.duckdb == 'OFF' + env: + CHRONOS_CLI_REQUIRE_FILESYSTEM: 'ON' + run: | + sudo apt-get install -y fuse-overlayfs fuse3 + sudo modprobe fuse + python -I -m pytest tests/test_cli_integration.py::TestWorkspaceFilesystemWorkflows -q --timeout=120 + - name: Test native ChronosFS CLI + if: matrix.filesystem == 'ON' + env: + CHRONOS_CLI_REQUIRE_CHRONOSFS: 'ON' + run: | + python -I -m pytest tests/test_cli.py -k native -q --timeout=60 + python -I -m pytest tests/test_cli_integration.py::TestNativeChronosFSWorkflows -q --timeout=120 + python -I -m pytest tests/test_chronosfs.py::test_shared_daemon_checkpoint_preserves_database_path \ + tests/test_chronosfs.py::test_existing_sqlite_adapter_preserves_database_path -q --timeout=60 + - name: Test workspace DuckDB and Qdrant together + if: matrix.duckdb == 'ON' + env: + CHRONOS_CLI_REQUIRE_DUCKDB: 'ON' + run: | + python -m pip install 'qdrant-client==1.20.0' + python -I -m pytest tests/test_cli_integration.py::TestWorkspaceQdrantWorkflows::test_duckdb_and_qdrant -q --timeout=60 + - name: Test Qdrant CLI in local and server modes + if: matrix.python == '3.12' && matrix.duckdb == 'OFF' + env: + CHRONOS_CLI_TEST_POSTGRES_ADMIN_URL: postgresql://postgres:postgres@127.0.0.1:5432/postgres + CHRONOS_CLI_TEST_QDRANT_URL: http://127.0.0.1:6333 + run: | + python -m pip install 'qdrant-client==1.20.0' + docker run --detach --rm --name chronos-cli-qdrant-test \ + -p 127.0.0.1:6333:6333 qdrant/qdrant:v1.20.0 + trap 'docker stop chronos-cli-qdrant-test > /dev/null' EXIT + python -I -m pytest tests/test_cli.py::TestWorkspaceQdrantCommands \ + tests/test_cli_integration.py::TestWorkspaceResources \ + tests/test_cli_integration.py::TestWorkspaceQdrantWorkflows \ + tests/test_cli_integration.py::TestWorkspaceFailureWorkflows -q --timeout=60 diff --git a/.gitignore b/.gitignore index 3019220..64d532b 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ dist/ .eggs/ # Local runtime state +/.chronos/ .chronos-code/ .chronos-runtime/ .chronos-overlay-test/ @@ -28,9 +29,11 @@ dist/ # Generated benchmark output .benchmarks/ -# Generated SQLite databases and journal files +# Generated local databases and journal files *.sqlite *.sqlite-* +*.duckdb +*.duckdb.wal # Literal local workspace state directory /$state/ diff --git a/docs/README.md b/docs/README.md index 389e273..6a10f79 100644 --- a/docs/README.md +++ b/docs/README.md @@ -13,6 +13,8 @@ page says otherwise, commands and APIs apply to Chronos 0.2.0a1. optional DuckDB, filesystem, and S3 features. - [Integration](integration.md) explains how to adopt existing tables and route application access through Chronos. +- [CLI](cli.md) wraps one or several workspace stores with JSON output for + pipes and files. - [Compatibility and limits](compatibility.md) states what is supported and where direct database access can bypass branch isolation. - [Branching introduction](branching-introduction.md) explains the data model diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 0000000..06cbe92 --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,335 @@ +# Chronos CLI + +The `chronos` command is a one-off wrapper around `ChronosWorkspaceContext` and +its named store APIs. A workspace can contain one store or several. Each command +opens its configured stores, calls the API, prints the returned data, and closes +connections. Stored data persists; the CLI keeps no background connection or +current-branch state. + +## Install + +The CLI ships with `chronos-core`. Follow [installation](installation.md) for +native build dependencies. From the repository root, install the workspace package: + +```sh +uv sync --package chronos-core +uv run --package chronos-core chronos --help +``` + +You do not need to activate the virtual environment when using `uv run`. +An installed wheel also provides `chronos`; `python -m chronos_core.cli` invokes +the same application. + +On macOS with Homebrew Python, build against the same SQLite library Python uses. +If Python uses Homebrew SQLite, explicitly select it when rebuilding: + +```sh +brew install sqlite libpq boost +CMAKE_PREFIX_PATH="$(brew --prefix libpq)" \ +uv sync --package chronos-core --reinstall-package chronos-core \ + -C "cmake.define.SQLite3_INCLUDE_DIR=$(brew --prefix sqlite)/include" \ + -C "cmake.define.SQLite3_LIBRARY=$(brew --prefix sqlite)/lib/libsqlite3.dylib" +``` + +Mixing Python's Homebrew SQLite connection with a native extension linked to +macOS's SQLite can crash the process. After a custom build, use +`uv run --no-sync --package chronos-core chronos ...` to run the installed build +without syncing it again. + +## Workspaces + +Configure named stores in `chronos.toml`. For example, one SQLite store: + +```toml +[stores.app] +kind = "sqlite" +database_url = "sqlite:///.chronos/app.sqlite" +``` + +Save the example as `chronos.toml` and create `.chronos/` first. Local database URLs and Qdrant paths resolve relative +to the TOML file. Configuration precedence is `--config PATH`, then +`CHRONOS_WORKSPACE_CONFIG`, then `chronos.toml` in the current directory. A +workspace name is not required. Connection strings may reference `${ENV_VAR}`; +unset references are errors. `.env` files are not loaded automatically. + +From the repository root, run the commands below. If `chronos` is already on +your PATH, you can use it directly in place of the `uv run` invocation. + +```sh +mkdir -p .chronos +uv run --package chronos-core chronos table create --store app \ + 'CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)' +uv run --package chronos-core chronos table register --store app items --primary-key id +uv run --package chronos-core chronos branch create experiment +uv run --package chronos-core chronos query --store app --branch experiment 'SELECT * FROM items' +``` + +SQL participants support `sqlite`, `postgresql`, and `duckdb` kinds, with +`database_url` and optional `metadata_url` (required for DuckDB). Qdrant uses +`kind = "qdrant"`, `metadata_url`, and exactly one of `path` or `url`; remote +connections also accept `api_key` and `timeout`. Select vector operations with +`chronos qdrant --store NAME ...`. + +Local database URLs use percent-encoded filenames, resolved relative to the +configuration file. For example, `sqlite:///data%23trial.sqlite` names +`data#trial.sqlite`; `%2520` names a literal `%20`. Characters in the config +directory are encoded automatically. DuckDB paths containing `?` are rejected +because the bundled DuckDB version cannot construct their WAL paths correctly. + +Branches, checkpoints, diffs, and merges operate across the workspace. With +independent metadata, `merge preview/apply` calls the coordinated API, which +can retain partial changes if a later participant fails. Atomic mode requires +`[workspace].shared_metadata_url` and matching metadata URLs for every store. +Multiple SQL participants sharing metadata require a nonempty `merge_tables` +scope for each store. See the atomic SQL example below. Use +`merge atomic-preview` to inspect the candidate changes and +`merge atomic-apply --operation-id ID` to publish them. Pass the returned +`--preview-token` to reject a changed preview. Ordinary `diff` can fail in +split-data workspaces because the Python API's diff does not use merge scopes. +See the filesystem sections below for OverlayFS and native ChronosFS setup. + +## Checked terminal demo + +After installation, run `bash examples/cli_workspace_demo.sh --help` from the +repository. The default demo uses PostgreSQL plus embedded Qdrant; set +`CHRONOS_DEMO_POSTGRES_URL` to a fresh demo database, then run: + +```sh +bash examples/cli_workspace_demo.sh +``` + +For SQLite plus embedded Qdrant without servers: + +```sh +bash examples/cli_workspace_demo.sh --sqlite +``` + +The script performs every Chronos operation through the CLI and uses Python +only to check its JSON output. It prints each check, verifies single-store +startup, adding Qdrant, branch isolation, checkpoint restoration, search, and +atomic merging, then removes its branches. It leaves the checkpoint and data +in place and prints the config path for further exploration. Local files go +into a fresh `.chronos/workspace-demo.*` directory; PostgreSQL data remains in +the supplied database. No servers are started or stopped by the script. + +## Store operations and output + +Prepare existing SQL tables with `table register --store NAME TABLE --primary-key KEY`; +repeat `--primary-key` for composite keys. `table create` runs one ordinary CREATE +TABLE statement before registration; it is not a branch-local schema change. +After registration, use the branch-session `query` and `execute` commands for +managed reads and writes. See [integration](integration.md) before adopting +application data; direct access to the original table bypasses Chronos. + +SQL accepts text, `--file PATH`, or `--stdin`. Piped input is read automatically +when no source is supplied. Sources are mutually exclusive; SQL is passed to the +API without splitting scripts. `--params` accepts a JSON object of named values. +Branch defaults are `main`. + +```sh +chronos execute --store app --branch experiment \ + 'INSERT INTO items (id, value) VALUES (:id, :value)' \ + --params '{"id": 1, "value": "candidate"}' +chronos query --store app --file query.sql +printf 'SELECT * FROM items' | chronos query --store app +``` + +Qdrant requires `chronos-core[qdrant]`. Register collections, then use point and +search commands on a selected store. Dense vectors and payloads are JSON; point +upserts and searches also accept `--file` and `--stdin`. + +```sh +chronos qdrant --store vectors collection register documents --dimensions 3 +chronos qdrant --store vectors point upsert documents one \ + '{"vector": [1, 0, 0], "payload": {"title": "Example"}}' --branch experiment +chronos qdrant --store vectors search documents '[1, 0, 0]' --branch experiment +``` + +Except for `fs run`'s default command-output mode described below, results are +the API's JSON values, including workspace store names and result +field names. `None` prints nothing. Dataclasses become objects, dates use ISO +strings, decimals become strings, and bytes become tagged base64 objects. +Expected operational errors go to stderr with exit code 1; invalid CLI arguments +use exit code 2. Help opens no connections. JSON output can be piped to `jq` or +redirected to a file. +Non-finite floats returned by the API fail clearly because JSON cannot represent +them; query execution and the API's returned values are not modified. + +## OverlayFS workspace + +The OverlayFS participant wraps `ChronosFilesystemStore` and requires Linux, +`fuse-overlayfs`, `fusermount3` (or `fusermount`), and permission to use `/dev/fuse`. +It can be the only participant or accompany named SQL/Qdrant stores. Configure +an existing base directory and an explicit persistent state directory, preferably +outside the base directory. Relative paths resolve against the TOML file: + +```toml +[filesystem] +root = "base" +state_dir = ".chronos/filesystem" +``` + +Branch, checkpoint, and diff commands include this participant automatically. +Run ordinary programs against a branch using the existing session `run()` API: + +```sh +chronos branch create experiment +chronos fs run --branch experiment -- sh -c 'printf "hello\n" > report.txt' +chronos fs run --branch experiment -- cat report.txt +chronos checkpoint create saved --branch experiment +chronos diff main experiment +chronos merge preview experiment main --policy api-default +chronos merge apply experiment main --policy api-default +``` + +`fs run` defaults to branch `main` and working directory `.`. Options must precede +`-- COMMAND...`. `--cwd` selects a directory within the branch; `--timeout` passes +a timeout in seconds; `--env '{"NAME":"value"}'` replaces the child environment +(the API inherits the current environment when this option is omitted). Commands +are passed as argument lists; shell syntax requires explicitly running a shell. +The mounted working directory is not a security sandbox for the child process. + +Output is captured by the API and printed to stdout/stderr after completion. +The CLI exits with the child's exit code (signals use the shell convention +`128 + signal`). `--json` instead prints the API result fields `args`, `returncode`, +`stdout`, and `stderr`, while retaining that exit status. These child-generated +outputs are passed through as-is. Startup failures and timeouts exit with code 1. + +`--policy api-default` passes `None` to the workspace API. This is required for +the current filesystem merge implementation, which applies file-level changes +without three-way conflict detection. The existing `abort_on_conflict` default +and explicit policies remain unchanged; the API rejects them for this filesystem. +The CLI's OverlayFS backend does not support shared-metadata atomic workspaces. +These configurations are rejected before opening stores. Coordinated merges do not promise all-store rollback. + +Each invocation releases its mounts while preserving state and versioned files. +Use short-lived commands; this interface does not keep a mount alive for later +external use or background processes. For a disposable demo, keep the base, +configuration, and state inside one dedicated directory, and remove that directory +only after commands finish and its mounts are released. + +Filesystem integration tests live in `TestWorkspaceFilesystemWorkflows` in +`tests/test_cli_integration.py`. Run them in Linux with the dependencies above: + +```sh +CHRONOS_CLI_REQUIRE_FILESYSTEM=ON python -m pytest \ + tests/test_cli_integration.py::TestWorkspaceFilesystemWorkflows -q +``` + +For Docker, the test container needs `/dev/fuse`, `SYS_ADMIN`, and a mount-permitting +security profile (for example `--device /dev/fuse --cap-add SYS_ADMIN +--security-opt apparmor=unconfined`). Keep test data on the container's Linux +filesystem. The required-test flag prevents missing dependencies from counting +as a successful skipped run. + +## Native ChronosFS workspace + +Native ChronosFS stores versioned files in Chronos. Use a Linux package built +with `CHRONOS_WITH_FILESYSTEM=ON`. File commands use the Python API directly +and do not need a FUSE mount. Select one filesystem backend per workspace. +Omitting `kind` keeps the existing OverlayFS configuration behavior. + +```toml +[filesystem] +kind = "chronosfs" +database_url = "sqlite:///files.sqlite" +``` + +```sh +chronos branch create experiment +printf 'hello\n' | chronos fs write /notes/report.txt --branch experiment --parents --stdin +chronos fs ls /notes --branch experiment +chronos fs read /notes/report.txt --branch experiment +``` + +`fs read` writes raw bytes to stdout. `fs write` accepts either `--file PATH` +or `--stdin` and prints the API's result as JSON. `fs run` is available with +the OverlayFS backend. + +For an atomic workspace with SQLite, configure the SQL store and native +ChronosFS to use the same database: + +```toml +[workspace] +shared_metadata_url = "sqlite:///app.sqlite" + +[stores.app] +kind = "sqlite" +database_url = "sqlite:///app.sqlite" + +[filesystem] +kind = "chronosfs" +database_url = "sqlite:///app.sqlite" +``` + +Use the existing `merge atomic-preview` and `merge atomic-apply` commands to +publish SQL and filesystem changes together. The API preview can show the +filesystem changes both as interval rows under the SQL participant and as +file changes under the filesystem participant. + +To keep SQL rows and native file data in separate databases, put the SQL +participant's table scope in the workspace TOML: + +```toml +[workspace] +shared_metadata_url = "sqlite:///app.sqlite" + +[stores.app] +kind = "sqlite" +database_url = "sqlite:///app.sqlite" +merge_tables = ["items"] + +[filesystem] +kind = "chronosfs" +database_url = "sqlite:///files.sqlite" +metadata_url = "sqlite:///app.sqlite" +``` + +Omitting `merge_tables` or setting it to `[]` keeps the API's all-tables +default. Split atomic configurations that require explicit table ownership +need a nonempty list. The CLI passes that list to `set_merge_table_scope()` +when it opens the SQL store. Register tables separately with `chronos table register`. +Update the scope when you want another registered table included in merges. +This setting controls merge scope and does not change workspace `diff`. + +On Linux with FUSE access, you can also mount a native branch into an existing +directory: + +```sh +mkdir -p ./chronos-mount +chronos fs mount ./chronos-mount --branch experiment +``` + +The command stays in the foreground. Use the directory from another terminal +and unmount it there with `fusermount3 -u ./chronos-mount` when finished. +Repeat `--option` (or `-o`) to pass individual FUSE options to the Python API. + +## Atomic merge across SQL stores + +Declare each SQL store in the workspace TOML and point each one at the shared +metadata database. Give each store an explicit `merge_tables` list containing +the tables it should handle during an atomic merge: + +```toml +[workspace] +shared_metadata_url = "sqlite:///metadata.sqlite" + +[stores.app] +kind = "sqlite" +database_url = "sqlite:///app.sqlite" +metadata_url = "sqlite:///metadata.sqlite" +merge_tables = ["items"] + +[stores.audit] +kind = "sqlite" +database_url = "sqlite:///audit.sqlite" +metadata_url = "sqlite:///metadata.sqlite" +merge_tables = ["events"] +``` + +Register each table in its owning store. Use `merge atomic-preview` to inspect +the changes per store and `merge atomic-apply` to publish them together. +`merge_tables` is forwarded to the Python API's merge scope. To compare a +candidate for publication in this split setup, use atomic preview. The +ordinary workspace `diff` API does not use merge scopes and can fail when it +looks for a table owned by a different data database. diff --git a/examples/cli_workspace_demo.sh b/examples/cli_workspace_demo.sh new file mode 100755 index 0000000..33906ae --- /dev/null +++ b/examples/cli_workspace_demo.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# Run from any directory after installing chronos-core with Qdrant support. +set -euo pipefail + +case "${1:---postgres}" in + --postgres) + : "${CHRONOS_DEMO_POSTGRES_URL:?Set CHRONOS_DEMO_POSTGRES_URL to a fresh PostgreSQL demo database}" + sql_kind="postgresql" + sql_url='${CHRONOS_DEMO_POSTGRES_URL}' + ;; + --sqlite) + sql_kind="sqlite" + sql_url="sqlite:///app.sqlite" + ;; + --help) + printf 'Usage: bash examples/cli_workspace_demo.sh [--postgres|--sqlite]\n' + printf 'Default: PostgreSQL (CHRONOS_DEMO_POSTGRES_URL) + embedded Qdrant.\n' + printf 'Use --sqlite for SQLite + embedded Qdrant with no servers.\n' + exit 0 + ;; + *) printf 'Unknown option: %s\n' "$1" >&2; exit 2 ;; +esac + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" +mkdir -p .chronos +demo_dir="$(mktemp -d "$repo_root/.chronos/workspace-demo.XXXXXX")" +export CHRONOS_WORKSPACE_CONFIG="$demo_dir/chronos.toml" +cli=(uv run --no-sync --package chronos-core chronos) +python=(uv run --no-sync --package chronos-core python) + +# Python only compares JSON output; every Chronos operation uses the CLI. +check() { + local label="$1" field="$2" expected="$3" actual + shift 3 + printf '\n%s\n' "$label" + actual="$("${cli[@]}" "$@")" + printf '%s\n' "$actual" + "${python[@]}" -c ' +import json, sys +value = json.loads(sys.argv[1] or "null") +for field in filter(None, sys.argv[2].split(".")): + value = value[int(field)] if isinstance(value, list) else value[field] +expected = json.loads(sys.argv[3]) +if value != expected: + raise SystemExit(f"FAIL: expected {expected!r}, got {value!r}") +print("PASS") +' "$actual" "$field" "$expected" +} + +printf 'Persistent demo directory: %s\n' "$demo_dir" +cat > "$CHRONOS_WORKSPACE_CONFIG" <> "$CHRONOS_WORKSPACE_CONFIG" <=3.1", "sqlglot>=25", + "typer>=0.25.1", + "tomli>=2.0; python_version < '3.11'", ] [project.optional-dependencies] @@ -44,6 +46,7 @@ Documentation = "https://github.com/mitdbg/chronos/blob/main/docs/README.md" chronos-mcp = "chronos_core.mcp.cli:main" chronos-mcp-deploy-codex = "chronos_core.mcp.deploy_codex:main" chronos-lake = "chronos_core.lake.cli:main" +chronos = "chronos_core.cli.main:app" [tool.scikit-build] wheel.packages = ["src/chronos_core"] diff --git a/packages/chronos-core/src/chronos_core/cli/__init__.py b/packages/chronos-core/src/chronos_core/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packages/chronos-core/src/chronos_core/cli/__main__.py b/packages/chronos-core/src/chronos_core/cli/__main__.py new file mode 100644 index 0000000..53aa0c1 --- /dev/null +++ b/packages/chronos-core/src/chronos_core/cli/__main__.py @@ -0,0 +1,4 @@ +"""Run the packaged CLI with python -m chronos_core.cli.""" +from chronos_core.cli.main import app + +app() diff --git a/packages/chronos-core/src/chronos_core/cli/_config.py b/packages/chronos-core/src/chronos_core/cli/_config.py new file mode 100644 index 0000000..cf38b3b --- /dev/null +++ b/packages/chronos-core/src/chronos_core/cli/_config.py @@ -0,0 +1,372 @@ +"""Workspace configuration and ownership of public store resources.""" +from __future__ import annotations + +import math +import os +import re +import sys +from collections.abc import Generator +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any, Literal +from urllib.parse import quote, unquote, urlsplit + +try: + import tomllib +except ModuleNotFoundError: # Python 3.10 + import tomli as tomllib # type: ignore[no-redef] + +if TYPE_CHECKING: + from chronos_core.workspace import ChronosWorkspaceContext + + +@dataclass(frozen=True) +class DatabaseConfig: + database_url: str + metadata_url: str | None = None + + +@dataclass(frozen=True) +class QdrantConfig: + metadata_url: str + path: Path | None = None + url: str | None = None + api_key: str | None = None + collection_prefix: str = "chronos_" + timeout: float | None = None + + +class WorkspaceConfigError(ValueError): + """Invalid workspace configuration, before connecting to any store.""" + + +@dataclass(frozen=True) +class StoreConfig: + kind: Literal["sqlite", "postgresql", "duckdb", "qdrant"] + connection: DatabaseConfig | QdrantConfig + merge_tables: tuple[str, ...] | None = None + + @property + def metadata_url(self) -> str: + connection = self.connection + if isinstance(connection, QdrantConfig): + return connection.metadata_url + return connection.metadata_url or connection.database_url + + +@dataclass(frozen=True) +class FilesystemConfig: + root: Path + state_dir: Path + + +@dataclass(frozen=True) +class WorkspaceConfig: + config_path: Path + stores: dict[str, StoreConfig] + shared_metadata_url: str | None = None + filesystem: FilesystemConfig | DatabaseConfig | None = None + + def store(self, name: str, *, qdrant: bool = False) -> StoreConfig: + store = self.stores.get(name) + if store is None: + raise WorkspaceConfigError(f"Unknown store {name!r}.") + if (store.kind == "qdrant") != qdrant: + expected = "Qdrant" if qdrant else "relational" + raise WorkspaceConfigError(f"Store {name!r} is not a {expected} store.") + return store + + +def _table(value: Any, field: str, allowed: set[str]) -> dict[str, Any]: + if not isinstance(value, dict): + raise WorkspaceConfigError(f"{field} must be a TOML table.") + if set(value) - allowed: + raise WorkspaceConfigError(f"{field} contains unsupported fields.") + return value + + +def _text(value: Any, field: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise WorkspaceConfigError(f"{field} must be a non-empty string.") + + def substitute(match: re.Match[str]) -> str: + name = match.group(1) + if name not in os.environ: + raise WorkspaceConfigError(f"{field} references unset environment variable {name!r}.") + return os.environ[name] + + result = re.sub(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}", substitute, value) + if not result.strip(): + raise WorkspaceConfigError(f"{field} must not resolve to an empty value.") + return result + + +def _local_path(value: str | Path, root: Path, field: str) -> Path: + try: + location = Path(value).expanduser() + return (location if location.is_absolute() else root / location).resolve() + except (OSError, RuntimeError, ValueError) as error: + raise WorkspaceConfigError(f"{field} must be a valid local path.") from error + + +def _metadata_identity(url: str, root: Path) -> str: + if url.startswith("sqlite:///"): + # Match the adapter's decoding for comparisons without rewriting the + # URL passed to the API (literal percent signs may themselves be encoded). + return f"sqlite:///{_local_path(unquote(url[len('sqlite:///'):]), root, 'metadata_url')}" + return url + + +def _database_url(value: Any, root: Path, field: str, *, kind: str | None = None) -> str: + url = _text(value, field) + try: + parsed = urlsplit(url) + except ValueError as error: + raise WorkspaceConfigError(f"{field} must be a valid database URL.") from error + scheme = "postgresql" if parsed.scheme == "postgres" else parsed.scheme + expected = {kind} if kind is not None else {"sqlite", "postgresql"} + if scheme not in expected: + raise WorkspaceConfigError(f"{field} has an unsupported database URL scheme.") + if scheme == "postgresql": + return "postgresql:" + url.split(":", 1)[1] + if parsed.netloc or parsed.query or parsed.fragment or not url.startswith(f"{scheme}:///"): + raise WorkspaceConfigError(f"{field} must be a local database file URL.") + local = url[len(f"{scheme}:///"):] + if not local or local == ":memory:": + raise WorkspaceConfigError(f"{field} requires persistent storage, not an in-memory database.") + # Decode the URL's filename before resolving it, then encode the complete + # path so characters in the config directory cannot become URL syntax. + decoded = unquote(local) + # An encoded leading tilde is a literal filename, not a home-directory + # request. Join it to the root before _local_path expands unescaped '~'. + location = _local_path(decoded if local.startswith("~") else root / decoded, root, field) + if scheme == "duckdb" and "?" in str(location): + raise WorkspaceConfigError(f"{field}: DuckDB database paths cannot contain '?'.") + return f"{scheme}:///{quote(str(location), safe='/')}" + + +def _store_config(value: Any, root: Path, field: str) -> StoreConfig: + if not isinstance(value, dict): + raise WorkspaceConfigError(f"{field} must be a TOML table.") + kind = value.get("kind") + if kind in ("sqlite", "postgresql", "duckdb"): + data = _table(value, field, {"kind", "database_url", "metadata_url", "merge_tables"}) + url = _database_url(data.get("database_url"), root, f"{field}.database_url", kind=kind) + metadata = ( + _database_url(data["metadata_url"], root, f"{field}.metadata_url") + if "metadata_url" in data else None + ) + if kind == "duckdb" and metadata is None: + raise WorkspaceConfigError(f"{field}.metadata_url is required for DuckDB.") + merge_tables = None + if "merge_tables" in data: + tables = data["merge_tables"] + if not isinstance(tables, list) or any( + not isinstance(table, str) or not table.strip() for table in tables + ): + raise WorkspaceConfigError(f"{field}.merge_tables must be an array of non-empty table names.") + merge_tables = tuple(tables) or None + return StoreConfig(kind, DatabaseConfig(url, metadata), merge_tables) + if kind != "qdrant": + raise WorkspaceConfigError(f"{field}.kind must be sqlite, postgresql, duckdb, or qdrant.") + data = _table(value, field, {"kind", "metadata_url", "path", "url", "api_key", "collection_prefix", "timeout"}) + metadata = _database_url(data.get("metadata_url"), root, f"{field}.metadata_url") + if ("path" in data) == ("url" in data): + raise WorkspaceConfigError(f"{field} requires exactly one of path or url.") + local_path = None + url = None + if "path" in data: + local = _text(data["path"], f"{field}.path") + if local == ":memory:": + raise WorkspaceConfigError(f"{field}.path requires persistent storage.") + local_path = _local_path(local, root, f"{field}.path") + if "api_key" in data or "timeout" in data: + raise WorkspaceConfigError(f"{field}: api_key and timeout require url.") + else: + url = _text(data["url"], f"{field}.url") + try: + remote = urlsplit(url) + except ValueError as error: + raise WorkspaceConfigError(f"{field}.url must be a valid HTTP or HTTPS URL.") from error + if remote.scheme not in {"http", "https"} or not remote.hostname: + raise WorkspaceConfigError(f"{field}.url must use HTTP or HTTPS.") + api_key = _text(data["api_key"], f"{field}.api_key") if "api_key" in data else None + prefix = _text(data.get("collection_prefix", "chronos_"), f"{field}.collection_prefix") + timeout = data.get("timeout") + try: + valid_timeout = timeout is None or ( + not isinstance(timeout, bool) and isinstance(timeout, (int, float)) + and math.isfinite(timeout) and timeout >= 0.001 + ) + except OverflowError: + valid_timeout = False + if not valid_timeout: + raise WorkspaceConfigError(f"{field}.timeout must be a finite number of at least 0.001 seconds.") + return StoreConfig("qdrant", QdrantConfig(metadata, local_path, url, api_key, prefix, timeout)) + + +def load_workspace_config(path: Path | None = None) -> WorkspaceConfig: + """Read explicit path, environment path, or ./chronos.toml without opening stores.""" + selected = path if path is not None else Path(os.environ.get("CHRONOS_WORKSPACE_CONFIG", "chronos.toml")) + config_path = _local_path(selected, Path.cwd(), "--config") + try: + raw = tomllib.loads(config_path.read_text(encoding="utf-8")) + except (OSError, UnicodeError) as error: + raise WorkspaceConfigError(f"Cannot read workspace configuration: {config_path}") from error + except tomllib.TOMLDecodeError as error: + # Decoder messages can contain configuration values, including secrets. + raise WorkspaceConfigError("Invalid TOML in workspace configuration.") from error + data = _table(raw, "configuration", {"workspace", "stores", "filesystem"}) + workspace = _table(data.get("workspace", {}), "workspace", {"shared_metadata_url"}) + shared = ( + _database_url(workspace["shared_metadata_url"], config_path.parent, "workspace.shared_metadata_url") + if "shared_metadata_url" in workspace else None + ) + filesystem = None + if "filesystem" in data: + fields = _table(data["filesystem"], "filesystem", {"kind", "root", "state_dir", "database_url", "metadata_url"}) + kind = fields.get("kind", "overlayfs") + if kind == "overlayfs": + _table(fields, "filesystem", {"kind", "root", "state_dir"}) + filesystem = FilesystemConfig(*( + _local_path(_text(fields.get(key), f"filesystem.{key}"), config_path.parent, f"filesystem.{key}") + for key in ("root", "state_dir") + )) + if shared is not None: + raise WorkspaceConfigError("The OverlayFS backend does not support shared-metadata atomic workspaces.") + elif kind == "chronosfs": + _table(fields, "filesystem", {"kind", "database_url", "metadata_url"}) + url = _database_url(fields.get("database_url"), config_path.parent, "filesystem.database_url") + metadata = ( + _database_url(fields["metadata_url"], config_path.parent, "filesystem.metadata_url") + if "metadata_url" in fields else None + ) + filesystem = DatabaseConfig(url, metadata) + else: + raise WorkspaceConfigError("filesystem.kind must be overlayfs or chronosfs.") + entries = data.get("stores", {}) + if not isinstance(entries, dict) or (not entries and filesystem is None): + raise WorkspaceConfigError("Configure at least one named store in [stores.NAME] or [filesystem].") + # These names cannot be used by WorkspaceBranchSession, even via its stores mapping. + from chronos_core.workspace.runtime import WorkspaceBranchSession + + reserved = {"branch_id", "fs", "stores", "transaction", *dir(WorkspaceBranchSession)} + stores = {} + for name, value in entries.items(): + if not name.strip() or name in reserved or (filesystem is not None and name == "filesystem"): + raise WorkspaceConfigError(f"Invalid or reserved workspace store name {name!r}.") + stores[name] = _store_config(value, config_path.parent, f"stores.{name}") + metadata_urls = [store.metadata_url for store in stores.values()] + if isinstance(filesystem, DatabaseConfig): + metadata_urls.append(filesystem.metadata_url or filesystem.database_url) + if shared is not None and any( + isinstance(store.connection, DatabaseConfig) + and store.connection.database_url != filesystem.database_url + and store.merge_tables is None + for store in stores.values() + ): + raise WorkspaceConfigError( + "SQL stores with separate data from native ChronosFS require nonempty merge_tables for atomic workspaces." + ) + if shared is not None and any(url != shared for url in metadata_urls): + raise WorkspaceConfigError("Every store must use workspace.shared_metadata_url for atomic operations.") + sql_stores = {name: store for name, store in stores.items() if store.kind != "qdrant"} + if shared is not None and len(sql_stores) > 1: + unscoped = [name for name, store in sql_stores.items() if store.merge_tables is None] + if unscoped: + raise WorkspaceConfigError( + "Multiple SQL stores sharing metadata require nonempty merge_tables for each store. " + f"Missing for: {', '.join(unscoped)}." + ) + identities = [_metadata_identity(url, config_path.parent) for url in metadata_urls] + if shared is None and len(set(identities)) != len(identities): + raise WorkspaceConfigError("Stores sharing metadata require workspace.shared_metadata_url.") + return WorkspaceConfig(config_path, stores, shared, filesystem) + + +@contextmanager +def open_workspace(config: WorkspaceConfig) -> Generator[ChronosWorkspaceContext, None, None]: + """Close every owned resource; never remove persistent data or replace an API error.""" + from chronos_core.branching import ChronosBranchContext + from chronos_core.workspace import ChronosWorkspaceContext + + errors: list[Exception] = [] + + def close(resource: Any) -> None: + try: + resource.close() + except Exception as error: + errors.append(error) + + try: + with ExitStack() as resources: + stores: dict[str, Any] = {} + contexts: dict[str, Any] = {} + # Open SQL first so Qdrant may borrow an existing row-store context, + # regardless of the order in which users declare their stores. + for name, entry in config.stores.items(): + database = entry.connection + if not isinstance(database, DatabaseConfig): + continue + if database.metadata_url is None or database.metadata_url == database.database_url: + context = ChronosBranchContext.connect(database.database_url) + contexts[database.database_url] = context + else: + context = ChronosBranchContext.connect_split(database.database_url, database.metadata_url) + resources.callback(close, context) + if entry.merge_tables is not None: + context.set_merge_table_scope(list(entry.merge_tables)) + stores[name] = context + for name, entry in config.stores.items(): + target = entry.connection + if not isinstance(target, QdrantConfig): + continue + from chronos_core.workspace.qdrant import ChronosQdrantStore + + factory = ChronosQdrantStore + options = {"collection_prefix": target.collection_prefix, "context": contexts.get(target.metadata_url)} + if target.path is not None: + store = factory.local(target.metadata_url, path=target.path, **options) + else: + store = factory.remote( + target.metadata_url, url=target.url, api_key=target.api_key, + timeout=target.timeout, **options, + ) + resources.callback(close, store) + stores[name] = store + filesystem_options: dict[str, Any] = {} + if isinstance(config.filesystem, FilesystemConfig): + from chronos_core.workspace import ChronosFilesystemStore + + filesystem = ChronosFilesystemStore(config.filesystem.root, state_dir=config.filesystem.state_dir) + resources.callback(close, filesystem) + filesystem_options["filesystem"] = filesystem + elif isinstance(config.filesystem, DatabaseConfig): + from chronos_core.workspace import ChronosFSStore + + target = config.filesystem + context = ( + ChronosBranchContext.connect_split(target.database_url, target.metadata_url) + if target.metadata_url and target.metadata_url != target.database_url + else ChronosBranchContext.connect(target.database_url) + ) + try: + filesystem = ChronosFSStore(context) + except BaseException: + close(context) + raise + resources.callback(close, filesystem) + filesystem.ensure() + filesystem_options["filesystem"] = filesystem + workspace = ChronosWorkspaceContext( + stores={name: stores[name] for name in config.stores}, + shared_metadata_url=config.shared_metadata_url, + **filesystem_options, + ) + # ExitStack owns the participants directly because workspace.close() + # stops at the first close failure instead of attempting all stores. + yield workspace + finally: + if errors and sys.exc_info()[0] is None: + raise errors[0] diff --git a/packages/chronos-core/src/chronos_core/cli/_filesystem.py b/packages/chronos-core/src/chronos_core/cli/_filesystem.py new file mode 100644 index 0000000..a0b9524 --- /dev/null +++ b/packages/chronos-core/src/chronos_core/cli/_filesystem.py @@ -0,0 +1,140 @@ +"""Terminal access to the workspace's public filesystem session API.""" +from __future__ import annotations + +import json +import math +import subprocess +from collections.abc import Generator +from contextlib import contextmanager +from pathlib import Path +from typing import TYPE_CHECKING + +import typer + +from chronos_core.cli._terminal import print_result +from chronos_core.cli._workspace import connect_workspace, workspace_session + +if TYPE_CHECKING: + from chronos_core.workspace import ChronosFSBranchSession, ChronosFSStore, ChronosWorkspaceContext + +app = typer.Typer(no_args_is_help=True) + + +@app.command("run", context_settings={"allow_interspersed_args": False}) +def run( + ctx: typer.Context, + argv: list[str] = typer.Argument(..., help="Command and arguments; use -- before the command."), + branch: str = typer.Option("main", "--branch"), + cwd: str = typer.Option(".", "--cwd", help="Working directory relative to the branch mount."), + env: str | None = typer.Option(None, "--env", help="JSON object replacing the child environment; omitted inherits it."), + timeout: float | None = typer.Option(None, "--timeout", min=0.001), + as_json: bool = typer.Option(False, "--json", help="Print args, returncode, stdout, and stderr as JSON."), +) -> None: + """Run a command, print captured output after completion, then release mounts.""" + if timeout is not None and not math.isfinite(timeout): + raise typer.BadParameter("Expected a finite timeout.", param_hint="--timeout") + environment = None + if env is not None: + try: + environment = json.loads(env) + except ValueError as error: + raise typer.BadParameter("Expected a JSON object of string values.", param_hint="--env") from error + if not isinstance(environment, dict) or not all(isinstance(value, str) for value in environment.values()): + raise typer.BadParameter("Expected a JSON object of string values.", param_hint="--env") + from chronos_core.workspace import ChronosFSStore + + with connect_workspace(ctx) as workspace: + if isinstance(workspace.filesystem, ChronosFSStore): + raise typer.BadParameter("fs run requires the OverlayFS backend.") + if workspace.filesystem is None: + raise typer.BadParameter("Configure [filesystem] to run filesystem commands.") + try: + with workspace_session(workspace, branch) as checked_out: + result = checked_out.fs.run(argv, cwd=cwd, env=environment, timeout=timeout) + except subprocess.TimeoutExpired: + # Avoid echoing argv: it can contain credentials supplied to the child. + typer.echo("Error: filesystem command timed out.", err=True) + raise typer.Exit(1) from None + if as_json: + print_result({"args": result.args, "returncode": result.returncode, + "stdout": result.stdout, "stderr": result.stderr}) + else: + typer.echo(result.stdout, nl=False, color=True) + typer.echo(result.stderr, err=True, nl=False, color=True) + # subprocess uses negative return codes for signals; shells use 128+signal. + status = result.returncode if result.returncode >= 0 else 128 - result.returncode + raise typer.Exit(status) + + +@contextmanager +def _native_store(ctx: typer.Context) -> Generator[tuple[ChronosWorkspaceContext, ChronosFSStore], None, None]: + from chronos_core.workspace import ChronosFSStore + + with connect_workspace(ctx) as workspace: + if not isinstance(workspace.filesystem, ChronosFSStore): + raise typer.BadParameter("This command requires [filesystem] with kind = 'chronosfs'.") + yield workspace, workspace.filesystem + + +@contextmanager +def _native_session(ctx: typer.Context, branch: str) -> Generator[ChronosFSBranchSession, None, None]: + with _native_store(ctx) as (workspace, _): + with workspace_session(workspace, branch) as session: + yield session.fs + + +@app.command("mount") +def mount( + ctx: typer.Context, + mountpoint: Path = typer.Argument(..., exists=True, file_okay=False, resolve_path=True), + branch: str = typer.Option("main", "--branch"), + options: list[str] | None = typer.Option(None, "--option", "-o", help="FUSE option. May be repeated."), +) -> None: + """Mount a native ChronosFS branch in the foreground until unmounted.""" + from chronos_core.workspace import mount_chronosfs + + with _native_store(ctx) as (_, filesystem): + mount_chronosfs(filesystem, mountpoint, branch_id=branch, options=set(options or ())) + + +@app.command("ls") +def list_directory( + ctx: typer.Context, + path: str = typer.Argument("/"), + branch: str = typer.Option("main", "--branch"), +) -> None: + """List entries in a native ChronosFS directory.""" + with _native_session(ctx, branch) as filesystem: + print_result(filesystem.listdir(path)) + + +@app.command("read") +def read_file( + ctx: typer.Context, + path: str, + branch: str = typer.Option("main", "--branch"), +) -> None: + """Write native ChronosFS file bytes to stdout.""" + with _native_session(ctx, branch) as filesystem: + data = filesystem.read_file(path) + typer.get_binary_stream("stdout").write(data) + + +@app.command("write") +def write_file( + ctx: typer.Context, + path: str, + branch: str = typer.Option("main", "--branch"), + file: Path | None = typer.Option(None, "--file", exists=True, dir_okay=False, readable=True), + stdin: bool = typer.Option(False, "--stdin"), + parents: bool = typer.Option(False, "--parents", help="Create missing parent directories."), +) -> None: + """Write bytes to a native ChronosFS file.""" + if (file is not None) == stdin: + raise typer.BadParameter("Choose exactly one of --file or --stdin.") + try: + data = file.read_bytes() if file is not None else typer.get_binary_stream("stdin").read() + except OSError as error: + raise typer.BadParameter("Could not read the input bytes.") from error + with _native_session(ctx, branch) as filesystem: + print_result(filesystem.write_file(path, data, parents=parents)) diff --git a/packages/chronos-core/src/chronos_core/cli/_qdrant.py b/packages/chronos-core/src/chronos_core/cli/_qdrant.py new file mode 100644 index 0000000..0705e66 --- /dev/null +++ b/packages/chronos-core/src/chronos_core/cli/_qdrant.py @@ -0,0 +1,155 @@ +"""JSON input and commands for named Qdrant workspace participants.""" +from __future__ import annotations + +import json +import math +import sys +from dataclasses import dataclass +from pathlib import Path +from typing import Any + +import typer + +from chronos_core.cli._terminal import print_result +from chronos_core.cli._workspace import connect_workspace, store_session + +@dataclass(frozen=True) +class QdrantSelection: + store: str | None + + +app = typer.Typer(no_args_is_help=True) +collection_app = typer.Typer(no_args_is_help=True) +point_app = typer.Typer(no_args_is_help=True) +app.add_typer(collection_app, name="collection", help="Register and inspect collections.") +app.add_typer(point_app, name="point", help="Write, retrieve, and delete branch-visible points.") + + +def read_json_input(text: str | None, file: Path | None, stdin: bool) -> Any: + if sum((text is not None, file is not None, stdin)) > 1: + raise typer.BadParameter("Choose JSON text, --file, or --stdin, not multiple sources.") + try: + if file is not None: + text = file.read_text(encoding="utf-8") + elif stdin or (text is None and not sys.stdin.isatty()): + text = sys.stdin.read() + except (OSError, UnicodeError) as error: + raise typer.BadParameter(f"Cannot read JSON input: {error}") from error + if text is None or not text.strip(): + raise typer.BadParameter("Provide JSON text, --file, or JSON on stdin.") + try: + return json.loads(text, parse_constant=_reject_constant) + except ValueError as error: + raise typer.BadParameter(f"Invalid JSON: {error}") from error + + +def _reject_constant(value: str) -> None: + raise ValueError(f"{value} is not a finite JSON number") + + +def dense_vector(value: Any) -> list[float]: + try: + valid = isinstance(value, list) and all( + isinstance(item, (int, float)) and not isinstance(item, bool) and math.isfinite(item) + for item in value + ) + except OverflowError: + valid = False + if not valid: + raise typer.BadParameter("Expected a JSON array of finite numbers for the vector.") + return value + + +def point_input(value: Any) -> tuple[list[float], dict[str, Any] | None]: + if not isinstance(value, dict) or "vector" not in value or set(value) - {"vector", "payload"}: + raise typer.BadParameter("Expected an object with vector and optional payload fields.") + payload = value.get("payload") + if payload is not None and not isinstance(payload, dict): + raise typer.BadParameter("The payload must be a JSON object or null.") + return dense_vector(value["vector"]), payload + + +@app.callback() +def select_qdrant(ctx: typer.Context, store: str | None = typer.Option(None, "--store", help="Named Qdrant store; required for operations.")) -> None: + ctx.obj = QdrantSelection(store) + + +def _qdrant_name(ctx: typer.Context) -> str: + selection = ctx.find_object(QdrantSelection) + if selection is None or not selection.store: + raise typer.BadParameter("Select a Qdrant store with --store.", param_hint="--store") + return selection.store + + +@collection_app.command("register") +def register_collection(ctx: typer.Context, name: str, + dimensions: int = typer.Option(..., "--dimensions", min=1), + distance: str = typer.Option("cosine", "--distance")) -> None: + name_in_config = _qdrant_name(ctx) + with connect_workspace(ctx, name_in_config, qdrant=True) as workspace: + print_result(workspace.stores[name_in_config].register_collection(name, dimensions, distance=distance)) + + +@collection_app.command("list") +def list_collections(ctx: typer.Context) -> None: + name = _qdrant_name(ctx) + with connect_workspace(ctx, name, qdrant=True) as workspace: + print_result(workspace.stores[name].list_collections()) + + +@collection_app.command("info") +def collection_info(ctx: typer.Context, name: str) -> None: + target = _qdrant_name(ctx) + with connect_workspace(ctx, target, qdrant=True) as workspace: + print_result(workspace.stores[target].collection_info(name)) + + +@point_app.command("upsert") +def upsert( + ctx: typer.Context, collection: str, point_id: str, data: str | None = typer.Argument(None), + file: Path | None = typer.Option(None, "--file"), stdin: bool = typer.Option(False, "--stdin"), + branch: str = typer.Option("main", "--branch"), revision: str | None = typer.Option(None, "--revision"), +) -> None: + vector, payload = point_input(read_json_input(data, file, stdin)) + name = _qdrant_name(ctx) + with connect_workspace(ctx, name, qdrant=True) as workspace: + with store_session(workspace, branch, name) as session: + print_result(session.upsert(collection, point_id, vector, payload, revision=revision)) + + +@point_app.command("get") +def get(ctx: typer.Context, collection: str, point_id: str, branch: str = typer.Option("main", "--branch")) -> None: + name = _qdrant_name(ctx) + with connect_workspace(ctx, name, qdrant=True) as workspace: + with store_session(workspace, branch, name) as session: + print_result(session.get(collection, point_id)) + + +@point_app.command("list") +def list_points(ctx: typer.Context, collection: str, branch: str = typer.Option("main", "--branch")) -> None: + name = _qdrant_name(ctx) + with connect_workspace(ctx, name, qdrant=True) as workspace: + with store_session(workspace, branch, name) as session: + print_result(session.list_points(collection)) + + +@point_app.command("delete") +def delete_point(ctx: typer.Context, collection: str, point_id: str, branch: str = typer.Option("main", "--branch")) -> None: + name = _qdrant_name(ctx) + with connect_workspace(ctx, name, qdrant=True) as workspace: + with store_session(workspace, branch, name) as session: + print_result(session.delete(collection, point_id)) + + +@app.command() +def search( + ctx: typer.Context, collection: str, vector: str | None = typer.Argument(None), + file: Path | None = typer.Option(None, "--file"), stdin: bool = typer.Option(False, "--stdin"), + branch: str = typer.Option("main", "--branch"), limit: int = typer.Option(10, "--limit"), + score_threshold: float | None = typer.Option(None, "--score-threshold"), +) -> None: + query_vector = dense_vector(read_json_input(vector, file, stdin)) + name = _qdrant_name(ctx) + with connect_workspace(ctx, name, qdrant=True) as workspace: + with store_session(workspace, branch, name) as session: + print_result(session.search(collection, query_vector, limit=limit, score_threshold=score_threshold)) diff --git a/packages/chronos-core/src/chronos_core/cli/_relational.py b/packages/chronos-core/src/chronos_core/cli/_relational.py new file mode 100644 index 0000000..1baf6df --- /dev/null +++ b/packages/chronos-core/src/chronos_core/cli/_relational.py @@ -0,0 +1,113 @@ +"""SQL input and commands for named relational workspace participants.""" +from __future__ import annotations + +import json +import sys +from pathlib import Path +from typing import Any + +import sqlglot +import typer +from sqlglot import exp + +from chronos_core.cli._terminal import print_result +from chronos_core.cli._workspace import connect_workspace, store_session + + +table_app = typer.Typer(no_args_is_help=True) + + +def _sql_input(sql: str | None, file: Path | None, stdin: bool) -> str: + if sum((sql is not None, file is not None, stdin)) > 1: + raise typer.BadParameter("Choose SQL text, --file, or --stdin, not multiple sources.") + if file is not None: + try: + sql = file.read_text(encoding="utf-8") + except (OSError, UnicodeError) as error: + raise typer.BadParameter(f"Cannot read SQL file: {error}", param_hint="--file") from error + elif stdin or (sql is None and not sys.stdin.isatty()): + try: + sql = sys.stdin.read() + except (OSError, UnicodeError) as error: + typer.echo(f"Error reading stdin: {error}", err=True) + raise typer.Exit(1) from error + if sql is None or not sql.strip(): + raise typer.BadParameter("Provide SQL text, --file, or SQL on stdin.") + return sql + + +def _parameters(params: str | None) -> dict[str, Any] | None: + if params is None: + return None + try: + value = json.loads(params) + except ValueError as error: + raise typer.BadParameter("Expected a JSON object.", param_hint="--params") from error + if not isinstance(value, dict): + raise typer.BadParameter("Expected a JSON object.", param_hint="--params") + return value + + +def _create_table_input(sql: str | None, file: Path | None, stdin: bool) -> str: + statement = _sql_input(sql, file, stdin) + try: + statements = sqlglot.parse(statement) + except sqlglot.errors.ParseError as error: + raise typer.BadParameter("Expected one CREATE TABLE statement.") from error + if ( + len(statements) != 1 + or not isinstance(statements[0], exp.Create) + or statements[0].kind != "TABLE" + ): + raise typer.BadParameter("Expected one CREATE TABLE statement.") + return statement + + +@table_app.command("create") +def create_table( + ctx: typer.Context, sql: str | None = typer.Argument(None), + store: str = typer.Option(..., "--store"), + file: Path | None = typer.Option(None, "--file"), stdin: bool = typer.Option(False, "--stdin"), +) -> None: + statement = _create_table_input(sql, file, stdin) + with connect_workspace(ctx, store) as workspace: + database = workspace.stores[store] + database.db.execute(statement) + print_result(database.db.commit()) + + +@table_app.command("register") +def register_table(ctx: typer.Context, table: str, store: str = typer.Option(..., "--store"), + primary_key: list[str] = typer.Option(..., "--primary-key")) -> None: + with connect_workspace(ctx, store) as workspace: + print_result(workspace.stores[store].register_table(table, primary_key)) + + +def query( + ctx: typer.Context, sql: str | None = typer.Argument(None), + store: str = typer.Option(..., "--store"), branch: str = typer.Option("main", "--branch"), + params: str | None = typer.Option(None, "--params"), + file: Path | None = typer.Option(None, "--file"), stdin: bool = typer.Option(False, "--stdin"), +) -> None: + statement, parameters = _sql_input(sql, file, stdin), _parameters(params) + with connect_workspace(ctx, store) as workspace: + with store_session(workspace, branch, store) as session: + print_result(session.query(statement, parameters)) + + +def execute( + ctx: typer.Context, sql: str | None = typer.Argument(None), + store: str = typer.Option(..., "--store"), branch: str = typer.Option("main", "--branch"), + params: str | None = typer.Option(None, "--params"), + file: Path | None = typer.Option(None, "--file"), stdin: bool = typer.Option(False, "--stdin"), +) -> None: + statement, parameters = _sql_input(sql, file, stdin), _parameters(params) + with connect_workspace(ctx, store) as workspace: + with store_session(workspace, branch, store) as session: + print_result(session.execute(statement, parameters)) + + +def register_commands(app: typer.Typer) -> None: + app.command()(query) + app.command()(execute) + app.add_typer(table_app, name="table", help="Prepare tables in a named relational store.") diff --git a/packages/chronos-core/src/chronos_core/cli/_terminal.py b/packages/chronos-core/src/chronos_core/cli/_terminal.py new file mode 100644 index 0000000..e6b1e07 --- /dev/null +++ b/packages/chronos-core/src/chronos_core/cli/_terminal.py @@ -0,0 +1,69 @@ +"""Shared terminal output, error reporting, and CLI policy choices.""" +from __future__ import annotations + +import base64 +import json +import re +from dataclasses import asdict, is_dataclass +from datetime import date, datetime, time +from decimal import Decimal +from enum import Enum +from typing import Any, NoReturn +from urllib.parse import parse_qsl, unquote, urlsplit + +import typer + + +class MergePolicy(str, Enum): + api_default = "api-default" + abort_on_conflict = "abort_on_conflict" + snapshot_isolation = "snapshot_isolation" + + @property + def api_value(self) -> str | None: + return None if self is MergePolicy.api_default else self.value + + +def _json_default(value: Any) -> Any: + if is_dataclass(value) and not isinstance(value, type): + return asdict(value) + if isinstance(value, (datetime, date, time)): + return value.isoformat() + if isinstance(value, Decimal): + return str(value) + if isinstance(value, bytes): + return {"encoding": "base64", "data": base64.b64encode(value).decode("ascii")} + raise TypeError(f"Unsupported JSON value type: {type(value).__name__}") + + +def print_result(result: Any) -> None: + if result is not None: + typer.echo(json.dumps(result, default=_json_default, ensure_ascii=False, indent=2, allow_nan=False)) + + +def report_error( + error: Exception, *, secrets: tuple[str | None, ...] = (), urls: tuple[str | None, ...] = (), +) -> NoReturn: + """Report an operational failure without exposing configured credentials.""" + message = str(error) + sensitive = [secret for secret in secrets if secret] + for url in urls: + if url: + message = message.replace(url, "") + try: + parsed = urlsplit(url) + except ValueError: + continue + if parsed.password: + sensitive.extend((parsed.password, unquote(parsed.password))) + sensitive.extend(value for key, value in parse_qsl(parsed.query) if key == "password" and value) + for item in parsed.query.split("&"): + key, separator, value = item.partition("=") + if separator and unquote(key) == "password" and value: + sensitive.append(value) + for secret in sorted(set(sensitive), key=len, reverse=True): + message = message.replace(secret, "") + message = re.sub(r"([a-zA-Z][\w+.-]*://)[^\s/@]+@", r"\1@", message) + message = re.sub(r"(?i)(password\s*=\s*)(?:'[^']*'|\"[^\"]*\"|[^\s]+)", r"\1", message) + typer.echo(f"Error: {message}", err=True) + raise typer.Exit(1) from error diff --git a/packages/chronos-core/src/chronos_core/cli/_workspace.py b/packages/chronos-core/src/chronos_core/cli/_workspace.py new file mode 100644 index 0000000..5458850 --- /dev/null +++ b/packages/chronos-core/src/chronos_core/cli/_workspace.py @@ -0,0 +1,181 @@ +"""Workspace lifecycle commands and invocation-owned connections and sessions.""" +from __future__ import annotations + +import sqlite3 +import sys +from collections.abc import Generator +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import psycopg +import typer + +from chronos_core.cli._config import DatabaseConfig, QdrantConfig, WorkspaceConfigError, load_workspace_config, open_workspace +from chronos_core.cli._terminal import MergePolicy, print_result, report_error + +if TYPE_CHECKING: + from chronos_core.workspace import ChronosWorkspaceContext + from chronos_core.workspace.runtime import WorkspaceBranchSession + + +@dataclass(frozen=True) +class WorkspaceOptions: + config_path: Path | None + + +branch_app = typer.Typer(no_args_is_help=True) +checkpoint_app = typer.Typer(no_args_is_help=True) +merge_app = typer.Typer(no_args_is_help=True) + + +@contextmanager +def connect_workspace( + ctx: typer.Context, store: str | None = None, *, qdrant: bool = False, +) -> Generator[ChronosWorkspaceContext, None, None]: + options = ctx.find_object(WorkspaceOptions) + try: + config = load_workspace_config(options.config_path if options else None) + except WorkspaceConfigError as error: + raise typer.BadParameter(str(error), param_hint="--config") from error + if store is not None: + try: + config.store(store, qdrant=qdrant) + except WorkspaceConfigError as error: + raise typer.BadParameter(str(error), param_hint="--store") from error + + from chronos_core.branching import BranchingError + from chronos_core.workspace import ChronosFSError, ChronosFSMountError, FilesystemStoreError + + errors = ( + ChronosFSError, ChronosFSMountError, FilesystemStoreError, BranchingError, + RuntimeError, ValueError, TypeError, OSError, sqlite3.Error, psycopg.Error, + ) + secrets: list[str | None] = [] + urls: list[str | None] = [] + if isinstance(config.filesystem, DatabaseConfig): + urls.extend((config.filesystem.database_url, config.filesystem.metadata_url)) + for entry in config.stores.values(): + target = entry.connection + urls.append(target.metadata_url) + if isinstance(target, QdrantConfig): + secrets.append(target.api_key) + urls.append(target.url) + else: + urls.append(target.database_url) + if any(entry.kind == "qdrant" for entry in config.stores.values()): + try: + import grpc + import httpx + from qdrant_client.http.exceptions import ResponseHandlingException, UnexpectedResponse + except ImportError: + report_error(RuntimeError("Qdrant support requires installing chronos-core[qdrant].")) + errors += (grpc.RpcError, httpx.HTTPError, ResponseHandlingException, UnexpectedResponse) + try: + with open_workspace(config) as workspace: + yield workspace + except errors as error: + report_error(error, secrets=tuple(secrets), urls=tuple(urls)) + + +@contextmanager +def workspace_session(workspace: ChronosWorkspaceContext, branch: str) -> Generator[WorkspaceBranchSession, None, None]: + checked_out = workspace.checkout(branch) + errors: list[Exception] = [] + try: + yield checked_out + finally: + # SQL sessions expose close(). Qdrant control sessions without a public + # close method are released by their owning context when it closes. + for session in checked_out.stores.values(): + close = getattr(session, "close", None) + if callable(close): + try: + close() + except Exception as error: + errors.append(error) + if errors and sys.exc_info()[0] is None: + raise errors[0] + + +@contextmanager +def store_session(workspace: ChronosWorkspaceContext, branch: str, store: str) -> Generator[Any, None, None]: + with workspace_session(workspace, branch) as checked_out: + yield checked_out.stores[store] + + +@branch_app.command("list") +def list_branches(ctx: typer.Context) -> None: + with connect_workspace(ctx) as workspace: + print_result(workspace.list_branches()) + + +@branch_app.command("create") +def create_branch(ctx: typer.Context, name: str, from_branch: str = typer.Option("main", "--from")) -> None: + with connect_workspace(ctx) as workspace: + print_result(workspace.create_branch(name, from_branch=from_branch)) + + +@branch_app.command("create-from-checkpoint") +def create_branch_from_checkpoint(ctx: typer.Context, name: str, checkpoint: str) -> None: + with connect_workspace(ctx) as workspace: + print_result(workspace.create_branch_from_checkpoint(name, checkpoint)) + + +@branch_app.command("delete") +def delete_branch(ctx: typer.Context, name: str) -> None: + with connect_workspace(ctx) as workspace: + print_result(workspace.delete_branch(name)) + + +@checkpoint_app.command("create") +def create_checkpoint(ctx: typer.Context, name: str, branch: str = typer.Option("main", "--branch")) -> None: + with connect_workspace(ctx) as workspace: + print_result(workspace.create_checkpoint(name, branch=branch)) + + +def diff(ctx: typer.Context, left: str, right: str) -> None: + with connect_workspace(ctx) as workspace: + print_result(workspace.diff(left, right)) + + +@merge_app.command("preview") +def merge_preview(ctx: typer.Context, source: str, target: str, + policy: MergePolicy = typer.Option(MergePolicy.abort_on_conflict, "--policy")) -> None: + with connect_workspace(ctx) as workspace: + print_result(workspace.merge_preview(source, target, policy=policy.api_value)) + + +@merge_app.command("apply") +def merge_apply(ctx: typer.Context, source: str, target: str, + policy: MergePolicy = typer.Option(MergePolicy.abort_on_conflict, "--policy")) -> None: + with connect_workspace(ctx) as workspace: + print_result(workspace.merge_apply(source, target, policy=policy.api_value)) + + +@merge_app.command("atomic-preview") +def merge_atomic_preview(ctx: typer.Context, source: str, target: str, + policy: MergePolicy = typer.Option(MergePolicy.abort_on_conflict, "--policy")) -> None: + with connect_workspace(ctx) as workspace: + print_result(workspace.merge_atomic_preview(source, target, policy=policy.api_value)) + + +@merge_app.command("atomic-apply") +def merge_atomic( + ctx: typer.Context, source: str, target: str, + operation_id: str = typer.Option(..., "--operation-id"), + preview_token: str | None = typer.Option(None, "--preview-token"), + policy: MergePolicy = typer.Option(MergePolicy.abort_on_conflict, "--policy"), +) -> None: + with connect_workspace(ctx) as workspace: + print_result(workspace.merge_atomic( + source, target, operation_id=operation_id, preview_token=preview_token, policy=policy.api_value, + )) + + +def register_commands(app: typer.Typer) -> None: + app.command()(diff) + app.add_typer(branch_app, name="branch", help="Manage branches across the workspace.") + app.add_typer(checkpoint_app, name="checkpoint", help="Checkpoint all participants.") + app.add_typer(merge_app, name="merge", help="Preview and apply coordinated or atomic merges.") diff --git a/packages/chronos-core/src/chronos_core/cli/main.py b/packages/chronos-core/src/chronos_core/cli/main.py new file mode 100644 index 0000000..57a0724 --- /dev/null +++ b/packages/chronos-core/src/chronos_core/cli/main.py @@ -0,0 +1,33 @@ +"""Assemble the workspace CLI and its root configuration option.""" +from __future__ import annotations + +from pathlib import Path + +import typer + +from chronos_core.cli._filesystem import app as filesystem_app +from chronos_core.cli._qdrant import app as qdrant_app +from chronos_core.cli._relational import register_commands as register_relational_commands +from chronos_core.cli._workspace import WorkspaceOptions, register_commands as register_workspace_commands + +app = typer.Typer(no_args_is_help=True, pretty_exceptions_enable=False) +app.add_typer(filesystem_app, name="fs", help="Use the configured workspace filesystem.") +register_workspace_commands(app) +register_relational_commands(app) +app.add_typer(qdrant_app, name="qdrant", help="Use a named Qdrant participant.") + + +@app.callback() +def main( + ctx: typer.Context, + config: Path | None = typer.Option( + None, "--config", envvar="CHRONOS_WORKSPACE_CONFIG", + help="Workspace TOML path; defaults to ./chronos.toml.", + ), +) -> None: + """Use configured Chronos stores together, including workspaces with just one store.""" + ctx.obj = WorkspaceOptions(config) + + +if __name__ == "__main__": + app() diff --git a/packages/chronos-core/src/chronos_core/native/chronos_s3.cpp b/packages/chronos-core/src/chronos_core/native/chronos_s3.cpp index 4f5af79..7681cdd 100644 --- a/packages/chronos-core/src/chronos_core/native/chronos_s3.cpp +++ b/packages/chronos-core/src/chronos_core/native/chronos_s3.cpp @@ -1,5 +1,6 @@ #include "chronos_s3.hpp" +#include "database_url_path.hpp" #include "interval_data_plane.hpp" #include @@ -66,16 +67,14 @@ const std::vector kObjectColumns = { }; const std::vector kObjectPrimaryKey = {"bucket", "object_key"}; +std::string url_encode(std::string_view input, bool preserve_slash); +std::string sqlite_file_path(const std::string &path); + std::string sqlite_url(const std::string &path) { std::string url = path; - std::string file_path = path; - const std::string prefix = "sqlite:///"; - if (path.rfind(prefix, 0) == 0) { - file_path = "/" + path.substr(prefix.size()); - } else if (path.rfind("sqlite://", 0) == 0) { - file_path = path.substr(std::strlen("sqlite://")); - } else { - url = "sqlite:///" + path; + const std::string file_path = sqlite_file_path(path); + if (path.rfind("sqlite://", 0) != 0) { + url = "sqlite:///" + url_encode(path, true); } sqlite3 *database = nullptr; if (sqlite3_open_v2( @@ -94,8 +93,12 @@ std::string sqlite_url(const std::string &path) { std::string sqlite_file_path(const std::string &path) { const std::string prefix = "sqlite:///"; - if (path.rfind(prefix, 0) == 0) return "/" + path.substr(prefix.size()); - if (path.rfind("sqlite://", 0) == 0) return path.substr(std::strlen("sqlite://")); + if (path.rfind(prefix, 0) == 0) { + return "/" + decode_database_url_path(path.substr(prefix.size())); + } + if (path.rfind("sqlite://", 0) == 0) { + return decode_database_url_path(path.substr(std::strlen("sqlite://"))); + } return path; } diff --git a/packages/chronos-core/src/chronos_core/native/chronosfs_fuse.cpp b/packages/chronos-core/src/chronos_core/native/chronosfs_fuse.cpp index 4d0b1a9..0663df6 100644 --- a/packages/chronos-core/src/chronos_core/native/chronosfs_fuse.cpp +++ b/packages/chronos-core/src/chronos_core/native/chronosfs_fuse.cpp @@ -2,6 +2,7 @@ #include "chronosfs_fuse.hpp" +#include "database_url_path.hpp" #include "interval_data_plane.hpp" #include @@ -436,7 +437,8 @@ std::string blob_sql_type(const chronos::native::NativeBranchStore &store) { std::string chronosfs_object_dir(const std::string &database_url) { constexpr const char *prefix = "sqlite:///"; if (database_url.rfind(prefix, 0) != 0) return {}; - std::string database_path = database_url.substr(std::strlen(prefix)); + std::string database_path = chronos::native::decode_database_url_path( + database_url.substr(std::strlen(prefix))); if (database_path.empty() || database_path == ":memory:") return {}; return database_path + ".chronosfs-objects"; } diff --git a/packages/chronos-core/src/chronos_core/native/database_url_path.hpp b/packages/chronos-core/src/chronos_core/native/database_url_path.hpp new file mode 100644 index 0000000..369f2c7 --- /dev/null +++ b/packages/chronos-core/src/chronos_core/native/database_url_path.hpp @@ -0,0 +1,38 @@ +#pragma once + +#include +#include +#include + +namespace chronos::native { + +// Database URLs use percent-encoded paths. Decode once, without treating '+' +// as a space; raw filesystem paths must not pass through this helper. +inline std::string decode_database_url_path(std::string_view path) { + const auto hex_digit = [](char ch) -> int { + if (ch >= '0' && ch <= '9') return ch - '0'; + if (ch >= 'a' && ch <= 'f') return ch - 'a' + 10; + if (ch >= 'A' && ch <= 'F') return ch - 'A' + 10; + return -1; + }; + std::string decoded; + decoded.reserve(path.size()); + for (std::size_t i = 0; i < path.size(); ++i) { + char ch = path[i]; + if (ch == '%' && i + 2 < path.size()) { + const int high = hex_digit(path[i + 1]); + const int low = hex_digit(path[i + 2]); + if (high >= 0 && low >= 0) { + ch = static_cast((high << 4) | low); + i += 2; + } + } + if (ch == '\0') { + throw std::invalid_argument("database path must not contain NUL bytes"); + } + decoded.push_back(ch); + } + return decoded; +} + +} // namespace chronos::native diff --git a/packages/chronos-core/src/chronos_core/native/interval_sql_runtime.cpp b/packages/chronos-core/src/chronos_core/native/interval_sql_runtime.cpp index e1cec03..616045b 100644 --- a/packages/chronos-core/src/chronos_core/native/interval_sql_runtime.cpp +++ b/packages/chronos-core/src/chronos_core/native/interval_sql_runtime.cpp @@ -1210,8 +1210,12 @@ py::dict stats_to_py_dict(const BulkUpsertStats &native_stats, std::size_t input std::string sqlite_path_from_url(const std::string &database_url) { const std::string prefix = "sqlite:///"; - if (database_url.rfind(prefix, 0) == 0) return "/" + database_url.substr(prefix.size()); - if (database_url.rfind("sqlite://", 0) == 0) return database_url.substr(std::string("sqlite://").size()); + if (database_url.rfind(prefix, 0) == 0) { + return "/" + decode_database_url_path(database_url.substr(prefix.size())); + } + if (database_url.rfind("sqlite://", 0) == 0) { + return decode_database_url_path(database_url.substr(std::string("sqlite://").size())); + } return database_url; } @@ -1324,16 +1328,16 @@ std::string duckdb_path_from_url(const std::string &database_url) { const std::string url_prefix = "duckdb://"; const std::string scheme_prefix = "duckdb:"; if (database_url.rfind(absolute_prefix, 0) == 0) { - std::string path = database_url.substr(absolute_prefix.size()); + std::string path = decode_database_url_path(database_url.substr(absolute_prefix.size())); if (path == ":memory:") return path; while (!path.empty() && path.front() == '/') path.erase(path.begin()); return "/" + path; } if (database_url.rfind(url_prefix, 0) == 0) { - return database_url.substr(url_prefix.size()); + return decode_database_url_path(database_url.substr(url_prefix.size())); } if (database_url.rfind(scheme_prefix, 0) == 0) { - return database_url.substr(scheme_prefix.size()); + return decode_database_url_path(database_url.substr(scheme_prefix.size())); } return database_url; } @@ -2923,6 +2927,11 @@ class NativeDuckDBDriver final : public NativeSqlDriver { public: explicit NativeDuckDBDriver(const std::string &database_url) : dialect_("duckdb") { path_ = duckdb_path_from_url(database_url); + // DuckDB 1.5.4 treats '?' as a query delimiter when constructing its + // WAL filename, even for local paths. Fail before creating any files. + if (path_.find('?') != std::string::npos) { + throw std::invalid_argument("DuckDB database paths cannot contain '?'"); + } if (path_ != ":memory:" && !path_.empty()) { std::filesystem::path db_path(path_); if (db_path.has_parent_path()) { diff --git a/packages/chronos-core/src/chronos_core/native/native_backends.cpp b/packages/chronos-core/src/chronos_core/native/native_backends.cpp index 5224c20..20802b3 100644 --- a/packages/chronos-core/src/chronos_core/native/native_backends.cpp +++ b/packages/chronos-core/src/chronos_core/native/native_backends.cpp @@ -46,6 +46,7 @@ #include #include "copy_branch_store.hpp" +#include "database_url_path.hpp" #include "interval_data_plane.hpp" namespace py = pybind11; diff --git a/packages/chronos-core/src/chronos_core/workspace/chronosfs/daemon.py b/packages/chronos-core/src/chronos_core/workspace/chronosfs/daemon.py index b494120..ad7fa2a 100644 --- a/packages/chronos-core/src/chronos_core/workspace/chronosfs/daemon.py +++ b/packages/chronos-core/src/chronos_core/workspace/chronosfs/daemon.py @@ -17,6 +17,7 @@ import time from pathlib import Path from typing import Any +from urllib.parse import unquote from chronos_core import _native_interval @@ -133,7 +134,7 @@ def _sqlite_database_path(database_url: str) -> str | None: prefix = "sqlite:///" if not database_url.startswith(prefix): return None - return "/" + database_url[len(prefix) :] + return "/" + unquote(database_url[len(prefix) :]) def _checkpoint_sqlite(database_url: str, state: _DaemonState) -> None: diff --git a/packages/chronos-core/src/chronos_core/workspace/chronosfs/fuse.py b/packages/chronos-core/src/chronos_core/workspace/chronosfs/fuse.py index 874ed24..27a9460 100644 --- a/packages/chronos-core/src/chronos_core/workspace/chronosfs/fuse.py +++ b/packages/chronos-core/src/chronos_core/workspace/chronosfs/fuse.py @@ -12,6 +12,7 @@ import time from contextlib import suppress from pathlib import Path +from urllib.parse import quote from chronos_core import _native_interval from chronos_core.workspace.chronosfs.store import ChronosFSStore @@ -207,7 +208,7 @@ def _database_url_for_mount(store: ChronosFSStore) -> str: if not database_url: database_path = getattr(store.context.db, "database_path", None) if database_path: - database_url = f"sqlite:///{database_path}" + database_url = f"sqlite:///{quote(str(database_path), safe='/')}" if not database_url: raise ChronosFSMountError( "native ChronosFS FUSE requires a file-backed database URL" @@ -221,7 +222,7 @@ def _database_urls_for_mount(store: ChronosFSStore) -> tuple[str, str]: if not metadata_url: metadata_path = getattr(store.context.metadata_db, "database_path", None) if metadata_path: - metadata_url = f"sqlite:///{metadata_path}" + metadata_url = f"sqlite:///{quote(str(metadata_path), safe='/')}" return database_url, str(metadata_url or database_url) diff --git a/packages/chronos-core/src/chronos_core/workspace/chronosfs/store.py b/packages/chronos-core/src/chronos_core/workspace/chronosfs/store.py index d361efa..4654a0b 100644 --- a/packages/chronos-core/src/chronos_core/workspace/chronosfs/store.py +++ b/packages/chronos-core/src/chronos_core/workspace/chronosfs/store.py @@ -9,6 +9,7 @@ from dataclasses import dataclass from pathlib import Path, PurePosixPath from typing import Any, Iterator, Literal +from urllib.parse import quote from chronos_core import _native_interval from chronos_core.branching import ( @@ -1460,7 +1461,7 @@ def _database_url_for_adapter(adapter: Any) -> str: return str(database_url) database_path = getattr(adapter, "database_path", None) if database_path and str(database_path) != ":memory:": - return f"sqlite:///{database_path}" + return f"sqlite:///{quote(str(database_path), safe='/')}" raise ChronosFSError( "native ChronosFS requires a file-backed SQLite or PostgreSQL database" ) diff --git a/tests/test_chronosfs.py b/tests/test_chronosfs.py index baac12b..ad01659 100644 --- a/tests/test_chronosfs.py +++ b/tests/test_chronosfs.py @@ -1292,6 +1292,50 @@ def test_shared_daemon_identity_is_store_scoped() -> None: ] +def test_shared_daemon_checkpoint_preserves_database_path(tmp_path: Path) -> None: + import sqlite3 + from urllib.parse import quote + + from chronos_core.workspace.chronosfs.daemon import _DaemonState, _checkpoint_sqlite + + database = tmp_path / "state #?%20+目录.sqlite" + with sqlite3.connect(database) as connection: + connection.execute("CREATE TABLE marker (id INTEGER PRIMARY KEY)") + connection.execute("INSERT INTO marker VALUES (7)") + state = _DaemonState() + state.request_shutdown() + _checkpoint_sqlite(f"sqlite:///{quote(str(database), safe='/')}", state) + assert sorted(path.name for path in tmp_path.iterdir()) == [database.name] + with sqlite3.connect(database) as connection: + assert connection.execute("SELECT id FROM marker").fetchall() == [(7,)] + + +def test_existing_sqlite_adapter_preserves_database_path(tmp_path: Path) -> None: + import sqlite3 + from urllib.parse import quote + + from chronos_core import _native_interval + from chronos_core.branching.sql_adapters import SQLiteDatabaseAdapter + from chronos_core.workspace.chronosfs.fuse import _database_urls_for_mount + + database = tmp_path / "state #?%20+目录.sqlite" + raw_connection = sqlite3.connect(database) + raw_connection.row_factory = sqlite3.Row + adapter = SQLiteDatabaseAdapter(raw_connection, str(database)) + context = ChronosBranchContext.from_database_adapter(adapter) + store = ChronosFSStore(context) + try: + store.ensure() + store.write_file("main", "/note", b"preserved") + url = f"sqlite:///{quote(str(database), safe='/')}" + assert store._database_url == url + assert _database_urls_for_mount(store) == (url, url) + connection = _native_interval.NativeSqlConnection(url) + assert connection.query_sql("SELECT count(*) FROM _chronosfs_metadata")[0][0] > 0 + finally: + store.close() + + def test_shared_daemon_shutdown_waits_for_mounts() -> None: from chronos_core.workspace.chronosfs.daemon import _DaemonState diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..76bc324 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,830 @@ +"""CLI argument forwarding, terminal behavior, and cleanup without real services.""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import textwrap +from dataclasses import dataclass +from datetime import date, datetime, time +from decimal import Decimal +from pathlib import Path +from urllib.parse import quote +from unittest.mock import MagicMock +from contextlib import contextmanager + +import pytest +import typer +from rich.text import Text +from typer.testing import CliRunner + +from chronos_core.branching import BranchingError, ChronosBranchContext +from chronos_core.cli import _workspace +from chronos_core.cli._terminal import print_result +from chronos_core.cli._config import ( + DatabaseConfig, QdrantConfig, WorkspaceConfigError, load_workspace_config, open_workspace, +) +from chronos_core.cli.main import app + +runner = CliRunner() + + +@pytest.fixture +def workspace_connection(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> MagicMock: + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("CHRONOS_WORKSPACE_CONFIG", raising=False) + (tmp_path / "chronos.toml").write_text( + '[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"\n' + '[stores.other]\nkind="sqlite"\ndatabase_url="sqlite:///other.sqlite"\n' + ) + workspace = MagicMock() + for method in ("create_branch", "create_branch_from_checkpoint", "delete_branch", "create_checkpoint", + "diff", "merge_preview", "merge_apply", "merge_atomic_preview", "merge_atomic"): + getattr(workspace, method).return_value = None + workspace.list_branches.return_value = ["main"] + workspace.stores = {"app": MagicMock(), "other": MagicMock()} + workspace.stores["app"].register_table.return_value = None + workspace.stores["app"].db.commit.return_value = None + session = MagicMock() + session.query.return_value = [{"id": 1}] + session.execute.return_value = None + workspace.checkout.return_value.stores = {"app": session, "other": MagicMock()} + + @contextmanager + def opened(config): + try: + yield workspace + finally: + workspace.close() + + monkeypatch.setattr(_workspace, "open_workspace", MagicMock(side_effect=opened)) + return workspace + + +class TestTerminalBehavior: + def test_json_values(self, capsys: pytest.CaptureFixture[str]) -> None: + @dataclass + class Result: + values: list + print_result(Result([date(2026, 1, 2), datetime(2026, 1, 2, 3, 4), time(3, 4), Decimal("1.20"), b"hi"])) + assert json.loads(capsys.readouterr().out) == {"values": [ + "2026-01-02", "2026-01-02T03:04:00", "03:04:00", "1.20", {"encoding": "base64", "data": "aGk="} + ]} + print_result(None) + assert capsys.readouterr().out == "" + with pytest.raises(TypeError, match="Unsupported JSON value"): + print_result(object()) + + +class TestWorkspaceConfiguration: + @pytest.mark.parametrize("source", [ + '[stores.vectors]\nkind="qdrant"\nmetadata_url="sqlite:///meta.sqlite"\nurl="http://localhost:6333"\ntimeout=' + '9' * 400, + '[stores.app]\nkind="duckdb"\ndatabase_url="duckdb:///query%3Fdir/data.duckdb"\nmetadata_url="sqlite:///metadata.sqlite"', + '[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///~chronos_no_such_user_987654/app.sqlite"', + '[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///shared.sqlite"\n[stores.vectors]\nkind="qdrant"\nmetadata_url="sqlite:///shared%2Esqlite"\npath="vectors"', + '[workspace]\nshared_metadata_url="sqlite:///shared.sqlite"\n[stores.first]\nkind="sqlite"\ndatabase_url="sqlite:///shared.sqlite"\n[stores.second]\nkind="sqlite"\ndatabase_url="sqlite:///shared.sqlite"', + '[workspace]\nshared_metadata_url="sqlite:///shared.sqlite"\n[stores.first]\nkind="duckdb"\ndatabase_url="duckdb:///first.duckdb"\nmetadata_url="sqlite:///shared.sqlite"\n[stores.second]\nkind="duckdb"\ndatabase_url="duckdb:///second.duckdb"\nmetadata_url="sqlite:///shared.sqlite"', + ]) + def test_reviewed_config_errors_fail_before_connecting(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, source: str) -> None: + config = tmp_path / "chronos.toml" + config.write_text(source) + opened = MagicMock() + monkeypatch.setattr(_workspace, "open_workspace", opened) + result = runner.invoke(app, ["--config", str(config), "branch", "list"]) + assert result.exit_code == 2, result.output + assert "Invalid value" in result.stderr + opened.assert_not_called() + assert list(tmp_path.iterdir()) == [config] + + def test_encoded_filename_preserves_api_url(self, tmp_path: Path) -> None: + config = tmp_path / "chronos.toml" + config.write_text('[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///literal%252Ename.sqlite"') + loaded = load_workspace_config(config) + assert loaded.stores["app"].connection.database_url == f"sqlite:///{tmp_path / 'literal%252Ename.sqlite'}" + + def test_encoded_tilde_is_a_literal_filename(self, tmp_path: Path) -> None: + config = tmp_path / "chronos.toml" + config.write_text('[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///%7Echronos_no_such_user_987654/app.sqlite"') + loaded = load_workspace_config(config) + assert loaded.stores["app"].connection.database_url == f"sqlite:///{tmp_path / '~chronos_no_such_user_987654/app.sqlite'}" + + @pytest.mark.parametrize("kind", ["sqlite", "duckdb"]) + @pytest.mark.parametrize("directory", ["space dir", "hash#dir", "query?dir", "literal%20dir", "plus+dir", "目录"]) + def test_local_database_urls_encode_config_directory(self, tmp_path: Path, kind: str, directory: str) -> None: + root = tmp_path / directory + root.mkdir() + config = root / "chronos.toml" + filename = "literal%20#.db" if kind == "duckdb" else "literal%20#?.db" + config.write_text( + f'[stores.app]\nkind="{kind}"\ndatabase_url="{kind}:///{quote(filename, safe="")}"\n' + + ('metadata_url="sqlite:///metadata.sqlite"\n' if kind == "duckdb" else '') + ) + if kind == "duckdb" and "?" in directory: + with pytest.raises(WorkspaceConfigError, match="DuckDB database paths"): + load_workspace_config(config) + return + loaded = load_workspace_config(config) + connection = loaded.stores["app"].connection + assert connection.database_url == f"{kind}:///{quote(str(root / filename), safe='/')}" + if kind == "duckdb": + assert connection.metadata_url == f"sqlite:///{quote(str(root / 'metadata.sqlite'), safe='/')}" + + def test_invalid_config_path_is_a_usage_error(self) -> None: + result = runner.invoke(app, ["--config", "~chronos_no_such_user_987654/chronos.toml", "branch", "list"]) + assert result.exit_code == 2, result.output + assert "valid local path" in result.stderr + + def test_path_precedence_and_relative_storage(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("CHRONOS_WORKSPACE_CONFIG", raising=False) + default = tmp_path / "chronos.toml" + default.write_text('[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///.chronos/app.sqlite"\n') + assert load_workspace_config().stores["app"].metadata_url == f"sqlite:///{tmp_path / '.chronos/app.sqlite'}" + other = tmp_path / "project" + other.mkdir() + configured = other / "workspace.toml" + configured.write_text('[stores.analytics]\nkind="duckdb"\ndatabase_url="duckdb:///data.duckdb"\nmetadata_url="sqlite:///meta.sqlite"\n') + monkeypatch.setenv("CHRONOS_WORKSPACE_CONFIG", str(configured)) + selected = load_workspace_config() + assert selected.config_path == configured + assert selected.stores["analytics"].connection == DatabaseConfig( + f"duckdb:///{other / 'data.duckdb'}", f"sqlite:///{other / 'meta.sqlite'}", + ) + assert load_workspace_config(default).config_path == default + assert not (tmp_path / ".chronos").exists() + assert sorted(item.name for item in other.iterdir()) == ["workspace.toml"] + + def test_environment_values_and_atomic_metadata(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("WORKSPACE_DATABASE", "postgres://user:secret@localhost/test") + monkeypatch.setenv("VECTOR_KEY", "vector-secret") + config = tmp_path / "chronos.toml" + config.write_text(''' +[workspace] +shared_metadata_url = "${WORKSPACE_DATABASE}" +[stores.app] +kind = "postgresql" +database_url = "${WORKSPACE_DATABASE}" +[stores.vectors] +kind = "qdrant" +metadata_url = "${WORKSPACE_DATABASE}" +url = "http://localhost:6333" +api_key = "${VECTOR_KEY}" +timeout = 2.5 +''') + loaded = load_workspace_config(config) + assert loaded.shared_metadata_url == "postgresql://user:secret@localhost/test" + assert loaded.stores["vectors"].connection == QdrantConfig( + loaded.shared_metadata_url, url="http://localhost:6333", api_key="vector-secret", timeout=2.5, + ) + monkeypatch.delenv("VECTOR_KEY") + with pytest.raises(WorkspaceConfigError, match="unset environment variable 'VECTOR_KEY'") as failure: + load_workspace_config(config) + assert "secret" not in str(failure.value) + + @pytest.mark.parametrize("source", [ + '', '[stores.app]\nkind="other"', + '[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///:memory:"', + '[stores.app]\nkind="sqlite"\ndatabase_url="postgresql://user:secret@host/test"', + '[stores.app]\nkind="duckdb"\ndatabase_url="duckdb:///app.duckdb"', + '[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"\ntypo="secret"', + '[stores.fs]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"', + '[stores.vectors]\nkind="qdrant"\nmetadata_url="sqlite:///meta.sqlite"', + '[stores.vectors]\nkind="qdrant"\nmetadata_url="sqlite:///meta.sqlite"\npath=":memory:"', + '[stores.vectors]\nkind="qdrant"\nmetadata_url="sqlite:///meta.sqlite"\npath="data"\nurl="http://localhost:6333"', + '[stores.vectors]\nkind="qdrant"\nmetadata_url="sqlite:///meta.sqlite"\npath="data"\napi_key="secret"', + '[stores.vectors]\nkind="qdrant"\nmetadata_url="sqlite:///meta.sqlite"\nurl="http://localhost:6333"\ntimeout=nan', + '[stores.vectors]\nkind="qdrant"\nmetadata_url="sqlite:///meta.sqlite"\nurl="http://localhost:6333"\ntimeout=true', + '[workspace]\nshared_metadata_url="sqlite:///shared.sqlite"\n[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///other.sqlite"', + '[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///same.sqlite"\n[stores.vectors]\nkind="qdrant"\nmetadata_url="sqlite:///same.sqlite"\npath="data"', + '[stores.app]\nkind="sqlite"\ndatabase_url="secret', + ]) + def test_invalid_configuration_does_not_create_storage(self, source: str, tmp_path: Path) -> None: + config = tmp_path / "chronos.toml" + config.write_text(source) + with pytest.raises(WorkspaceConfigError) as failure: + load_workspace_config(config) + assert "secret" not in str(failure.value) + assert list(tmp_path.iterdir()) == [config] + + def test_store_selection(self, tmp_path: Path) -> None: + config = tmp_path / "chronos.toml" + config.write_text('[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"') + loaded = load_workspace_config(config) + assert loaded.store("app").kind == "sqlite" + with pytest.raises(WorkspaceConfigError, match="Unknown store"): + loaded.store("missing") + with pytest.raises(WorkspaceConfigError, match="not a Qdrant store"): + loaded.store("app", qdrant=True) + + @pytest.mark.parametrize("failure_stage", ["none", "open", "workspace", "operation", "close"]) + def test_resource_ownership(self, failure_stage: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + import chronos_core.workspace as workspace_api + from chronos_core.workspace.qdrant import ChronosQdrantStore + + config_path = tmp_path / "chronos.toml" + # Declaration order must not affect shared-context construction or cleanup. + config_path.write_text(''' +[workspace] +shared_metadata_url="sqlite:///app.sqlite" +[stores.vectors] +kind="qdrant" +metadata_url="sqlite:///app.sqlite" +path="vectors" +[stores.app] +kind="sqlite" +database_url="sqlite:///app.sqlite" +''') + loaded = load_workspace_config(config_path) + database = MagicMock() + vectors = MagicMock() + factory = MagicMock() + factory.local.return_value = vectors + workspace_factory = MagicMock() + monkeypatch.setattr(ChronosBranchContext, "connect", MagicMock(return_value=database)) + monkeypatch.setattr(ChronosQdrantStore, "local", factory.local) + monkeypatch.setattr(ChronosQdrantStore, "remote", factory.remote) + monkeypatch.setattr(workspace_api, "ChronosWorkspaceContext", workspace_factory) + events = [] + database.close.side_effect = lambda: events.append("database") + + def close_vectors() -> None: + events.append("vectors") + if failure_stage in {"operation", "close"}: + raise RuntimeError("close failed") + + vectors.close.side_effect = close_vectors + if failure_stage == "open": + factory.local.side_effect = RuntimeError("open failed") + if failure_stage == "workspace": + workspace_factory.side_effect = RuntimeError("workspace failed") + if failure_stage == "none": + with open_workspace(loaded) as workspace: + assert workspace is workspace_factory.return_value + else: + expected = "operation failed" if failure_stage == "operation" else f"{failure_stage} failed" + with pytest.raises(RuntimeError, match=expected): + with open_workspace(loaded): + if failure_stage == "operation": + raise RuntimeError("operation failed") + assert events == (["database"] if failure_stage == "open" else ["vectors", "database"]) + if failure_stage != "open": + factory.local.assert_called_once_with( + loaded.shared_metadata_url, path=tmp_path / "vectors", collection_prefix="chronos_", context=database, + ) + workspace_factory.assert_called_once_with( + stores={"vectors": vectors, "app": database}, shared_metadata_url=loaded.shared_metadata_url, + ) + workspace_factory.return_value.close.assert_not_called() + + def test_split_sql_and_remote_qdrant(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + import chronos_core.workspace as workspace_api + from chronos_core.workspace.qdrant import ChronosQdrantStore + + config_file = tmp_path / "chronos.toml" + config_file.write_text(''' +[workspace] +shared_metadata_url="postgresql://localhost/metadata" +[stores.analytics] +kind="duckdb" +database_url="duckdb:///analytics.duckdb" +metadata_url="postgresql://localhost/metadata" +[stores.vectors] +kind="qdrant" +metadata_url="postgresql://localhost/metadata" +url="http://localhost:6333" +api_key="test-key" +collection_prefix="project_" +timeout=3 +''') + config = load_workspace_config(config_file) + split = MagicMock() + monkeypatch.setattr(ChronosBranchContext, "connect_split", split) + factory = MagicMock() + monkeypatch.setattr(ChronosQdrantStore, "local", factory.local) + monkeypatch.setattr(ChronosQdrantStore, "remote", factory.remote) + workspace_factory = MagicMock() + monkeypatch.setattr(workspace_api, "ChronosWorkspaceContext", workspace_factory) + with open_workspace(config): + pass + split.assert_called_once_with(f"duckdb:///{tmp_path / 'analytics.duckdb'}", "postgresql://localhost/metadata") + factory.remote.assert_called_once_with( + "postgresql://localhost/metadata", url="http://localhost:6333", api_key="test-key", + collection_prefix="project_", timeout=3, context=None, + ) + split.return_value.close.assert_called_once_with() + factory.remote.return_value.close.assert_called_once_with() + + def test_missing_file(self, tmp_path: Path) -> None: + with pytest.raises(WorkspaceConfigError, match="Cannot read workspace configuration"): + load_workspace_config(tmp_path / "missing.toml") + + +class TestWorkspaceCommands: + @pytest.mark.parametrize("value", [float("inf"), float("-inf"), float("nan")]) + def test_nonfinite_query_result_fails_clearly(self, workspace_connection: MagicMock, value: float) -> None: + session = workspace_connection.checkout.return_value.stores["app"] + session.query.return_value = [{"value": value}] + result = runner.invoke(app, ["query", "--store", "app", "SELECT 1"]) + assert result.exit_code == 1, result.output + assert result.stdout == "" + assert "JSON compliant" in result.stderr + session.close.assert_called_once_with() + workspace_connection.close.assert_called_once_with() + + @pytest.mark.parametrize("credentials", [ + 'review_user:review_secret%zz@127.0.0.1/db', + 'review_user:review%40secret%zz@127.0.0.1/db', + 'review_user@127.0.0.1/db?password=review%40secret%zz', + ]) + def test_postgres_parser_error_hides_password(self, tmp_path: Path, credentials: str) -> None: + config = tmp_path / "chronos.toml" + config.write_text(f'[stores.app]\nkind="postgresql"\ndatabase_url="postgresql://{credentials}"') + # Invalid percent escapes fail in the real driver's parser, before any network call. + result = runner.invoke(app, ["--config", str(config), "branch", "list"]) + assert result.exit_code == 1, result.output + assert result.stdout == "" + assert "invalid percent-encoded token" in result.stderr + assert "secret" not in result.stderr + assert "" in result.stderr + + def test_installed_entrypoints(self) -> None: + executable = Path(sys.executable).parent / "chronos" + for args in ([str(executable), "--help"], [sys.executable, "-m", "chronos_core.cli", "--help"]): + environment = {key: value for key, value in os.environ.items() if not key.startswith("CHRONOS_")} + result = subprocess.run(args, capture_output=True, text=True, env=environment, timeout=30) + assert result.returncode == 0, result.stderr + help_text = Text.from_ansi(result.stdout).plain + assert "--config" in help_text + assert "--database-url" not in help_text + + def test_help_and_sql_without_qdrant_dependency(self, tmp_path: Path) -> None: + code = ''' + import importlib.abc + import sys + class Block(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == "qdrant_client" or fullname.startswith("qdrant_client."): + raise ModuleNotFoundError("simulated missing qdrant-client") + sys.meta_path.insert(0, Block()) + from chronos_core.cli.main import app + sys.argv = ["chronos", *sys.argv[1:]] + app() + ''' + environment = {key: value for key, value in os.environ.items() if not key.startswith("CHRONOS_")} + for args in (["--help"], ["qdrant", "--help"], ["qdrant", "point", "upsert", "--help"]): + result = subprocess.run([sys.executable, "-c", textwrap.dedent(code), *args], capture_output=True, text=True, env=environment) + assert result.returncode == 0, result.stderr + config = tmp_path / "chronos.toml" + config.write_text('[stores.vectors]\nkind="qdrant"\nmetadata_url="sqlite:///meta.sqlite"\npath="vectors"\n') + result = subprocess.run( + [sys.executable, "-c", textwrap.dedent(code), "--config", str(config), "qdrant", "--store", "vectors", "collection", "list"], + capture_output=True, text=True, env=environment, + ) + assert result.returncode == 1 + assert "chronos-core[qdrant]" in result.stderr + assert list(tmp_path.iterdir()) == [config] + config.write_text('[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"\n') + result = subprocess.run( + [sys.executable, "-c", textwrap.dedent(code), "--config", str(config), "branch", "list"], + capture_output=True, text=True, env=environment, + ) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout) == ["main"] + + def test_open_failure_reports_credentials_safely(self, workspace_connection: MagicMock) -> None: + _workspace.open_workspace.side_effect = RuntimeError("Backend unavailable at postgresql://user:secret@host/meta") + result = runner.invoke(app, ["branch", "list"]) + assert result.exit_code == 1 + assert result.stdout == "" + assert "secret" not in result.stderr + assert "Backend unavailable" in result.stderr + workspace_connection.close.assert_not_called() + + def test_unsupported_result_closes_workspace(self, workspace_connection: MagicMock) -> None: + workspace_connection.diff.return_value = object() + result = runner.invoke(app, ["diff", "main", "trial"]) + assert result.exit_code == 1 + assert result.stdout == "" + assert "Unsupported JSON value type" in result.stderr + workspace_connection.close.assert_called_once_with() + + @pytest.mark.parametrize(("args", "method", "expected_args", "expected_kwargs"), [ + (["branch", "list"], "list_branches", (), {}), + (["branch", "create", "trial", "--from", "parent"], "create_branch", ("trial",), {"from_branch": "parent"}), + (["branch", "create-from-checkpoint", "trial", "saved"], "create_branch_from_checkpoint", ("trial", "saved"), {}), + (["branch", "delete", "trial"], "delete_branch", ("trial",), {}), + (["checkpoint", "create", "saved", "--branch", "trial"], "create_checkpoint", ("saved",), {"branch": "trial"}), + (["diff", "main", "trial"], "diff", ("main", "trial"), {}), + (["merge", "preview", "trial", "main"], "merge_preview", ("trial", "main"), {"policy": "abort_on_conflict"}), + (["merge", "apply", "trial", "main", "--policy", "snapshot_isolation"], "merge_apply", ("trial", "main"), {"policy": "snapshot_isolation"}), + (["merge", "atomic-preview", "trial", "main"], "merge_atomic_preview", ("trial", "main"), {"policy": "abort_on_conflict"}), + (["merge", "atomic-apply", "trial", "main", "--operation-id", "merge-1", "--preview-token", "reviewed"], + "merge_atomic", ("trial", "main"), {"policy": "abort_on_conflict", "operation_id": "merge-1", "preview_token": "reviewed"}), + ]) + def test_workspace_forwarding(self, workspace_connection: MagicMock, args, method, expected_args, expected_kwargs) -> None: + result = runner.invoke(app, [*args]) + assert result.exit_code == 0, result.output + getattr(workspace_connection, method).assert_called_once_with(*expected_args, **expected_kwargs) + workspace_connection.close.assert_called_once_with() + if method == "list_branches": + assert json.loads(result.stdout) == ["main"] + else: + assert result.stdout == "" + + def test_named_relational_commands(self, workspace_connection: MagicMock) -> None: + workspace = workspace_connection + result = runner.invoke(app, ["table", "create", "--store", "app", "CREATE TABLE items (id INTEGER)"]) + assert result.exit_code == 0, result.output + workspace.stores["app"].db.execute.assert_called_once_with("CREATE TABLE items (id INTEGER)") + workspace.stores["app"].db.commit.assert_called_once_with() + result = runner.invoke(app, ["table", "register", "items", "--store", "app", "--primary-key", "id", "--primary-key", "tenant"]) + assert result.exit_code == 0, result.output + workspace.stores["app"].register_table.assert_called_once_with("items", ["id", "tenant"]) + result = runner.invoke(app, ["query", "SELECT :id", "--store", "app", "--branch", "trial", "--params", '{"id":1}']) + assert result.exit_code == 0, result.output + assert json.loads(result.stdout) == [{"id": 1}] + session = workspace.checkout.return_value.stores["app"] + session.query.assert_called_once_with("SELECT :id", {"id": 1}) + workspace.checkout.assert_called_once_with("trial") + for checked_out in workspace.checkout.return_value.stores.values(): + checked_out.close.assert_called_once_with() + result = runner.invoke(app, ["execute", "UPDATE items SET id = :id", "--store", "app", "--params", '{"id":2}']) + assert result.exit_code == 0, result.output + session.execute.assert_called_once_with("UPDATE items SET id = :id", {"id": 2}) + + def test_sql_input_modes(self, workspace_connection: MagicMock, tmp_path: Path) -> None: + sql_file = tmp_path / "query.sql" + sql_file.write_text("SELECT :id") + session = workspace_connection.checkout.return_value.stores["app"] + for args, input_text in [(["--file", str(sql_file)], None), (["--stdin"], "SELECT :id"), ([], "SELECT :id")]: + result = runner.invoke(app, ["query", "--store", "app", "--params", '{"id":3}', *args], input=input_text) + assert result.exit_code == 0, result.output + assert session.query.call_args.args == ("SELECT :id", {"id": 3}) + + @pytest.mark.parametrize("args", [ + ["query", "SELECT 1", "--store", "missing"], + ["query", "SELECT 1", "--store", "app", "--params", "[]"], + ["execute", "SELECT 1", "--store", "app", "--params", "broken"], + ["query", "SELECT 1", "--store", "app", "--file", "missing.sql"], + ["query", "--store", "app", "--file", "missing.sql"], + ["query", "SELECT 1", "--store", "app", "--stdin"], + ["query", "--store", "app", "--stdin"], + ["table", "create", "--store", "app", "DELETE FROM items"], + ["table", "create", "--store", "app", "CREATE INDEX idx ON items(id)"], + ["table", "create", "--store", "app", "CREATE TABLE items (id INTEGER); DROP TABLE items"], + ["table", "create", "--store", "app", "not sql"], + ["table", "register", "--store", "app", "items"], + ["qdrant", "--store", "app", "collection", "list"], + ["qdrant", "collection", "list"], + ["merge", "atomic-apply", "trial", "main"], + ["merge", "apply", "trial", "main", "--policy", "unknown"], + ]) + def test_invalid_arguments_do_not_open_stores(self, workspace_connection: MagicMock, args) -> None: + result = runner.invoke(app, [*args], input="") + assert result.exit_code == 2, result.output + _workspace.open_workspace.assert_not_called() + + def test_help_and_configuration_precedence(self, workspace_connection: MagicMock, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + for args in ([], ["branch"], ["query"], ["qdrant"], ["qdrant", "point"], ["qdrant", "point", "upsert"], ["merge", "atomic-apply"]): + result = runner.invoke(app, ["--config", "missing.toml", *args, "--help"]) + assert result.exit_code == 0, result.output + _workspace.open_workspace.assert_not_called() + monkeypatch.setenv("CHRONOS_WORKSPACE_CONFIG", str(tmp_path / "missing.toml")) + result = runner.invoke(app, ["branch", "list"]) + assert result.exit_code == 2 + _workspace.open_workspace.assert_not_called() + result = runner.invoke(app, ["--config", str(tmp_path / "chronos.toml"), "branch", "list"]) + assert result.exit_code == 0, result.output + assert _workspace.open_workspace.call_args.args[0].config_path == tmp_path / "chronos.toml" + + def test_api_error_and_session_cleanup(self, workspace_connection: MagicMock) -> None: + workspace = workspace_connection + session = workspace.checkout.return_value.stores["app"] + session.query.side_effect = BranchingError("query failed") + session.close.side_effect = RuntimeError("close failed") + result = runner.invoke(app, ["query", "SELECT 1", "--store", "app"]) + assert result.exit_code == 1 + assert result.stdout == "" + assert "query failed" in result.stderr + assert "close failed" not in result.stderr + session.close.assert_called_once_with() + workspace.checkout.return_value.stores["other"].close.assert_called_once_with() + workspace.close.assert_called_once_with() + + def test_api_result_field_names_are_preserved(self, workspace_connection: MagicMock) -> None: + from chronos_core.workspace import AtomicMergeResult, WorkspaceBranchToken + token = WorkspaceBranchToken("main", 1) + result = AtomicMergeResult("merge-1", "trial", "main", "committed", token, token, token, 2, 0, {"app": 1, "other": 1}) + workspace_connection.merge_atomic.return_value = result + invoked = runner.invoke(app, ["merge", "atomic-apply", "trial", "main", "--operation-id", "merge-1"]) + assert invoked.exit_code == 0, invoked.output + assert json.loads(invoked.stdout) == { + "operation_id": "merge-1", "source": "trial", "target": "main", "status": "committed", + "source_token": {"branch_id": "main", "current_segment_id": 1}, + "old_target_token": {"branch_id": "main", "current_segment_id": 1}, + "new_target_token": {"branch_id": "main", "current_segment_id": 1}, + "selected": 2, "skipped": 0, "stores": {"app": 1, "other": 1}, + } + + +class TestWorkspaceQdrantCommands: + @pytest.fixture + def workspace(self, workspace_connection: MagicMock, tmp_path: Path) -> MagicMock: + pytest.importorskip("qdrant_client") + (tmp_path / "chronos.toml").write_text(''' +[stores.app] +kind="sqlite" +database_url="sqlite:///app.sqlite" +[stores.vectors] +kind="qdrant" +metadata_url="sqlite:///vectors.sqlite" +path="vectors" +''') + vector_store = MagicMock() + for method in ("register_collection", "list_collections", "collection_info"): + getattr(vector_store, method).return_value = None + vector_session = MagicMock() + for method in ("upsert", "get", "list_points", "delete", "search"): + getattr(vector_session, method).return_value = None + workspace_connection.stores["vectors"] = vector_store + workspace_connection.checkout.return_value.stores["vectors"] = vector_session + return workspace_connection + + @pytest.mark.parametrize(("args", "method", "positional", "keywords", "session"), [ + (["collection", "register", "docs", "--dimensions", "3", "--distance", "dot"], "register_collection", ("docs", 3), {"distance": "dot"}, False), + (["collection", "list"], "list_collections", (), {}, False), + (["collection", "info", "docs"], "collection_info", ("docs",), {}, False), + (["point", "upsert", "docs", "one", '{"vector":[1,0,0],"payload":{"value":"one"}}', "--revision", "v1"], + "upsert", ("docs", "one", [1, 0, 0], {"value": "one"}), {"revision": "v1"}, True), + (["point", "get", "docs", "one"], "get", ("docs", "one"), {}, True), + (["point", "list", "docs"], "list_points", ("docs",), {}, True), + (["point", "delete", "docs", "one"], "delete", ("docs", "one"), {}, True), + (["search", "docs", "[1,0,0]", "--limit", "3", "--score-threshold", "0.5"], "search", ("docs", [1, 0, 0]), {"limit": 3, "score_threshold": 0.5}, True), + ]) + def test_forwarding(self, workspace: MagicMock, args, method, positional, keywords, session: bool) -> None: + result = runner.invoke(app, ["qdrant", "--store", "vectors", *args, *(["--branch", "trial"] if session else [])]) + assert result.exit_code == 0, result.output + assert result.stdout == "" + target = workspace.checkout.return_value.stores["vectors"] if session else workspace.stores["vectors"] + getattr(target, method).assert_called_once_with(*positional, **keywords) + if session: + workspace.checkout.assert_called_once_with("trial") + workspace.close.assert_called_once_with() + + @pytest.mark.parametrize("data", ['{"vector":[true]}', '{"vector":[NaN]}', '{"vector":[1],"payload":[]}']) + def test_invalid_point_does_not_connect(self, workspace: MagicMock, data: str) -> None: + result = runner.invoke(app, ["qdrant", "--store", "vectors", "point", "upsert", "docs", "one", data]) + assert result.exit_code == 2, result.output + _workspace.open_workspace.assert_not_called() + + def test_transport_error_redacts_credentials(self, workspace: MagicMock, tmp_path: Path) -> None: + import httpx + (tmp_path / "chronos.toml").write_text(''' +[stores.app] +kind="postgresql" +database_url="postgresql://user:sql-secret@localhost/app" +[stores.vectors] +kind="qdrant" +metadata_url="sqlite:///vectors.sqlite" +url="http://localhost:6333" +api_key="vector-secret" +''') + workspace.checkout.return_value.stores["vectors"].search.side_effect = httpx.ConnectError( + "Failed using vector-secret and postgresql://user:sql-secret@localhost/app" + ) + result = runner.invoke(app, ["qdrant", "--store", "vectors", "search", "docs", "[1,0,0]"]) + assert result.exit_code == 1 + assert result.stdout == "" + assert "secret" not in result.stderr + assert "" in result.stderr + workspace.close.assert_called_once_with() + + def test_json_file_and_stdin(self, workspace: MagicMock, tmp_path: Path) -> None: + data = tmp_path / "point.json" + data.write_text('{"vector":[1,0,0],"payload":{"source":"file"}}') + result = runner.invoke(app, ["qdrant", "--store", "vectors", "point", "upsert", "docs", "one", "--file", str(data)]) + assert result.exit_code == 0, result.output + session = workspace.checkout.return_value.stores["vectors"] + session.upsert.assert_called_once_with("docs", "one", [1, 0, 0], {"source": "file"}, revision=None) + for args in (["--stdin"], []): + result = runner.invoke(app, ["qdrant", "--store", "vectors", "search", "docs", *args], input="[0,1,0]") + assert result.exit_code == 0, result.output + assert session.search.call_args.args == ("docs", [0, 1, 0]) + data.write_text("[1,0,0]") + result = runner.invoke(app, ["qdrant", "--store", "vectors", "search", "docs", "--file", str(data)]) + assert result.exit_code == 0, result.output + assert session.search.call_args.args == ("docs", [1, 0, 0]) + + +class TestWorkspaceFilesystemCommands: + def test_config_and_resource_ownership(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from chronos_core import workspace as api + + config_file = tmp_path / 'chronos.toml' + config_file.write_text('[filesystem]\nroot="base"\nstate_dir="state"\n') + config = load_workspace_config(config_file) + assert config.stores == {} + assert config.filesystem.root == tmp_path / 'base' + assert config.filesystem.state_dir == tmp_path / 'state' + factory = MagicMock() + monkeypatch.setattr(api, 'ChronosFilesystemStore', factory) + with open_workspace(config) as workspace: + assert workspace.filesystem is factory.return_value + factory.assert_called_once_with(tmp_path / 'base', state_dir=tmp_path / 'state') + factory.return_value.close.assert_called_once_with() + factory.return_value.cleanup.assert_not_called() + + @pytest.mark.parametrize('extra', [ + '[workspace]\nshared_metadata_url="sqlite:///meta.sqlite"\n', + '[stores.filesystem]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"\n', + ]) + def test_unsupported_config(self, tmp_path: Path, extra: str) -> None: + config_file = tmp_path / 'chronos.toml' + config_file.write_text('[filesystem]\nroot="base"\nstate_dir="state"\n' + extra) + with pytest.raises(WorkspaceConfigError): + load_workspace_config(config_file) + + def test_state_directory_is_required(self, tmp_path: Path) -> None: + config_file = tmp_path / 'chronos.toml' + config_file.write_text('[filesystem]\nroot="base"\n') + with pytest.raises(WorkspaceConfigError, match='state_dir'): + load_workspace_config(config_file) + + @pytest.mark.parametrize('as_json', [False, True]) + def test_run_forwards_arguments_and_output(self, workspace_connection: MagicMock, as_json: bool) -> None: + workspace = workspace_connection + command = ['example', '--child-option', 'two words'] + filesystem = workspace.checkout.return_value.fs + filesystem.run.return_value = subprocess.CompletedProcess(command, 7, '\x1b[32moutput\x1b[0m\n', '\x1b[31merror\x1b[0m\n') + args = ['fs', 'run', '--branch', 'feature', '--cwd', 'nested', '--env', '{"MODE":"test"}', '--timeout', '2'] + if as_json: + args.append('--json') + result = runner.invoke(app, [*args, '--', *command]) + assert result.exit_code == 7, result.output + filesystem.run.assert_called_once_with(command, cwd='nested', env={'MODE': 'test'}, timeout=2.0) + workspace.checkout.assert_called_once_with('feature') + if as_json: + assert json.loads(result.stdout) == dict(args=command, returncode=7, stdout='\x1b[32moutput\x1b[0m\n', stderr='\x1b[31merror\x1b[0m\n') + assert result.stderr == '' + else: + assert result.stdout == '\x1b[32moutput\x1b[0m\n' + assert result.stderr == '\x1b[31merror\x1b[0m\n' + for session in workspace.checkout.return_value.stores.values(): + session.close.assert_called_once_with() + workspace.close.assert_called_once_with() + + @pytest.mark.parametrize('args', [[], ['--env', '[]', '--', 'echo'], ['--env', '{"X":1}', '--', 'echo'], + ['--timeout', 'nan', '--', 'echo']]) + def test_invalid_input_before_connection(self, workspace_connection: MagicMock, args: list[str]) -> None: + result = runner.invoke(app, ['fs', 'run', *args]) + assert result.exit_code == 2 + _workspace.open_workspace.assert_not_called() + + @pytest.mark.parametrize('error', [FileNotFoundError('missing command'), subprocess.TimeoutExpired(['secret'], 1)]) + def test_run_failure_closes_resources(self, workspace_connection: MagicMock, error: Exception) -> None: + workspace_connection.checkout.return_value.fs.run.side_effect = error + result = runner.invoke(app, ['fs', 'run', '--', 'secret']) + assert result.exit_code == 1 + assert 'secret' not in result.stderr + workspace_connection.close.assert_called_once_with() + for session in workspace_connection.checkout.return_value.stores.values(): + session.close.assert_called_once_with() + + def test_no_filesystem(self, workspace_connection: MagicMock) -> None: + workspace_connection.filesystem = None + result = runner.invoke(app, ['fs', 'run', '--', 'echo']) + assert result.exit_code == 2 + workspace_connection.checkout.assert_not_called() + workspace_connection.close.assert_called_once_with() + + @pytest.mark.parametrize('operation', ['preview', 'apply', 'atomic-preview', 'atomic-apply']) + def test_api_default_merge_policy(self, workspace_connection: MagicMock, operation: str) -> None: + args = ['merge', operation, 'feature', 'main', '--policy', 'api-default'] + if operation == 'atomic-apply': + args.extend(['--operation-id', 'operation']) + result = runner.invoke(app, args) + assert result.exit_code == 0, result.output + name = {'preview': 'merge_preview', 'apply': 'merge_apply', + 'atomic-preview': 'merge_atomic_preview', 'atomic-apply': 'merge_atomic'}[operation] + assert getattr(workspace_connection, name).call_args.kwargs['policy'] is None + + def test_signal_exit_and_api_defaults(self, workspace_connection: MagicMock) -> None: + filesystem = workspace_connection.checkout.return_value.fs + filesystem.run.return_value = subprocess.CompletedProcess(['command'], -15, '', '') + result = runner.invoke(app, ['fs', 'run', '--', 'command']) + assert result.exit_code == 143 + filesystem.run.assert_called_once_with(['command'], cwd='.', env=None, timeout=None) + workspace_connection.checkout.assert_called_once_with('main') + + def test_help_needs_no_config(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.chdir(tmp_path) + monkeypatch.delenv('CHRONOS_WORKSPACE_CONFIG', raising=False) + result = runner.invoke(app, ['fs', 'run', '--help']) + assert result.exit_code == 0 + assert "--json" in Text.from_ansi(result.stdout).plain + + def test_filesystem_error_is_reported(self, workspace_connection: MagicMock) -> None: + from chronos_core.workspace.filesystem import FilesystemStoreError + + workspace_connection.checkout.side_effect = FilesystemStoreError('failed to mount branch') + result = runner.invoke(app, ['fs', 'run', '--', 'command']) + assert result.exit_code == 1 + assert 'failed to mount branch' in result.stderr + workspace_connection.close.assert_called_once_with() + + +def test_native_filesystem_commands(workspace_connection: MagicMock, tmp_path: Path) -> None: + from chronos_core.workspace import ChronosFSStore + + workspace_connection.filesystem = MagicMock(spec=ChronosFSStore) + session = workspace_connection.checkout.return_value.fs + payload = b"\x00\xff\n" + source = tmp_path / "payload" + source.write_bytes(payload) + session.write_file.return_value = 12 + result = runner.invoke(app, ["fs", "write", "/dir/file", "--branch", "draft", "--file", str(source), "--parents"]) + assert result.exit_code == 0, result.output + assert json.loads(result.stdout) == 12 + workspace_connection.checkout.assert_called_with("draft") + session.write_file.assert_called_once_with("/dir/file", payload, parents=True) + + session.read_file.return_value = payload + result = runner.invoke(app, ["fs", "read", "/dir/file"]) + assert result.exit_code == 0, result.output + assert result.stdout_bytes == payload + session.read_file.assert_called_once_with("/dir/file") + + session.listdir.return_value = ["file"] + result = runner.invoke(app, ["fs", "ls", "/dir"]) + assert result.exit_code == 0, result.output + assert json.loads(result.stdout) == ["file"] + session.listdir.assert_called_once_with("/dir") + + result = runner.invoke(app, ["fs", "run", "--", "true"]) + assert result.exit_code == 2 + assert "requires the OverlayFS backend" in result.stderr + + +@pytest.mark.parametrize("args", [[], ["--file", __file__, "--stdin"]]) +def test_native_write_requires_one_input(workspace_connection: MagicMock, args: list[str]) -> None: + result = runner.invoke(app, ["fs", "write", "/file", *args]) + assert result.exit_code == 2 + workspace_connection.checkout.assert_not_called() + + +def test_native_filesystem_requires_matching_atomic_metadata(tmp_path: Path) -> None: + config = tmp_path / "chronos.toml" + config.write_text( + '[workspace]\nshared_metadata_url="sqlite:///shared.sqlite"\n' + '[filesystem]\nkind="chronosfs"\ndatabase_url="sqlite:///files.sqlite"\n' + ) + with pytest.raises(WorkspaceConfigError, match="Every store must use"): + load_workspace_config(config) + + +@pytest.mark.parametrize("scope", ["", "merge_tables=[]\n"]) +def test_native_atomic_split_sql_requires_explicit_scope(tmp_path: Path, scope: str) -> None: + config = tmp_path / "chronos.toml" + config.write_text( + '[workspace]\nshared_metadata_url="sqlite:///app.sqlite"\n' + '[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"\n' + + scope + '[filesystem]\nkind="chronosfs"\ndatabase_url="sqlite:///files.sqlite"\n' + 'metadata_url="sqlite:///app.sqlite"\n' + ) + with pytest.raises(WorkspaceConfigError, match="require nonempty merge_tables"): + load_workspace_config(config) + + +@pytest.mark.parametrize("scope", ["", "merge_tables=[]\n"]) +def test_default_merge_scope_includes_all_tables(tmp_path: Path, scope: str) -> None: + config = tmp_path / "chronos.toml" + config.write_text('[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"\n' + scope) + assert load_workspace_config(config).stores["app"].merge_tables is None + + +@pytest.mark.parametrize("mount_error", [False, True]) +def test_native_mount_delegates_and_closes( + workspace_connection: MagicMock, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, mount_error: bool, +) -> None: + from chronos_core import workspace as api + + filesystem = MagicMock(spec=api.ChronosFSStore) + workspace_connection.filesystem = filesystem + mountpoint = tmp_path / "mount" + mountpoint.mkdir() + failure = api.ChronosFSMountError("mount failed") if mount_error else None + mount = MagicMock(side_effect=failure) + monkeypatch.setattr(api, "mount_chronosfs", mount) + + result = runner.invoke(app, [ + "fs", "mount", str(mountpoint), "--branch", "draft", "-o", "ro", "--option", "default_permissions", + ]) + assert result.exit_code == (1 if mount_error else 0), result.output + if mount_error: + assert "mount failed" in result.stderr + mount.assert_called_once_with( + filesystem, mountpoint.resolve(), branch_id="draft", options={"ro", "default_permissions"}, + ) + workspace_connection.checkout.assert_not_called() + workspace_connection.close.assert_called_once_with() diff --git a/tests/test_cli_integration.py b/tests/test_cli_integration.py new file mode 100644 index 0000000..670aaff --- /dev/null +++ b/tests/test_cli_integration.py @@ -0,0 +1,859 @@ +"""Real CLI workflows across independent relational and vector stores.""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import textwrap +import time +from dataclasses import asdict +from functools import partial +from pathlib import Path +from tempfile import TemporaryDirectory +from typing import Any, Iterator +from urllib.parse import quote, urlsplit, urlunsplit +from uuid import uuid4 + +import psycopg +import pytest +from typer.testing import CliRunner +from psycopg import sql as postgres_sql + +from chronos_core import _native_interval +from chronos_core.cli._config import DatabaseConfig, QdrantConfig, load_workspace_config, open_workspace + + +def _run_cli(arguments: list[str], *, environment: dict[str, str], + ok: bool = True, input_text: str | None = None) -> Any: + result = subprocess.run( + [sys.executable, "-X", "faulthandler", "-m", "chronos_core.cli", *arguments], + env=environment, input=input_text, capture_output=True, text=True, timeout=30, + ) + assert result.returncode == (0 if ok else 1), result.stderr + assert "Traceback" not in result.stderr + if not ok: + assert result.stdout == "" + return json.loads(result.stdout) if result.stdout.strip() else None + + +@pytest.fixture(scope="session") +def postgres_cli_database_url() -> Iterator[str]: + """Create one owned database for the run, without resetting the admin database.""" + admin_url = os.environ.get("CHRONOS_CLI_TEST_POSTGRES_ADMIN_URL") + if not admin_url: + pytest.skip("Set CHRONOS_CLI_TEST_POSTGRES_ADMIN_URL to test PostgreSQL") + parsed = urlsplit(admin_url) + if parsed.scheme not in {"postgres", "postgresql"}: + pytest.fail("CHRONOS_CLI_TEST_POSTGRES_ADMIN_URL must be a PostgreSQL URL") + name = f"chronos_cli_test_{uuid4().hex}" + with psycopg.connect(admin_url, autocommit=True, connect_timeout=10) as admin: + admin.execute(postgres_sql.SQL("CREATE DATABASE {}").format(postgres_sql.Identifier(name))) + try: + yield urlunsplit(parsed._replace(path=f"/{name}")) + finally: + admin.execute( + postgres_sql.SQL("DROP DATABASE {} WITH (FORCE)").format(postgres_sql.Identifier(name)) + ) + + +@pytest.fixture(params=["sqlite", "postgres", "duckdb-sqlite", "duckdb-postgres"]) +def cli_database(request: pytest.FixtureRequest, tmp_path: Path) -> DatabaseConfig: + kind = request.param + if kind.startswith("duckdb") and not _native_interval.has_duckdb: + if os.environ.get("CHRONOS_CLI_REQUIRE_DUCKDB") == "ON": + pytest.fail("The CLI test matrix requires a DuckDB-enabled native build") + pytest.skip("DuckDB disabled in this native build") + if kind == "sqlite": + return DatabaseConfig(f"sqlite:///{tmp_path / 'workflow.sqlite'}") + if kind == "duckdb-sqlite": + return DatabaseConfig( + f"duckdb:///{tmp_path / 'workflow.duckdb'}", + f"sqlite:///{tmp_path / 'metadata.sqlite'}", + ) + url = request.getfixturevalue("postgres_cli_database_url") + # This URL belongs only to the database created by our session fixture. + with psycopg.connect(url, autocommit=True, connect_timeout=10) as database: + database.execute("DROP SCHEMA public CASCADE") + database.execute("CREATE SCHEMA public") + return DatabaseConfig( + f"duckdb:///{tmp_path / 'workflow.duckdb'}", url, + ) if kind == "duckdb-postgres" else DatabaseConfig(url) + + +@pytest.fixture(params=["local-sqlite", "remote-sqlite", "local-postgres", "remote-postgres"]) +def qdrant_target(request: pytest.FixtureRequest): + client_type = pytest.importorskip("qdrant_client").QdrantClient + from qdrant_client.http.exceptions import ResponseHandlingException, UnexpectedResponse + + kind = request.param + remote_url = os.environ.get("CHRONOS_CLI_TEST_QDRANT_URL") + api_key = os.environ.get("CHRONOS_CLI_TEST_QDRANT_API_KEY") + if kind.startswith("remote") and not remote_url: + pytest.skip("Set CHRONOS_CLI_TEST_QDRANT_URL to test the Qdrant server") + client = None + prefix = f"cli_test_{uuid4().hex}_" + if kind.startswith("remote"): + client = client_type(url=remote_url, api_key=api_key, timeout=2) + deadline = time.monotonic() + 30 + while True: + try: + client.get_collections() + break + except (ResponseHandlingException, UnexpectedResponse): + if time.monotonic() >= deadline: + client.close() + pytest.fail("Configured Qdrant test server is not ready after 30 seconds") + time.sleep(0.5) + try: + with TemporaryDirectory(prefix="chronos-cli-qdrant-") as directory: + root = Path(directory) + metadata = f"sqlite:///{root / 'metadata.sqlite'}" + if kind.endswith("postgres"): + metadata = request.getfixturevalue("postgres_cli_database_url") + # Only reset the database that our session fixture owns. + with psycopg.connect(metadata, autocommit=True) as admin: + admin.execute("DROP SCHEMA public CASCADE") + admin.execute("CREATE SCHEMA public") + yield QdrantConfig( + metadata_url=metadata, + path=None if kind.startswith("remote") else root / "vectors", + url=remote_url if kind.startswith("remote") else None, + api_key=api_key if kind.startswith("remote") else None, + collection_prefix=prefix, + ) + finally: + if client is not None: + try: + for collection in client.get_collections().collections: + if collection.name.startswith(prefix): + client.delete_collection(collection.name) + finally: + client.close() + + +def _open_qdrant_api(target: QdrantConfig): + from chronos_core.workspace.qdrant import ChronosQdrantStore + + if target.path is not None: + return ChronosQdrantStore.local( + target.metadata_url, path=target.path, collection_prefix=target.collection_prefix, + ) + return ChronosQdrantStore.remote( + target.metadata_url, url=target.url, api_key=target.api_key, + collection_prefix=target.collection_prefix, + ) + + +def _workspace_config_file(root: Path, database: DatabaseConfig, vectors: QdrantConfig | None = None, *, atomic: bool = False) -> Path: + lines = [] + if atomic: + lines.extend(["[workspace]", f"shared_metadata_url={json.dumps(database.metadata_url or database.database_url)}"]) + lines.extend(["[stores.app]", f"kind={json.dumps(urlsplit(database.database_url).scheme)}", + f"database_url={json.dumps(database.database_url)}"]) + if database.metadata_url: + lines.append(f"metadata_url={json.dumps(database.metadata_url)}") + if vectors is not None: + lines.extend(["[stores.vectors]", 'kind="qdrant"', f"metadata_url={json.dumps(vectors.metadata_url)}", + f"collection_prefix={json.dumps(vectors.collection_prefix)}"]) + if vectors.path is not None: + lines.append(f"path={json.dumps(str(vectors.path))}") + else: + lines.append(f"url={json.dumps(vectors.url)}") + if vectors.api_key: + lines.append(f"api_key={json.dumps(vectors.api_key)}") + config = root / "chronos.toml" + config.write_text("\n".join(lines) + "\n") + return config + + +def _invoke_workspace(config: Path, *args: str, ok: bool = True, input_text: str | None = None) -> Any: + environment = {key: value for key, value in os.environ.items() if not key.startswith("CHRONOS_")} + environment["CHRONOS_WORKSPACE_CONFIG"] = str(config) + return _run_cli([*args], environment=environment, ok=ok, input_text=input_text) + + +def _prepare_workspace_merge(root: Path, *, atomic: bool) -> Path: + pytest.importorskip("qdrant_client") + database = DatabaseConfig(f"sqlite:///{root / 'app.sqlite'}") + target = QdrantConfig( + metadata_url=database.database_url if atomic else f"sqlite:///{root / 'vectors.sqlite'}", + path=root / "vectors", + ) + config = _workspace_config_file(root, database, target, atomic=atomic) + _invoke_workspace(config, "table", "create", "CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)", "--store", "app") + _invoke_workspace(config, "table", "register", "items", "--primary-key", "id", "--store", "app") + _invoke_workspace(config, "execute", "INSERT INTO items (id, value) VALUES (1, 'base')", "--store", "app") + _invoke_workspace(config, "qdrant", "--store", "vectors", "collection", "register", "documents", "--dimensions", "3") + _invoke_workspace(config, "branch", "create", "experiment") + _invoke_workspace(config, "execute", "UPDATE items SET value = 'candidate' WHERE id = 1", "--store", "app", "--branch", "experiment") + _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "upsert", "documents", "one", '{"vector":[1,0,0]}', "--branch", "experiment") + return config + + +def _qdrant_workspace_file(root: Path, target: QdrantConfig) -> Path: + config = _workspace_config_file(root, DatabaseConfig(target.metadata_url), target) + # A Qdrant-only workspace still has its own relational metadata context. + text = config.read_text() + config.write_text(text[text.index("[stores.vectors]"):]) + return config + + +class TestWorkspaceResources: + @pytest.mark.parametrize("kind", ["sqlite", "sqlite-split", "duckdb-sqlite"]) + def test_database_paths_preserve_reserved_characters(self, tmp_path: Path, kind: str) -> None: + if kind == "duckdb-sqlite" and not _native_interval.has_duckdb: + if os.environ.get("CHRONOS_CLI_REQUIRE_DUCKDB") == "ON": + pytest.fail("This test run requires a DuckDB-enabled native build") + pytest.skip("DuckDB disabled in this native build") + root = tmp_path / ("workspace #%20+目录" if kind == "duckdb-sqlite" else "workspace #?%20+目录") + root.mkdir() + scheme = "duckdb" if kind == "duckdb-sqlite" else "sqlite" + data = root / ("data%20#.duckdb" if scheme == "duckdb" else "data%20#?.sqlite") + metadata = root / "metadata%20#?.sqlite" + config = root / "chronos.toml" + config.write_text( + f'[stores.app]\nkind="{scheme}"\ndatabase_url="{scheme}:///{quote(data.name, safe="")}"\n' + + (f'metadata_url="sqlite:///{quote(metadata.name, safe="")}"\n' if kind != "sqlite" else '') + ) + loaded = load_workspace_config(config) + with open_workspace(loaded) as workspace: + app = workspace.stores["app"] + app.db.execute("CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)") + app.db.commit() + app.register_table("items", ["id"]) + with app.checkout("main") as session: + session.execute("INSERT INTO items (id, value) VALUES (1, 'persisted')") + assert data.is_file() + if kind != "sqlite": + assert metadata.is_file() + with open_workspace(loaded) as workspace: + with workspace.stores["app"].checkout("main") as session: + assert session.query("SELECT value FROM items") == [{"value": "persisted"}] + assert sorted(path.name for path in tmp_path.iterdir()) == [root.name] + + def test_hash_directories_keep_workspace_state_separate(self, tmp_path: Path) -> None: + configs = [] + for name in ("workspace#one", "workspace#two"): + root = tmp_path / name + root.mkdir() + config = root / "chronos.toml" + config.write_text('[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"\n') + configs.append(config) + _invoke_workspace(configs[0], "branch", "create", "private-to-one") + assert _invoke_workspace(configs[1], "branch", "list") == ["main"] + assert sorted(path.name for path in tmp_path.iterdir()) == ["workspace#one", "workspace#two"] + + def test_sqlite_workspace_without_qdrant_dependency(self, tmp_path: Path) -> None: + config_file = tmp_path / "chronos.toml" + config_file.write_text('[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"\n') + code = ''' + import importlib.abc + import json + import sys + from pathlib import Path + + class BlockQdrant(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == "qdrant_client" or fullname.startswith("qdrant_client."): + raise ModuleNotFoundError("simulated missing qdrant-client") + + sys.meta_path.insert(0, BlockQdrant()) + from chronos_core.cli._config import load_workspace_config, open_workspace + with open_workspace(load_workspace_config(Path(sys.argv[1]))) as workspace: + print(json.dumps(workspace.list_branches())) + ''' + result = subprocess.run( + [sys.executable, "-X", "faulthandler", "-c", textwrap.dedent(code), str(config_file)], + capture_output=True, text=True, timeout=30, + ) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout) == ["main"] + + def test_sqlite_workspace_persists_after_close(self, tmp_path: Path) -> None: + config_file = tmp_path / "chronos.toml" + config_file.write_text('[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"\n') + config = load_workspace_config(config_file) + with open_workspace(config) as workspace: + database = workspace.stores["app"] + database.db.execute("CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)") + database.db.commit() + database.register_table("items", ["id"]) + session = workspace.checkout().stores["app"] + session.execute("INSERT INTO items (id, value) VALUES (1, 'persisted')") + assert (tmp_path / "app.sqlite").is_file() + with open_workspace(config) as workspace: + assert workspace.checkout().stores["app"].query("SELECT * FROM items") == [ + {"id": 1, "value": "persisted"}, + ] + + def test_add_qdrant_to_existing_sqlite_workspace(self, tmp_path: Path) -> None: + pytest.importorskip("qdrant_client") + config_file = tmp_path / "chronos.toml" + sqlite_config = '[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"\n' + config_file.write_text(sqlite_config) + with open_workspace(load_workspace_config(config_file)) as workspace: + database = workspace.stores["app"] + database.db.execute("CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)") + database.db.commit() + database.register_table("items", ["id"]) + workspace.checkout().stores["app"].execute("INSERT INTO items (id, value) VALUES (1, 'base')") + workspace.create_branch("experiment") + workspace.checkout("experiment").stores["app"].execute("UPDATE items SET value = 'candidate' WHERE id = 1") + config_file.write_text(sqlite_config + ''' +[workspace] +shared_metadata_url="sqlite:///app.sqlite" +[stores.vectors] +kind="qdrant" +metadata_url="sqlite:///app.sqlite" +path="vectors" +''') + config = load_workspace_config(config_file) + with open_workspace(config) as workspace: + assert set(workspace.list_branches()) == {"main", "experiment"} + workspace.stores["vectors"].register_collection("documents", 2) + main = workspace.checkout("main") + experiment = workspace.checkout("experiment") + assert main.stores["app"].query("SELECT * FROM items") == [{"id": 1, "value": "base"}] + assert experiment.stores["app"].query("SELECT * FROM items") == [{"id": 1, "value": "candidate"}] + experiment.stores["vectors"].upsert("documents", "one", [1.0, 0.0], {"value": "candidate"}) + assert main.stores["vectors"].get("documents", "one") is None + result = workspace.merge_atomic( + "experiment", "main", operation_id="added-qdrant", policy="abort_on_conflict", + ) + assert result.status == "committed" + assert result.stores == {"app": 1, "vectors": 1} + with open_workspace(config) as workspace: + main = workspace.checkout("main") + assert main.stores["app"].query("SELECT * FROM items") == [{"id": 1, "value": "candidate"}] + assert main.stores["vectors"].get("documents", "one").payload == {"value": "candidate"} + + @pytest.mark.parametrize("atomic", [False, True]) + def test_sqlite_qdrant_workspace_reopens(self, tmp_path: Path, atomic: bool) -> None: + pytest.importorskip("qdrant_client") + config_file = tmp_path / "chronos.toml" + header = '[workspace]\nshared_metadata_url="sqlite:///app.sqlite"\n' if atomic else "" + metadata = "app.sqlite" if atomic else "vectors.sqlite" + config_file.write_text(header + f''' +[stores.app] +kind="sqlite" +database_url="sqlite:///app.sqlite" +[stores.vectors] +kind="qdrant" +metadata_url="sqlite:///{metadata}" +path="vectors" +''') + config = load_workspace_config(config_file) + with open_workspace(config) as workspace: + database = workspace.stores["app"] + database.db.execute("CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)") + database.db.commit() + database.register_table("items", ["id"]) + workspace.stores["vectors"].register_collection("documents", 2) + workspace.create_branch("experiment") + session = workspace.checkout("experiment") + session.stores["app"].execute("INSERT INTO items (id, value) VALUES (1, 'candidate')") + session.stores["vectors"].upsert("documents", "one", [1.0, 0.0], {"value": "candidate"}) + workspace.create_checkpoint("saved", branch="experiment") + assert (tmp_path / "app.sqlite").is_file() + assert (tmp_path / "vectors").is_dir() + with open_workspace(config) as workspace: + main = workspace.checkout("main") + assert main.stores["app"].query("SELECT * FROM items") == [] + assert main.stores["vectors"].get("documents", "one") is None + assert {name: len(diff.changes) for name, diff in workspace.diff("main", "experiment").items()} == { + "app": 1, "vectors": 1, + } + if atomic: + preview = workspace.merge_atomic_preview("experiment", "main", policy="abort_on_conflict") + result = workspace.merge_atomic( + "experiment", "main", policy="abort_on_conflict", + operation_id="workspace-persistence", preview_token=preview.preview_token, + ) + assert result.status == "committed" + assert result.stores == {"app": 1, "vectors": 1} + else: + assert set(workspace.merge_apply("experiment", "main", policy="abort_on_conflict")) == {"app", "vectors"} + with open_workspace(config) as workspace: + main = workspace.checkout("main") + assert main.stores["app"].query("SELECT * FROM items") == [{"id": 1, "value": "candidate"}] + assert main.stores["vectors"].get("documents", "one").payload == {"value": "candidate"} + workspace.delete_branch("experiment") + + +class TestWorkspaceSQLWorkflows: + def test_atomic_merge_across_separate_sql_databases(self, tmp_path: Path) -> None: + config = tmp_path / "chronos.toml" + config.write_text(textwrap.dedent(""" + [workspace] + shared_metadata_url = "sqlite:///metadata.sqlite" + + [stores.app] + kind = "sqlite" + database_url = "sqlite:///app.sqlite" + metadata_url = "sqlite:///metadata.sqlite" + merge_tables = ["items"] + + [stores.audit] + kind = "sqlite" + database_url = "sqlite:///audit.sqlite" + metadata_url = "sqlite:///metadata.sqlite" + merge_tables = ["events"] + """)) + invoke = partial(_invoke_workspace, config) + tables = {"app": "items", "audit": "events"} + for store, table in tables.items(): + invoke("table", "create", f"CREATE TABLE {table} (id INTEGER PRIMARY KEY)", "--store", store) + invoke("table", "register", table, "--store", store, "--primary-key", "id") + invoke("branch", "create", "draft") + for store, table in tables.items(): + invoke("execute", f"INSERT INTO {table} (id) VALUES (1)", "--store", store, "--branch", "draft") + assert invoke("query", f"SELECT id FROM {table}", "--store", store) == [] + + preview = invoke("merge", "atomic-preview", "draft", "main") + assert { + store: [change["table"] for change in result["changes"]] + for store, result in preview["stores"].items() + } == {"app": ["items"], "audit": ["events"]} + result = invoke( + "merge", "atomic-apply", "draft", "main", "--operation-id", "separate-sql", + "--preview-token", preview["preview_token"], + ) + assert result["status"] == "committed" + assert result["stores"] == {"app": 1, "audit": 1} + for store, table in tables.items(): + assert invoke("query", f"SELECT id FROM {table}", "--store", store) == [{"id": 1}] + + def test_nonfinite_api_result_is_not_printed_as_invalid_json(self, tmp_path: Path) -> None: + config = _workspace_config_file(tmp_path, DatabaseConfig(f"sqlite:///{tmp_path / 'app.sqlite'}")) + with open_workspace(load_workspace_config(config)) as workspace: + with workspace.stores["app"].checkout("main") as session: + assert session.query("SELECT 1e999 AS value") == [{"value": float("inf")}] + _invoke_workspace(config, "query", "SELECT 1e999 AS value", "--store", "app", ok=False) + assert _invoke_workspace(config, "query", "SELECT 1 AS value", "--store", "app") == [{"value": 1}] + + def test_relational_workspace(self, cli_database: DatabaseConfig, tmp_path: Path) -> None: + config = _workspace_config_file(tmp_path, cli_database) + _invoke_workspace(config, "table", "create", "CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)", "--store", "app") + _invoke_workspace(config, "table", "register", "items", "--primary-key", "id", "--store", "app") + _invoke_workspace(config, "execute", "INSERT INTO items (id, value) VALUES (:id, :value)", "--store", "app", "--params", '{"id":1,"value":"base"}') + # A second independently backed SQL store needs no special adapter in the CLI. + audit_url = f"sqlite:///{tmp_path / 'audit.sqlite'}" + with config.open("a") as stream: + stream.write(f'\n[stores.audit]\nkind="sqlite"\ndatabase_url={json.dumps(audit_url)}\n') + _invoke_workspace(config, "table", "create", "CREATE TABLE audit (id INTEGER PRIMARY KEY, value TEXT)", "--store", "audit") + _invoke_workspace(config, "table", "register", "audit", "--primary-key", "id", "--store", "audit") + _invoke_workspace(config, "branch", "create", "experiment") + _invoke_workspace(config, "execute", "UPDATE items SET value = 'candidate' WHERE id = 1", "--store", "app", "--branch", "experiment") + _invoke_workspace(config, "execute", "INSERT INTO audit (id, value) VALUES (1, 'candidate')", "--store", "audit", "--branch", "experiment") + assert _invoke_workspace(config, "query", "SELECT value FROM items", "--store", "app") == [{"value": "base"}] + assert _invoke_workspace(config, "query", "SELECT * FROM audit", "--store", "audit") == [] + checkpoint = _invoke_workspace(config, "checkpoint", "create", "saved", "--branch", "experiment") + assert set(checkpoint) == {"app", "audit"} + _invoke_workspace(config, "branch", "create-from-checkpoint", "restored", "saved") + assert _invoke_workspace(config, "query", "SELECT value FROM items", "--store", "app", "--branch", "restored") == [{"value": "candidate"}] + difference = _invoke_workspace(config, "diff", "main", "experiment") + assert {name: len(result["changes"]) for name, result in difference.items()} == {"app": 1, "audit": 1} + assert set(_invoke_workspace(config, "merge", "preview", "experiment", "main")) == {"app", "audit"} + _invoke_workspace(config, "merge", "atomic-preview", "experiment", "main", ok=False) + assert set(_invoke_workspace(config, "merge", "apply", "experiment", "main")) == {"app", "audit"} + assert _invoke_workspace(config, "query", "SELECT value FROM items", "--store", "app") == [{"value": "candidate"}] + assert _invoke_workspace(config, "query", "SELECT value FROM audit", "--store", "audit") == [{"value": "candidate"}] + _invoke_workspace(config, "branch", "create", "conflicting") + _invoke_workspace(config, "execute", "UPDATE items SET value = 'source' WHERE id = 1", "--store", "app", "--branch", "conflicting") + _invoke_workspace(config, "execute", "UPDATE items SET value = 'target' WHERE id = 1", "--store", "app") + assert len(_invoke_workspace(config, "merge", "preview", "conflicting", "main")["app"]["conflicts"]) == 1 + for policy in ("abort_on_conflict", "snapshot_isolation"): + _invoke_workspace(config, "merge", "apply", "conflicting", "main", "--policy", policy, ok=False) + assert _invoke_workspace(config, "query", "SELECT value FROM items", "--store", "app") == [{"value": "target"}] + _invoke_workspace(config, "branch", "delete", "conflicting") + _invoke_workspace(config, "query", "SELECT * FROM items", "--store", "app", "--branch", "missing", ok=False) + _invoke_workspace(config, "execute", "DROP TABLE items", "--store", "app", ok=False) + _invoke_workspace(config, "branch", "delete", "restored") + _invoke_workspace(config, "branch", "delete", "experiment") + assert _invoke_workspace(config, "branch", "list") == ["main"] + + def test_relational_sql_inputs_and_rollback(self, cli_database: DatabaseConfig, tmp_path: Path) -> None: + config = _workspace_config_file(tmp_path, cli_database) + invoke = partial(_invoke_workspace, config) + statement = tmp_path / "statement.sql" + statement.write_text( + "CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)", encoding="utf-8", + ) + invoke("table", "create", "--store", "app", "--file", str(statement)) + invoke("table", "register", "--store", "app", "items", "--primary-key", "id") + statement.write_text( + "INSERT INTO items (id, value) VALUES (:id, :value)", encoding="utf-8", + ) + assert invoke("execute", "--store", "app", "--file", str(statement), "--params", + '{"id": 1, "value": "100% done"}') == {"rowcount": 1} + assert invoke("query", "--store", "app", "--stdin", "--params", '{"pattern": "100%"}', + input_text="SELECT * FROM items WHERE value LIKE :pattern") == [ + {"id": 1, "value": "100% done"}, + ] + assert invoke("execute", "--store", "app", "--stdin", "--params", '{"id": 1, "value": "updated"}', + input_text="UPDATE items SET value = :value WHERE id = :id") == {"rowcount": 1} + statement.write_text("SELECT * FROM items", encoding="utf-8") + assert invoke("query", "--store", "app", "--file", str(statement)) == [{"id": 1, "value": "updated"}] + # A failed statement must not leave its earlier rows committed. + invoke("execute", "--store", "app", "INSERT INTO items (id, value) VALUES (2, 'partial'), (1, 'duplicate')", ok=False) + assert invoke("query", "--store", "app", input_text="SELECT * FROM items ORDER BY id") == [ + {"id": 1, "value": "updated"}, + ] + + def test_relational_aggregation(self, cli_database: DatabaseConfig, tmp_path: Path) -> None: + config = _workspace_config_file(tmp_path, cli_database) + invoke = partial(_invoke_workspace, config) + invoke("table", "create", "--store", "app", "CREATE TABLE sales (id INTEGER PRIMARY KEY, category TEXT, amount BIGINT)") + invoke("table", "register", "--store", "app", "sales", "--primary-key", "id") + invoke("execute", "--store", "app", "INSERT INTO sales (id, category, amount) VALUES (1, 'books', 10), (2, 'books', 20), (3, 'games', 50)") + invoke("branch", "create", "forecast") + invoke("execute", "--store", "app", "UPDATE sales SET amount = amount * :factor WHERE category = :category", + "--branch", "forecast", "--params", '{"factor": 2, "category": "books"}') + query = "SELECT category, SUM(amount) AS total FROM sales GROUP BY category ORDER BY category" + # PostgreSQL SUM(BIGINT) returns NUMERIC, serialized as a precision-preserving string. + total = str if urlsplit(cli_database.database_url).scheme in {"postgres", "postgresql"} else int + assert invoke("query", "--store", "app", query) == [ + {"category": "books", "total": total(30)}, {"category": "games", "total": total(50)}, + ] + assert invoke("query", "--store", "app", query, "--branch", "forecast") == [ + {"category": "books", "total": total(60)}, {"category": "games", "total": total(50)}, + ] + + +class TestWorkspaceQdrantWorkflows: + def test_duckdb_and_qdrant(self, tmp_path: Path) -> None: + if not _native_interval.has_duckdb: + if os.environ.get("CHRONOS_CLI_REQUIRE_DUCKDB") == "ON": + pytest.fail("The CLI test matrix requires a DuckDB-enabled native build") + pytest.skip("DuckDB disabled in this native build") + pytest.importorskip("qdrant_client") + metadata = f"sqlite:///{tmp_path / 'metadata.sqlite'}" + database = DatabaseConfig(f"duckdb:///{tmp_path / 'data.duckdb'}", metadata) + vectors = QdrantConfig(metadata, path=tmp_path / "vectors") + config = _workspace_config_file(tmp_path, database, vectors, atomic=True) + _invoke_workspace(config, "table", "create", "CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)", "--store", "app") + _invoke_workspace(config, "table", "register", "items", "--primary-key", "id", "--store", "app") + _invoke_workspace(config, "qdrant", "--store", "vectors", "collection", "register", "documents", "--dimensions", "3") + _invoke_workspace(config, "branch", "create", "experiment") + _invoke_workspace(config, "execute", "INSERT INTO items (id, value) VALUES (1, 'candidate')", "--store", "app", "--branch", "experiment") + _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "upsert", "documents", "one", '{"vector":[1,0,0]}', "--branch", "experiment") + assert _invoke_workspace(config, "query", "SELECT * FROM items", "--store", "app") == [] + assert _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "get", "documents", "one") is None + preview = _invoke_workspace(config, "merge", "atomic-preview", "experiment", "main") + assert {name: len(result["changes"]) for name, result in preview["stores"].items()} == {"app": 1, "vectors": 1} + result = _invoke_workspace(config, "merge", "atomic-apply", "experiment", "main", "--operation-id", "duckdb-qdrant", "--preview-token", preview["preview_token"]) + assert result["status"] == "committed" + assert result["stores"] == {"app": 1, "vectors": 1} + assert _invoke_workspace(config, "query", "SELECT * FROM items", "--store", "app") == [{"id": 1, "value": "candidate"}] + assert _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "get", "documents", "one")["vector"] == [1, 0, 0] + + @pytest.mark.parametrize("atomic", [False, True]) + def test_two_store_workflow(self, qdrant_target: QdrantConfig, tmp_path: Path, atomic: bool) -> None: + target = qdrant_target + database = DatabaseConfig(target.metadata_url if atomic else f"sqlite:///{tmp_path / 'app.sqlite'}") + # Begin with SQLite/PostgreSQL alone, then add Qdrant to existing data. + config = _workspace_config_file(tmp_path, database) + _invoke_workspace(config, "table", "create", "CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)", "--store", "app") + _invoke_workspace(config, "table", "register", "items", "--primary-key", "id", "--store", "app") + _invoke_workspace(config, "execute", "INSERT INTO items (id, value) VALUES (1, 'base')", "--store", "app") + config = _workspace_config_file(tmp_path, database, target, atomic=atomic) + _invoke_workspace(config, "qdrant", "--store", "vectors", "collection", "register", "documents", "--dimensions", "3") + assert _invoke_workspace(config, "qdrant", "--store", "vectors", "collection", "info", "documents")["dimensions"] == 3 + _invoke_workspace(config, "branch", "create", "experiment") + _invoke_workspace(config, "execute", "UPDATE items SET value = 'candidate' WHERE id = 1", "--store", "app", "--branch", "experiment") + _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "upsert", "documents", "one", "--stdin", "--branch", "experiment", input_text='{"vector":[1,0,0],"payload":{"value":"candidate"}}') + assert _invoke_workspace(config, "query", "--store", "app", "--stdin", input_text="SELECT value FROM items") == [{"value": "base"}] + assert _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "get", "documents", "one") is None + assert _invoke_workspace(config, "qdrant", "--store", "vectors", "search", "documents", "[1,0,0]", "--branch", "experiment")[0]["id"] == "one" + _invoke_workspace(config, "checkpoint", "create", "saved", "--branch", "experiment") + _invoke_workspace(config, "branch", "create-from-checkpoint", "restored", "saved") + assert _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "get", "documents", "one", "--branch", "restored")["payload"] == {"value": "candidate"} + assert {name: len(result["changes"]) for name, result in _invoke_workspace(config, "diff", "main", "experiment").items()} == {"app": 1, "vectors": 1} + if atomic: + preview = _invoke_workspace(config, "merge", "atomic-preview", "experiment", "main") + _invoke_workspace(config, "merge", "apply", "experiment", "main", ok=False) + result = _invoke_workspace(config, "merge", "atomic-apply", "experiment", "main", "--operation-id", "workspace-demo", "--preview-token", preview["preview_token"]) + assert result["status"] == "committed" + assert result["stores"] == {"app": 1, "vectors": 1} + else: + assert set(_invoke_workspace(config, "merge", "preview", "experiment", "main")) == {"app", "vectors"} + _invoke_workspace(config, "merge", "apply", "experiment", "main") + assert _invoke_workspace(config, "query", "SELECT value FROM items", "--store", "app") == [{"value": "candidate"}] + assert _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "get", "documents", "one")["payload"] == {"value": "candidate"} + _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "delete", "documents", "one") + assert _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "list", "documents") == [] + _invoke_workspace(config, "branch", "delete", "restored") + _invoke_workspace(config, "branch", "delete", "experiment") + assert _invoke_workspace(config, "branch", "list") == ["main"] + + def test_qdrant_cli_conflicts_and_failed_operations(self, qdrant_target: QdrantConfig, tmp_path: Path) -> None: + config = _qdrant_workspace_file(tmp_path, qdrant_target) + vectors = partial(_invoke_workspace, config, "qdrant", "--store", "vectors") + workspace = partial(_invoke_workspace, config) + vectors("collection", "register", "docs", "--dimensions", "3") + vectors("point", "upsert", "docs", "one", '{"vector":[1,0,0],"payload":{"value":"base"}}') + workspace("branch", "create", "trial") + workspace("branch", "create", "trial", ok=False) + vectors("point", "upsert", "docs", "one", '{"vector":[0,1,0],"payload":{"value":"trial"}}', "--branch", "trial") + vectors("point", "upsert", "docs", "one", '{"vector":[0,0,1],"payload":{"value":"main"}}') + assert len(workspace("merge", "preview", "trial", "main")["vectors"]["conflicts"]) == 1 + workspace("merge", "apply", "trial", "main", ok=False) + workspace("merge", "apply", "trial", "main", "--policy", "snapshot_isolation", ok=False) + vectors("point", "upsert", "docs", "bad", '{"vector":[1,0]}', ok=False) + assert vectors("point", "get", "docs", "bad") is None + assert vectors("point", "get", "docs", "one")["payload"] == {"value": "main"} + vectors("point", "get", "docs", "one", "--branch", "missing", ok=False) + vectors("collection", "info", "missing", ok=False) + + def test_qdrant_cli_json_files_and_stdin(self, qdrant_target: QdrantConfig, tmp_path: Path) -> None: + config = _qdrant_workspace_file(tmp_path, qdrant_target) + vectors = partial(_invoke_workspace, config, "qdrant", "--store", "vectors") + workspace = partial(_invoke_workspace, config) + vectors("collection", "register", "docs", "--dimensions", "3") + data = tmp_path / "point.json" + data.write_text('{"vector":[1,0,0],"payload":{"source":"file"}}', encoding="utf-8") + result = vectors("point", "upsert", "docs", "one", "--file", str(data), "--revision", "initial") + assert result["payload"] == {"source": "file"} + store = _open_qdrant_api(qdrant_target) + try: + with store.context.checkout("main") as control: + session = store.checkout_control(control) + assert result == asdict(session.get("docs", "one")) + expected_search = [asdict(item) for item in session.search("docs", [1, 0, 0], limit=1)] + finally: + store.close() + assert vectors("search", "docs", "[1,0,0]", "--limit", "1") == expected_search + assert vectors("search", "docs", "--stdin", input_text="[1,0,0]")[0]["id"] == "one" + vectors("point", "upsert", "docs", "two", "--stdin", input_text='{"vector":[0,1,0]}') + data.write_text("[0,1,0]", encoding="utf-8") + assert vectors("search", "docs", "--file", str(data))[0]["id"] == "two" + assert vectors("search", "docs", input_text="[1,0,0]")[0]["id"] == "one" + + +class TestWorkspaceFailureWorkflows: + def test_atomic_stale_preview_is_rejected(self, tmp_path: Path) -> None: + config = _prepare_workspace_merge(tmp_path, atomic=True) + preview = _invoke_workspace(config, "merge", "atomic-preview", "experiment", "main") + _invoke_workspace(config, "execute", "UPDATE items SET value = 'changed after preview' WHERE id = 1", "--store", "app", "--branch", "experiment") + _invoke_workspace(config, "merge", "atomic-apply", "experiment", "main", "--operation-id", "stale", "--preview-token", preview["preview_token"], ok=False) + assert _invoke_workspace(config, "query", "SELECT value FROM items", "--store", "app") == [{"value": "base"}] + assert _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "get", "documents", "one") is None + + @pytest.mark.parametrize("atomic", [False, True]) + def test_merge_failure_retains_api_guarantees(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, atomic: bool) -> None: + config = _prepare_workspace_merge(tmp_path, atomic=atomic) + from chronos_core.cli.main import app + from chronos_core.workspace.qdrant import ChronosQdrantStore + + def fail_vector_store(*args, **kwargs): + raise RuntimeError("injected vector failure") + + # Fail the second participant after SQL has applied/staged its changes. + method = "stage_branch_transaction_changes" if atomic else "merge_apply" + monkeypatch.setattr(ChronosQdrantStore, method, fail_vector_store) + command = "atomic-apply" if atomic else "apply" + arguments = ["--config", str(config), "merge", command, "experiment", "main"] + if atomic: + arguments.extend(["--operation-id", "failed-merge"]) + result = CliRunner().invoke(app, arguments) + assert result.exit_code == 1, result.output + assert result.stdout == "" + assert "injected vector failure" in result.stderr + assert "Traceback" not in result.stderr + # Atomic mode never publishes the staged SQL change; coordinated mode + # preserves the API's partial apply instead of pretending to roll it back. + expected = "base" if atomic else "candidate" + assert _invoke_workspace(config, "query", "SELECT value FROM items", "--store", "app") == [{"value": expected}] + assert _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "get", "documents", "one") is None + assert _invoke_workspace(config, "query", "SELECT value FROM items", "--store", "app", "--branch", "experiment") == [{"value": "candidate"}] + + def test_conflict_blocks_atomic_merge(self, tmp_path: Path) -> None: + config = _prepare_workspace_merge(tmp_path, atomic=True) + _invoke_workspace(config, "execute", "UPDATE items SET value = 'main change' WHERE id = 1", "--store", "app") + preview = _invoke_workspace(config, "merge", "atomic-preview", "experiment", "main") + assert len(preview["stores"]["app"]["conflicts"]) == 1 + _invoke_workspace(config, "merge", "atomic-apply", "experiment", "main", "--operation-id", "conflicting", ok=False) + assert _invoke_workspace(config, "query", "SELECT value FROM items", "--store", "app") == [{"value": "main change"}] + assert _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "get", "documents", "one") is None + + +class TestWorkspaceFilesystemWorkflows: + @pytest.mark.parametrize('with_sql', [False, True]) + def test_persistent_workspace(self, tmp_path: Path, with_sql: bool) -> None: + import shutil + + available = (sys.platform == 'linux' and shutil.which('fuse-overlayfs') + and (shutil.which('fusermount3') or shutil.which('fusermount')) + and Path('/dev/fuse').exists()) + if not available: + if os.environ.get('CHRONOS_CLI_REQUIRE_FILESYSTEM') == 'ON': + pytest.fail('Filesystem integration requires Linux, /dev/fuse, and fuse-overlayfs') + pytest.skip('Filesystem integration requires Linux FUSE') + base = tmp_path / 'base' + base.mkdir() + (base / 'report.txt').write_text('original') + config = tmp_path / 'chronos.toml' + config.write_text('[filesystem]\nroot="base"\nstate_dir="state"\n' + ( + '[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"\n' if with_sql else '' + )) + + def run(branch: str, code: str) -> dict[str, Any]: + return _invoke_workspace(config, 'fs', 'run', '--branch', branch, '--json', '--', sys.executable, '-c', code) + + assert _invoke_workspace(config, 'branch', 'list') == ['main'] + if with_sql: + _invoke_workspace(config, 'table', 'create', '--store', 'app', 'CREATE TABLE docs (id INTEGER PRIMARY KEY)') + _invoke_workspace(config, 'table', 'register', '--store', 'app', 'docs', '--primary-key', 'id') + _invoke_workspace(config, 'branch', 'create', 'feature') + run('feature', 'from pathlib import Path; Path("report.txt").write_text("changed")') + assert run('main', 'print(open("report.txt").read())')['stdout'] == 'original\n' + assert run('feature', 'print(open("report.txt").read())')['stdout'] == 'changed\n' + if with_sql: + _invoke_workspace(config, 'execute', '--store', 'app', '--branch', 'feature', 'INSERT INTO docs (id) VALUES (1)') + checkpoint = _invoke_workspace(config, 'checkpoint', 'create', 'saved', '--branch', 'feature') + assert 'filesystem' in checkpoint + _invoke_workspace(config, 'branch', 'create-from-checkpoint', 'restored', 'saved') + assert run('restored', 'print(open("report.txt").read())')['stdout'] == 'changed\n' + diff = _invoke_workspace(config, 'diff', 'main', 'feature') + with open_workspace(load_workspace_config(config)) as workspace: + assert diff == json.loads(json.dumps({key: asdict(value) for key, value in workspace.diff('main', 'feature').items()})) + assert diff['filesystem']['changes'][0]['path'] == 'report.txt' + # Preserve the API's refusal of policy-aware filesystem merging. + _invoke_workspace(config, 'merge', 'preview', 'feature', 'main', ok=False) + _invoke_workspace(config, 'merge', 'apply', 'feature', 'main', ok=False) + preview = _invoke_workspace(config, 'merge', 'preview', 'feature', 'main', '--policy', 'api-default') + assert preview['filesystem'] == diff['filesystem'] + merged = _invoke_workspace(config, 'merge', 'apply', 'feature', 'main', '--policy', 'api-default') + assert merged['filesystem']['applied'] == 1 + assert run('main', 'print(open("report.txt").read())')['stdout'] == 'changed\n' + if with_sql: + assert _invoke_workspace(config, 'query', '--store', 'app', 'SELECT * FROM docs') == [{'id': 1}] + _invoke_workspace(config, 'branch', 'delete', 'restored') + _invoke_workspace(config, 'branch', 'delete', 'feature') + assert _invoke_workspace(config, 'branch', 'list') == ['main'] + assert (base / 'report.txt').read_text() == 'original' + assert (tmp_path / 'state' / 'metadata.json').exists() + assert str(tmp_path / 'state' / 'mounts') not in Path('/proc/self/mountinfo').read_text() + + environment = {key: value for key, value in os.environ.items() if not key.startswith('CHRONOS_')} + command = [sys.executable, '-m', 'chronos_core.cli', '--config', str(config), 'fs', 'run'] + failed = subprocess.run([*command, '--', sys.executable, '-c', + 'import sys; print("out"); print("err", file=sys.stderr); sys.exit(7)'], + capture_output=True, text=True, env=environment, timeout=30) + assert (failed.returncode, failed.stdout, failed.stderr) == (7, 'out\n', 'err\n') + for args in [ ['--cwd', '..', '--', 'true'], ['--timeout', '0.01', '--', sys.executable, '-c', 'import time; time.sleep(1)'] ]: + failed = subprocess.run([*command, *args], capture_output=True, text=True, env=environment, timeout=30) + assert failed.returncode == 1 + assert 'Traceback' not in failed.stderr + assert str(tmp_path / 'state' / 'mounts') not in Path('/proc/self/mountinfo').read_text() + + +class TestNativeChronosFSWorkflows: + @pytest.fixture(autouse=True) + def require_native_filesystem(self) -> None: + if not hasattr(_native_interval, "NativeChronosFSStore"): + if os.environ.get("CHRONOS_CLI_REQUIRE_CHRONOSFS") == "ON": + pytest.fail("This test run requires a native ChronosFS build") + pytest.skip("Native ChronosFS is disabled in this build") + + @staticmethod + def file_command(config: Path, *args: str, payload: bytes | None = None) -> subprocess.CompletedProcess[bytes]: + return subprocess.run( + [sys.executable, "-m", "chronos_core.cli", "--config", str(config), "fs", *args], + input=payload, capture_output=True, timeout=30, + ) + + def test_native_only_persists_binary_file(self, tmp_path: Path) -> None: + config = tmp_path / "chronos.toml" + config.write_text('[filesystem]\nkind="chronosfs"\ndatabase_url="sqlite:///files.sqlite"\n') + payload = b"\x00\xff\n" + written = self.file_command(config, "write", "/dir/file", "--stdin", "--parents", payload=payload) + assert written.returncode == 0, written.stderr + assert _invoke_workspace(config, "fs", "ls", "/dir") == ["file"] + read = self.file_command(config, "read", "/dir/file") + assert read.returncode == 0, read.stderr + assert read.stdout == payload + + @pytest.mark.parametrize("split", [False, True], ids=["unified", "split-scoped"]) + def test_sql_and_native_filesystem_atomic_merge(self, tmp_path: Path, split: bool) -> None: + config = tmp_path / "chronos.toml" + sql_config = '[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"\n' + fs_config = '[filesystem]\nkind="chronosfs"\n' + if split: + sql_config += 'merge_tables=["items"]\n' + fs_config += 'database_url="sqlite:///files.sqlite"\nmetadata_url="sqlite:///app.sqlite"\n' + else: + fs_config += 'database_url="sqlite:///app.sqlite"\n' + config.write_text( + '[workspace]\nshared_metadata_url="sqlite:///app.sqlite"\n' + sql_config + fs_config + ) + invoke = partial(_invoke_workspace, config) + invoke("table", "create", "CREATE TABLE items (id INTEGER PRIMARY KEY)", "--store", "app") + invoke("table", "register", "items", "--primary-key", "id", "--store", "app") + invoke("branch", "create", "draft") + invoke("execute", "INSERT INTO items (id) VALUES (1)", "--store", "app", "--branch", "draft") + written = self.file_command(config, "write", "/report", "--branch", "draft", "--stdin", payload=b"draft\n") + assert written.returncode == 0, written.stderr + assert invoke("query", "SELECT * FROM items", "--store", "app") == [] + assert "report" not in invoke("fs", "ls") + preview = invoke("merge", "atomic-preview", "draft", "main") + assert all(preview["stores"][name]["changes"] for name in ("app", "filesystem")) + if split: + assert [change["table"] for change in preview["stores"]["app"]["changes"]] == ["items"] + result = invoke("merge", "atomic-apply", "draft", "main", + "--operation-id", "native-filesystem", "--preview-token", preview["preview_token"]) + assert result["status"] == "committed" + assert invoke("query", "SELECT * FROM items", "--store", "app") == [{"id": 1}] + read = self.file_command(config, "read", "/report") + assert read.returncode == 0, read.stderr + assert read.stdout == b"draft\n" + + @pytest.mark.parametrize("directory", ["plain", "workspace #?%20+目录"]) + def test_mount_writes_to_selected_branch(self, tmp_path: Path, directory: str) -> None: + import shutil + + unmount = shutil.which("fusermount3") + if unmount is None or not Path("/dev/fuse").exists(): + if os.environ.get("CHRONOS_CLI_REQUIRE_CHRONOSFS") == "ON": + pytest.fail("This test requires /dev/fuse and fusermount3") + pytest.skip("Native mount requires /dev/fuse and fusermount3") + + tmp_path = tmp_path / directory + tmp_path.mkdir() + config = tmp_path / "chronos.toml" + config.write_text('[filesystem]\nkind="chronosfs"\ndatabase_url="sqlite:///files.sqlite"\n') + mountpoint = tmp_path / "mount" + mountpoint.mkdir() + _invoke_workspace(config, "branch", "create", "draft") + command = [ + sys.executable, "-m", "chronos_core.cli", "--config", str(config), + "fs", "mount", str(mountpoint), "--branch", "draft", + ] + with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as process: + try: + deadline = time.monotonic() + 15 + while not mountpoint.is_mount(): + if process.poll() is not None: + _, stderr = process.communicate() + pytest.fail(f"Mount command exited early: {stderr.decode(errors='replace')}") + if time.monotonic() >= deadline: + pytest.fail("Timed out waiting for the native mount") + time.sleep(0.05) + payload = b"mounted draft\n" * 8192 + (mountpoint / "report").write_bytes(payload) + subprocess.run([unmount, "-u", str(mountpoint)], check=True, timeout=10) + _, stderr = process.communicate(timeout=15) + assert process.returncode == 0, stderr + finally: + if mountpoint.is_mount(): + subprocess.run([unmount, "-uz", str(mountpoint)], timeout=10) + if process.poll() is None: + process.kill() + process.wait(timeout=10) + + assert "report" not in _invoke_workspace(config, "fs", "ls") + read = self.file_command(config, "read", "/report", "--branch", "draft") + assert read.returncode == 0, read.stderr + assert read.stdout == payload + assert (tmp_path / "files.sqlite.chronosfs-objects").is_dir() diff --git a/tests/test_native_interval.py b/tests/test_native_interval.py index d9f948a..94b469e 100644 --- a/tests/test_native_interval.py +++ b/tests/test_native_interval.py @@ -1,4 +1,5 @@ import sqlite3 +from urllib.parse import quote import pytest import chronos_core._native_interval as native_interval @@ -59,6 +60,59 @@ def test_native_connectors_smoke(tmp_path): assert native_interval.supports_connection_dialect("postgres") is True +@pytest.mark.parametrize("encoded", [False, True], ids=["raw-path", "url"]) +def test_native_sqlite_connection_preserves_path_characters(tmp_path, encoded): + path = tmp_path / "data #?%20+目录.sqlite" + with sqlite3.connect(path) as db: + db.execute("CREATE TABLE items (id INTEGER PRIMARY KEY)") + db.execute("INSERT INTO items VALUES (7)") + location = f"sqlite:///{quote(str(path), safe='/')}" if encoded else str(path) + conn = native_interval.NativeSqlConnection(location) + assert conn.query_sql("SELECT id FROM items") == [[7]] + + +def test_native_sqlite_connection_rejects_encoded_nul(tmp_path): + path = tmp_path / "data.sqlite" + sqlite3.connect(path).close() + with pytest.raises(ValueError, match="NUL"): + native_interval.NativeSqlConnection(f"sqlite:///{path}%00ignored") + + +@pytest.mark.skipif(not native_interval.has_duckdb, reason="DuckDB driver disabled in this build") +def test_native_duckdb_connection_preserves_path_characters(tmp_path): + path = tmp_path / "data #%20+目录.duckdb" + conn = native_interval.NativeSqlConnection(f"duckdb:///{quote(str(path), safe='/')}") + conn.execute_sql("CREATE TABLE items (id INTEGER PRIMARY KEY)") + conn.execute_sql("INSERT INTO items VALUES (7)") + conn.commit() + assert conn.query_sql("SELECT id FROM items") == [[7]] + assert path.is_file() + + +@pytest.mark.skipif(not native_interval.has_duckdb, reason="DuckDB driver disabled in this build") +@pytest.mark.parametrize("relative", ["data?.duckdb", "query?dir/data.duckdb"]) +def test_native_duckdb_rejects_question_mark_before_creating_files(tmp_path, relative): + path = tmp_path / relative + with pytest.raises(ValueError, match="DuckDB database paths"): + native_interval.NativeSqlConnection(f"duckdb:///{quote(str(path), safe='/')}") + assert list(tmp_path.iterdir()) == [] + + +@pytest.mark.skipif(not hasattr(native_interval, "NativeChronosLakeServer"), reason="S3 support disabled in this build") +@pytest.mark.parametrize("encoded", [False, True], ids=["raw-path", "url"]) +def test_native_lake_metadata_preserves_path_characters(tmp_path, encoded): + path = tmp_path / "metadata #?%20+目录.sqlite" + location = f"sqlite:///{quote(str(path), safe='/')}" if encoded else str(path) + server = native_interval.NativeChronosLakeServer(location, "http://127.0.0.1:9", worker_threads=1) + try: + assert server.branches() == ["main"] + server.create_branch("trial", "main") + assert sorted(server.branches()) == ["main", "trial"] + assert path.is_file() + finally: + server.stop() + + def test_native_sqlite_connection_uses_chronos_wal_checkpoint_default(tmp_path, monkeypatch): monkeypatch.delenv("CHRONOS_NATIVE_SQLITE_WAL_AUTOCHECKPOINT_PAGES", raising=False) db_path = tmp_path / "wal_default.sqlite" diff --git a/uv.lock b/uv.lock index e0d0e15..3efc8bf 100644 --- a/uv.lock +++ b/uv.lock @@ -190,6 +190,8 @@ source = { editable = "packages/chronos-core" } dependencies = [ { name = "psycopg", extra = ["binary"] }, { name = "sqlglot" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, + { name = "typer" }, ] [package.optional-dependencies] @@ -226,8 +228,10 @@ requires-dist = [ { name = "qdrant-client", marker = "extra == 'qdrant'", specifier = ">=1.18,<2" }, { name = "qdrant-client", marker = "extra == 'test'", specifier = ">=1.18,<2" }, { name = "sqlglot", specifier = ">=25" }, + { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0" }, { name = "tomli", marker = "python_full_version < '3.11' and extra == 'mcp'", specifier = ">=2.0" }, { name = "tomli", marker = "python_full_version < '3.11' and extra == 'test'", specifier = ">=2.0" }, + { name = "typer", specifier = ">=0.25.1" }, ] provides-extras = ["duckdb", "qdrant", "mcp", "test"] @@ -359,7 +363,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [