From 563cffdae951c8a6fcdcf91450830b5cb3bad660 Mon Sep 17 00:00:00 2001 From: Seth Rojas Date: Tue, 27 Oct 2026 22:26:39 -0700 Subject: [PATCH 01/11] Add relational API command-line interface Wrap public branching, SQL, checkpoint, diff, and merge APIs with Typer commands. Support shared database configuration, SQL text/file/stdin input, JSON results, table setup, and connection cleanup. Add usage documentation and 45 passing CLI tests, including isolated SQLite workflows, conflict handling, and installed entry points. PostgreSQL integration and the full repository suite remain unverified. --- docs/README.md | 2 + docs/cli.md | 171 ++++++++++++ packages/chronos-core/README.md | 2 + packages/chronos-core/pyproject.toml | 2 + .../src/chronos_core/cli/__init__.py | 0 .../src/chronos_core/cli/__main__.py | 4 + .../src/chronos_core/cli/_support.py | 70 +++++ .../chronos-core/src/chronos_core/cli/main.py | 204 ++++++++++++++ tests/test_cli.py | 254 ++++++++++++++++++ tests/test_cli_support.py | 41 +++ uv.lock | 4 +- 11 files changed, 753 insertions(+), 1 deletion(-) create mode 100644 docs/cli.md create mode 100644 packages/chronos-core/src/chronos_core/cli/__init__.py create mode 100644 packages/chronos-core/src/chronos_core/cli/__main__.py create mode 100644 packages/chronos-core/src/chronos_core/cli/_support.py create mode 100644 packages/chronos-core/src/chronos_core/cli/main.py create mode 100644 tests/test_cli.py create mode 100644 tests/test_cli_support.py diff --git a/docs/README.md b/docs/README.md index 389e273..616f3fa 100644 --- a/docs/README.md +++ b/docs/README.md @@ -13,6 +13,8 @@ page says otherwise, commands and APIs apply to Chronos 0.2.0a1. optional DuckDB, filesystem, and S3 features. - [Integration](integration.md) explains how to adopt existing tables and route application access through Chronos. +- [CLI](cli.md) wraps the relational API with branch, query, checkpoint, diff, + and merge commands, including JSON output for pipes and files. - [Compatibility and limits](compatibility.md) states what is supported and where direct database access can bypass branch isolation. - [Branching introduction](branching-introduction.md) explains the data model diff --git a/docs/cli.md b/docs/cli.md new file mode 100644 index 0000000..8e40391 --- /dev/null +++ b/docs/cli.md @@ -0,0 +1,171 @@ +# Chronos CLI + +The `chronos` command is a one-off wrapper around the public relational Python +API. Each invocation opens the selected database, calls the API, prints its +return value, and closes the connection. It does not remember a current branch +or keep a background connection open. + +## Install and choose a database + +The CLI ships with `chronos-core`. Follow [installation](installation.md) for +native build dependencies. From the repository root, install the workspace package: + +```sh +uv sync --package chronos-core +uv run --package chronos-core chronos --help +``` + +You do not need to activate the virtual environment when using `uv run`. +An installed wheel also provides `chronos`; `python -m chronos_core.cli` invokes +the same application. + +On macOS with Homebrew Python, build against the same SQLite library Python uses. +If Python uses Homebrew SQLite, explicitly select it when rebuilding: + +```sh +brew install sqlite libpq boost +CMAKE_PREFIX_PATH="$(brew --prefix libpq)" \ +uv sync --package chronos-core --reinstall-package chronos-core \ + -C "cmake.define.SQLite3_INCLUDE_DIR=$(brew --prefix sqlite)/include" \ + -C "cmake.define.SQLite3_LIBRARY=$(brew --prefix sqlite)/lib/libsqlite3.dylib" +``` + +Mixing Python's Homebrew SQLite connection with a native extension linked to +macOS's SQLite can crash the process. After a custom build, use +`uv run --no-sync --package chronos-core chronos ...` to run the installed build +without syncing it again. + +Set the database URL in your terminal. This example uses a persistent local file: + +```sh +mkdir -p .chronos +export CHRONOS_DATABASE_URL="sqlite:///$PWD/.chronos/demo.sqlite" +``` + +This keeps the demo in `.chronos/` under your current directory. The absolute URL +continues to select that file if you change directories. Files ending in `.sqlite` +and their SQLite journal files are ignored by this repository's Git configuration. +SQLite creates a missing database file; its parent directory must already exist. +PostgreSQL URLs can also be supplied, for example through an environment variable +populated by your secret-management tooling. No PostgreSQL server is started by +the CLI. A `.env` file is not loaded automatically. + +Override the environment variable by placing the shared flag before the command: + +```sh +uv run --package chronos-core chronos \ + --database-url "sqlite:///$PWD/.chronos/other.sqlite" branch list +``` + +Help does not require a database URL and does not open a connection. + +## Create and register a table + +For a fresh database, create an ordinary table directly through the CLI, then +register it with Chronos: + +```sh +uv run --package chronos-core chronos table create \ + 'CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)' +uv run --package chronos-core chronos table register items --primary-key id +``` + +`table create` wraps the public database adapter's `execute()` and `commit()` +methods for initial setup. It accepts one CREATE TABLE statement directly, through +`--file`, or through stdin. This creates an ordinary database table; it is not a +branch-local schema change. Run it once before registration and managed access. +The branch-session `execute` command continues to use the API's default behavior. + +If your database already contains the application table, skip `table create` and +register it directly. For composite keys, repeat `--primary-key`. Register each +table once. Registration copies existing data into Chronos-managed storage; +subsequent managed reads and writes must use branch sessions. Direct access to +the original table bypasses Chronos. See [integration](integration.md) before +adopting application data. + +## Branch, query, checkpoint, and merge + +Run this walkthrough against the newly prepared database: + +```sh +uv run --package chronos-core chronos branch list +uv run --package chronos-core chronos branch create experiment --from main + +uv run --package chronos-core chronos execute \ + 'INSERT INTO items (id, value) VALUES (:id, :value)' \ + --branch experiment --params '{"id": 1, "value": "hello"}' + +uv run --package chronos-core chronos query \ + 'SELECT * FROM items WHERE id = :id' \ + --branch experiment --params '{"id": 1}' + +uv run --package chronos-core chronos checkpoint create saved --branch experiment +uv run --package chronos-core chronos checkpoint list --branch experiment +uv run --package chronos-core chronos diff main experiment +uv run --package chronos-core chronos diff main experiment --table items +uv run --package chronos-core chronos merge preview experiment main +uv run --package chronos-core chronos merge apply experiment main +uv run --package chronos-core chronos query 'SELECT * FROM items' +uv run --package chronos-core chronos branch delete experiment +``` + +Queries, writes, and checkpoint creation default to branch `main`. Branch creation +defaults to `--from main`. Checkpoint listing without `--branch` lists all +checkpoints. Both merge commands accept `--policy abort_on_conflict` (the default) +or `--policy snapshot_isolation`. These are API policies; snapshot isolation does +not mean force-overwriting conflicts. API validation and conflict behavior apply +unchanged. + +## SQL input + +Both `query` and `execute` accept one SQL source: + +```sh +# Direct text, including multiline text inside shell quotes. +uv run --package chronos-core chronos query 'SELECT * FROM items' + +# A UTF-8 file. +uv run --package chronos-core chronos query --file query.sql + +# Piped input, automatically detected when no other source is supplied. +printf 'SELECT * FROM items\n' | uv run --package chronos-core chronos query + +# Explicit stdin; type SQL and finish with Ctrl-D in an interactive terminal. +uv run --package chronos-core chronos query --stdin +``` + +Empty SQL and conflicting explicit sources are rejected. With no SQL source in an +interactive terminal, the CLI shows an input error rather than opening an editor. +`--params` must be a JSON object of named parameter values. SQL and parameters are +passed directly to `session.query()` or `session.execute()`; files are not split +into a script of multiple statements. Supported SQL is determined by the API. + +## Results and errors + +Non-`None` API return values are printed as JSON to stdout: + +- Queries produce an array of row objects; no rows produces `[]`. +- Writes produce the API's result fields, such as `{"rowcount": 1}`. +- Lists, checkpoints, diffs, and merges preserve the API result field names. +- Methods returning `None`, including successful branch creation/deletion and + table registration, produce no output. + +Dates and times use ISO strings, decimals use strings to preserve precision, and +bytes use `{"encoding": "base64", "data": "..."}` objects. Unsupported values +produce an error instead of silently changing their representation. No query +results are cached or saved unless you redirect stdout: + +```sh +uv run --package chronos-core chronos query 'SELECT * FROM items' > rows.json +uv run --package chronos-core chronos query 'SELECT * FROM items' | python -m json.tool +``` + +Exit code 0 means the API call and output completed successfully. Expected +operational failures are reported on stderr with exit code 1; invalid command-line +input uses exit code 2. A merge preview may successfully return conflicts as data; +merge application follows the API's rejection behavior. Errors do not emit a JSON +success result. + +This first version uses the default interval backend with one database per +invocation. Workspace coordination, a persistent interactive shell, advanced +connection tuning, and custom Python merge callbacks are not exposed. diff --git a/packages/chronos-core/README.md b/packages/chronos-core/README.md index 702c687..9c43f57 100644 --- a/packages/chronos-core/README.md +++ b/packages/chronos-core/README.md @@ -27,5 +27,7 @@ Optional builds use `-Ccmake.define.CHRONOS_WITH_FILESYSTEM=ON`, Python extras do not change compiled capabilities. See the repository's docs/installation.md, docs/integration.md, and docs/tutorials. +The installed `chronos` command wraps the relational API; run `chronos --help` +and see [the CLI guide](../../docs/cli.md) for setup and a complete workflow. This is a Python library, not a PostgreSQL-wire proxy. Managed access must pass through branch sessions. Version 0.2.0a1 is an experimental release candidate. diff --git a/packages/chronos-core/pyproject.toml b/packages/chronos-core/pyproject.toml index 3ac5353..5a5e3fd 100644 --- a/packages/chronos-core/pyproject.toml +++ b/packages/chronos-core/pyproject.toml @@ -13,6 +13,7 @@ requires-python = ">=3.10" dependencies = [ "psycopg[binary]>=3.1", "sqlglot>=25", + "typer>=0.25.1", ] [project.optional-dependencies] @@ -44,6 +45,7 @@ Documentation = "https://github.com/mitdbg/chronos/blob/main/docs/README.md" chronos-mcp = "chronos_core.mcp.cli:main" chronos-mcp-deploy-codex = "chronos_core.mcp.deploy_codex:main" chronos-lake = "chronos_core.lake.cli:main" +chronos = "chronos_core.cli.main:app" [tool.scikit-build] wheel.packages = ["src/chronos_core"] diff --git a/packages/chronos-core/src/chronos_core/cli/__init__.py b/packages/chronos-core/src/chronos_core/cli/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/packages/chronos-core/src/chronos_core/cli/__main__.py b/packages/chronos-core/src/chronos_core/cli/__main__.py new file mode 100644 index 0000000..53aa0c1 --- /dev/null +++ b/packages/chronos-core/src/chronos_core/cli/__main__.py @@ -0,0 +1,4 @@ +"""Run the packaged CLI with python -m chronos_core.cli.""" +from chronos_core.cli.main import app + +app() diff --git a/packages/chronos-core/src/chronos_core/cli/_support.py b/packages/chronos-core/src/chronos_core/cli/_support.py new file mode 100644 index 0000000..51253e9 --- /dev/null +++ b/packages/chronos-core/src/chronos_core/cli/_support.py @@ -0,0 +1,70 @@ +"""Shared configuration, connection lifecycle, and JSON output.""" +from __future__ import annotations + +import base64 +import json +import re +import sqlite3 +from contextlib import contextmanager +from dataclasses import asdict, dataclass, is_dataclass +from datetime import date, datetime, time +from decimal import Decimal +from typing import TYPE_CHECKING, Any, Iterator + +import psycopg +import typer + +if TYPE_CHECKING: + from chronos_core.branching import ChronosBranchContext + + +@dataclass(frozen=True) +class DatabaseConfig: + database_url: str + + +def database_config(ctx: typer.Context) -> DatabaseConfig: + database = ctx.find_object(DatabaseConfig) + if database is None or not database.database_url: + raise typer.BadParameter( + "Provide --database-url or set CHRONOS_DATABASE_URL.", + param_hint="--database-url", + ) + return database + + +def _json_default(value: Any) -> Any: + if is_dataclass(value) and not isinstance(value, type): + return asdict(value) + if isinstance(value, (datetime, date, time)): + return value.isoformat() + if isinstance(value, Decimal): + return str(value) + if isinstance(value, bytes): + return {"encoding": "base64", "data": base64.b64encode(value).decode("ascii")} + raise TypeError(f"Unsupported JSON value type: {type(value).__name__}") + + +def print_result(result: Any) -> None: + if result is not None: + typer.echo(json.dumps(result, default=_json_default, ensure_ascii=False, indent=2)) + + +@contextmanager +def connect(database: DatabaseConfig) -> Iterator[ChronosBranchContext]: + from chronos_core.branching import BranchingError, ChronosBranchContext + + try: + context = ChronosBranchContext.connect(database.database_url) + try: + yield context + finally: + context.close() + except (BranchingError, RuntimeError, sqlite3.Error, psycopg.Error, OSError, ValueError, TypeError) as error: + message = str(error) + if database.database_url: + message = message.replace(database.database_url, "") + message = re.sub(r"([a-zA-Z][\w+.-]*://)[^\s/@]+@", r"\1@", message) + message = re.sub(r"(?i)(password\s*=\s*)(?:'[^']*'|\"[^\"]*\"|[^\s]+)", r"\1", message) + typer.echo(f"Error: {message}", err=True) + raise typer.Exit(1) from error diff --git a/packages/chronos-core/src/chronos_core/cli/main.py b/packages/chronos-core/src/chronos_core/cli/main.py new file mode 100644 index 0000000..df3ff30 --- /dev/null +++ b/packages/chronos-core/src/chronos_core/cli/main.py @@ -0,0 +1,204 @@ +"""One-off terminal commands wrapping the public relational API.""" +from __future__ import annotations + +import json +import sys +from enum import Enum +from pathlib import Path +from typing import Any + +import typer +import sqlglot +from sqlglot import exp + +from chronos_core.cli._support import DatabaseConfig, connect, database_config, print_result + +app = typer.Typer(no_args_is_help=True, pretty_exceptions_enable=False) +branch_app = typer.Typer(no_args_is_help=True) +table_app = typer.Typer(no_args_is_help=True) +checkpoint_app = typer.Typer(no_args_is_help=True) +merge_app = typer.Typer(no_args_is_help=True) +app.add_typer(branch_app, name="branch", help="Manage branches.") +app.add_typer(table_app, name="table", help="Create ordinary tables and register them with Chronos.") +app.add_typer(checkpoint_app, name="checkpoint", help="Create and list checkpoints.") +app.add_typer(merge_app, name="merge", help="Preview and apply merges.") + + +class MergePolicy(str, Enum): + abort_on_conflict = "abort_on_conflict" + snapshot_isolation = "snapshot_isolation" + + +@app.callback() +def main( + ctx: typer.Context, + database_url: str | None = typer.Option( + None, "--database-url", envvar="CHRONOS_DATABASE_URL", help="Database connection URL." + ), +) -> None: + ctx.obj = DatabaseConfig(database_url) if database_url else None + + +@branch_app.command("list") +def list_branches(ctx: typer.Context) -> None: + with connect(database_config(ctx)) as database: + print_result(database.list_branches()) + + +@branch_app.command("create") +def create_branch( + ctx: typer.Context, name: str, + from_branch: str = typer.Option("main", "--from"), +) -> None: + with connect(database_config(ctx)) as database: + print_result(database.create_branch(name, from_branch=from_branch)) + + +@branch_app.command("delete") +def delete_branch(ctx: typer.Context, name: str) -> None: + with connect(database_config(ctx)) as database: + print_result(database.delete_branch(name)) + + +@table_app.command("register") +def register_table( + ctx: typer.Context, table: str, + primary_key: list[str] = typer.Option(..., "--primary-key"), +) -> None: + with connect(database_config(ctx)) as database: + print_result(database.register_table(table, primary_key)) + + +def _sql_input(sql: str | None, file: Path | None, stdin: bool) -> str: + if sum((sql is not None, file is not None, stdin)) > 1: + raise typer.BadParameter("Choose SQL text, --file, or --stdin, not multiple sources.") + if file is not None: + try: + sql = file.read_text(encoding="utf-8") + except (OSError, UnicodeError) as error: + raise typer.BadParameter(f"Cannot read SQL file: {error}", param_hint="--file") from error + elif stdin or (sql is None and not sys.stdin.isatty()): + try: + sql = sys.stdin.read() + except (OSError, UnicodeError) as error: + typer.echo(f"Error reading stdin: {error}", err=True) + raise typer.Exit(1) from error + if sql is None or not sql.strip(): + raise typer.BadParameter("Provide SQL text, --file, or SQL on stdin.") + return sql + + +def _parameters(params: str | None) -> dict[str, Any] | None: + if params is None: + return None + try: + value = json.loads(params) + except ValueError as error: + raise typer.BadParameter("Expected a JSON object.", param_hint="--params") from error + if not isinstance(value, dict): + raise typer.BadParameter("Expected a JSON object.", param_hint="--params") + return value + + +@table_app.command("create") +def create_table( + ctx: typer.Context, + sql: str | None = typer.Argument(None), + file: Path | None = typer.Option(None, "--file"), + stdin: bool = typer.Option(False, "--stdin"), +) -> None: + """Create an ordinary table for subsequent table register; not a branch-local write.""" + statement = _sql_input(sql, file, stdin) + try: + statements = sqlglot.parse(statement) + except sqlglot.errors.ParseError as error: + raise typer.BadParameter("Expected one CREATE TABLE statement.") from error + if ( + len(statements) != 1 + or not isinstance(statements[0], exp.Create) + or statements[0].kind != "TABLE" + ): + raise typer.BadParameter("Expected one CREATE TABLE statement.") + with connect(database_config(ctx)) as database: + database.db.execute(statement) + print_result(database.db.commit()) + + +@app.command() +def query( + ctx: typer.Context, + sql: str | None = typer.Argument(None), + branch: str = typer.Option("main", "--branch"), + params: str | None = typer.Option(None, "--params"), + file: Path | None = typer.Option(None, "--file"), + stdin: bool = typer.Option(False, "--stdin"), +) -> None: + """Call the branch session's query API with SQL and optional named parameters.""" + statement, parameters = _sql_input(sql, file, stdin), _parameters(params) + with connect(database_config(ctx)) as database: + with database.checkout(branch) as session: + print_result(session.query(statement, parameters)) + + +@app.command() +def execute( + ctx: typer.Context, + sql: str | None = typer.Argument(None), + branch: str = typer.Option("main", "--branch"), + params: str | None = typer.Option(None, "--params"), + file: Path | None = typer.Option(None, "--file"), + stdin: bool = typer.Option(False, "--stdin"), +) -> None: + """Call the branch session's execute API; SQL is not split into a script.""" + statement, parameters = _sql_input(sql, file, stdin), _parameters(params) + with connect(database_config(ctx)) as database: + with database.checkout(branch) as session: + print_result(session.execute(statement, parameters)) + + +@checkpoint_app.command("create") +def create_checkpoint( + ctx: typer.Context, name: str, + branch: str = typer.Option("main", "--branch"), +) -> None: + with connect(database_config(ctx)) as database: + print_result(database.create_checkpoint(name, branch=branch)) + + +@checkpoint_app.command("list") +def list_checkpoints( + ctx: typer.Context, branch: str | None = typer.Option(None, "--branch"), +) -> None: + with connect(database_config(ctx)) as database: + print_result(database.list_checkpoints(branch=branch)) + + +@app.command() +def diff( + ctx: typer.Context, left: str, right: str, + table: str | None = typer.Option(None, "--table"), +) -> None: + with connect(database_config(ctx)) as database: + print_result(database.diff(left, right) if table is None else database.diff_rows(left, right, table)) + + +@merge_app.command("preview") +def merge_preview( + ctx: typer.Context, source: str, target: str, + policy: MergePolicy = typer.Option(MergePolicy.abort_on_conflict, "--policy"), +) -> None: + with connect(database_config(ctx)) as database: + print_result(database.merge_preview(source, target, policy=policy.value)) + + +@merge_app.command("apply") +def merge_apply( + ctx: typer.Context, source: str, target: str, + policy: MergePolicy = typer.Option(MergePolicy.abort_on_conflict, "--policy"), +) -> None: + with connect(database_config(ctx)) as database: + print_result(database.merge_apply(source, target, policy=policy.value)) + + +if __name__ == "__main__": + app() diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..0568431 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,254 @@ +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +from typer.testing import CliRunner + +from chronos_core.branching import BranchingError, ChronosBranchContext +from chronos_core.cli.main import app + +runner = CliRunner() + + +@pytest.fixture +def connection(monkeypatch: pytest.MonkeyPatch) -> MagicMock: + database = MagicMock() + database.list_branches.return_value = [] + database.checkout.return_value.__enter__.return_value.query.return_value = [] + monkeypatch.setattr(ChronosBranchContext, "connect", MagicMock(return_value=database)) + monkeypatch.delenv("CHRONOS_DATABASE_URL", raising=False) + return database + + +def test_configuration_and_help(connection: MagicMock) -> None: + for args in (["--help"], ["branch", "--help"], ["query", "--help"]): + assert runner.invoke(app, args).exit_code == 0 + ChronosBranchContext.connect.assert_not_called() + result = runner.invoke(app, ["branch", "list"]) + assert result.exit_code == 2 + ChronosBranchContext.connect.assert_not_called() + result = runner.invoke(app, ["branch", "list"], env={"CHRONOS_DATABASE_URL": "sqlite:///env.db"}) + assert result.exit_code == 0 + ChronosBranchContext.connect.assert_called_with("sqlite:///env.db") + result = runner.invoke(app, ["--database-url", "sqlite:///flag.db", "branch", "list"], + env={"CHRONOS_DATABASE_URL": "sqlite:///env.db"}) + assert result.exit_code == 0 + ChronosBranchContext.connect.assert_called_with("sqlite:///flag.db") + assert json.loads(result.stdout) == [] + assert connection.close.call_count == 2 + + +@pytest.mark.parametrize(("args", "method", "expected_args", "expected_kwargs"), [ + (["branch", "list"], "list_branches", (), {}), + (["branch", "create", "trial"], "create_branch", ("trial",), {"from_branch": "main"}), + (["branch", "create", "trial", "--from", "parent"], "create_branch", ("trial",), {"from_branch": "parent"}), + (["branch", "delete", "trial"], "delete_branch", ("trial",), {}), + (["table", "register", "items", "--primary-key", "id", "--primary-key", "tenant"], "register_table", ("items", ["id", "tenant"]), {}), + (["checkpoint", "create", "saved", "--branch", "trial"], "create_checkpoint", ("saved",), {"branch": "trial"}), + (["checkpoint", "list"], "list_checkpoints", (), {"branch": None}), + (["checkpoint", "list", "--branch", "trial"], "list_checkpoints", (), {"branch": "trial"}), + (["diff", "main", "trial"], "diff", ("main", "trial"), {}), + (["diff", "main", "trial", "--table", "items"], "diff_rows", ("main", "trial", "items"), {}), + (["merge", "preview", "trial", "main"], "merge_preview", ("trial", "main"), {"policy": "abort_on_conflict"}), + (["merge", "apply", "trial", "main", "--policy", "snapshot_isolation"], "merge_apply", ("trial", "main"), {"policy": "snapshot_isolation"}), +]) +def test_api_forwarding(connection: MagicMock, args: list[str], method: str, + expected_args: tuple, expected_kwargs: dict) -> None: + getattr(connection, method).return_value = None + result = runner.invoke(app, ["--database-url", "sqlite:///test.db", *args]) + assert result.exit_code == 0, result.output + getattr(connection, method).assert_called_once_with(*expected_args, **expected_kwargs) + assert result.stdout == "" + connection.close.assert_called_once() + + +@pytest.mark.parametrize("command", ["query", "execute"]) +@pytest.mark.parametrize("source", ["string", "file", "stdin", "pipe"]) +def test_sql_sources(connection: MagicMock, tmp_path: Path, command: str, source: str) -> None: + sql = "SELECT *\nFROM items WHERE id = :id\n" + args = ["--database-url", "sqlite:///test.db", command, "--branch", "trial", "--params", '{"id": 3}'] + input_text = None + if source == "string": + args.append(sql) + elif source == "file": + path = tmp_path / "query.sql" + path.write_text(sql, encoding="utf-8") + args.extend(["--file", str(path)]) + else: + input_text = sql + if source == "stdin": + args.append("--stdin") + session = connection.checkout.return_value.__enter__.return_value + getattr(session, command).return_value = [{"id": 3}] if command == "query" else {"rowcount": 1} + result = runner.invoke(app, args, input=input_text) + assert result.exit_code == 0, result.output + connection.checkout.assert_called_once_with("trial") + getattr(session, command).assert_called_once_with(sql, {"id": 3}) + assert json.loads(result.stdout) == getattr(session, command).return_value + connection.checkout.return_value.__exit__.assert_called_once() + connection.close.assert_called_once() + + +@pytest.mark.parametrize("args", [ + ["query"], ["query", " "], ["query", "SELECT 1", "--stdin"], + ["query", "SELECT 1", "--file", "missing.sql"], + ["query", "--file", "missing.sql"], + ["query", "SELECT 1", "--params", "[]"], + ["execute", "SELECT 1", "--params", "broken"], + ["merge", "apply", "a", "b", "--policy", "unknown"], + ["table", "register", "items"], +]) +def test_bad_inputs_do_not_connect(connection: MagicMock, args: list[str]) -> None: + result = runner.invoke(app, ["--database-url", "sqlite:///test.db", *args]) + assert result.exit_code == 2, result.output + ChronosBranchContext.connect.assert_not_called() + + +def test_interactive_missing_sql(connection: MagicMock, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sys.stdin, "isatty", lambda: True) + from chronos_core.cli.main import _sql_input + import typer + with pytest.raises(typer.BadParameter, match="Provide SQL"): + _sql_input(None, None, False) + ChronosBranchContext.connect.assert_not_called() + + +def test_cleanup_and_redaction(connection: MagicMock) -> None: + url = "postgresql://user:secret@localhost/db" + session = connection.checkout.return_value.__enter__.return_value + session.query.side_effect = BranchingError(f"Cannot use {url}; password=secret") + result = runner.invoke(app, ["--database-url", url, "query", "SELECT 1"]) + assert result.exit_code == 1 + assert "secret" not in result.output + assert "Error:" in result.stderr + connection.checkout.return_value.__exit__.assert_called_once() + connection.close.assert_called_once() + + +def test_connection_failure_does_not_close_unassigned_context(connection: MagicMock) -> None: + ChronosBranchContext.connect.side_effect = ValueError("unsupported database") + result = runner.invoke(app, ["--database-url", "bad", "branch", "list"]) + assert result.exit_code == 1 + assert "unsupported database" in result.stderr + connection.close.assert_not_called() + + + +def test_sqlite_workflow(tmp_path: Path) -> None: + url = f"sqlite:///{tmp_path / 'workflow.db'}" + database = ChronosBranchContext.connect(url) + try: + database.db.execute("CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)") + database.db.commit() + finally: + database.close() + + def invoke(*args: str, ok: bool = True): + result = subprocess.run( + [sys.executable, "-X", "faulthandler", "-m", "chronos_core.cli", "--database-url", url, *args], + capture_output=True, text=True, timeout=30, + ) + assert result.returncode == (0 if ok else 1), result.stderr + return json.loads(result.stdout) if result.stdout.strip() else None + + assert invoke("table", "register", "items", "--primary-key", "id") is None + assert invoke("branch", "create", "trial") is None + invoke("branch", "create", "trial", ok=False) + assert {branch["branch_id"] for branch in invoke("branch", "list")} == {"main", "trial"} + assert invoke("execute", "INSERT INTO items (id, value) VALUES (:id, :value)", "--branch", "trial", + "--params", '{"id": 1, "value": "hello"}') == {"rowcount": 1} + assert invoke("query", "SELECT * FROM items", "--branch", "trial") == [{"id": 1, "value": "hello"}] + assert invoke("query", "SELECT * FROM items") == [] + assert invoke("checkpoint", "create", "saved", "--branch", "trial")["checkpoint_id"] == "saved" + assert len(invoke("checkpoint", "list", "--branch", "trial")) == 1 + assert len(invoke("diff", "main", "trial")["changes"]) == 1 + assert len(invoke("diff", "main", "trial", "--table", "items")) == 1 + assert invoke("merge", "preview", "trial", "main")["conflicts"] == [] + assert invoke("merge", "apply", "trial", "main")["applied"] == 1 + assert invoke("query", "SELECT * FROM items") == [{"id": 1, "value": "hello"}] + invoke("branch", "create", "conflicting") + invoke("execute", "UPDATE items SET value = 'source' WHERE id = 1", "--branch", "conflicting") + invoke("execute", "UPDATE items SET value = 'target' WHERE id = 1") + assert len(invoke("merge", "preview", "conflicting", "main")["conflicts"]) == 1 + invoke("merge", "apply", "conflicting", "main", ok=False) + invoke("merge", "apply", "conflicting", "main", "--policy", "snapshot_isolation", ok=False) + assert invoke("query", "SELECT value FROM items") == [{"value": "target"}] + invoke("branch", "delete", "conflicting") + invoke("query", "SELECT * FROM items", "--branch", "missing", ok=False) + invoke("execute", "DROP TABLE items", ok=False) + assert invoke("branch", "delete", "trial") is None + assert [branch["branch_id"] for branch in invoke("branch", "list")] == ["main"] + + +def test_installed_entrypoints() -> None: + executable = Path(sys.executable).parent / "chronos" + for args in ([str(executable), "--help"], [sys.executable, "-m", "chronos_core.cli", "--help"]): + env = dict(os.environ) + env.pop("CHRONOS_DATABASE_URL", None) + result = subprocess.run(args, capture_output=True, text=True, env=env, timeout=30) + assert result.returncode == 0, result.stderr + assert "--database-url" in result.stdout + + +def test_create_table_forwarding(connection: MagicMock) -> None: + connection.db.commit.return_value = None + statement = "CREATE TABLE items (id INTEGER PRIMARY KEY)" + result = runner.invoke(app, ["--database-url", "sqlite:///test.db", "table", "create", statement]) + assert result.exit_code == 0, result.output + connection.db.execute.assert_called_once_with(statement) + connection.db.commit.assert_called_once() + connection.checkout.assert_not_called() + assert result.stdout == "" + + +@pytest.mark.parametrize("statement", [ + "DELETE FROM items", "CREATE INDEX idx ON items(id)", + "CREATE TABLE items (id INTEGER); DROP TABLE items", "not sql", +]) +def test_create_table_rejects_other_sql(connection: MagicMock, statement: str) -> None: + result = runner.invoke(app, ["--database-url", "sqlite:///test.db", "table", "create", statement]) + assert result.exit_code == 2, result.output + ChronosBranchContext.connect.assert_not_called() + + +def test_native_error_is_reported_without_traceback(connection: MagicMock) -> None: + session = connection.checkout.return_value.__enter__.return_value + session.execute.side_effect = RuntimeError("INSERT must specify a column list") + result = runner.invoke(app, [ + "--database-url", "sqlite:///test.db", "execute", "INSERT INTO items VALUES (1)", + ]) + assert result.exit_code == 1 + assert result.stdout == "" + assert result.stderr.strip() == "Error: INSERT must specify a column list" + connection.checkout.return_value.__exit__.assert_called_once() + connection.close.assert_called_once() + + +def test_cli_only_sqlite_setup(tmp_path: Path) -> None: + url = f"sqlite:///{tmp_path / 'cli-only.sqlite'}" + + def invoke(*args: str): + result = subprocess.run( + [sys.executable, "-X", "faulthandler", "-m", "chronos_core.cli", "--database-url", url, *args], + capture_output=True, text=True, timeout=30, + ) + assert result.returncode == 0, result.stderr + return json.loads(result.stdout) if result.stdout.strip() else None + + assert invoke("table", "create", + "CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)") is None + invoke("table", "register", "items", "--primary-key", "id") + invoke("branch", "create", "experiment") + assert invoke("execute", "INSERT INTO items (id, value) VALUES (1, 'hello')", + "--branch", "experiment") == {"rowcount": 1} + assert invoke("query", "SELECT * FROM items", "--branch", "experiment") == [{"id": 1, "value": "hello"}] + assert invoke("query", "SELECT * FROM items") == [] + assert len(invoke("diff", "main", "experiment")["changes"]) == 1 + assert invoke("merge", "apply", "experiment", "main")["applied"] == 1 + assert invoke("query", "SELECT * FROM items") == [{"id": 1, "value": "hello"}] diff --git a/tests/test_cli_support.py b/tests/test_cli_support.py new file mode 100644 index 0000000..3aa1c37 --- /dev/null +++ b/tests/test_cli_support.py @@ -0,0 +1,41 @@ +from __future__ import annotations + +import json +from dataclasses import dataclass +from datetime import date, datetime, time +from decimal import Decimal +from unittest.mock import MagicMock + +import pytest +import typer + +from chronos_core.branching import ChronosBranchContext +from chronos_core.cli._support import DatabaseConfig, connect, print_result + + +def test_json_values(capsys: pytest.CaptureFixture[str]) -> None: + @dataclass + class Result: + values: list + print_result(Result([date(2026, 1, 2), datetime(2026, 1, 2, 3, 4), time(3, 4), Decimal("1.20"), b"hi"])) + assert json.loads(capsys.readouterr().out) == {"values": [ + "2026-01-02", "2026-01-02T03:04:00", "03:04:00", "1.20", {"encoding": "base64", "data": "aGk="} + ]} + print_result(None) + assert capsys.readouterr().out == "" + with pytest.raises(TypeError, match="Unsupported JSON value"): + print_result(object()) + + +@pytest.mark.parametrize("failure", [False, True]) +def test_connection_cleanup(monkeypatch: pytest.MonkeyPatch, failure: bool) -> None: + database = MagicMock() + monkeypatch.setattr(ChronosBranchContext, "connect", MagicMock(return_value=database)) + if failure: + with pytest.raises(typer.Exit): + with connect(DatabaseConfig("sqlite:///:memory:")): + raise ValueError("operation failed") + else: + with connect(DatabaseConfig("sqlite:///:memory:")) as context: + assert context is database + database.close.assert_called_once() diff --git a/uv.lock b/uv.lock index e0d0e15..16d097e 100644 --- a/uv.lock +++ b/uv.lock @@ -190,6 +190,7 @@ source = { editable = "packages/chronos-core" } dependencies = [ { name = "psycopg", extra = ["binary"] }, { name = "sqlglot" }, + { name = "typer" }, ] [package.optional-dependencies] @@ -228,6 +229,7 @@ requires-dist = [ { name = "sqlglot", specifier = ">=25" }, { name = "tomli", marker = "python_full_version < '3.11' and extra == 'mcp'", specifier = ">=2.0" }, { name = "tomli", marker = "python_full_version < '3.11' and extra == 'test'", specifier = ">=2.0" }, + { name = "typer", specifier = ">=0.25.1" }, ] provides-extras = ["duckdb", "qdrant", "mcp", "test"] @@ -359,7 +361,7 @@ name = "exceptiongroup" version = "1.3.1" source = { registry = "https://pypi.org/simple" } dependencies = [ - { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + { name = "typing-extensions" }, ] sdist = { url = "https://files.pythonhosted.org/packages/50/79/66800aadf48771f6b62f7eb014e352e5d06856655206165d775e675a02c9/exceptiongroup-1.3.1.tar.gz", hash = "sha256:8b412432c6055b0b7d14c310000ae93352ed6754f70fa8f7c34141f91c4e3219", size = 30371, upload-time = "2025-11-21T23:01:54.787Z" } wheels = [ From 1ec56e1a15638c55f8fd31d83c959d1f863a79db Mon Sep 17 00:00:00 2001 From: Seth Rojas Date: Wed, 28 Oct 2026 21:20:49 -0700 Subject: [PATCH 02/11] test(cli): verify relational workflows on PostgreSQL Separate real database workflows from command-level tests. Reuse one temporary PostgreSQL database per test session, verify SQL inputs and rollback, and run the installed CLI tests in CI. Document the disposable Docker setup. Validation: 49 CLI tests passed against SQLite and PostgreSQL 16. The GitHub Actions matrix has been configured but has not run for this commit. --- .github/workflows/relational-package.yml | 18 +++ docs/cli.md | 55 ++++++++ tests/test_cli.py | 71 ----------- tests/test_cli_integration.py | 152 +++++++++++++++++++++++ 4 files changed, 225 insertions(+), 71 deletions(-) create mode 100644 tests/test_cli_integration.py diff --git a/.github/workflows/relational-package.yml b/.github/workflows/relational-package.yml index 0e6ebf4..77e4578 100644 --- a/.github/workflows/relational-package.yml +++ b/.github/workflows/relational-package.yml @@ -63,6 +63,18 @@ jobs: PYTHONPATH=examples/verl python -m pytest examples/verl/test_e2b_episode.py -q wheel: runs-on: ubuntu-22.04 + services: + postgres: + image: postgres:16-alpine + env: + POSTGRES_PASSWORD: postgres + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U postgres" + --health-interval 5s + --health-timeout 5s + --health-retries 10 strategy: matrix: python: ['3.10', '3.12'] @@ -81,3 +93,9 @@ jobs: run: | python -I -m pytest tests/test_build_features.py tests/test_native_interval.py tests/test_interval_allocator.py tests/test_checkpoint_forks.py tests/test_reused_session_transactions.py -k 'not postgres' -q --timeout=60 python -I examples/single_store_branching.py + - name: Test the installed CLI against SQLite and PostgreSQL + env: + CHRONOS_CLI_TEST_POSTGRES_ADMIN_URL: postgresql://postgres:postgres@127.0.0.1:5432/postgres + run: | + python -I -m pytest tests/test_cli.py tests/test_cli_support.py \ + tests/test_cli_integration.py -q --timeout=60 diff --git a/docs/cli.md b/docs/cli.md index 8e40391..75a226d 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -169,3 +169,58 @@ success result. This first version uses the default interval backend with one database per invocation. Workspace coordination, a persistent interactive shell, advanced connection tuning, and custom Python merge callbacks are not exposed. + +## Test PostgreSQL locally + +The same relational commands work with SQLite and PostgreSQL. To run the CLI +integration tests, start one disposable PostgreSQL server using Docker: + +```sh +docker run --detach --rm --name chronos-cli-postgres-test \ + -e POSTGRES_PASSWORD=chronos_test \ + -p 127.0.0.1:55434:5432 \ + --health-cmd 'pg_isready -U postgres' \ + --health-interval 1s --health-timeout 3s --health-retries 30 \ + postgres:16-alpine + +docker inspect --format '{{.State.Health.Status}}' chronos-cli-postgres-test +``` + +Wait until the health status is `healthy`. Port `55434` on your computer forwards +to PostgreSQL's port `5432` inside the container; it is bound only to localhost. +This port must be free before starting the container. Reuse the existing +container if you already started it; do not start one per test. + +With the package and pytest installed in your environment, run from the +repository root: + +```sh +CHRONOS_CLI_TEST_POSTGRES_ADMIN_URL=postgresql://postgres:chronos_test@127.0.0.1:55434/postgres \ + uv run --no-sync --package chronos-core python -m pytest \ + tests/test_cli.py tests/test_cli_support.py tests/test_cli_integration.py -v +``` + +The PostgreSQL account must have permission to create and drop databases. The +test session creates one uniquely named test database and reuses it, clearing its +`public` schema between cases. It drops that database on completion and never +resets the database named in the admin URL. No additional servers are started by +pytest. Without the environment variable, PostgreSQL cases are reported as skips; +an explicitly configured server that cannot be reached causes a test failure. + +The tests cover CLI-only setup, branch isolation, parameterized writes, SQL files +and stdin, checkpoints, diffs, successful merges, conflicts, and rollback after a +failed write. CI is configured to run these against the installed wheel on Python +3.10 and 3.12. + +`test_cli.py` checks command parsing, API arguments, errors, and entry points with +fake connections. `test_cli_support.py` checks JSON conversion and cleanup. +`test_cli_integration.py` runs real CLI processes against SQLite and PostgreSQL. + +Stop the disposable server when finished: + +```sh +docker stop chronos-cli-postgres-test +``` + +Because the container uses `--rm`, stopping it removes the container and its +temporary database storage. No Docker volume is retained for this test server. diff --git a/tests/test_cli.py b/tests/test_cli.py index 0568431..8acd9d5 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -139,53 +139,6 @@ def test_connection_failure_does_not_close_unassigned_context(connection: MagicM connection.close.assert_not_called() - -def test_sqlite_workflow(tmp_path: Path) -> None: - url = f"sqlite:///{tmp_path / 'workflow.db'}" - database = ChronosBranchContext.connect(url) - try: - database.db.execute("CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)") - database.db.commit() - finally: - database.close() - - def invoke(*args: str, ok: bool = True): - result = subprocess.run( - [sys.executable, "-X", "faulthandler", "-m", "chronos_core.cli", "--database-url", url, *args], - capture_output=True, text=True, timeout=30, - ) - assert result.returncode == (0 if ok else 1), result.stderr - return json.loads(result.stdout) if result.stdout.strip() else None - - assert invoke("table", "register", "items", "--primary-key", "id") is None - assert invoke("branch", "create", "trial") is None - invoke("branch", "create", "trial", ok=False) - assert {branch["branch_id"] for branch in invoke("branch", "list")} == {"main", "trial"} - assert invoke("execute", "INSERT INTO items (id, value) VALUES (:id, :value)", "--branch", "trial", - "--params", '{"id": 1, "value": "hello"}') == {"rowcount": 1} - assert invoke("query", "SELECT * FROM items", "--branch", "trial") == [{"id": 1, "value": "hello"}] - assert invoke("query", "SELECT * FROM items") == [] - assert invoke("checkpoint", "create", "saved", "--branch", "trial")["checkpoint_id"] == "saved" - assert len(invoke("checkpoint", "list", "--branch", "trial")) == 1 - assert len(invoke("diff", "main", "trial")["changes"]) == 1 - assert len(invoke("diff", "main", "trial", "--table", "items")) == 1 - assert invoke("merge", "preview", "trial", "main")["conflicts"] == [] - assert invoke("merge", "apply", "trial", "main")["applied"] == 1 - assert invoke("query", "SELECT * FROM items") == [{"id": 1, "value": "hello"}] - invoke("branch", "create", "conflicting") - invoke("execute", "UPDATE items SET value = 'source' WHERE id = 1", "--branch", "conflicting") - invoke("execute", "UPDATE items SET value = 'target' WHERE id = 1") - assert len(invoke("merge", "preview", "conflicting", "main")["conflicts"]) == 1 - invoke("merge", "apply", "conflicting", "main", ok=False) - invoke("merge", "apply", "conflicting", "main", "--policy", "snapshot_isolation", ok=False) - assert invoke("query", "SELECT value FROM items") == [{"value": "target"}] - invoke("branch", "delete", "conflicting") - invoke("query", "SELECT * FROM items", "--branch", "missing", ok=False) - invoke("execute", "DROP TABLE items", ok=False) - assert invoke("branch", "delete", "trial") is None - assert [branch["branch_id"] for branch in invoke("branch", "list")] == ["main"] - - def test_installed_entrypoints() -> None: executable = Path(sys.executable).parent / "chronos" for args in ([str(executable), "--help"], [sys.executable, "-m", "chronos_core.cli", "--help"]): @@ -228,27 +181,3 @@ def test_native_error_is_reported_without_traceback(connection: MagicMock) -> No assert result.stderr.strip() == "Error: INSERT must specify a column list" connection.checkout.return_value.__exit__.assert_called_once() connection.close.assert_called_once() - - -def test_cli_only_sqlite_setup(tmp_path: Path) -> None: - url = f"sqlite:///{tmp_path / 'cli-only.sqlite'}" - - def invoke(*args: str): - result = subprocess.run( - [sys.executable, "-X", "faulthandler", "-m", "chronos_core.cli", "--database-url", url, *args], - capture_output=True, text=True, timeout=30, - ) - assert result.returncode == 0, result.stderr - return json.loads(result.stdout) if result.stdout.strip() else None - - assert invoke("table", "create", - "CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)") is None - invoke("table", "register", "items", "--primary-key", "id") - invoke("branch", "create", "experiment") - assert invoke("execute", "INSERT INTO items (id, value) VALUES (1, 'hello')", - "--branch", "experiment") == {"rowcount": 1} - assert invoke("query", "SELECT * FROM items", "--branch", "experiment") == [{"id": 1, "value": "hello"}] - assert invoke("query", "SELECT * FROM items") == [] - assert len(invoke("diff", "main", "experiment")["changes"]) == 1 - assert invoke("merge", "apply", "experiment", "main")["applied"] == 1 - assert invoke("query", "SELECT * FROM items") == [{"id": 1, "value": "hello"}] diff --git a/tests/test_cli_integration.py b/tests/test_cli_integration.py new file mode 100644 index 0000000..14e9aff --- /dev/null +++ b/tests/test_cli_integration.py @@ -0,0 +1,152 @@ +"""Real CLI processes exercising the same workflows on SQLite and PostgreSQL.""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +from functools import partial +from pathlib import Path +from typing import Any, Iterator +from urllib.parse import urlsplit, urlunsplit +from uuid import uuid4 + +import psycopg +import pytest +from psycopg import sql as postgres_sql + +from chronos_core.branching import ChronosBranchContext + + +def _invoke_cli( + database_url: str, *args: str, ok: bool = True, input_text: str | None = None, +) -> Any: + result = subprocess.run( + [sys.executable, "-X", "faulthandler", "-m", "chronos_core.cli", + "--database-url", database_url, *args], + input=input_text, capture_output=True, text=True, timeout=30, + ) + assert result.returncode == (0 if ok else 1), result.stderr + if not ok: + assert result.stdout == "" + assert "Traceback" not in result.stderr + return json.loads(result.stdout) if result.stdout.strip() else None + + +@pytest.fixture(scope="session") +def postgres_cli_database_url() -> Iterator[str]: + """Create one owned database for the run, without resetting the admin database.""" + admin_url = os.environ.get("CHRONOS_CLI_TEST_POSTGRES_ADMIN_URL") + if not admin_url: + pytest.skip("Set CHRONOS_CLI_TEST_POSTGRES_ADMIN_URL to test PostgreSQL") + parsed = urlsplit(admin_url) + if parsed.scheme not in {"postgres", "postgresql"}: + pytest.fail("CHRONOS_CLI_TEST_POSTGRES_ADMIN_URL must be a PostgreSQL URL") + name = f"chronos_cli_test_{uuid4().hex}" + with psycopg.connect(admin_url, autocommit=True, connect_timeout=10) as admin: + admin.execute(postgres_sql.SQL("CREATE DATABASE {}").format(postgres_sql.Identifier(name))) + try: + yield urlunsplit(parsed._replace(path=f"/{name}")) + finally: + admin.execute( + postgres_sql.SQL("DROP DATABASE {} WITH (FORCE)").format(postgres_sql.Identifier(name)) + ) + + +@pytest.fixture(params=["sqlite", "postgres"]) +def cli_database_url(request: pytest.FixtureRequest, tmp_path: Path) -> str: + if request.param == "sqlite": + return f"sqlite:///{tmp_path / 'workflow.sqlite'}" + url = request.getfixturevalue("postgres_cli_database_url") + # This URL belongs only to the database created by our session fixture. + with psycopg.connect(url, autocommit=True, connect_timeout=10) as database: + database.execute("DROP SCHEMA public CASCADE") + database.execute("CREATE SCHEMA public") + return url + + +def test_relational_workflow(cli_database_url: str) -> None: + url = cli_database_url + database = ChronosBranchContext.connect(url) + try: + database.db.execute("CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)") + database.db.commit() + finally: + database.close() + + invoke = partial(_invoke_cli, url) + + assert invoke("table", "register", "items", "--primary-key", "id") is None + assert invoke("branch", "create", "trial") is None + invoke("branch", "create", "trial", ok=False) + assert {branch["branch_id"] for branch in invoke("branch", "list")} == {"main", "trial"} + assert invoke("execute", "INSERT INTO items (id, value) VALUES (:id, :value)", "--branch", "trial", + "--params", '{"id": 1, "value": "hello"}') == {"rowcount": 1} + assert invoke("query", "SELECT * FROM items", "--branch", "trial") == [{"id": 1, "value": "hello"}] + assert invoke("query", "SELECT * FROM items") == [] + assert invoke("checkpoint", "create", "saved", "--branch", "trial")["checkpoint_id"] == "saved" + assert len(invoke("checkpoint", "list", "--branch", "trial")) == 1 + assert len(invoke("diff", "main", "trial")["changes"]) == 1 + assert len(invoke("diff", "main", "trial", "--table", "items")) == 1 + assert invoke("merge", "preview", "trial", "main")["conflicts"] == [] + assert invoke("merge", "apply", "trial", "main")["applied"] == 1 + assert invoke("query", "SELECT * FROM items") == [{"id": 1, "value": "hello"}] + invoke("branch", "create", "conflicting") + invoke("execute", "UPDATE items SET value = 'source' WHERE id = 1", "--branch", "conflicting") + invoke("execute", "UPDATE items SET value = 'target' WHERE id = 1") + assert len(invoke("merge", "preview", "conflicting", "main")["conflicts"]) == 1 + invoke("merge", "apply", "conflicting", "main", ok=False) + invoke("merge", "apply", "conflicting", "main", "--policy", "snapshot_isolation", ok=False) + assert invoke("query", "SELECT value FROM items") == [{"value": "target"}] + invoke("branch", "delete", "conflicting") + invoke("query", "SELECT * FROM items", "--branch", "missing", ok=False) + invoke("execute", "DROP TABLE items", ok=False) + assert invoke("branch", "delete", "trial") is None + assert [branch["branch_id"] for branch in invoke("branch", "list")] == ["main"] + + +def test_cli_only_setup(cli_database_url: str) -> None: + url = cli_database_url + + invoke = partial(_invoke_cli, url) + + assert invoke("table", "create", + "CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)") is None + invoke("table", "register", "items", "--primary-key", "id") + invoke("branch", "create", "experiment") + assert invoke("execute", "INSERT INTO items (id, value) VALUES (1, 'hello')", + "--branch", "experiment") == {"rowcount": 1} + assert invoke("query", "SELECT * FROM items", "--branch", "experiment") == [{"id": 1, "value": "hello"}] + assert invoke("query", "SELECT * FROM items") == [] + assert len(invoke("diff", "main", "experiment")["changes"]) == 1 + assert invoke("merge", "apply", "experiment", "main")["applied"] == 1 + assert invoke("query", "SELECT * FROM items") == [{"id": 1, "value": "hello"}] + + +def test_relational_sql_inputs_and_rollback(cli_database_url: str, tmp_path: Path) -> None: + invoke = partial(_invoke_cli, cli_database_url) + statement = tmp_path / "statement.sql" + statement.write_text( + "CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)", encoding="utf-8", + ) + invoke("table", "create", "--file", str(statement)) + invoke("table", "register", "items", "--primary-key", "id") + statement.write_text( + "INSERT INTO items (id, value) VALUES (:id, :value)", encoding="utf-8", + ) + assert invoke("execute", "--file", str(statement), "--params", + '{"id": 1, "value": "100% done"}') == {"rowcount": 1} + assert invoke("query", "--stdin", "--params", '{"pattern": "100%"}', + input_text="SELECT * FROM items WHERE value LIKE :pattern") == [ + {"id": 1, "value": "100% done"}, + ] + assert invoke("execute", "--stdin", "--params", '{"id": 1, "value": "updated"}', + input_text="UPDATE items SET value = :value WHERE id = :id") == {"rowcount": 1} + statement.write_text("SELECT * FROM items", encoding="utf-8") + assert invoke("query", "--file", str(statement)) == [{"id": 1, "value": "updated"}] + # A failed statement must not leave its earlier rows committed. + invoke("execute", "INSERT INTO items (id, value) VALUES (2, 'partial'), (1, 'duplicate')", ok=False) + assert invoke("query", input_text="SELECT * FROM items ORDER BY id") == [ + {"id": 1, "value": "updated"}, + ] + From 644be787806c6dde6bba56ddfe0e1773f187c159 Mon Sep 17 00:00:00 2001 From: Seth Rojas Date: Wed, 28 Oct 2026 21:44:54 -0700 Subject: [PATCH 03/11] feat(cli): support DuckDB with separate branch metadata Expose --metadata-url and CHRONOS_METADATA_URL through the existing connect_split API, retaining relational command behavior. Validate required DuckDB metadata, redact both connection URLs, and use the macOS loader path for packaged native libraries. Extend integration coverage to DuckDB with SQLite and PostgreSQL metadata, add analytical queries, configure an optional DuckDB CI build, and document setup. Local validation: 67 passed and one expected disabled-driver test skipped. GitHub CI has not yet validated this commit. --- .github/workflows/relational-package.yml | 13 ++- .gitignore | 4 +- docs/cli.md | 83 +++++++++++++++++-- packages/chronos-core/CMakeLists.txt | 8 +- .../src/chronos_core/cli/_support.py | 17 +++- .../chronos-core/src/chronos_core/cli/main.py | 6 +- tests/test_cli.py | 50 +++++++++++ tests/test_cli_integration.py | 79 ++++++++++++------ 8 files changed, 216 insertions(+), 44 deletions(-) diff --git a/.github/workflows/relational-package.yml b/.github/workflows/relational-package.yml index 77e4578..bb38ce7 100644 --- a/.github/workflows/relational-package.yml +++ b/.github/workflows/relational-package.yml @@ -77,7 +77,13 @@ jobs: --health-retries 10 strategy: matrix: - python: ['3.10', '3.12'] + include: + - python: '3.10' + duckdb: 'OFF' + - python: '3.12' + duckdb: 'OFF' + - python: '3.12' + duckdb: 'ON' steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 @@ -87,15 +93,16 @@ jobs: - run: python -m pip install build pytest pytest-asyncio pytest-timeout - name: Build from source distribution and install the wheel run: | - python -m build packages/chronos-core --outdir /tmp/chronos-wheel -Cbuild-dir=/tmp/chronos-build -Cbuild.tool-args=-j2 + python -m build packages/chronos-core --outdir /tmp/chronos-wheel -Cbuild-dir=/tmp/chronos-build -Cbuild.tool-args=-j2 -Ccmake.define.CHRONOS_WITH_DUCKDB=${{ matrix.duckdb }} python -m pip install /tmp/chronos-wheel/*.whl - name: Test the installed relational artifact run: | python -I -m pytest tests/test_build_features.py tests/test_native_interval.py tests/test_interval_allocator.py tests/test_checkpoint_forks.py tests/test_reused_session_transactions.py -k 'not postgres' -q --timeout=60 python -I examples/single_store_branching.py - - name: Test the installed CLI against SQLite and PostgreSQL + - name: Test the installed CLI against the enabled relational stores env: CHRONOS_CLI_TEST_POSTGRES_ADMIN_URL: postgresql://postgres:postgres@127.0.0.1:5432/postgres + CHRONOS_CLI_REQUIRE_DUCKDB: ${{ matrix.duckdb }} run: | python -I -m pytest tests/test_cli.py tests/test_cli_support.py \ tests/test_cli_integration.py -q --timeout=60 diff --git a/.gitignore b/.gitignore index 3019220..26fd921 100644 --- a/.gitignore +++ b/.gitignore @@ -28,9 +28,11 @@ dist/ # Generated benchmark output .benchmarks/ -# Generated SQLite databases and journal files +# Generated local databases and journal files *.sqlite *.sqlite-* +*.duckdb +*.duckdb.wal # Literal local workspace state directory /$state/ diff --git a/docs/cli.md b/docs/cli.md index 75a226d..b4a2ab7 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -166,14 +166,79 @@ input uses exit code 2. A merge preview may successfully return conflicts as dat merge application follows the API's rejection behavior. Errors do not emit a JSON success result. -This first version uses the default interval backend with one database per -invocation. Workspace coordination, a persistent interactive shell, advanced -connection tuning, and custom Python merge callbacks are not exposed. +The CLI uses the default interval backend with one relational data store per +invocation. DuckDB uses a separate metadata store as described below. Workspace +coordination, a persistent interactive shell, advanced connection tuning, and +custom Python merge callbacks are not exposed. + +## DuckDB data with separate metadata + +DuckDB is an in-process SQL database designed for analytical queries such as +aggregations and joins. The CLI opens a local DuckDB file; no DuckDB server or +port is required. Chronos stores branch metadata separately in SQLite or +PostgreSQL, using the existing `ChronosBranchContext.connect_split()` API. + +The native extension must be built with DuckDB enabled. Installing the Python +`duckdb` package alone does not enable Chronos's native driver. On Linux x86-64, +the build downloads the pinned DuckDB library: + +```sh +uv sync --inexact --package chronos-core --reinstall-package chronos-core \ + -Ccmake.define.CHRONOS_WITH_DUCKDB=ON +``` + +On macOS or other architectures, supply `-Ccmake.define.CHRONOS_DUCKDB_ROOT=PATH` +with a directory containing `duckdb.h` and the compatible `libduckdb` library. +The repository pins DuckDB 1.5.4 for its automatic Linux build. For example, after +extracting the [official release's macOS universal library](https://github.com/duckdb/duckdb/releases/tag/v1.5.4) into +`packages/chronos-core/build/duckdb-sdk`, rebuild with: + +```sh +CMAKE_PREFIX_PATH="$(brew --prefix libpq)" \ +uv sync --inexact --package chronos-core --reinstall-package chronos-core \ + -C "cmake.define.SQLite3_INCLUDE_DIR=$(brew --prefix sqlite)/include" \ + -C "cmake.define.SQLite3_LIBRARY=$(brew --prefix sqlite)/lib/libsqlite3.dylib" \ + -Ccmake.define.CHRONOS_WITH_DUCKDB=ON \ + -C "cmake.define.CHRONOS_DUCKDB_ROOT=$PWD/packages/chronos-core/build/duckdb-sdk" +``` + +Use the same table, branch, query, checkpoint, diff, and merge commands with two +connection URLs. Both databases must persist between one-off commands: + +```sh +mkdir -p .chronos +export CHRONOS_DATABASE_URL="duckdb:///$PWD/.chronos/analytics.duckdb" +export CHRONOS_METADATA_URL="sqlite:///$PWD/.chronos/analytics-metadata.sqlite" + +uv run --no-sync --package chronos-core chronos table create \ + 'CREATE TABLE sales (id INTEGER PRIMARY KEY, amount BIGINT)' +uv run --no-sync --package chronos-core chronos table register sales --primary-key id +uv run --no-sync --package chronos-core chronos execute \ + 'INSERT INTO sales (id, amount) VALUES (1, 10), (2, 20)' +uv run --no-sync --package chronos-core chronos branch create forecast +uv run --no-sync --package chronos-core chronos execute \ + 'UPDATE sales SET amount = amount * 2' --branch forecast +uv run --no-sync --package chronos-core chronos query \ + 'SELECT SUM(amount) AS total FROM sales' --branch forecast +``` + +The forecast totals 60 while `main` still totals 30. You can substitute a +PostgreSQL URL for `CHRONOS_METADATA_URL`; no multi-store workspace is required. +The explicit `--metadata-url` flag precedes the command name and overrides its +environment variable, just like `--database-url`. Missing DuckDB metadata produces +a usage error before connecting. An unavailable native driver produces the API's +build-option error. + +Each invocation must use the same data/metadata pair to access the same branches. +Unset `CHRONOS_METADATA_URL` when returning to the default single-database SQLite +or PostgreSQL mode. This wrapper preserves the API's split-store semantics; it +does not add cross-database transaction guarantees. DuckDB's normal file locking +also applies, so use sequential invocations for this one-off CLI workflow. ## Test PostgreSQL locally -The same relational commands work with SQLite and PostgreSQL. To run the CLI -integration tests, start one disposable PostgreSQL server using Docker: +The same relational commands work with SQLite, PostgreSQL, and DuckDB. To run the +CLI integration tests, start one disposable PostgreSQL server using Docker: ```sh docker run --detach --rm --name chronos-cli-postgres-test \ @@ -207,10 +272,14 @@ resets the database named in the admin URL. No additional servers are started by pytest. Without the environment variable, PostgreSQL cases are reported as skips; an explicitly configured server that cannot be reached causes a test failure. +A DuckDB-enabled build also runs the workflows with SQLite and PostgreSQL +metadata independently. Without its native driver those cases are skipped. Set +`CHRONOS_CLI_REQUIRE_DUCKDB=ON` to fail rather than skip if the driver is missing. + The tests cover CLI-only setup, branch isolation, parameterized writes, SQL files and stdin, checkpoints, diffs, successful merges, conflicts, and rollback after a -failed write. CI is configured to run these against the installed wheel on Python -3.10 and 3.12. +failed write. CI is configured to test the installed wheel on Python 3.10 and 3.12, with +an additional DuckDB-enabled Python 3.12 build. `test_cli.py` checks command parsing, API arguments, errors, and entry points with fake connections. `test_cli_support.py` checks JSON conversion and cleanup. diff --git a/packages/chronos-core/CMakeLists.txt b/packages/chronos-core/CMakeLists.txt index c974622..107da94 100644 --- a/packages/chronos-core/CMakeLists.txt +++ b/packages/chronos-core/CMakeLists.txt @@ -133,9 +133,11 @@ if(CHRONOS_BUILD_PYTHON_MODULE) pybind11_add_module(_native_interval src/chronos_core/native/interval_module.cpp) target_compile_features(_native_interval PRIVATE cxx_std_17) target_link_libraries(_native_interval PRIVATE chronos_native_backend) - set_target_properties(_native_interval PROPERTIES - INSTALL_RPATH "$ORIGIN" - ) + if(APPLE) + set_target_properties(_native_interval PROPERTIES INSTALL_RPATH "@loader_path") + else() + set_target_properties(_native_interval PROPERTIES INSTALL_RPATH "$ORIGIN") + endif() install(TARGETS _native_interval LIBRARY DESTINATION chronos_core) endif() diff --git a/packages/chronos-core/src/chronos_core/cli/_support.py b/packages/chronos-core/src/chronos_core/cli/_support.py index 51253e9..71b03d2 100644 --- a/packages/chronos-core/src/chronos_core/cli/_support.py +++ b/packages/chronos-core/src/chronos_core/cli/_support.py @@ -10,6 +10,7 @@ from datetime import date, datetime, time from decimal import Decimal from typing import TYPE_CHECKING, Any, Iterator +from urllib.parse import urlsplit import psycopg import typer @@ -21,6 +22,7 @@ @dataclass(frozen=True) class DatabaseConfig: database_url: str + metadata_url: str | None = None def database_config(ctx: typer.Context) -> DatabaseConfig: @@ -30,6 +32,11 @@ def database_config(ctx: typer.Context) -> DatabaseConfig: "Provide --database-url or set CHRONOS_DATABASE_URL.", param_hint="--database-url", ) + if urlsplit(database.database_url).scheme == "duckdb" and not database.metadata_url: + raise typer.BadParameter( + "DuckDB requires --metadata-url or CHRONOS_METADATA_URL for SQLite or PostgreSQL metadata.", + param_hint="--metadata-url", + ) return database @@ -55,15 +62,19 @@ def connect(database: DatabaseConfig) -> Iterator[ChronosBranchContext]: from chronos_core.branching import BranchingError, ChronosBranchContext try: - context = ChronosBranchContext.connect(database.database_url) + if database.metadata_url is None: + context = ChronosBranchContext.connect(database.database_url) + else: + context = ChronosBranchContext.connect_split(database.database_url, database.metadata_url) try: yield context finally: context.close() except (BranchingError, RuntimeError, sqlite3.Error, psycopg.Error, OSError, ValueError, TypeError) as error: message = str(error) - if database.database_url: - message = message.replace(database.database_url, "") + for url in (database.database_url, database.metadata_url): + if url: + message = message.replace(url, "") message = re.sub(r"([a-zA-Z][\w+.-]*://)[^\s/@]+@", r"\1@", message) message = re.sub(r"(?i)(password\s*=\s*)(?:'[^']*'|\"[^\"]*\"|[^\s]+)", r"\1", message) typer.echo(f"Error: {message}", err=True) diff --git a/packages/chronos-core/src/chronos_core/cli/main.py b/packages/chronos-core/src/chronos_core/cli/main.py index df3ff30..c3ea892 100644 --- a/packages/chronos-core/src/chronos_core/cli/main.py +++ b/packages/chronos-core/src/chronos_core/cli/main.py @@ -35,8 +35,12 @@ def main( database_url: str | None = typer.Option( None, "--database-url", envvar="CHRONOS_DATABASE_URL", help="Database connection URL." ), + metadata_url: str | None = typer.Option( + None, "--metadata-url", envvar="CHRONOS_METADATA_URL", + help="Separate SQLite or PostgreSQL metadata URL; required for DuckDB.", + ), ) -> None: - ctx.obj = DatabaseConfig(database_url) if database_url else None + ctx.obj = DatabaseConfig(database_url, metadata_url) if database_url else None @branch_app.command("list") diff --git a/tests/test_cli.py b/tests/test_cli.py index 8acd9d5..9aa1030 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -23,6 +23,7 @@ def connection(monkeypatch: pytest.MonkeyPatch) -> MagicMock: database.checkout.return_value.__enter__.return_value.query.return_value = [] monkeypatch.setattr(ChronosBranchContext, "connect", MagicMock(return_value=database)) monkeypatch.delenv("CHRONOS_DATABASE_URL", raising=False) + monkeypatch.delenv("CHRONOS_METADATA_URL", raising=False) return database @@ -144,6 +145,7 @@ def test_installed_entrypoints() -> None: for args in ([str(executable), "--help"], [sys.executable, "-m", "chronos_core.cli", "--help"]): env = dict(os.environ) env.pop("CHRONOS_DATABASE_URL", None) + env.pop("CHRONOS_METADATA_URL", None) result = subprocess.run(args, capture_output=True, text=True, env=env, timeout=30) assert result.returncode == 0, result.stderr assert "--database-url" in result.stdout @@ -181,3 +183,51 @@ def test_native_error_is_reported_without_traceback(connection: MagicMock) -> No assert result.stderr.strip() == "Error: INSERT must specify a column list" connection.checkout.return_value.__exit__.assert_called_once() connection.close.assert_called_once() + + +@pytest.mark.parametrize("source", ["flag", "environment", "override"]) +def test_split_connection_configuration(connection: MagicMock, monkeypatch: pytest.MonkeyPatch, source: str) -> None: + split = MagicMock(return_value=connection) + monkeypatch.setattr(ChronosBranchContext, "connect_split", split) + args = ["--database-url", "duckdb:////tmp/data.duckdb"] + environment = {} + if source != "flag": + environment["CHRONOS_METADATA_URL"] = "sqlite:////tmp/env.sqlite" + if source != "environment": + args.extend(["--metadata-url", "sqlite:////tmp/flag.sqlite"]) + result = runner.invoke(app, [*args, "branch", "list"], env=environment) + assert result.exit_code == 0, result.output + expected_metadata = "sqlite:////tmp/env.sqlite" if source == "environment" else "sqlite:////tmp/flag.sqlite" + split.assert_called_once_with("duckdb:////tmp/data.duckdb", expected_metadata) + ChronosBranchContext.connect.assert_not_called() + connection.close.assert_called_once() + + +def test_duckdb_requires_metadata_before_connecting(connection: MagicMock) -> None: + result = runner.invoke(app, ["--database-url", "duckdb:////tmp/data.duckdb", "branch", "list"]) + assert result.exit_code == 2 + assert "--metadata-url" in result.output + ChronosBranchContext.connect.assert_not_called() + + +def test_split_connection_redacts_metadata_credentials(connection: MagicMock, monkeypatch: pytest.MonkeyPatch) -> None: + metadata = "postgresql://user:secret@localhost/metadata" + split = MagicMock(side_effect=ValueError(f"Cannot connect to {metadata}")) + monkeypatch.setattr(ChronosBranchContext, "connect_split", split) + result = runner.invoke(app, ["--database-url", "duckdb:////tmp/data.duckdb", + "--metadata-url", metadata, "branch", "list"]) + assert result.exit_code == 1 + assert "secret" not in result.output + assert "" in result.stderr + connection.close.assert_not_called() + + +def test_unavailable_duckdb_driver_reports_api_error(connection: MagicMock, monkeypatch: pytest.MonkeyPatch) -> None: + split = MagicMock(side_effect=RuntimeError("DuckDB support requires CHRONOS_WITH_DUCKDB=ON")) + monkeypatch.setattr(ChronosBranchContext, "connect_split", split) + result = runner.invoke(app, ["--database-url", "duckdb:////tmp/data.duckdb", + "--metadata-url", "sqlite:////tmp/metadata.sqlite", "branch", "list"]) + assert result.exit_code == 1 + assert result.stdout == "" + assert "CHRONOS_WITH_DUCKDB=ON" in result.stderr + connection.close.assert_not_called() diff --git a/tests/test_cli_integration.py b/tests/test_cli_integration.py index 14e9aff..203a280 100644 --- a/tests/test_cli_integration.py +++ b/tests/test_cli_integration.py @@ -15,16 +15,22 @@ import pytest from psycopg import sql as postgres_sql -from chronos_core.branching import ChronosBranchContext +from chronos_core import _native_interval +from chronos_core.cli._support import DatabaseConfig def _invoke_cli( - database_url: str, *args: str, ok: bool = True, input_text: str | None = None, + database: DatabaseConfig, *args: str, ok: bool = True, input_text: str | None = None, ) -> Any: + connection_args = ["--database-url", database.database_url] + if database.metadata_url is not None: + connection_args.extend(["--metadata-url", database.metadata_url]) + environment = dict(os.environ) + environment.pop("CHRONOS_METADATA_URL", None) result = subprocess.run( [sys.executable, "-X", "faulthandler", "-m", "chronos_core.cli", - "--database-url", database_url, *args], - input=input_text, capture_output=True, text=True, timeout=30, + *connection_args, *args], + input=input_text, capture_output=True, text=True, timeout=30, env=environment, ) assert result.returncode == (0 if ok else 1), result.stderr if not ok: @@ -53,29 +59,33 @@ def postgres_cli_database_url() -> Iterator[str]: ) -@pytest.fixture(params=["sqlite", "postgres"]) -def cli_database_url(request: pytest.FixtureRequest, tmp_path: Path) -> str: - if request.param == "sqlite": - return f"sqlite:///{tmp_path / 'workflow.sqlite'}" +@pytest.fixture(params=["sqlite", "postgres", "duckdb-sqlite", "duckdb-postgres"]) +def cli_database(request: pytest.FixtureRequest, tmp_path: Path) -> DatabaseConfig: + kind = request.param + if kind.startswith("duckdb") and not _native_interval.has_duckdb: + if os.environ.get("CHRONOS_CLI_REQUIRE_DUCKDB") == "ON": + pytest.fail("The CLI test matrix requires a DuckDB-enabled native build") + pytest.skip("DuckDB disabled in this native build") + if kind == "sqlite": + return DatabaseConfig(f"sqlite:///{tmp_path / 'workflow.sqlite'}") + if kind == "duckdb-sqlite": + return DatabaseConfig( + f"duckdb:///{tmp_path / 'workflow.duckdb'}", + f"sqlite:///{tmp_path / 'metadata.sqlite'}", + ) url = request.getfixturevalue("postgres_cli_database_url") # This URL belongs only to the database created by our session fixture. with psycopg.connect(url, autocommit=True, connect_timeout=10) as database: database.execute("DROP SCHEMA public CASCADE") database.execute("CREATE SCHEMA public") - return url + return DatabaseConfig( + f"duckdb:///{tmp_path / 'workflow.duckdb'}", url, + ) if kind == "duckdb-postgres" else DatabaseConfig(url) -def test_relational_workflow(cli_database_url: str) -> None: - url = cli_database_url - database = ChronosBranchContext.connect(url) - try: - database.db.execute("CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)") - database.db.commit() - finally: - database.close() - - invoke = partial(_invoke_cli, url) - +def test_relational_workflow(cli_database: DatabaseConfig) -> None: + invoke = partial(_invoke_cli, cli_database) + invoke("table", "create", "CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)") assert invoke("table", "register", "items", "--primary-key", "id") is None assert invoke("branch", "create", "trial") is None invoke("branch", "create", "trial", ok=False) @@ -105,10 +115,8 @@ def test_relational_workflow(cli_database_url: str) -> None: assert [branch["branch_id"] for branch in invoke("branch", "list")] == ["main"] -def test_cli_only_setup(cli_database_url: str) -> None: - url = cli_database_url - - invoke = partial(_invoke_cli, url) +def test_cli_only_setup(cli_database: DatabaseConfig) -> None: + invoke = partial(_invoke_cli, cli_database) assert invoke("table", "create", "CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)") is None @@ -123,8 +131,8 @@ def test_cli_only_setup(cli_database_url: str) -> None: assert invoke("query", "SELECT * FROM items") == [{"id": 1, "value": "hello"}] -def test_relational_sql_inputs_and_rollback(cli_database_url: str, tmp_path: Path) -> None: - invoke = partial(_invoke_cli, cli_database_url) +def test_relational_sql_inputs_and_rollback(cli_database: DatabaseConfig, tmp_path: Path) -> None: + invoke = partial(_invoke_cli, cli_database) statement = tmp_path / "statement.sql" statement.write_text( "CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)", encoding="utf-8", @@ -150,3 +158,22 @@ def test_relational_sql_inputs_and_rollback(cli_database_url: str, tmp_path: Pat {"id": 1, "value": "updated"}, ] + + +def test_relational_aggregation(cli_database: DatabaseConfig) -> None: + invoke = partial(_invoke_cli, cli_database) + invoke("table", "create", "CREATE TABLE sales (id INTEGER PRIMARY KEY, category TEXT, amount BIGINT)") + invoke("table", "register", "sales", "--primary-key", "id") + invoke("execute", "INSERT INTO sales (id, category, amount) VALUES (1, 'books', 10), (2, 'books', 20), (3, 'games', 50)") + invoke("branch", "create", "forecast") + invoke("execute", "UPDATE sales SET amount = amount * :factor WHERE category = :category", + "--branch", "forecast", "--params", '{"factor": 2, "category": "books"}') + query = "SELECT category, SUM(amount) AS total FROM sales GROUP BY category ORDER BY category" + # PostgreSQL SUM(BIGINT) returns NUMERIC, serialized as a precision-preserving string. + total = str if urlsplit(cli_database.database_url).scheme in {"postgres", "postgresql"} else int + assert invoke("query", query) == [ + {"category": "books", "total": total(30)}, {"category": "games", "total": total(50)}, + ] + assert invoke("query", query, "--branch", "forecast") == [ + {"category": "books", "total": total(60)}, {"category": "games", "total": total(50)}, + ] From 4d149cf450b272ddf2fe660eb3a47bd1f66b8478 Mon Sep 17 00:00:00 2001 From: Seth Rojas Date: Sun, 1 Nov 2026 21:33:58 -0700 Subject: [PATCH 04/11] feat(cli): wrap Qdrant collections points and branch operations Add local and server Qdrant configuration, dense-vector commands, branch lifecycle, checkpoints, diffs, and merges through the public Chronos APIs. Preserve API results as JSON, manage resource cleanup, and keep the dependency optional. Add command and integration tests, pinned server CI coverage, and concise usage documentation. Local validation: 56 Qdrant CLI tests passed; relational/build regressions had 67 passes and one expected skip. The unchanged library suite separately had a deep-branch interval exhaustion failure and a ChronosFS test requiring an unavailable Linux build. GitHub CI has not run for this commit. --- .github/workflows/relational-package.yml | 12 + .gitignore | 1 + docs/README.md | 4 +- docs/cli.md | 37 ++- packages/chronos-core/README.md | 4 +- .../src/chronos_core/cli/_qdrant_support.py | 135 +++++++++++ .../src/chronos_core/cli/_support.py | 37 ++- .../chronos-core/src/chronos_core/cli/main.py | 12 +- .../src/chronos_core/cli/qdrant.py | 167 ++++++++++++++ tests/test_cli_qdrant.py | 218 ++++++++++++++++++ tests/test_cli_qdrant_integration.py | 186 +++++++++++++++ 11 files changed, 789 insertions(+), 24 deletions(-) create mode 100644 packages/chronos-core/src/chronos_core/cli/_qdrant_support.py create mode 100644 packages/chronos-core/src/chronos_core/cli/qdrant.py create mode 100644 tests/test_cli_qdrant.py create mode 100644 tests/test_cli_qdrant_integration.py diff --git a/.github/workflows/relational-package.yml b/.github/workflows/relational-package.yml index bb38ce7..905962a 100644 --- a/.github/workflows/relational-package.yml +++ b/.github/workflows/relational-package.yml @@ -106,3 +106,15 @@ jobs: run: | python -I -m pytest tests/test_cli.py tests/test_cli_support.py \ tests/test_cli_integration.py -q --timeout=60 + - name: Test Qdrant CLI in local and server modes + if: matrix.python == '3.12' && matrix.duckdb == 'OFF' + env: + CHRONOS_CLI_TEST_POSTGRES_ADMIN_URL: postgresql://postgres:postgres@127.0.0.1:5432/postgres + CHRONOS_CLI_TEST_QDRANT_URL: http://127.0.0.1:6333 + run: | + python -m pip install 'qdrant-client==1.20.0' + docker run --detach --rm --name chronos-cli-qdrant-test \ + -p 127.0.0.1:6333:6333 qdrant/qdrant:v1.20.0 + trap 'docker stop chronos-cli-qdrant-test > /dev/null' EXIT + python -I -m pytest tests/test_cli_qdrant.py \ + tests/test_cli_qdrant_integration.py -q --timeout=60 diff --git a/.gitignore b/.gitignore index 26fd921..64d532b 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ dist/ .eggs/ # Local runtime state +/.chronos/ .chronos-code/ .chronos-runtime/ .chronos-overlay-test/ diff --git a/docs/README.md b/docs/README.md index 616f3fa..a770c25 100644 --- a/docs/README.md +++ b/docs/README.md @@ -13,8 +13,8 @@ page says otherwise, commands and APIs apply to Chronos 0.2.0a1. optional DuckDB, filesystem, and S3 features. - [Integration](integration.md) explains how to adopt existing tables and route application access through Chronos. -- [CLI](cli.md) wraps the relational API with branch, query, checkpoint, diff, - and merge commands, including JSON output for pipes and files. +- [CLI](cli.md) wraps the relational and Qdrant APIs with JSON output for + pipes and files. - [Compatibility and limits](compatibility.md) states what is supported and where direct database access can bypass branch isolation. - [Branching introduction](branching-introduction.md) explains the data model diff --git a/docs/cli.md b/docs/cli.md index b4a2ab7..a270f4d 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1,7 +1,7 @@ # Chronos CLI -The `chronos` command is a one-off wrapper around the public relational Python -API. Each invocation opens the selected database, calls the API, prints its +The `chronos` command is a one-off wrapper around the public relational and Qdrant +Python APIs. Each invocation opens the selected store, calls the API, prints its return value, and closes the connection. It does not remember a current branch or keep a background connection open. @@ -171,6 +171,39 @@ invocation. DuckDB uses a separate metadata store as described below. Workspace coordination, a persistent interactive shell, advanced connection tuning, and custom Python merge callbacks are not exposed. +## Qdrant + +The `qdrant` command group wraps `ChronosQdrantStore` and its branch sessions. +Install the optional Qdrant dependency; for an existing checkout environment: + +```sh +uv pip install 'qdrant-client>=1.18,<2' +mkdir -p .chronos/qdrant-demo +export CHRONOS_QDRANT_PATH="$PWD/.chronos/qdrant-demo/vectors" +export CHRONOS_METADATA_URL="sqlite:///$PWD/.chronos/qdrant-demo/metadata.sqlite" + +uv run --no-sync --package chronos-core chronos qdrant collection register documents --dimensions 3 +uv run --no-sync --package chronos-core chronos qdrant point upsert documents example \ + '{"vector":[1,0,0],"payload":{"title":"Example"}}' +uv run --no-sync --package chronos-core chronos qdrant search documents '[1,0,0]' +``` + +Choose persistent local storage with `--path` / `CHRONOS_QDRANT_PATH`, or a server +with `--url` / `CHRONOS_QDRANT_URL`, not both. Server authentication uses +`--api-key` / `CHRONOS_QDRANT_API_KEY`. Metadata requires SQLite or PostgreSQL via +`--metadata-url` / `CHRONOS_METADATA_URL`. Explicit Qdrant options go after +`qdrant` and before the subcommand; flags override their environment variables. + +Point upsert and search also accept JSON from `--file`, `--stdin`, or a pipe. +Use `--branch` to select a branch; `main` is the default. The group also exposes +branch, checkpoint, diff, and merge operations through the public store APIs. +Run `chronos qdrant --help` or subcommand help for available options. Results use +the JSON conventions above; search consumes vectors rather than generating them. + +The example stores everything under `.chronos/qdrant-demo/`. After closing clients, +remove that folder to delete both its vectors and metadata. Unset the demo +environment variables before switching stores. No storage is created by help. + ## DuckDB data with separate metadata DuckDB is an in-process SQL database designed for analytical queries such as diff --git a/packages/chronos-core/README.md b/packages/chronos-core/README.md index 9c43f57..0ec4c70 100644 --- a/packages/chronos-core/README.md +++ b/packages/chronos-core/README.md @@ -27,7 +27,7 @@ Optional builds use `-Ccmake.define.CHRONOS_WITH_FILESYSTEM=ON`, Python extras do not change compiled capabilities. See the repository's docs/installation.md, docs/integration.md, and docs/tutorials. -The installed `chronos` command wraps the relational API; run `chronos --help` -and see [the CLI guide](../../docs/cli.md) for setup and a complete workflow. +The installed `chronos` command wraps the relational and Qdrant APIs; run +`chronos --help` and see [the CLI guide](../../docs/cli.md) for setup and usage. This is a Python library, not a PostgreSQL-wire proxy. Managed access must pass through branch sessions. Version 0.2.0a1 is an experimental release candidate. diff --git a/packages/chronos-core/src/chronos_core/cli/_qdrant_support.py b/packages/chronos-core/src/chronos_core/cli/_qdrant_support.py new file mode 100644 index 0000000..60dfd93 --- /dev/null +++ b/packages/chronos-core/src/chronos_core/cli/_qdrant_support.py @@ -0,0 +1,135 @@ +"""Qdrant configuration, JSON input, and ownership of public API resources.""" +from __future__ import annotations + +import json +import math +import sqlite3 +import sys +from collections.abc import Generator +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import psycopg +import typer + +from chronos_core.cli._support import report_error + +if TYPE_CHECKING: + from chronos_core.workspace.qdrant import ChronosQdrantStore, QdrantBranchSession + + +@dataclass(frozen=True) +class QdrantConfig: + metadata_url: str + path: Path | None = None + url: str | None = None + api_key: str | None = None + collection_prefix: str = "chronos_" + timeout: float | None = None + + +def qdrant_config(ctx: typer.Context) -> QdrantConfig: + config = ctx.find_object(QdrantConfig) + if config is None or not config.metadata_url: + raise typer.BadParameter("Provide --metadata-url or CHRONOS_METADATA_URL.", param_hint="--metadata-url") + if (config.path is None) == (config.url is None): + raise typer.BadParameter("Choose exactly one of --path or --url.") + if config.metadata_url in {":memory:", "sqlite:///:memory:"} or str(config.path) == ":memory:": + raise typer.BadParameter("One-off commands require persistent storage, not :memory:.") + if config.path is not None and (config.api_key is not None or config.timeout is not None): + raise typer.BadParameter("--api-key and --timeout require --url.") + return config + + +def _store_class() -> type[ChronosQdrantStore]: + # Optional dependencies must not be imported when showing help or using SQL. + from chronos_core.workspace.qdrant import ChronosQdrantStore + return ChronosQdrantStore + + +@contextmanager +def open_qdrant(config: QdrantConfig) -> Generator[ChronosQdrantStore, None, None]: + try: + import grpc + import httpx + from qdrant_client.http.exceptions import ResponseHandlingException, UnexpectedResponse + from chronos_core.branching import BranchingError + except ImportError: + report_error(RuntimeError("Qdrant support requires installing chronos-core[qdrant].")) + + try: + factory = _store_class() + if config.path is not None: + store = factory.local( + config.metadata_url, path=config.path, collection_prefix=config.collection_prefix, + ) + elif config.url is not None: + store = factory.remote( + config.metadata_url, url=config.url, api_key=config.api_key, + collection_prefix=config.collection_prefix, timeout=config.timeout, + ) + else: + raise ValueError("Choose exactly one of --path or --url.") + try: + yield store + finally: + store.close() + except ( + BranchingError, RuntimeError, ValueError, TypeError, OSError, sqlite3.Error, + psycopg.Error, httpx.HTTPError, grpc.RpcError, ResponseHandlingException, UnexpectedResponse, + ) as error: + report_error(error, secrets=(config.api_key,), urls=(config.metadata_url, config.url)) + + +@contextmanager +def qdrant_session(store: ChronosQdrantStore, branch: str) -> Generator[QdrantBranchSession, None, None]: + # Qdrant sessions wrap a relational control session but have no close method. + # Both of these are public APIs; keep control-session ownership explicit. + with store.context.checkout(branch) as control: + yield store.checkout_control(control) + + +def read_json_input(text: str | None, file: Path | None, stdin: bool) -> Any: + if sum((text is not None, file is not None, stdin)) > 1: + raise typer.BadParameter("Choose JSON text, --file, or --stdin, not multiple sources.") + try: + if file is not None: + text = file.read_text(encoding="utf-8") + elif stdin or (text is None and not sys.stdin.isatty()): + text = sys.stdin.read() + except (OSError, UnicodeError) as error: + raise typer.BadParameter(f"Cannot read JSON input: {error}") from error + if text is None or not text.strip(): + raise typer.BadParameter("Provide JSON text, --file, or JSON on stdin.") + try: + return json.loads(text, parse_constant=_reject_constant) + except ValueError as error: + raise typer.BadParameter(f"Invalid JSON: {error}") from error + + +def _reject_constant(value: str) -> None: + raise ValueError(f"{value} is not a finite JSON number") + + +def dense_vector(value: Any) -> list[float]: + try: + valid = isinstance(value, list) and all( + isinstance(item, (int, float)) and not isinstance(item, bool) and math.isfinite(item) + for item in value + ) + except OverflowError: + valid = False + if not valid: + raise typer.BadParameter("Expected a JSON array of finite numbers for the vector.") + return value + + +def point_input(value: Any) -> tuple[list[float], dict[str, Any] | None]: + if not isinstance(value, dict) or "vector" not in value or set(value) - {"vector", "payload"}: + raise typer.BadParameter("Expected an object with vector and optional payload fields.") + payload = value.get("payload") + if payload is not None and not isinstance(payload, dict): + raise typer.BadParameter("The payload must be a JSON object or null.") + return dense_vector(value["vector"]), payload diff --git a/packages/chronos-core/src/chronos_core/cli/_support.py b/packages/chronos-core/src/chronos_core/cli/_support.py index 71b03d2..c0078a5 100644 --- a/packages/chronos-core/src/chronos_core/cli/_support.py +++ b/packages/chronos-core/src/chronos_core/cli/_support.py @@ -5,11 +5,13 @@ import json import re import sqlite3 +from collections.abc import Generator from contextlib import contextmanager from dataclasses import asdict, dataclass, is_dataclass from datetime import date, datetime, time from decimal import Decimal -from typing import TYPE_CHECKING, Any, Iterator +from enum import Enum +from typing import TYPE_CHECKING, Any, NoReturn from urllib.parse import urlsplit import psycopg @@ -25,6 +27,11 @@ class DatabaseConfig: metadata_url: str | None = None +class MergePolicy(str, Enum): + abort_on_conflict = "abort_on_conflict" + snapshot_isolation = "snapshot_isolation" + + def database_config(ctx: typer.Context) -> DatabaseConfig: database = ctx.find_object(DatabaseConfig) if database is None or not database.database_url: @@ -57,8 +64,25 @@ def print_result(result: Any) -> None: typer.echo(json.dumps(result, default=_json_default, ensure_ascii=False, indent=2)) +def report_error( + error: Exception, *, secrets: tuple[str | None, ...] = (), urls: tuple[str | None, ...] = (), +) -> NoReturn: + """Report an operational failure without exposing configured credentials.""" + message = str(error) + for url in urls: + if url: + message = message.replace(url, "") + for secret in secrets: + if secret: + message = message.replace(secret, "") + message = re.sub(r"([a-zA-Z][\w+.-]*://)[^\s/@]+@", r"\1@", message) + message = re.sub(r"(?i)(password\s*=\s*)(?:'[^']*'|\"[^\"]*\"|[^\s]+)", r"\1", message) + typer.echo(f"Error: {message}", err=True) + raise typer.Exit(1) from error + + @contextmanager -def connect(database: DatabaseConfig) -> Iterator[ChronosBranchContext]: +def connect(database: DatabaseConfig) -> Generator[ChronosBranchContext, None, None]: from chronos_core.branching import BranchingError, ChronosBranchContext try: @@ -71,11 +95,4 @@ def connect(database: DatabaseConfig) -> Iterator[ChronosBranchContext]: finally: context.close() except (BranchingError, RuntimeError, sqlite3.Error, psycopg.Error, OSError, ValueError, TypeError) as error: - message = str(error) - for url in (database.database_url, database.metadata_url): - if url: - message = message.replace(url, "") - message = re.sub(r"([a-zA-Z][\w+.-]*://)[^\s/@]+@", r"\1@", message) - message = re.sub(r"(?i)(password\s*=\s*)(?:'[^']*'|\"[^\"]*\"|[^\s]+)", r"\1", message) - typer.echo(f"Error: {message}", err=True) - raise typer.Exit(1) from error + report_error(error, urls=(database.database_url, database.metadata_url)) diff --git a/packages/chronos-core/src/chronos_core/cli/main.py b/packages/chronos-core/src/chronos_core/cli/main.py index c3ea892..1ce767e 100644 --- a/packages/chronos-core/src/chronos_core/cli/main.py +++ b/packages/chronos-core/src/chronos_core/cli/main.py @@ -1,9 +1,8 @@ -"""One-off terminal commands wrapping the public relational API.""" +"""One-off terminal commands wrapping public Chronos APIs.""" from __future__ import annotations import json import sys -from enum import Enum from pathlib import Path from typing import Any @@ -11,7 +10,8 @@ import sqlglot from sqlglot import exp -from chronos_core.cli._support import DatabaseConfig, connect, database_config, print_result +from chronos_core.cli._support import DatabaseConfig, MergePolicy, connect, database_config, print_result +from chronos_core.cli.qdrant import app as qdrant_app app = typer.Typer(no_args_is_help=True, pretty_exceptions_enable=False) branch_app = typer.Typer(no_args_is_help=True) @@ -22,11 +22,7 @@ app.add_typer(table_app, name="table", help="Create ordinary tables and register them with Chronos.") app.add_typer(checkpoint_app, name="checkpoint", help="Create and list checkpoints.") app.add_typer(merge_app, name="merge", help="Preview and apply merges.") - - -class MergePolicy(str, Enum): - abort_on_conflict = "abort_on_conflict" - snapshot_isolation = "snapshot_isolation" +app.add_typer(qdrant_app, name="qdrant", help="Branch-aware vector storage and search.") @app.callback() diff --git a/packages/chronos-core/src/chronos_core/cli/qdrant.py b/packages/chronos-core/src/chronos_core/cli/qdrant.py new file mode 100644 index 0000000..0d46e41 --- /dev/null +++ b/packages/chronos-core/src/chronos_core/cli/qdrant.py @@ -0,0 +1,167 @@ +"""Terminal commands wrapping Chronos's public Qdrant store API.""" +from __future__ import annotations + +from pathlib import Path + +import typer + +from chronos_core.cli._qdrant_support import ( + QdrantConfig, + dense_vector, + open_qdrant, + point_input, + qdrant_config, + qdrant_session, + read_json_input, +) +from chronos_core.cli._support import MergePolicy, print_result + +app = typer.Typer(no_args_is_help=True) +collection_app = typer.Typer(no_args_is_help=True) +point_app = typer.Typer(no_args_is_help=True) +branch_app = typer.Typer(no_args_is_help=True) +checkpoint_app = typer.Typer(no_args_is_help=True) +merge_app = typer.Typer(no_args_is_help=True) +app.add_typer(collection_app, name="collection", help="Register and inspect collections.") +app.add_typer(point_app, name="point", help="Write, retrieve, and delete branch-visible points.") +app.add_typer(branch_app, name="branch", help="Manage Qdrant branches.") +app.add_typer(checkpoint_app, name="checkpoint", help="Create and list checkpoints.") +app.add_typer(merge_app, name="merge", help="Preview and apply Qdrant merges.") + + +@app.callback() +def main( + ctx: typer.Context, + metadata_url: str | None = typer.Option(None, "--metadata-url", envvar="CHRONOS_METADATA_URL"), + path: Path | None = typer.Option(None, "--path", envvar="CHRONOS_QDRANT_PATH", file_okay=False), + url: str | None = typer.Option(None, "--url", envvar="CHRONOS_QDRANT_URL"), + api_key: str | None = typer.Option(None, "--api-key", envvar="CHRONOS_QDRANT_API_KEY", show_default=False), + collection_prefix: str = typer.Option("chronos_", "--collection-prefix", envvar="CHRONOS_QDRANT_COLLECTION_PREFIX"), + timeout: float | None = typer.Option(None, "--timeout", min=0.001, help="Remote transport timeout in seconds."), +) -> None: + """Use persistent local Qdrant (--path) or a server (--url) with Chronos metadata.""" + ctx.obj = QdrantConfig(metadata_url or "", path, url, api_key, collection_prefix, timeout) + + +@collection_app.command("register") +def register_collection( + ctx: typer.Context, name: str, + dimensions: int = typer.Option(..., "--dimensions", min=1), + distance: str = typer.Option("cosine", "--distance", help="cosine, dot, euclid, or manhattan."), +) -> None: + with open_qdrant(qdrant_config(ctx)) as store: + print_result(store.register_collection(name, dimensions, distance=distance)) + + +@collection_app.command("list") +def list_collections(ctx: typer.Context) -> None: + with open_qdrant(qdrant_config(ctx)) as store: + print_result(store.list_collections()) + + +@collection_app.command("info") +def collection_info(ctx: typer.Context, name: str) -> None: + with open_qdrant(qdrant_config(ctx)) as store: + print_result(store.collection_info(name)) + + +@point_app.command("upsert") +def upsert( + ctx: typer.Context, collection: str, point_id: str, + data: str | None = typer.Argument(None, help='JSON: {"vector": [...], "payload": {...}}'), + file: Path | None = typer.Option(None, "--file"), + stdin: bool = typer.Option(False, "--stdin"), + branch: str = typer.Option("main", "--branch"), + revision: str | None = typer.Option(None, "--revision"), +) -> None: + vector, payload = point_input(read_json_input(data, file, stdin)) + with open_qdrant(qdrant_config(ctx)) as store: + with qdrant_session(store, branch) as session: + print_result(session.upsert(collection, point_id, vector, payload, revision=revision)) + + +@point_app.command("get") +def get(ctx: typer.Context, collection: str, point_id: str, branch: str = typer.Option("main", "--branch")) -> None: + with open_qdrant(qdrant_config(ctx)) as store: + with qdrant_session(store, branch) as session: + print_result(session.get(collection, point_id)) + + +@point_app.command("list") +def list_points(ctx: typer.Context, collection: str, branch: str = typer.Option("main", "--branch")) -> None: + with open_qdrant(qdrant_config(ctx)) as store: + with qdrant_session(store, branch) as session: + print_result(session.list_points(collection)) + + +@point_app.command("delete") +def delete(ctx: typer.Context, collection: str, point_id: str, branch: str = typer.Option("main", "--branch")) -> None: + with open_qdrant(qdrant_config(ctx)) as store: + with qdrant_session(store, branch) as session: + print_result(session.delete(collection, point_id)) + + +@app.command() +def search( + ctx: typer.Context, collection: str, + vector: str | None = typer.Argument(None, help="Query vector as a JSON array."), + file: Path | None = typer.Option(None, "--file"), + stdin: bool = typer.Option(False, "--stdin"), + branch: str = typer.Option("main", "--branch"), + limit: int = typer.Option(10, "--limit"), + score_threshold: float | None = typer.Option(None, "--score-threshold"), +) -> None: + query_vector = dense_vector(read_json_input(vector, file, stdin)) + with open_qdrant(qdrant_config(ctx)) as store: + with qdrant_session(store, branch) as session: + print_result(session.search(collection, query_vector, limit=limit, score_threshold=score_threshold)) + + +@branch_app.command("list") +def list_branches(ctx: typer.Context) -> None: + with open_qdrant(qdrant_config(ctx)) as store: + print_result(store.context.list_branches()) + + +@branch_app.command("create") +def create_branch(ctx: typer.Context, name: str, from_branch: str = typer.Option("main", "--from")) -> None: + with open_qdrant(qdrant_config(ctx)) as store: + print_result(store.create_branch(name, from_branch=from_branch)) + + +@branch_app.command("delete") +def delete_branch(ctx: typer.Context, name: str) -> None: + with open_qdrant(qdrant_config(ctx)) as store: + print_result(store.delete_branch(name)) + + +@checkpoint_app.command("create") +def create_checkpoint(ctx: typer.Context, name: str, branch: str = typer.Option("main", "--branch")) -> None: + with open_qdrant(qdrant_config(ctx)) as store: + print_result(store.create_checkpoint(name, branch=branch)) + + +@checkpoint_app.command("list") +def list_checkpoints(ctx: typer.Context, branch: str | None = typer.Option(None, "--branch")) -> None: + with open_qdrant(qdrant_config(ctx)) as store: + print_result(store.context.list_checkpoints(branch=branch)) + + +@app.command() +def diff(ctx: typer.Context, left: str, right: str) -> None: + with open_qdrant(qdrant_config(ctx)) as store: + print_result(store.diff(left, right)) + + +@merge_app.command("preview") +def merge_preview(ctx: typer.Context, source: str, target: str, + policy: MergePolicy = typer.Option(MergePolicy.abort_on_conflict, "--policy")) -> None: + with open_qdrant(qdrant_config(ctx)) as store: + print_result(store.merge_preview(source, target, policy=policy.value)) + + +@merge_app.command("apply") +def merge_apply(ctx: typer.Context, source: str, target: str, + policy: MergePolicy = typer.Option(MergePolicy.abort_on_conflict, "--policy")) -> None: + with open_qdrant(qdrant_config(ctx)) as store: + print_result(store.merge_apply(source, target, policy=policy.value)) diff --git a/tests/test_cli_qdrant.py b/tests/test_cli_qdrant.py new file mode 100644 index 0000000..a698c9e --- /dev/null +++ b/tests/test_cli_qdrant.py @@ -0,0 +1,218 @@ +"""Command parsing, API forwarding, errors, and resource ownership for Qdrant.""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +from typer.testing import CliRunner + +from chronos_core.cli import _qdrant_support as support +from chronos_core.cli.main import app + +runner = CliRunner() + + +@pytest.fixture +def factory(monkeypatch: pytest.MonkeyPatch) -> MagicMock: + pytest.importorskip("qdrant_client") + for name in ("CHRONOS_METADATA_URL", "CHRONOS_QDRANT_PATH", "CHRONOS_QDRANT_URL", + "CHRONOS_QDRANT_API_KEY", "CHRONOS_QDRANT_COLLECTION_PREFIX"): + monkeypatch.delenv(name, raising=False) + factory = MagicMock() + factory.remote.return_value = factory.local.return_value + monkeypatch.setattr(support, "_store_class", lambda: factory) + return factory + + +def invoke(*args: str, **kwargs): + return runner.invoke(app, ["qdrant", "--path", "vectors", "--metadata-url", "sqlite:///metadata.sqlite", *args], **kwargs) + + +def test_help_does_not_connect(factory: MagicMock) -> None: + for args in (["--help"], ["qdrant", "--help"], ["qdrant", "point", "--help"], + ["qdrant", "search", "--help"]): + assert runner.invoke(app, args).exit_code == 0 + factory.local.assert_not_called() + factory.remote.assert_not_called() + + +@pytest.mark.parametrize(("args", "method", "positional", "keywords"), [ + (["collection", "register", "docs", "--dimensions", "3"], "register_collection", ("docs", 3), {"distance": "cosine"}), + (["collection", "register", "docs", "--dimensions", "3", "--distance", "dot"], "register_collection", ("docs", 3), {"distance": "dot"}), + (["collection", "list"], "list_collections", (), {}), + (["collection", "info", "docs"], "collection_info", ("docs",), {}), + (["branch", "create", "trial", "--from", "parent"], "create_branch", ("trial",), {"from_branch": "parent"}), + (["branch", "delete", "trial"], "delete_branch", ("trial",), {}), + (["checkpoint", "create", "saved", "--branch", "trial"], "create_checkpoint", ("saved",), {"branch": "trial"}), + (["diff", "main", "trial"], "diff", ("main", "trial"), {}), + (["merge", "preview", "trial", "main"], "merge_preview", ("trial", "main"), {"policy": "abort_on_conflict"}), + (["merge", "apply", "trial", "main", "--policy", "snapshot_isolation"], "merge_apply", ("trial", "main"), {"policy": "snapshot_isolation"}), +]) +def test_store_api_forwarding(factory: MagicMock, args, method, positional, keywords) -> None: + store = factory.local.return_value + getattr(store, method).return_value = {"result": "from API"} + result = invoke(*args) + assert result.exit_code == 0, result.output + getattr(store, method).assert_called_once_with(*positional, **keywords) + assert json.loads(result.stdout) == {"result": "from API"} + store.close.assert_called_once() + + +@pytest.mark.parametrize(("args", "method", "keywords"), [ + (["branch", "list"], "list_branches", {}), + (["checkpoint", "list", "--branch", "trial"], "list_checkpoints", {"branch": "trial"}), +]) +def test_metadata_context_calls(factory: MagicMock, args, method, keywords) -> None: + store = factory.local.return_value + getattr(store.context, method).return_value = [] + result = invoke(*args) + assert result.exit_code == 0, result.output + getattr(store.context, method).assert_called_once_with(**keywords) + assert json.loads(result.stdout) == [] + store.close.assert_called_once() + + +@pytest.mark.parametrize(("command", "method", "positional"), [ + (["point", "get", "docs", "one"], "get", ("docs", "one")), + (["point", "list", "docs"], "list_points", ("docs",)), + (["point", "delete", "docs", "one"], "delete", ("docs", "one")), +]) +def test_point_session_calls(factory: MagicMock, command, method, positional) -> None: + store = factory.local.return_value + session = store.checkout_control.return_value + getattr(session, method).return_value = False if method == "delete" else None + result = invoke(*command, "--branch", "trial") + assert result.exit_code == 0, result.output + store.context.checkout.assert_called_once_with("trial") + store.checkout_control.assert_called_once_with(store.context.checkout.return_value.__enter__.return_value) + getattr(session, method).assert_called_once_with(*positional) + assert result.stdout == ("false\n" if method == "delete" else "") + store.context.checkout.return_value.__exit__.assert_called_once() + store.close.assert_called_once() + + +@pytest.mark.parametrize("source", ["text", "file", "stdin", "pipe"]) +@pytest.mark.parametrize("command", ["upsert", "search"]) +def test_json_sources(factory: MagicMock, tmp_path: Path, source: str, command: str) -> None: + text = '{"vector": [1, 0, 0], "payload": {"title": "hello"}}' if command == "upsert" else "[1, 0, 0]" + args = ["point", "upsert", "docs", "one", "--revision", "r1"] if command == "upsert" else ["search", "docs", "--limit", "2", "--score-threshold", "0.5"] + input_text = None + if source == "text": + args.append(text) + elif source == "file": + path = tmp_path / "input.json" + path.write_text(text, encoding="utf-8") + args.extend(["--file", str(path)]) + else: + input_text = text + if source == "stdin": + args.append("--stdin") + store = factory.local.return_value + session = store.checkout_control.return_value + getattr(session, command).return_value = [] + result = invoke(*args, input=input_text) + assert result.exit_code == 0, result.output + if command == "upsert": + session.upsert.assert_called_once_with("docs", "one", [1, 0, 0], {"title": "hello"}, revision="r1") + else: + session.search.assert_called_once_with("docs", [1, 0, 0], limit=2, score_threshold=0.5) + assert json.loads(result.stdout) == [] + store.close.assert_called_once() + + +def test_connection_flags_and_environment(factory: MagicMock) -> None: + store = factory.local.return_value + store.list_collections.return_value = [] + env = {"CHRONOS_METADATA_URL": "sqlite:///env.sqlite", "CHRONOS_QDRANT_URL": "http://localhost:6333", + "CHRONOS_QDRANT_API_KEY": "test-secret"} + result = runner.invoke(app, ["qdrant", "--metadata-url", "sqlite:///flag.sqlite", "--collection-prefix", "demo_", + "--timeout", "12", "collection", "list"], env=env) + assert result.exit_code == 0, result.output + factory.remote.assert_called_once_with("sqlite:///flag.sqlite", url="http://localhost:6333", + api_key="test-secret", collection_prefix="demo_", timeout=12.0) + factory.local.assert_not_called() + store.close.assert_called_once() + + +@pytest.mark.parametrize("args", [ + ["collection", "list"], + ["--metadata-url", "sqlite:///m.sqlite", "collection", "list"], + ["--metadata-url", "sqlite:///m.sqlite", "--path", "vectors", "--url", "http://localhost:6333", "collection", "list"], + ["--metadata-url", "sqlite:///:memory:", "--path", "vectors", "collection", "list"], + ["--metadata-url", "sqlite:///m.sqlite", "--path", ":memory:", "collection", "list"], + ["--metadata-url", "sqlite:///m.sqlite", "--path", "vectors", "--api-key", "secret", "collection", "list"], +]) +def test_invalid_configuration_does_not_connect(factory: MagicMock, args) -> None: + assert runner.invoke(app, ["qdrant", *args]).exit_code == 2 + factory.local.assert_not_called() + factory.remote.assert_not_called() + + +@pytest.mark.parametrize("args", [ + ["search", "docs"], ["search", "docs", "not JSON"], ["search", "docs", "[NaN]"], + ["search", "docs", "[true]"], ["search", "docs", '{}'], ["search", "docs", "[1]", "--stdin"], + ["search", "docs", "--file", "missing.json"], + ["point", "upsert", "docs", "one", '{"vector": [1], "payload": []}'], + ["point", "upsert", "docs", "one", '{"vector": [1], "unexpected": 1}'], + ["collection", "register", "docs", "--dimensions", "0"], +]) +def test_invalid_input_does_not_connect(factory: MagicMock, args) -> None: + result = invoke(*args) + assert result.exit_code == 2, result.output + factory.local.assert_not_called() + + +def test_failure_closes_session_before_store_and_redacts_secrets(factory: MagicMock) -> None: + store = factory.local.return_value + session = store.checkout_control.return_value + events = [] + store.context.checkout.return_value.__exit__.side_effect = lambda *args: events.append("session") + store.close.side_effect = lambda: events.append("store") + session.search.side_effect = RuntimeError("Failed with api-key test-secret at postgresql://user:pw@host/meta") + result = runner.invoke(app, ["qdrant", "--url", "http://localhost:6333", "--api-key", "test-secret", + "--metadata-url", "postgresql://user:pw@host/meta", "search", "docs", "[1,0,0]"]) + assert result.exit_code == 1 + assert result.stdout == "" + assert "test-secret" not in result.stderr + assert "user:pw" not in result.stderr + assert events == ["session", "store"] + + +def test_factory_failure_does_not_close_unassigned_store(factory: MagicMock) -> None: + factory.local.side_effect = ValueError("Cannot open storage") + result = invoke("collection", "list") + assert result.exit_code == 1 + factory.local.return_value.close.assert_not_called() + + +def test_optional_dependency_is_not_needed_for_help(tmp_path: Path) -> None: + code = ''' +import importlib.abc +import sys +class Block(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == "qdrant_client" or fullname.startswith("qdrant_client."): + raise ModuleNotFoundError("simulated missing qdrant-client") +sys.meta_path.insert(0, Block()) +from chronos_core.cli.main import app +sys.argv = ["chronos", *sys.argv[1:]] +app() +''' + env = {key: value for key, value in os.environ.items() if not key.startswith("CHRONOS_")} + for args in (["--help"], ["qdrant", "--help"]): + result = subprocess.run([sys.executable, "-c", code, *args], capture_output=True, text=True, env=env) + assert result.returncode == 0, result.stderr + result = subprocess.run([sys.executable, "-c", code, "qdrant", "--metadata-url", f"sqlite:///{tmp_path / 'metadata.sqlite'}", + "--path", str(tmp_path / "vectors"), "collection", "list"], capture_output=True, text=True, env=env) + assert result.returncode == 1 + assert "chronos-core[qdrant]" in result.stderr + assert list(tmp_path.iterdir()) == [] + result = subprocess.run([sys.executable, "-c", code, "--database-url", f"sqlite:///{tmp_path / 'sql.sqlite'}", + "branch", "list"], capture_output=True, text=True, env=env) + assert result.returncode == 0, result.stderr + assert [branch["branch_id"] for branch in json.loads(result.stdout)] == ["main"] diff --git a/tests/test_cli_qdrant_integration.py b/tests/test_cli_qdrant_integration.py new file mode 100644 index 0000000..4d01a6a --- /dev/null +++ b/tests/test_cli_qdrant_integration.py @@ -0,0 +1,186 @@ +"""Persistent local and server-backed Qdrant through separate CLI processes.""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import time +from dataclasses import asdict +from pathlib import Path +from tempfile import TemporaryDirectory +from uuid import uuid4 + +import psycopg +import pytest + +pytest.importorskip("qdrant_client") +from qdrant_client import QdrantClient +from qdrant_client.http.exceptions import ResponseHandlingException, UnexpectedResponse + +from chronos_core.cli._qdrant_support import QdrantConfig +from chronos_core.workspace.qdrant import ChronosQdrantStore +# Reuse the existing isolated, session-owned PostgreSQL database fixture. +from tests.test_cli_integration import postgres_cli_database_url # noqa: F401 + + +@pytest.fixture(params=["local-sqlite", "remote-sqlite", "local-postgres", "remote-postgres"]) +def qdrant_target(request: pytest.FixtureRequest): + kind = request.param + remote_url = os.environ.get("CHRONOS_CLI_TEST_QDRANT_URL") + api_key = os.environ.get("CHRONOS_CLI_TEST_QDRANT_API_KEY") + if kind.startswith("remote") and not remote_url: + pytest.skip("Set CHRONOS_CLI_TEST_QDRANT_URL to test the Qdrant server") + client = None + prefix = f"cli_test_{uuid4().hex}_" + if kind.startswith("remote"): + client = QdrantClient(url=remote_url, api_key=api_key, timeout=2) + deadline = time.monotonic() + 30 + while True: + try: + client.get_collections() + break + except (ResponseHandlingException, UnexpectedResponse): + if time.monotonic() >= deadline: + client.close() + pytest.fail("Configured Qdrant test server is not ready after 30 seconds") + time.sleep(0.5) + try: + with TemporaryDirectory(prefix="chronos-cli-qdrant-") as directory: + root = Path(directory) + metadata = f"sqlite:///{root / 'metadata.sqlite'}" + if kind.endswith("postgres"): + metadata = request.getfixturevalue("postgres_cli_database_url") + # Only reset the database that our session fixture owns. + with psycopg.connect(metadata, autocommit=True) as admin: + admin.execute("DROP SCHEMA public CASCADE") + admin.execute("CREATE SCHEMA public") + yield QdrantConfig( + metadata_url=metadata, + path=None if kind.startswith("remote") else root / "vectors", + url=remote_url if kind.startswith("remote") else None, + api_key=api_key if kind.startswith("remote") else None, + collection_prefix=prefix, + ) + finally: + if client is not None: + try: + for collection in client.get_collections().collections: + if collection.name.startswith(prefix): + client.delete_collection(collection.name) + finally: + client.close() + + +def invoke(target: QdrantConfig, *args: str, ok: bool = True, input_text: str | None = None): + env = {key: value for key, value in os.environ.items() if not key.startswith("CHRONOS_")} + env["CHRONOS_METADATA_URL"] = target.metadata_url + env["CHRONOS_QDRANT_COLLECTION_PREFIX"] = target.collection_prefix + if target.path is not None: + env["CHRONOS_QDRANT_PATH"] = str(target.path) + else: + env["CHRONOS_QDRANT_URL"] = target.url + if target.api_key is not None: + env["CHRONOS_QDRANT_API_KEY"] = target.api_key + result = subprocess.run( + [sys.executable, "-m", "chronos_core.cli", "qdrant", *args], + env=env, input=input_text, capture_output=True, text=True, timeout=30, + ) + assert result.returncode == (0 if ok else 1), result.stderr + assert "Traceback" not in result.stderr + if not ok: + assert result.stdout == "" + return json.loads(result.stdout) if result.stdout.strip() else None + + +def open_api(target: QdrantConfig): + if target.path is not None: + return ChronosQdrantStore.local( + target.metadata_url, path=target.path, collection_prefix=target.collection_prefix, + ) + return ChronosQdrantStore.remote( + target.metadata_url, url=target.url, api_key=target.api_key, + collection_prefix=target.collection_prefix, + ) + + +def test_qdrant_cli_workflow(qdrant_target: QdrantConfig) -> None: + target = qdrant_target + info = invoke(target, "collection", "register", "documents", "--dimensions", "3") + assert info["name"] == "documents" + assert info["dimensions"] == 3 + assert invoke(target, "collection", "info", "documents") == info + assert invoke(target, "collection", "list") == [info] + first = {"vector": [1, 0, 0], "payload": {"title": "Runtime"}} + second = {"vector": [0, 1, 0], "payload": {"title": "Billing"}} + point = invoke(target, "point", "upsert", "documents", "runtime", json.dumps(first)) + invoke(target, "point", "upsert", "documents", "billing", json.dumps(second)) + assert invoke(target, "point", "get", "documents", "runtime") == point + assert {item["id"] for item in invoke(target, "point", "list", "documents")} == {"runtime", "billing"} + assert invoke(target, "point", "get", "documents", "missing") is None + results = invoke(target, "search", "documents", "[1,0,0]", "--limit", "2") + assert [item["id"] for item in results] == ["runtime", "billing"] + assert results[0]["score"] == pytest.approx(1.0, abs=1e-6) + assert results[1]["score"] == pytest.approx(0.0, abs=1e-6) + assert len(invoke(target, "search", "documents", "[1,0,0]", "--score-threshold", "0.9")) == 1 + + # Independent expected data plus a direct public-API comparison. + store = open_api(target) + try: + with store.context.checkout("main") as control: + session = store.checkout_control(control) + assert point == asdict(session.get("documents", "runtime")) + assert results == [asdict(item) for item in session.search("documents", [1, 0, 0], limit=2)] + finally: + store.close() + + invoke(target, "branch", "create", "experiment") + assert {item["branch_id"] for item in invoke(target, "branch", "list")} == {"main", "experiment"} + changed = {"vector": [0, 0, 1], "payload": {"title": "Experiment"}} + invoke(target, "point", "upsert", "documents", "runtime", json.dumps(changed), "--branch", "experiment") + assert invoke(target, "point", "delete", "documents", "billing", "--branch", "experiment") is True + assert invoke(target, "point", "get", "documents", "billing", "--branch", "experiment") is None + assert invoke(target, "point", "get", "documents", "billing")["payload"] == second["payload"] + assert invoke(target, "point", "get", "documents", "runtime")["payload"] == first["payload"] + assert invoke(target, "checkpoint", "create", "saved", "--branch", "experiment")["checkpoint_id"] == "saved" + assert len(invoke(target, "checkpoint", "list", "--branch", "experiment")) == 1 + changes = invoke(target, "diff", "main", "experiment")["changes"] + assert {(item["key"]["id"], item["change"]) for item in changes} == {("runtime", "modified"), ("billing", "deleted")} + assert invoke(target, "merge", "preview", "experiment", "main")["conflicts"] == [] + assert invoke(target, "merge", "apply", "experiment", "main")["applied"] == 2 + assert invoke(target, "point", "get", "documents", "runtime")["payload"] == changed["payload"] + assert invoke(target, "point", "get", "documents", "billing") is None + assert invoke(target, "point", "delete", "documents", "missing") is False + invoke(target, "branch", "delete", "experiment") + + +def test_qdrant_cli_conflicts_and_failed_operations(qdrant_target: QdrantConfig) -> None: + target = qdrant_target + invoke(target, "collection", "register", "docs", "--dimensions", "3") + invoke(target, "point", "upsert", "docs", "one", '{"vector":[1,0,0],"payload":{"value":"base"}}') + invoke(target, "branch", "create", "trial") + invoke(target, "branch", "create", "trial", ok=False) + invoke(target, "point", "upsert", "docs", "one", '{"vector":[0,1,0],"payload":{"value":"trial"}}', "--branch", "trial") + invoke(target, "point", "upsert", "docs", "one", '{"vector":[0,0,1],"payload":{"value":"main"}}') + assert len(invoke(target, "merge", "preview", "trial", "main")["conflicts"]) == 1 + invoke(target, "merge", "apply", "trial", "main", ok=False) + invoke(target, "merge", "apply", "trial", "main", "--policy", "snapshot_isolation", ok=False) + invoke(target, "point", "upsert", "docs", "bad", '{"vector":[1,0]}', ok=False) + assert invoke(target, "point", "get", "docs", "bad") is None + assert invoke(target, "point", "get", "docs", "one")["payload"] == {"value": "main"} + invoke(target, "point", "get", "docs", "one", "--branch", "missing", ok=False) + invoke(target, "collection", "info", "missing", ok=False) + + +def test_qdrant_cli_json_files_and_stdin(qdrant_target: QdrantConfig, tmp_path: Path) -> None: + target = qdrant_target + invoke(target, "collection", "register", "docs", "--dimensions", "3") + data = tmp_path / "point.json" + data.write_text('{"vector":[1,0,0],"payload":{"source":"file"}}', encoding="utf-8") + result = invoke(target, "point", "upsert", "docs", "one", "--file", str(data), "--revision", "initial") + assert result["payload"] == {"source": "file"} + assert invoke(target, "search", "docs", "--stdin", input_text="[1,0,0]")[0]["id"] == "one" + invoke(target, "point", "upsert", "docs", "two", "--stdin", input_text='{"vector":[0,1,0]}') + data.write_text("[0,1,0]", encoding="utf-8") + assert invoke(target, "search", "docs", "--file", str(data))[0]["id"] == "two" + assert invoke(target, "search", "docs", input_text="[1,0,0]")[0]["id"] == "one" From 5f05c860b3d2fbfe53b430276444d5b3ac8d2468 Mon Sep 17 00:00:00 2001 From: Seth Rojas Date: Sun, 1 Nov 2026 21:55:10 -0700 Subject: [PATCH 05/11] refactor(cli): flatten command modules and consolidate tests Keep root assembly in main.py, co-locate relational and Qdrant helpers with their command implementations, and isolate shared terminal behavior. Consolidate five CLI test files into command and integration suites, removing only the duplicated setup workflow. Validation: 119 tests passed and one expected build-feature test skipped. All 39 help screens are unchanged; moved function bodies and signatures were compared with the feature baseline. Update CI selections and existing documentation references without changing public CLI behavior. --- .github/workflows/relational-package.yml | 9 +- docs/cli.md | 9 +- .../cli/{qdrant.py => _qdrant.py} | 141 +++- .../src/chronos_core/cli/_qdrant_support.py | 135 ---- .../src/chronos_core/cli/_relational.py | 235 +++++++ .../src/chronos_core/cli/_support.py | 98 --- .../src/chronos_core/cli/_terminal.py | 52 ++ .../chronos-core/src/chronos_core/cli/main.py | 183 +----- tests/test_cli.py | 622 ++++++++++++------ tests/test_cli_integration.py | 361 +++++++--- tests/test_cli_qdrant.py | 218 ------ tests/test_cli_qdrant_integration.py | 186 ------ tests/test_cli_support.py | 41 -- 13 files changed, 1104 insertions(+), 1186 deletions(-) rename packages/chronos-core/src/chronos_core/cli/{qdrant.py => _qdrant.py} (56%) delete mode 100644 packages/chronos-core/src/chronos_core/cli/_qdrant_support.py create mode 100644 packages/chronos-core/src/chronos_core/cli/_relational.py delete mode 100644 packages/chronos-core/src/chronos_core/cli/_support.py create mode 100644 packages/chronos-core/src/chronos_core/cli/_terminal.py delete mode 100644 tests/test_cli_qdrant.py delete mode 100644 tests/test_cli_qdrant_integration.py delete mode 100644 tests/test_cli_support.py diff --git a/.github/workflows/relational-package.yml b/.github/workflows/relational-package.yml index 905962a..3778aeb 100644 --- a/.github/workflows/relational-package.yml +++ b/.github/workflows/relational-package.yml @@ -104,8 +104,9 @@ jobs: CHRONOS_CLI_TEST_POSTGRES_ADMIN_URL: postgresql://postgres:postgres@127.0.0.1:5432/postgres CHRONOS_CLI_REQUIRE_DUCKDB: ${{ matrix.duckdb }} run: | - python -I -m pytest tests/test_cli.py tests/test_cli_support.py \ - tests/test_cli_integration.py -q --timeout=60 + python -I -m pytest tests/test_cli.py::TestRelationalCommands \ + tests/test_cli.py::TestTerminalBehavior \ + tests/test_cli_integration.py::TestRelationalWorkflows -q --timeout=60 - name: Test Qdrant CLI in local and server modes if: matrix.python == '3.12' && matrix.duckdb == 'OFF' env: @@ -116,5 +117,5 @@ jobs: docker run --detach --rm --name chronos-cli-qdrant-test \ -p 127.0.0.1:6333:6333 qdrant/qdrant:v1.20.0 trap 'docker stop chronos-cli-qdrant-test > /dev/null' EXIT - python -I -m pytest tests/test_cli_qdrant.py \ - tests/test_cli_qdrant_integration.py -q --timeout=60 + python -I -m pytest tests/test_cli.py::TestQdrantCommands \ + tests/test_cli_integration.py::TestQdrantWorkflows -q --timeout=60 diff --git a/docs/cli.md b/docs/cli.md index a270f4d..1dc26c1 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -295,7 +295,7 @@ repository root: ```sh CHRONOS_CLI_TEST_POSTGRES_ADMIN_URL=postgresql://postgres:chronos_test@127.0.0.1:55434/postgres \ uv run --no-sync --package chronos-core python -m pytest \ - tests/test_cli.py tests/test_cli_support.py tests/test_cli_integration.py -v + tests/test_cli.py tests/test_cli_integration.py -v ``` The PostgreSQL account must have permission to create and drop databases. The @@ -314,9 +314,10 @@ and stdin, checkpoints, diffs, successful merges, conflicts, and rollback after failed write. CI is configured to test the installed wheel on Python 3.10 and 3.12, with an additional DuckDB-enabled Python 3.12 build. -`test_cli.py` checks command parsing, API arguments, errors, and entry points with -fake connections. `test_cli_support.py` checks JSON conversion and cleanup. -`test_cli_integration.py` runs real CLI processes against SQLite and PostgreSQL. +`test_cli.py` groups relational, Qdrant, and shared terminal checks, using fake +connections where appropriate. `test_cli_integration.py` groups real relational +and Qdrant workflows. Optional stores and services are skipped when unavailable; +SQL tests do not require the Qdrant dependency. Stop the disposable server when finished: diff --git a/packages/chronos-core/src/chronos_core/cli/qdrant.py b/packages/chronos-core/src/chronos_core/cli/_qdrant.py similarity index 56% rename from packages/chronos-core/src/chronos_core/cli/qdrant.py rename to packages/chronos-core/src/chronos_core/cli/_qdrant.py index 0d46e41..55819e2 100644 --- a/packages/chronos-core/src/chronos_core/cli/qdrant.py +++ b/packages/chronos-core/src/chronos_core/cli/_qdrant.py @@ -1,20 +1,139 @@ -"""Terminal commands wrapping Chronos's public Qdrant store API.""" +"""Qdrant commands, JSON input, and ownership of public API resources.""" from __future__ import annotations +import json +import math +import sqlite3 +import sys +from collections.abc import Generator +from contextlib import contextmanager +from dataclasses import dataclass from pathlib import Path +from typing import TYPE_CHECKING, Any +import psycopg import typer -from chronos_core.cli._qdrant_support import ( - QdrantConfig, - dense_vector, - open_qdrant, - point_input, - qdrant_config, - qdrant_session, - read_json_input, -) -from chronos_core.cli._support import MergePolicy, print_result +from chronos_core.cli._terminal import MergePolicy, print_result, report_error + +if TYPE_CHECKING: + from chronos_core.workspace.qdrant import ChronosQdrantStore, QdrantBranchSession + + +@dataclass(frozen=True) +class QdrantConfig: + metadata_url: str + path: Path | None = None + url: str | None = None + api_key: str | None = None + collection_prefix: str = "chronos_" + timeout: float | None = None + + +def qdrant_config(ctx: typer.Context) -> QdrantConfig: + config = ctx.find_object(QdrantConfig) + if config is None or not config.metadata_url: + raise typer.BadParameter("Provide --metadata-url or CHRONOS_METADATA_URL.", param_hint="--metadata-url") + if (config.path is None) == (config.url is None): + raise typer.BadParameter("Choose exactly one of --path or --url.") + if config.metadata_url in {":memory:", "sqlite:///:memory:"} or str(config.path) == ":memory:": + raise typer.BadParameter("One-off commands require persistent storage, not :memory:.") + if config.path is not None and (config.api_key is not None or config.timeout is not None): + raise typer.BadParameter("--api-key and --timeout require --url.") + return config + + +def _store_class() -> type[ChronosQdrantStore]: + # Optional dependencies must not be imported when showing help or using SQL. + from chronos_core.workspace.qdrant import ChronosQdrantStore + return ChronosQdrantStore + + +@contextmanager +def open_qdrant(config: QdrantConfig) -> Generator[ChronosQdrantStore, None, None]: + try: + import grpc + import httpx + from qdrant_client.http.exceptions import ResponseHandlingException, UnexpectedResponse + from chronos_core.branching import BranchingError + except ImportError: + report_error(RuntimeError("Qdrant support requires installing chronos-core[qdrant].")) + + try: + factory = _store_class() + if config.path is not None: + store = factory.local( + config.metadata_url, path=config.path, collection_prefix=config.collection_prefix, + ) + elif config.url is not None: + store = factory.remote( + config.metadata_url, url=config.url, api_key=config.api_key, + collection_prefix=config.collection_prefix, timeout=config.timeout, + ) + else: + raise ValueError("Choose exactly one of --path or --url.") + try: + yield store + finally: + store.close() + except ( + BranchingError, RuntimeError, ValueError, TypeError, OSError, sqlite3.Error, + psycopg.Error, httpx.HTTPError, grpc.RpcError, ResponseHandlingException, UnexpectedResponse, + ) as error: + report_error(error, secrets=(config.api_key,), urls=(config.metadata_url, config.url)) + + +@contextmanager +def qdrant_session(store: ChronosQdrantStore, branch: str) -> Generator[QdrantBranchSession, None, None]: + # Qdrant sessions wrap a relational control session but have no close method. + # Both of these are public APIs; keep control-session ownership explicit. + with store.context.checkout(branch) as control: + yield store.checkout_control(control) + + +def read_json_input(text: str | None, file: Path | None, stdin: bool) -> Any: + if sum((text is not None, file is not None, stdin)) > 1: + raise typer.BadParameter("Choose JSON text, --file, or --stdin, not multiple sources.") + try: + if file is not None: + text = file.read_text(encoding="utf-8") + elif stdin or (text is None and not sys.stdin.isatty()): + text = sys.stdin.read() + except (OSError, UnicodeError) as error: + raise typer.BadParameter(f"Cannot read JSON input: {error}") from error + if text is None or not text.strip(): + raise typer.BadParameter("Provide JSON text, --file, or JSON on stdin.") + try: + return json.loads(text, parse_constant=_reject_constant) + except ValueError as error: + raise typer.BadParameter(f"Invalid JSON: {error}") from error + + +def _reject_constant(value: str) -> None: + raise ValueError(f"{value} is not a finite JSON number") + + +def dense_vector(value: Any) -> list[float]: + try: + valid = isinstance(value, list) and all( + isinstance(item, (int, float)) and not isinstance(item, bool) and math.isfinite(item) + for item in value + ) + except OverflowError: + valid = False + if not valid: + raise typer.BadParameter("Expected a JSON array of finite numbers for the vector.") + return value + + +def point_input(value: Any) -> tuple[list[float], dict[str, Any] | None]: + if not isinstance(value, dict) or "vector" not in value or set(value) - {"vector", "payload"}: + raise typer.BadParameter("Expected an object with vector and optional payload fields.") + payload = value.get("payload") + if payload is not None and not isinstance(payload, dict): + raise typer.BadParameter("The payload must be a JSON object or null.") + return dense_vector(value["vector"]), payload + app = typer.Typer(no_args_is_help=True) collection_app = typer.Typer(no_args_is_help=True) diff --git a/packages/chronos-core/src/chronos_core/cli/_qdrant_support.py b/packages/chronos-core/src/chronos_core/cli/_qdrant_support.py deleted file mode 100644 index 60dfd93..0000000 --- a/packages/chronos-core/src/chronos_core/cli/_qdrant_support.py +++ /dev/null @@ -1,135 +0,0 @@ -"""Qdrant configuration, JSON input, and ownership of public API resources.""" -from __future__ import annotations - -import json -import math -import sqlite3 -import sys -from collections.abc import Generator -from contextlib import contextmanager -from dataclasses import dataclass -from pathlib import Path -from typing import TYPE_CHECKING, Any - -import psycopg -import typer - -from chronos_core.cli._support import report_error - -if TYPE_CHECKING: - from chronos_core.workspace.qdrant import ChronosQdrantStore, QdrantBranchSession - - -@dataclass(frozen=True) -class QdrantConfig: - metadata_url: str - path: Path | None = None - url: str | None = None - api_key: str | None = None - collection_prefix: str = "chronos_" - timeout: float | None = None - - -def qdrant_config(ctx: typer.Context) -> QdrantConfig: - config = ctx.find_object(QdrantConfig) - if config is None or not config.metadata_url: - raise typer.BadParameter("Provide --metadata-url or CHRONOS_METADATA_URL.", param_hint="--metadata-url") - if (config.path is None) == (config.url is None): - raise typer.BadParameter("Choose exactly one of --path or --url.") - if config.metadata_url in {":memory:", "sqlite:///:memory:"} or str(config.path) == ":memory:": - raise typer.BadParameter("One-off commands require persistent storage, not :memory:.") - if config.path is not None and (config.api_key is not None or config.timeout is not None): - raise typer.BadParameter("--api-key and --timeout require --url.") - return config - - -def _store_class() -> type[ChronosQdrantStore]: - # Optional dependencies must not be imported when showing help or using SQL. - from chronos_core.workspace.qdrant import ChronosQdrantStore - return ChronosQdrantStore - - -@contextmanager -def open_qdrant(config: QdrantConfig) -> Generator[ChronosQdrantStore, None, None]: - try: - import grpc - import httpx - from qdrant_client.http.exceptions import ResponseHandlingException, UnexpectedResponse - from chronos_core.branching import BranchingError - except ImportError: - report_error(RuntimeError("Qdrant support requires installing chronos-core[qdrant].")) - - try: - factory = _store_class() - if config.path is not None: - store = factory.local( - config.metadata_url, path=config.path, collection_prefix=config.collection_prefix, - ) - elif config.url is not None: - store = factory.remote( - config.metadata_url, url=config.url, api_key=config.api_key, - collection_prefix=config.collection_prefix, timeout=config.timeout, - ) - else: - raise ValueError("Choose exactly one of --path or --url.") - try: - yield store - finally: - store.close() - except ( - BranchingError, RuntimeError, ValueError, TypeError, OSError, sqlite3.Error, - psycopg.Error, httpx.HTTPError, grpc.RpcError, ResponseHandlingException, UnexpectedResponse, - ) as error: - report_error(error, secrets=(config.api_key,), urls=(config.metadata_url, config.url)) - - -@contextmanager -def qdrant_session(store: ChronosQdrantStore, branch: str) -> Generator[QdrantBranchSession, None, None]: - # Qdrant sessions wrap a relational control session but have no close method. - # Both of these are public APIs; keep control-session ownership explicit. - with store.context.checkout(branch) as control: - yield store.checkout_control(control) - - -def read_json_input(text: str | None, file: Path | None, stdin: bool) -> Any: - if sum((text is not None, file is not None, stdin)) > 1: - raise typer.BadParameter("Choose JSON text, --file, or --stdin, not multiple sources.") - try: - if file is not None: - text = file.read_text(encoding="utf-8") - elif stdin or (text is None and not sys.stdin.isatty()): - text = sys.stdin.read() - except (OSError, UnicodeError) as error: - raise typer.BadParameter(f"Cannot read JSON input: {error}") from error - if text is None or not text.strip(): - raise typer.BadParameter("Provide JSON text, --file, or JSON on stdin.") - try: - return json.loads(text, parse_constant=_reject_constant) - except ValueError as error: - raise typer.BadParameter(f"Invalid JSON: {error}") from error - - -def _reject_constant(value: str) -> None: - raise ValueError(f"{value} is not a finite JSON number") - - -def dense_vector(value: Any) -> list[float]: - try: - valid = isinstance(value, list) and all( - isinstance(item, (int, float)) and not isinstance(item, bool) and math.isfinite(item) - for item in value - ) - except OverflowError: - valid = False - if not valid: - raise typer.BadParameter("Expected a JSON array of finite numbers for the vector.") - return value - - -def point_input(value: Any) -> tuple[list[float], dict[str, Any] | None]: - if not isinstance(value, dict) or "vector" not in value or set(value) - {"vector", "payload"}: - raise typer.BadParameter("Expected an object with vector and optional payload fields.") - payload = value.get("payload") - if payload is not None and not isinstance(payload, dict): - raise typer.BadParameter("The payload must be a JSON object or null.") - return dense_vector(value["vector"]), payload diff --git a/packages/chronos-core/src/chronos_core/cli/_relational.py b/packages/chronos-core/src/chronos_core/cli/_relational.py new file mode 100644 index 0000000..0f86ed0 --- /dev/null +++ b/packages/chronos-core/src/chronos_core/cli/_relational.py @@ -0,0 +1,235 @@ +"""Relational commands, SQL input, and connection ownership.""" +from __future__ import annotations + +import json +import sqlite3 +import sys +from collections.abc import Generator +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any +from urllib.parse import urlsplit + +import psycopg +import sqlglot +import typer +from sqlglot import exp + +from chronos_core.cli._terminal import MergePolicy, print_result, report_error + +if TYPE_CHECKING: + from chronos_core.branching import ChronosBranchContext + + +@dataclass(frozen=True) +class DatabaseConfig: + database_url: str + metadata_url: str | None = None + + +def database_config(ctx: typer.Context) -> DatabaseConfig: + database = ctx.find_object(DatabaseConfig) + if database is None or not database.database_url: + raise typer.BadParameter( + "Provide --database-url or set CHRONOS_DATABASE_URL.", + param_hint="--database-url", + ) + if urlsplit(database.database_url).scheme == "duckdb" and not database.metadata_url: + raise typer.BadParameter( + "DuckDB requires --metadata-url or CHRONOS_METADATA_URL for SQLite or PostgreSQL metadata.", + param_hint="--metadata-url", + ) + return database + + +@contextmanager +def connect(database: DatabaseConfig) -> Generator[ChronosBranchContext, None, None]: + from chronos_core.branching import BranchingError, ChronosBranchContext + + try: + if database.metadata_url is None: + context = ChronosBranchContext.connect(database.database_url) + else: + context = ChronosBranchContext.connect_split(database.database_url, database.metadata_url) + try: + yield context + finally: + context.close() + except (BranchingError, RuntimeError, sqlite3.Error, psycopg.Error, OSError, ValueError, TypeError) as error: + report_error(error, urls=(database.database_url, database.metadata_url)) + + +def _sql_input(sql: str | None, file: Path | None, stdin: bool) -> str: + if sum((sql is not None, file is not None, stdin)) > 1: + raise typer.BadParameter("Choose SQL text, --file, or --stdin, not multiple sources.") + if file is not None: + try: + sql = file.read_text(encoding="utf-8") + except (OSError, UnicodeError) as error: + raise typer.BadParameter(f"Cannot read SQL file: {error}", param_hint="--file") from error + elif stdin or (sql is None and not sys.stdin.isatty()): + try: + sql = sys.stdin.read() + except (OSError, UnicodeError) as error: + typer.echo(f"Error reading stdin: {error}", err=True) + raise typer.Exit(1) from error + if sql is None or not sql.strip(): + raise typer.BadParameter("Provide SQL text, --file, or SQL on stdin.") + return sql + + +def _parameters(params: str | None) -> dict[str, Any] | None: + if params is None: + return None + try: + value = json.loads(params) + except ValueError as error: + raise typer.BadParameter("Expected a JSON object.", param_hint="--params") from error + if not isinstance(value, dict): + raise typer.BadParameter("Expected a JSON object.", param_hint="--params") + return value + + +branch_app = typer.Typer(no_args_is_help=True) +table_app = typer.Typer(no_args_is_help=True) +checkpoint_app = typer.Typer(no_args_is_help=True) +merge_app = typer.Typer(no_args_is_help=True) + + +@branch_app.command("list") +def list_branches(ctx: typer.Context) -> None: + with connect(database_config(ctx)) as database: + print_result(database.list_branches()) + + +@branch_app.command("create") +def create_branch( + ctx: typer.Context, name: str, + from_branch: str = typer.Option("main", "--from"), +) -> None: + with connect(database_config(ctx)) as database: + print_result(database.create_branch(name, from_branch=from_branch)) + + +@branch_app.command("delete") +def delete_branch(ctx: typer.Context, name: str) -> None: + with connect(database_config(ctx)) as database: + print_result(database.delete_branch(name)) + + +@table_app.command("register") +def register_table( + ctx: typer.Context, table: str, + primary_key: list[str] = typer.Option(..., "--primary-key"), +) -> None: + with connect(database_config(ctx)) as database: + print_result(database.register_table(table, primary_key)) + + +@table_app.command("create") +def create_table( + ctx: typer.Context, + sql: str | None = typer.Argument(None), + file: Path | None = typer.Option(None, "--file"), + stdin: bool = typer.Option(False, "--stdin"), +) -> None: + """Create an ordinary table for subsequent table register; not a branch-local write.""" + statement = _sql_input(sql, file, stdin) + try: + statements = sqlglot.parse(statement) + except sqlglot.errors.ParseError as error: + raise typer.BadParameter("Expected one CREATE TABLE statement.") from error + if ( + len(statements) != 1 + or not isinstance(statements[0], exp.Create) + or statements[0].kind != "TABLE" + ): + raise typer.BadParameter("Expected one CREATE TABLE statement.") + with connect(database_config(ctx)) as database: + database.db.execute(statement) + print_result(database.db.commit()) + + +def query( + ctx: typer.Context, + sql: str | None = typer.Argument(None), + branch: str = typer.Option("main", "--branch"), + params: str | None = typer.Option(None, "--params"), + file: Path | None = typer.Option(None, "--file"), + stdin: bool = typer.Option(False, "--stdin"), +) -> None: + """Call the branch session's query API with SQL and optional named parameters.""" + statement, parameters = _sql_input(sql, file, stdin), _parameters(params) + with connect(database_config(ctx)) as database: + with database.checkout(branch) as session: + print_result(session.query(statement, parameters)) + + +def execute( + ctx: typer.Context, + sql: str | None = typer.Argument(None), + branch: str = typer.Option("main", "--branch"), + params: str | None = typer.Option(None, "--params"), + file: Path | None = typer.Option(None, "--file"), + stdin: bool = typer.Option(False, "--stdin"), +) -> None: + """Call the branch session's execute API; SQL is not split into a script.""" + statement, parameters = _sql_input(sql, file, stdin), _parameters(params) + with connect(database_config(ctx)) as database: + with database.checkout(branch) as session: + print_result(session.execute(statement, parameters)) + + +@checkpoint_app.command("create") +def create_checkpoint( + ctx: typer.Context, name: str, + branch: str = typer.Option("main", "--branch"), +) -> None: + with connect(database_config(ctx)) as database: + print_result(database.create_checkpoint(name, branch=branch)) + + +@checkpoint_app.command("list") +def list_checkpoints( + ctx: typer.Context, branch: str | None = typer.Option(None, "--branch"), +) -> None: + with connect(database_config(ctx)) as database: + print_result(database.list_checkpoints(branch=branch)) + + +def diff( + ctx: typer.Context, left: str, right: str, + table: str | None = typer.Option(None, "--table"), +) -> None: + with connect(database_config(ctx)) as database: + print_result(database.diff(left, right) if table is None else database.diff_rows(left, right, table)) + + +@merge_app.command("preview") +def merge_preview( + ctx: typer.Context, source: str, target: str, + policy: MergePolicy = typer.Option(MergePolicy.abort_on_conflict, "--policy"), +) -> None: + with connect(database_config(ctx)) as database: + print_result(database.merge_preview(source, target, policy=policy.value)) + + +@merge_app.command("apply") +def merge_apply( + ctx: typer.Context, source: str, target: str, + policy: MergePolicy = typer.Option(MergePolicy.abort_on_conflict, "--policy"), +) -> None: + with connect(database_config(ctx)) as database: + print_result(database.merge_apply(source, target, policy=policy.value)) + + +def register_commands(app: typer.Typer) -> None: + """Attach relational commands to the root without introducing a SQL subcommand.""" + app.command()(query) + app.command()(execute) + app.command()(diff) + app.add_typer(branch_app, name="branch", help="Manage branches.") + app.add_typer(table_app, name="table", help="Create ordinary tables and register them with Chronos.") + app.add_typer(checkpoint_app, name="checkpoint", help="Create and list checkpoints.") + app.add_typer(merge_app, name="merge", help="Preview and apply merges.") diff --git a/packages/chronos-core/src/chronos_core/cli/_support.py b/packages/chronos-core/src/chronos_core/cli/_support.py deleted file mode 100644 index c0078a5..0000000 --- a/packages/chronos-core/src/chronos_core/cli/_support.py +++ /dev/null @@ -1,98 +0,0 @@ -"""Shared configuration, connection lifecycle, and JSON output.""" -from __future__ import annotations - -import base64 -import json -import re -import sqlite3 -from collections.abc import Generator -from contextlib import contextmanager -from dataclasses import asdict, dataclass, is_dataclass -from datetime import date, datetime, time -from decimal import Decimal -from enum import Enum -from typing import TYPE_CHECKING, Any, NoReturn -from urllib.parse import urlsplit - -import psycopg -import typer - -if TYPE_CHECKING: - from chronos_core.branching import ChronosBranchContext - - -@dataclass(frozen=True) -class DatabaseConfig: - database_url: str - metadata_url: str | None = None - - -class MergePolicy(str, Enum): - abort_on_conflict = "abort_on_conflict" - snapshot_isolation = "snapshot_isolation" - - -def database_config(ctx: typer.Context) -> DatabaseConfig: - database = ctx.find_object(DatabaseConfig) - if database is None or not database.database_url: - raise typer.BadParameter( - "Provide --database-url or set CHRONOS_DATABASE_URL.", - param_hint="--database-url", - ) - if urlsplit(database.database_url).scheme == "duckdb" and not database.metadata_url: - raise typer.BadParameter( - "DuckDB requires --metadata-url or CHRONOS_METADATA_URL for SQLite or PostgreSQL metadata.", - param_hint="--metadata-url", - ) - return database - - -def _json_default(value: Any) -> Any: - if is_dataclass(value) and not isinstance(value, type): - return asdict(value) - if isinstance(value, (datetime, date, time)): - return value.isoformat() - if isinstance(value, Decimal): - return str(value) - if isinstance(value, bytes): - return {"encoding": "base64", "data": base64.b64encode(value).decode("ascii")} - raise TypeError(f"Unsupported JSON value type: {type(value).__name__}") - - -def print_result(result: Any) -> None: - if result is not None: - typer.echo(json.dumps(result, default=_json_default, ensure_ascii=False, indent=2)) - - -def report_error( - error: Exception, *, secrets: tuple[str | None, ...] = (), urls: tuple[str | None, ...] = (), -) -> NoReturn: - """Report an operational failure without exposing configured credentials.""" - message = str(error) - for url in urls: - if url: - message = message.replace(url, "") - for secret in secrets: - if secret: - message = message.replace(secret, "") - message = re.sub(r"([a-zA-Z][\w+.-]*://)[^\s/@]+@", r"\1@", message) - message = re.sub(r"(?i)(password\s*=\s*)(?:'[^']*'|\"[^\"]*\"|[^\s]+)", r"\1", message) - typer.echo(f"Error: {message}", err=True) - raise typer.Exit(1) from error - - -@contextmanager -def connect(database: DatabaseConfig) -> Generator[ChronosBranchContext, None, None]: - from chronos_core.branching import BranchingError, ChronosBranchContext - - try: - if database.metadata_url is None: - context = ChronosBranchContext.connect(database.database_url) - else: - context = ChronosBranchContext.connect_split(database.database_url, database.metadata_url) - try: - yield context - finally: - context.close() - except (BranchingError, RuntimeError, sqlite3.Error, psycopg.Error, OSError, ValueError, TypeError) as error: - report_error(error, urls=(database.database_url, database.metadata_url)) diff --git a/packages/chronos-core/src/chronos_core/cli/_terminal.py b/packages/chronos-core/src/chronos_core/cli/_terminal.py new file mode 100644 index 0000000..adea51c --- /dev/null +++ b/packages/chronos-core/src/chronos_core/cli/_terminal.py @@ -0,0 +1,52 @@ +"""Shared terminal output, error reporting, and CLI policy choices.""" +from __future__ import annotations + +import base64 +import json +import re +from dataclasses import asdict, is_dataclass +from datetime import date, datetime, time +from decimal import Decimal +from enum import Enum +from typing import Any, NoReturn + +import typer + + +class MergePolicy(str, Enum): + abort_on_conflict = "abort_on_conflict" + snapshot_isolation = "snapshot_isolation" + + +def _json_default(value: Any) -> Any: + if is_dataclass(value) and not isinstance(value, type): + return asdict(value) + if isinstance(value, (datetime, date, time)): + return value.isoformat() + if isinstance(value, Decimal): + return str(value) + if isinstance(value, bytes): + return {"encoding": "base64", "data": base64.b64encode(value).decode("ascii")} + raise TypeError(f"Unsupported JSON value type: {type(value).__name__}") + + +def print_result(result: Any) -> None: + if result is not None: + typer.echo(json.dumps(result, default=_json_default, ensure_ascii=False, indent=2)) + + +def report_error( + error: Exception, *, secrets: tuple[str | None, ...] = (), urls: tuple[str | None, ...] = (), +) -> NoReturn: + """Report an operational failure without exposing configured credentials.""" + message = str(error) + for url in urls: + if url: + message = message.replace(url, "") + for secret in secrets: + if secret: + message = message.replace(secret, "") + message = re.sub(r"([a-zA-Z][\w+.-]*://)[^\s/@]+@", r"\1@", message) + message = re.sub(r"(?i)(password\s*=\s*)(?:'[^']*'|\"[^\"]*\"|[^\s]+)", r"\1", message) + typer.echo(f"Error: {message}", err=True) + raise typer.Exit(1) from error diff --git a/packages/chronos-core/src/chronos_core/cli/main.py b/packages/chronos-core/src/chronos_core/cli/main.py index 1ce767e..33a715f 100644 --- a/packages/chronos-core/src/chronos_core/cli/main.py +++ b/packages/chronos-core/src/chronos_core/cli/main.py @@ -1,27 +1,13 @@ -"""One-off terminal commands wrapping public Chronos APIs.""" +"""Assemble the public Chronos CLI and its root connection options.""" from __future__ import annotations -import json -import sys -from pathlib import Path -from typing import Any - import typer -import sqlglot -from sqlglot import exp -from chronos_core.cli._support import DatabaseConfig, MergePolicy, connect, database_config, print_result -from chronos_core.cli.qdrant import app as qdrant_app +from chronos_core.cli._qdrant import app as qdrant_app +from chronos_core.cli._relational import DatabaseConfig, register_commands app = typer.Typer(no_args_is_help=True, pretty_exceptions_enable=False) -branch_app = typer.Typer(no_args_is_help=True) -table_app = typer.Typer(no_args_is_help=True) -checkpoint_app = typer.Typer(no_args_is_help=True) -merge_app = typer.Typer(no_args_is_help=True) -app.add_typer(branch_app, name="branch", help="Manage branches.") -app.add_typer(table_app, name="table", help="Create ordinary tables and register them with Chronos.") -app.add_typer(checkpoint_app, name="checkpoint", help="Create and list checkpoints.") -app.add_typer(merge_app, name="merge", help="Preview and apply merges.") +register_commands(app) app.add_typer(qdrant_app, name="qdrant", help="Branch-aware vector storage and search.") @@ -39,166 +25,5 @@ def main( ctx.obj = DatabaseConfig(database_url, metadata_url) if database_url else None -@branch_app.command("list") -def list_branches(ctx: typer.Context) -> None: - with connect(database_config(ctx)) as database: - print_result(database.list_branches()) - - -@branch_app.command("create") -def create_branch( - ctx: typer.Context, name: str, - from_branch: str = typer.Option("main", "--from"), -) -> None: - with connect(database_config(ctx)) as database: - print_result(database.create_branch(name, from_branch=from_branch)) - - -@branch_app.command("delete") -def delete_branch(ctx: typer.Context, name: str) -> None: - with connect(database_config(ctx)) as database: - print_result(database.delete_branch(name)) - - -@table_app.command("register") -def register_table( - ctx: typer.Context, table: str, - primary_key: list[str] = typer.Option(..., "--primary-key"), -) -> None: - with connect(database_config(ctx)) as database: - print_result(database.register_table(table, primary_key)) - - -def _sql_input(sql: str | None, file: Path | None, stdin: bool) -> str: - if sum((sql is not None, file is not None, stdin)) > 1: - raise typer.BadParameter("Choose SQL text, --file, or --stdin, not multiple sources.") - if file is not None: - try: - sql = file.read_text(encoding="utf-8") - except (OSError, UnicodeError) as error: - raise typer.BadParameter(f"Cannot read SQL file: {error}", param_hint="--file") from error - elif stdin or (sql is None and not sys.stdin.isatty()): - try: - sql = sys.stdin.read() - except (OSError, UnicodeError) as error: - typer.echo(f"Error reading stdin: {error}", err=True) - raise typer.Exit(1) from error - if sql is None or not sql.strip(): - raise typer.BadParameter("Provide SQL text, --file, or SQL on stdin.") - return sql - - -def _parameters(params: str | None) -> dict[str, Any] | None: - if params is None: - return None - try: - value = json.loads(params) - except ValueError as error: - raise typer.BadParameter("Expected a JSON object.", param_hint="--params") from error - if not isinstance(value, dict): - raise typer.BadParameter("Expected a JSON object.", param_hint="--params") - return value - - -@table_app.command("create") -def create_table( - ctx: typer.Context, - sql: str | None = typer.Argument(None), - file: Path | None = typer.Option(None, "--file"), - stdin: bool = typer.Option(False, "--stdin"), -) -> None: - """Create an ordinary table for subsequent table register; not a branch-local write.""" - statement = _sql_input(sql, file, stdin) - try: - statements = sqlglot.parse(statement) - except sqlglot.errors.ParseError as error: - raise typer.BadParameter("Expected one CREATE TABLE statement.") from error - if ( - len(statements) != 1 - or not isinstance(statements[0], exp.Create) - or statements[0].kind != "TABLE" - ): - raise typer.BadParameter("Expected one CREATE TABLE statement.") - with connect(database_config(ctx)) as database: - database.db.execute(statement) - print_result(database.db.commit()) - - -@app.command() -def query( - ctx: typer.Context, - sql: str | None = typer.Argument(None), - branch: str = typer.Option("main", "--branch"), - params: str | None = typer.Option(None, "--params"), - file: Path | None = typer.Option(None, "--file"), - stdin: bool = typer.Option(False, "--stdin"), -) -> None: - """Call the branch session's query API with SQL and optional named parameters.""" - statement, parameters = _sql_input(sql, file, stdin), _parameters(params) - with connect(database_config(ctx)) as database: - with database.checkout(branch) as session: - print_result(session.query(statement, parameters)) - - -@app.command() -def execute( - ctx: typer.Context, - sql: str | None = typer.Argument(None), - branch: str = typer.Option("main", "--branch"), - params: str | None = typer.Option(None, "--params"), - file: Path | None = typer.Option(None, "--file"), - stdin: bool = typer.Option(False, "--stdin"), -) -> None: - """Call the branch session's execute API; SQL is not split into a script.""" - statement, parameters = _sql_input(sql, file, stdin), _parameters(params) - with connect(database_config(ctx)) as database: - with database.checkout(branch) as session: - print_result(session.execute(statement, parameters)) - - -@checkpoint_app.command("create") -def create_checkpoint( - ctx: typer.Context, name: str, - branch: str = typer.Option("main", "--branch"), -) -> None: - with connect(database_config(ctx)) as database: - print_result(database.create_checkpoint(name, branch=branch)) - - -@checkpoint_app.command("list") -def list_checkpoints( - ctx: typer.Context, branch: str | None = typer.Option(None, "--branch"), -) -> None: - with connect(database_config(ctx)) as database: - print_result(database.list_checkpoints(branch=branch)) - - -@app.command() -def diff( - ctx: typer.Context, left: str, right: str, - table: str | None = typer.Option(None, "--table"), -) -> None: - with connect(database_config(ctx)) as database: - print_result(database.diff(left, right) if table is None else database.diff_rows(left, right, table)) - - -@merge_app.command("preview") -def merge_preview( - ctx: typer.Context, source: str, target: str, - policy: MergePolicy = typer.Option(MergePolicy.abort_on_conflict, "--policy"), -) -> None: - with connect(database_config(ctx)) as database: - print_result(database.merge_preview(source, target, policy=policy.value)) - - -@merge_app.command("apply") -def merge_apply( - ctx: typer.Context, source: str, target: str, - policy: MergePolicy = typer.Option(MergePolicy.abort_on_conflict, "--policy"), -) -> None: - with connect(database_config(ctx)) as database: - print_result(database.merge_apply(source, target, policy=policy.value)) - - if __name__ == "__main__": app() diff --git a/tests/test_cli.py b/tests/test_cli.py index 9aa1030..2a4c3db 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -1,16 +1,25 @@ +"""CLI argument forwarding, terminal behavior, and cleanup without real services.""" from __future__ import annotations import json import os import subprocess import sys +import textwrap +from dataclasses import dataclass +from datetime import date, datetime, time +from decimal import Decimal from pathlib import Path from unittest.mock import MagicMock import pytest +import typer from typer.testing import CliRunner from chronos_core.branching import BranchingError, ChronosBranchContext +from chronos_core.cli import _qdrant +from chronos_core.cli._relational import DatabaseConfig, connect +from chronos_core.cli._terminal import print_result from chronos_core.cli.main import app runner = CliRunner() @@ -27,207 +36,414 @@ def connection(monkeypatch: pytest.MonkeyPatch) -> MagicMock: return database -def test_configuration_and_help(connection: MagicMock) -> None: - for args in (["--help"], ["branch", "--help"], ["query", "--help"]): - assert runner.invoke(app, args).exit_code == 0 - ChronosBranchContext.connect.assert_not_called() - result = runner.invoke(app, ["branch", "list"]) - assert result.exit_code == 2 - ChronosBranchContext.connect.assert_not_called() - result = runner.invoke(app, ["branch", "list"], env={"CHRONOS_DATABASE_URL": "sqlite:///env.db"}) - assert result.exit_code == 0 - ChronosBranchContext.connect.assert_called_with("sqlite:///env.db") - result = runner.invoke(app, ["--database-url", "sqlite:///flag.db", "branch", "list"], - env={"CHRONOS_DATABASE_URL": "sqlite:///env.db"}) - assert result.exit_code == 0 - ChronosBranchContext.connect.assert_called_with("sqlite:///flag.db") - assert json.loads(result.stdout) == [] - assert connection.close.call_count == 2 - - -@pytest.mark.parametrize(("args", "method", "expected_args", "expected_kwargs"), [ - (["branch", "list"], "list_branches", (), {}), - (["branch", "create", "trial"], "create_branch", ("trial",), {"from_branch": "main"}), - (["branch", "create", "trial", "--from", "parent"], "create_branch", ("trial",), {"from_branch": "parent"}), - (["branch", "delete", "trial"], "delete_branch", ("trial",), {}), - (["table", "register", "items", "--primary-key", "id", "--primary-key", "tenant"], "register_table", ("items", ["id", "tenant"]), {}), - (["checkpoint", "create", "saved", "--branch", "trial"], "create_checkpoint", ("saved",), {"branch": "trial"}), - (["checkpoint", "list"], "list_checkpoints", (), {"branch": None}), - (["checkpoint", "list", "--branch", "trial"], "list_checkpoints", (), {"branch": "trial"}), - (["diff", "main", "trial"], "diff", ("main", "trial"), {}), - (["diff", "main", "trial", "--table", "items"], "diff_rows", ("main", "trial", "items"), {}), - (["merge", "preview", "trial", "main"], "merge_preview", ("trial", "main"), {"policy": "abort_on_conflict"}), - (["merge", "apply", "trial", "main", "--policy", "snapshot_isolation"], "merge_apply", ("trial", "main"), {"policy": "snapshot_isolation"}), -]) -def test_api_forwarding(connection: MagicMock, args: list[str], method: str, - expected_args: tuple, expected_kwargs: dict) -> None: - getattr(connection, method).return_value = None - result = runner.invoke(app, ["--database-url", "sqlite:///test.db", *args]) - assert result.exit_code == 0, result.output - getattr(connection, method).assert_called_once_with(*expected_args, **expected_kwargs) - assert result.stdout == "" - connection.close.assert_called_once() - - -@pytest.mark.parametrize("command", ["query", "execute"]) -@pytest.mark.parametrize("source", ["string", "file", "stdin", "pipe"]) -def test_sql_sources(connection: MagicMock, tmp_path: Path, command: str, source: str) -> None: - sql = "SELECT *\nFROM items WHERE id = :id\n" - args = ["--database-url", "sqlite:///test.db", command, "--branch", "trial", "--params", '{"id": 3}'] - input_text = None - if source == "string": - args.append(sql) - elif source == "file": - path = tmp_path / "query.sql" - path.write_text(sql, encoding="utf-8") - args.extend(["--file", str(path)]) - else: - input_text = sql - if source == "stdin": - args.append("--stdin") - session = connection.checkout.return_value.__enter__.return_value - getattr(session, command).return_value = [{"id": 3}] if command == "query" else {"rowcount": 1} - result = runner.invoke(app, args, input=input_text) - assert result.exit_code == 0, result.output - connection.checkout.assert_called_once_with("trial") - getattr(session, command).assert_called_once_with(sql, {"id": 3}) - assert json.loads(result.stdout) == getattr(session, command).return_value - connection.checkout.return_value.__exit__.assert_called_once() - connection.close.assert_called_once() - - -@pytest.mark.parametrize("args", [ - ["query"], ["query", " "], ["query", "SELECT 1", "--stdin"], - ["query", "SELECT 1", "--file", "missing.sql"], - ["query", "--file", "missing.sql"], - ["query", "SELECT 1", "--params", "[]"], - ["execute", "SELECT 1", "--params", "broken"], - ["merge", "apply", "a", "b", "--policy", "unknown"], - ["table", "register", "items"], -]) -def test_bad_inputs_do_not_connect(connection: MagicMock, args: list[str]) -> None: - result = runner.invoke(app, ["--database-url", "sqlite:///test.db", *args]) - assert result.exit_code == 2, result.output - ChronosBranchContext.connect.assert_not_called() - - -def test_interactive_missing_sql(connection: MagicMock, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(sys.stdin, "isatty", lambda: True) - from chronos_core.cli.main import _sql_input - import typer - with pytest.raises(typer.BadParameter, match="Provide SQL"): - _sql_input(None, None, False) - ChronosBranchContext.connect.assert_not_called() - - -def test_cleanup_and_redaction(connection: MagicMock) -> None: - url = "postgresql://user:secret@localhost/db" - session = connection.checkout.return_value.__enter__.return_value - session.query.side_effect = BranchingError(f"Cannot use {url}; password=secret") - result = runner.invoke(app, ["--database-url", url, "query", "SELECT 1"]) - assert result.exit_code == 1 - assert "secret" not in result.output - assert "Error:" in result.stderr - connection.checkout.return_value.__exit__.assert_called_once() - connection.close.assert_called_once() - - -def test_connection_failure_does_not_close_unassigned_context(connection: MagicMock) -> None: - ChronosBranchContext.connect.side_effect = ValueError("unsupported database") - result = runner.invoke(app, ["--database-url", "bad", "branch", "list"]) - assert result.exit_code == 1 - assert "unsupported database" in result.stderr - connection.close.assert_not_called() - - -def test_installed_entrypoints() -> None: - executable = Path(sys.executable).parent / "chronos" - for args in ([str(executable), "--help"], [sys.executable, "-m", "chronos_core.cli", "--help"]): - env = dict(os.environ) - env.pop("CHRONOS_DATABASE_URL", None) - env.pop("CHRONOS_METADATA_URL", None) - result = subprocess.run(args, capture_output=True, text=True, env=env, timeout=30) - assert result.returncode == 0, result.stderr - assert "--database-url" in result.stdout - - -def test_create_table_forwarding(connection: MagicMock) -> None: - connection.db.commit.return_value = None - statement = "CREATE TABLE items (id INTEGER PRIMARY KEY)" - result = runner.invoke(app, ["--database-url", "sqlite:///test.db", "table", "create", statement]) - assert result.exit_code == 0, result.output - connection.db.execute.assert_called_once_with(statement) - connection.db.commit.assert_called_once() - connection.checkout.assert_not_called() - assert result.stdout == "" - - -@pytest.mark.parametrize("statement", [ - "DELETE FROM items", "CREATE INDEX idx ON items(id)", - "CREATE TABLE items (id INTEGER); DROP TABLE items", "not sql", -]) -def test_create_table_rejects_other_sql(connection: MagicMock, statement: str) -> None: - result = runner.invoke(app, ["--database-url", "sqlite:///test.db", "table", "create", statement]) - assert result.exit_code == 2, result.output - ChronosBranchContext.connect.assert_not_called() - - -def test_native_error_is_reported_without_traceback(connection: MagicMock) -> None: - session = connection.checkout.return_value.__enter__.return_value - session.execute.side_effect = RuntimeError("INSERT must specify a column list") - result = runner.invoke(app, [ - "--database-url", "sqlite:///test.db", "execute", "INSERT INTO items VALUES (1)", +@pytest.fixture +def factory(monkeypatch: pytest.MonkeyPatch) -> MagicMock: + pytest.importorskip("qdrant_client") + for name in ("CHRONOS_METADATA_URL", "CHRONOS_QDRANT_PATH", "CHRONOS_QDRANT_URL", + "CHRONOS_QDRANT_API_KEY", "CHRONOS_QDRANT_COLLECTION_PREFIX"): + monkeypatch.delenv(name, raising=False) + factory = MagicMock() + factory.remote.return_value = factory.local.return_value + monkeypatch.setattr(_qdrant, "_store_class", lambda: factory) + return factory + + +def _invoke_qdrant(*args: str, **kwargs): + return runner.invoke(app, ["qdrant", "--path", "vectors", "--metadata-url", "sqlite:///metadata.sqlite", *args], **kwargs) + + +class TestRelationalCommands: + def test_configuration_and_help(self, connection: MagicMock) -> None: + for args in (["--help"], ["branch", "--help"], ["query", "--help"]): + assert runner.invoke(app, args).exit_code == 0 + ChronosBranchContext.connect.assert_not_called() + result = runner.invoke(app, ["branch", "list"]) + assert result.exit_code == 2 + ChronosBranchContext.connect.assert_not_called() + result = runner.invoke(app, ["branch", "list"], env={"CHRONOS_DATABASE_URL": "sqlite:///env.db"}) + assert result.exit_code == 0 + ChronosBranchContext.connect.assert_called_with("sqlite:///env.db") + result = runner.invoke(app, ["--database-url", "sqlite:///flag.db", "branch", "list"], + env={"CHRONOS_DATABASE_URL": "sqlite:///env.db"}) + assert result.exit_code == 0 + ChronosBranchContext.connect.assert_called_with("sqlite:///flag.db") + assert json.loads(result.stdout) == [] + assert connection.close.call_count == 2 + + @pytest.mark.parametrize(("args", "method", "expected_args", "expected_kwargs"), [ + (["branch", "list"], "list_branches", (), {}), + (["branch", "create", "trial"], "create_branch", ("trial",), {"from_branch": "main"}), + (["branch", "create", "trial", "--from", "parent"], "create_branch", ("trial",), {"from_branch": "parent"}), + (["branch", "delete", "trial"], "delete_branch", ("trial",), {}), + (["table", "register", "items", "--primary-key", "id", "--primary-key", "tenant"], "register_table", ("items", ["id", "tenant"]), {}), + (["checkpoint", "create", "saved", "--branch", "trial"], "create_checkpoint", ("saved",), {"branch": "trial"}), + (["checkpoint", "list"], "list_checkpoints", (), {"branch": None}), + (["checkpoint", "list", "--branch", "trial"], "list_checkpoints", (), {"branch": "trial"}), + (["diff", "main", "trial"], "diff", ("main", "trial"), {}), + (["diff", "main", "trial", "--table", "items"], "diff_rows", ("main", "trial", "items"), {}), + (["merge", "preview", "trial", "main"], "merge_preview", ("trial", "main"), {"policy": "abort_on_conflict"}), + (["merge", "apply", "trial", "main", "--policy", "snapshot_isolation"], "merge_apply", ("trial", "main"), {"policy": "snapshot_isolation"}), + ]) + def test_api_forwarding(self, connection: MagicMock, args: list[str], method: str, + expected_args: tuple, expected_kwargs: dict) -> None: + getattr(connection, method).return_value = None + result = runner.invoke(app, ["--database-url", "sqlite:///test.db", *args]) + assert result.exit_code == 0, result.output + getattr(connection, method).assert_called_once_with(*expected_args, **expected_kwargs) + assert result.stdout == "" + connection.close.assert_called_once() + + @pytest.mark.parametrize("command", ["query", "execute"]) + @pytest.mark.parametrize("source", ["string", "file", "stdin", "pipe"]) + def test_sql_sources(self, connection: MagicMock, tmp_path: Path, command: str, source: str) -> None: + sql = "SELECT *\nFROM items WHERE id = :id\n" + args = ["--database-url", "sqlite:///test.db", command, "--branch", "trial", "--params", '{"id": 3}'] + input_text = None + if source == "string": + args.append(sql) + elif source == "file": + path = tmp_path / "query.sql" + path.write_text(sql, encoding="utf-8") + args.extend(["--file", str(path)]) + else: + input_text = sql + if source == "stdin": + args.append("--stdin") + session = connection.checkout.return_value.__enter__.return_value + getattr(session, command).return_value = [{"id": 3}] if command == "query" else {"rowcount": 1} + result = runner.invoke(app, args, input=input_text) + assert result.exit_code == 0, result.output + connection.checkout.assert_called_once_with("trial") + getattr(session, command).assert_called_once_with(sql, {"id": 3}) + assert json.loads(result.stdout) == getattr(session, command).return_value + connection.checkout.return_value.__exit__.assert_called_once() + connection.close.assert_called_once() + + @pytest.mark.parametrize("args", [ + ["query"], ["query", " "], ["query", "SELECT 1", "--stdin"], + ["query", "SELECT 1", "--file", "missing.sql"], + ["query", "--file", "missing.sql"], + ["query", "SELECT 1", "--params", "[]"], + ["execute", "SELECT 1", "--params", "broken"], + ["merge", "apply", "a", "b", "--policy", "unknown"], + ["table", "register", "items"], + ]) + def test_bad_inputs_do_not_connect(self, connection: MagicMock, args: list[str]) -> None: + result = runner.invoke(app, ["--database-url", "sqlite:///test.db", *args]) + assert result.exit_code == 2, result.output + ChronosBranchContext.connect.assert_not_called() + + def test_interactive_missing_sql(self, connection: MagicMock, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setattr(sys.stdin, "isatty", lambda: True) + from chronos_core.cli._relational import _sql_input + import typer + with pytest.raises(typer.BadParameter, match="Provide SQL"): + _sql_input(None, None, False) + ChronosBranchContext.connect.assert_not_called() + + def test_cleanup_and_redaction(self, connection: MagicMock) -> None: + url = "postgresql://user:secret@localhost/db" + session = connection.checkout.return_value.__enter__.return_value + session.query.side_effect = BranchingError(f"Cannot use {url}; password=secret") + result = runner.invoke(app, ["--database-url", url, "query", "SELECT 1"]) + assert result.exit_code == 1 + assert "secret" not in result.output + assert "Error:" in result.stderr + connection.checkout.return_value.__exit__.assert_called_once() + connection.close.assert_called_once() + + def test_connection_failure_does_not_close_unassigned_context(self, connection: MagicMock) -> None: + ChronosBranchContext.connect.side_effect = ValueError("unsupported database") + result = runner.invoke(app, ["--database-url", "bad", "branch", "list"]) + assert result.exit_code == 1 + assert "unsupported database" in result.stderr + connection.close.assert_not_called() + + def test_installed_entrypoints(self) -> None: + executable = Path(sys.executable).parent / "chronos" + for args in ([str(executable), "--help"], [sys.executable, "-m", "chronos_core.cli", "--help"]): + env = dict(os.environ) + env.pop("CHRONOS_DATABASE_URL", None) + env.pop("CHRONOS_METADATA_URL", None) + result = subprocess.run(args, capture_output=True, text=True, env=env, timeout=30) + assert result.returncode == 0, result.stderr + assert "--database-url" in result.stdout + + def test_create_table_forwarding(self, connection: MagicMock) -> None: + connection.db.commit.return_value = None + statement = "CREATE TABLE items (id INTEGER PRIMARY KEY)" + result = runner.invoke(app, ["--database-url", "sqlite:///test.db", "table", "create", statement]) + assert result.exit_code == 0, result.output + connection.db.execute.assert_called_once_with(statement) + connection.db.commit.assert_called_once() + connection.checkout.assert_not_called() + assert result.stdout == "" + + @pytest.mark.parametrize("statement", [ + "DELETE FROM items", "CREATE INDEX idx ON items(id)", + "CREATE TABLE items (id INTEGER); DROP TABLE items", "not sql", + ]) + def test_create_table_rejects_other_sql(self, connection: MagicMock, statement: str) -> None: + result = runner.invoke(app, ["--database-url", "sqlite:///test.db", "table", "create", statement]) + assert result.exit_code == 2, result.output + ChronosBranchContext.connect.assert_not_called() + + def test_native_error_is_reported_without_traceback(self, connection: MagicMock) -> None: + session = connection.checkout.return_value.__enter__.return_value + session.execute.side_effect = RuntimeError("INSERT must specify a column list") + result = runner.invoke(app, [ + "--database-url", "sqlite:///test.db", "execute", "INSERT INTO items VALUES (1)", + ]) + assert result.exit_code == 1 + assert result.stdout == "" + assert result.stderr.strip() == "Error: INSERT must specify a column list" + connection.checkout.return_value.__exit__.assert_called_once() + connection.close.assert_called_once() + + @pytest.mark.parametrize("source", ["flag", "environment", "override"]) + def test_split_connection_configuration(self, connection: MagicMock, monkeypatch: pytest.MonkeyPatch, source: str) -> None: + split = MagicMock(return_value=connection) + monkeypatch.setattr(ChronosBranchContext, "connect_split", split) + args = ["--database-url", "duckdb:////tmp/data.duckdb"] + environment = {} + if source != "flag": + environment["CHRONOS_METADATA_URL"] = "sqlite:////tmp/env.sqlite" + if source != "environment": + args.extend(["--metadata-url", "sqlite:////tmp/flag.sqlite"]) + result = runner.invoke(app, [*args, "branch", "list"], env=environment) + assert result.exit_code == 0, result.output + expected_metadata = "sqlite:////tmp/env.sqlite" if source == "environment" else "sqlite:////tmp/flag.sqlite" + split.assert_called_once_with("duckdb:////tmp/data.duckdb", expected_metadata) + ChronosBranchContext.connect.assert_not_called() + connection.close.assert_called_once() + + def test_duckdb_requires_metadata_before_connecting(self, connection: MagicMock) -> None: + result = runner.invoke(app, ["--database-url", "duckdb:////tmp/data.duckdb", "branch", "list"]) + assert result.exit_code == 2 + assert "--metadata-url" in result.output + ChronosBranchContext.connect.assert_not_called() + + def test_split_connection_redacts_metadata_credentials(self, connection: MagicMock, monkeypatch: pytest.MonkeyPatch) -> None: + metadata = "postgresql://user:secret@localhost/metadata" + split = MagicMock(side_effect=ValueError(f"Cannot connect to {metadata}")) + monkeypatch.setattr(ChronosBranchContext, "connect_split", split) + result = runner.invoke(app, ["--database-url", "duckdb:////tmp/data.duckdb", + "--metadata-url", metadata, "branch", "list"]) + assert result.exit_code == 1 + assert "secret" not in result.output + assert "" in result.stderr + connection.close.assert_not_called() + + def test_unavailable_duckdb_driver_reports_api_error(self, connection: MagicMock, monkeypatch: pytest.MonkeyPatch) -> None: + split = MagicMock(side_effect=RuntimeError("DuckDB support requires CHRONOS_WITH_DUCKDB=ON")) + monkeypatch.setattr(ChronosBranchContext, "connect_split", split) + result = runner.invoke(app, ["--database-url", "duckdb:////tmp/data.duckdb", + "--metadata-url", "sqlite:////tmp/metadata.sqlite", "branch", "list"]) + assert result.exit_code == 1 + assert result.stdout == "" + assert "CHRONOS_WITH_DUCKDB=ON" in result.stderr + connection.close.assert_not_called() + + @pytest.mark.parametrize("failure", [False, True]) + def test_connection_cleanup(self, monkeypatch: pytest.MonkeyPatch, failure: bool) -> None: + database = MagicMock() + monkeypatch.setattr(ChronosBranchContext, "connect", MagicMock(return_value=database)) + if failure: + with pytest.raises(typer.Exit): + with connect(DatabaseConfig("sqlite:///:memory:")): + raise ValueError("operation failed") + else: + with connect(DatabaseConfig("sqlite:///:memory:")) as context: + assert context is database + database.close.assert_called_once() + + +class TestQdrantCommands: + def test_help_does_not_connect(self, factory: MagicMock) -> None: + for args in (["--help"], ["qdrant", "--help"], ["qdrant", "point", "--help"], + ["qdrant", "search", "--help"]): + assert runner.invoke(app, args).exit_code == 0 + factory.local.assert_not_called() + factory.remote.assert_not_called() + + @pytest.mark.parametrize(("args", "method", "positional", "keywords"), [ + (["collection", "register", "docs", "--dimensions", "3"], "register_collection", ("docs", 3), {"distance": "cosine"}), + (["collection", "register", "docs", "--dimensions", "3", "--distance", "dot"], "register_collection", ("docs", 3), {"distance": "dot"}), + (["collection", "list"], "list_collections", (), {}), + (["collection", "info", "docs"], "collection_info", ("docs",), {}), + (["branch", "create", "trial", "--from", "parent"], "create_branch", ("trial",), {"from_branch": "parent"}), + (["branch", "delete", "trial"], "delete_branch", ("trial",), {}), + (["checkpoint", "create", "saved", "--branch", "trial"], "create_checkpoint", ("saved",), {"branch": "trial"}), + (["diff", "main", "trial"], "diff", ("main", "trial"), {}), + (["merge", "preview", "trial", "main"], "merge_preview", ("trial", "main"), {"policy": "abort_on_conflict"}), + (["merge", "apply", "trial", "main", "--policy", "snapshot_isolation"], "merge_apply", ("trial", "main"), {"policy": "snapshot_isolation"}), ]) - assert result.exit_code == 1 - assert result.stdout == "" - assert result.stderr.strip() == "Error: INSERT must specify a column list" - connection.checkout.return_value.__exit__.assert_called_once() - connection.close.assert_called_once() - - -@pytest.mark.parametrize("source", ["flag", "environment", "override"]) -def test_split_connection_configuration(connection: MagicMock, monkeypatch: pytest.MonkeyPatch, source: str) -> None: - split = MagicMock(return_value=connection) - monkeypatch.setattr(ChronosBranchContext, "connect_split", split) - args = ["--database-url", "duckdb:////tmp/data.duckdb"] - environment = {} - if source != "flag": - environment["CHRONOS_METADATA_URL"] = "sqlite:////tmp/env.sqlite" - if source != "environment": - args.extend(["--metadata-url", "sqlite:////tmp/flag.sqlite"]) - result = runner.invoke(app, [*args, "branch", "list"], env=environment) - assert result.exit_code == 0, result.output - expected_metadata = "sqlite:////tmp/env.sqlite" if source == "environment" else "sqlite:////tmp/flag.sqlite" - split.assert_called_once_with("duckdb:////tmp/data.duckdb", expected_metadata) - ChronosBranchContext.connect.assert_not_called() - connection.close.assert_called_once() - - -def test_duckdb_requires_metadata_before_connecting(connection: MagicMock) -> None: - result = runner.invoke(app, ["--database-url", "duckdb:////tmp/data.duckdb", "branch", "list"]) - assert result.exit_code == 2 - assert "--metadata-url" in result.output - ChronosBranchContext.connect.assert_not_called() - - -def test_split_connection_redacts_metadata_credentials(connection: MagicMock, monkeypatch: pytest.MonkeyPatch) -> None: - metadata = "postgresql://user:secret@localhost/metadata" - split = MagicMock(side_effect=ValueError(f"Cannot connect to {metadata}")) - monkeypatch.setattr(ChronosBranchContext, "connect_split", split) - result = runner.invoke(app, ["--database-url", "duckdb:////tmp/data.duckdb", - "--metadata-url", metadata, "branch", "list"]) - assert result.exit_code == 1 - assert "secret" not in result.output - assert "" in result.stderr - connection.close.assert_not_called() - - -def test_unavailable_duckdb_driver_reports_api_error(connection: MagicMock, monkeypatch: pytest.MonkeyPatch) -> None: - split = MagicMock(side_effect=RuntimeError("DuckDB support requires CHRONOS_WITH_DUCKDB=ON")) - monkeypatch.setattr(ChronosBranchContext, "connect_split", split) - result = runner.invoke(app, ["--database-url", "duckdb:////tmp/data.duckdb", - "--metadata-url", "sqlite:////tmp/metadata.sqlite", "branch", "list"]) - assert result.exit_code == 1 - assert result.stdout == "" - assert "CHRONOS_WITH_DUCKDB=ON" in result.stderr - connection.close.assert_not_called() + def test_store_api_forwarding(self, factory: MagicMock, args, method, positional, keywords) -> None: + store = factory.local.return_value + getattr(store, method).return_value = {"result": "from API"} + result = _invoke_qdrant(*args) + assert result.exit_code == 0, result.output + getattr(store, method).assert_called_once_with(*positional, **keywords) + assert json.loads(result.stdout) == {"result": "from API"} + store.close.assert_called_once() + + @pytest.mark.parametrize(("args", "method", "keywords"), [ + (["branch", "list"], "list_branches", {}), + (["checkpoint", "list", "--branch", "trial"], "list_checkpoints", {"branch": "trial"}), + ]) + def test_metadata_context_calls(self, factory: MagicMock, args, method, keywords) -> None: + store = factory.local.return_value + getattr(store.context, method).return_value = [] + result = _invoke_qdrant(*args) + assert result.exit_code == 0, result.output + getattr(store.context, method).assert_called_once_with(**keywords) + assert json.loads(result.stdout) == [] + store.close.assert_called_once() + + @pytest.mark.parametrize(("command", "method", "positional"), [ + (["point", "get", "docs", "one"], "get", ("docs", "one")), + (["point", "list", "docs"], "list_points", ("docs",)), + (["point", "delete", "docs", "one"], "delete", ("docs", "one")), + ]) + def test_point_session_calls(self, factory: MagicMock, command, method, positional) -> None: + store = factory.local.return_value + session = store.checkout_control.return_value + getattr(session, method).return_value = False if method == "delete" else None + result = _invoke_qdrant(*command, "--branch", "trial") + assert result.exit_code == 0, result.output + store.context.checkout.assert_called_once_with("trial") + store.checkout_control.assert_called_once_with(store.context.checkout.return_value.__enter__.return_value) + getattr(session, method).assert_called_once_with(*positional) + assert result.stdout == ("false\n" if method == "delete" else "") + store.context.checkout.return_value.__exit__.assert_called_once() + store.close.assert_called_once() + + @pytest.mark.parametrize("source", ["text", "file", "stdin", "pipe"]) + @pytest.mark.parametrize("command", ["upsert", "search"]) + def test_json_sources(self, factory: MagicMock, tmp_path: Path, source: str, command: str) -> None: + text = '{"vector": [1, 0, 0], "payload": {"title": "hello"}}' if command == "upsert" else "[1, 0, 0]" + args = ["point", "upsert", "docs", "one", "--revision", "r1"] if command == "upsert" else ["search", "docs", "--limit", "2", "--score-threshold", "0.5"] + input_text = None + if source == "text": + args.append(text) + elif source == "file": + path = tmp_path / "input.json" + path.write_text(text, encoding="utf-8") + args.extend(["--file", str(path)]) + else: + input_text = text + if source == "stdin": + args.append("--stdin") + store = factory.local.return_value + session = store.checkout_control.return_value + getattr(session, command).return_value = [] + result = _invoke_qdrant(*args, input=input_text) + assert result.exit_code == 0, result.output + if command == "upsert": + session.upsert.assert_called_once_with("docs", "one", [1, 0, 0], {"title": "hello"}, revision="r1") + else: + session.search.assert_called_once_with("docs", [1, 0, 0], limit=2, score_threshold=0.5) + assert json.loads(result.stdout) == [] + store.close.assert_called_once() + + def test_connection_flags_and_environment(self, factory: MagicMock) -> None: + store = factory.local.return_value + store.list_collections.return_value = [] + env = {"CHRONOS_METADATA_URL": "sqlite:///env.sqlite", "CHRONOS_QDRANT_URL": "http://localhost:6333", + "CHRONOS_QDRANT_API_KEY": "test-secret"} + result = runner.invoke(app, ["qdrant", "--metadata-url", "sqlite:///flag.sqlite", "--collection-prefix", "demo_", + "--timeout", "12", "collection", "list"], env=env) + assert result.exit_code == 0, result.output + factory.remote.assert_called_once_with("sqlite:///flag.sqlite", url="http://localhost:6333", + api_key="test-secret", collection_prefix="demo_", timeout=12.0) + factory.local.assert_not_called() + store.close.assert_called_once() + + @pytest.mark.parametrize("args", [ + ["collection", "list"], + ["--metadata-url", "sqlite:///m.sqlite", "collection", "list"], + ["--metadata-url", "sqlite:///m.sqlite", "--path", "vectors", "--url", "http://localhost:6333", "collection", "list"], + ["--metadata-url", "sqlite:///:memory:", "--path", "vectors", "collection", "list"], + ["--metadata-url", "sqlite:///m.sqlite", "--path", ":memory:", "collection", "list"], + ["--metadata-url", "sqlite:///m.sqlite", "--path", "vectors", "--api-key", "secret", "collection", "list"], + ]) + def test_invalid_configuration_does_not_connect(self, factory: MagicMock, args) -> None: + assert runner.invoke(app, ["qdrant", *args]).exit_code == 2 + factory.local.assert_not_called() + factory.remote.assert_not_called() + + @pytest.mark.parametrize("args", [ + ["search", "docs"], ["search", "docs", "not JSON"], ["search", "docs", "[NaN]"], + ["search", "docs", "[true]"], ["search", "docs", '{}'], ["search", "docs", "[1]", "--stdin"], + ["search", "docs", "--file", "missing.json"], + ["point", "upsert", "docs", "one", '{"vector": [1], "payload": []}'], + ["point", "upsert", "docs", "one", '{"vector": [1], "unexpected": 1}'], + ["collection", "register", "docs", "--dimensions", "0"], + ]) + def test_invalid_input_does_not_connect(self, factory: MagicMock, args) -> None: + result = _invoke_qdrant(*args) + assert result.exit_code == 2, result.output + factory.local.assert_not_called() + + def test_failure_closes_session_before_store_and_redacts_secrets(self, factory: MagicMock) -> None: + store = factory.local.return_value + session = store.checkout_control.return_value + events = [] + store.context.checkout.return_value.__exit__.side_effect = lambda *args: events.append("session") + store.close.side_effect = lambda: events.append("store") + session.search.side_effect = RuntimeError("Failed with api-key test-secret at postgresql://user:pw@host/meta") + result = runner.invoke(app, ["qdrant", "--url", "http://localhost:6333", "--api-key", "test-secret", + "--metadata-url", "postgresql://user:pw@host/meta", "search", "docs", "[1,0,0]"]) + assert result.exit_code == 1 + assert result.stdout == "" + assert "test-secret" not in result.stderr + assert "user:pw" not in result.stderr + assert events == ["session", "store"] + + def test_factory_failure_does_not_close_unassigned_store(self, factory: MagicMock) -> None: + factory.local.side_effect = ValueError("Cannot open storage") + result = _invoke_qdrant("collection", "list") + assert result.exit_code == 1 + factory.local.return_value.close.assert_not_called() + + def test_optional_dependency_is_not_needed_for_help(self, tmp_path: Path) -> None: + code = ''' + import importlib.abc + import sys + class Block(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == "qdrant_client" or fullname.startswith("qdrant_client."): + raise ModuleNotFoundError("simulated missing qdrant-client") + sys.meta_path.insert(0, Block()) + from chronos_core.cli.main import app + sys.argv = ["chronos", *sys.argv[1:]] + app() + ''' + env = {key: value for key, value in os.environ.items() if not key.startswith("CHRONOS_")} + for args in (["--help"], ["qdrant", "--help"]): + result = subprocess.run([sys.executable, "-c", textwrap.dedent(code), *args], capture_output=True, text=True, env=env) + assert result.returncode == 0, result.stderr + result = subprocess.run([sys.executable, "-c", textwrap.dedent(code), "qdrant", "--metadata-url", f"sqlite:///{tmp_path / 'metadata.sqlite'}", + "--path", str(tmp_path / "vectors"), "collection", "list"], capture_output=True, text=True, env=env) + assert result.returncode == 1 + assert "chronos-core[qdrant]" in result.stderr + assert list(tmp_path.iterdir()) == [] + result = subprocess.run([sys.executable, "-c", textwrap.dedent(code), "--database-url", f"sqlite:///{tmp_path / 'sql.sqlite'}", + "branch", "list"], capture_output=True, text=True, env=env) + assert result.returncode == 0, result.stderr + assert [branch["branch_id"] for branch in json.loads(result.stdout)] == ["main"] + + +class TestTerminalBehavior: + def test_json_values(self, capsys: pytest.CaptureFixture[str]) -> None: + @dataclass + class Result: + values: list + print_result(Result([date(2026, 1, 2), datetime(2026, 1, 2, 3, 4), time(3, 4), Decimal("1.20"), b"hi"])) + assert json.loads(capsys.readouterr().out) == {"values": [ + "2026-01-02", "2026-01-02T03:04:00", "03:04:00", "1.20", {"encoding": "base64", "data": "aGk="} + ]} + print_result(None) + assert capsys.readouterr().out == "" + with pytest.raises(TypeError, match="Unsupported JSON value"): + print_result(object()) diff --git a/tests/test_cli_integration.py b/tests/test_cli_integration.py index 203a280..2167d06 100644 --- a/tests/test_cli_integration.py +++ b/tests/test_cli_integration.py @@ -1,12 +1,15 @@ -"""Real CLI processes exercising the same workflows on SQLite and PostgreSQL.""" +"""Real CLI workflows across independent relational and vector stores.""" from __future__ import annotations import json import os import subprocess import sys +import time +from dataclasses import asdict from functools import partial from pathlib import Path +from tempfile import TemporaryDirectory from typing import Any, Iterator from urllib.parse import urlsplit, urlunsplit from uuid import uuid4 @@ -16,29 +19,46 @@ from psycopg import sql as postgres_sql from chronos_core import _native_interval -from chronos_core.cli._support import DatabaseConfig +from chronos_core.cli._qdrant import QdrantConfig +from chronos_core.cli._relational import DatabaseConfig -def _invoke_cli( - database: DatabaseConfig, *args: str, ok: bool = True, input_text: str | None = None, -) -> Any: - connection_args = ["--database-url", database.database_url] - if database.metadata_url is not None: - connection_args.extend(["--metadata-url", database.metadata_url]) - environment = dict(os.environ) - environment.pop("CHRONOS_METADATA_URL", None) +def _run_cli(arguments: list[str], *, environment: dict[str, str], + ok: bool = True, input_text: str | None = None) -> Any: result = subprocess.run( - [sys.executable, "-X", "faulthandler", "-m", "chronos_core.cli", - *connection_args, *args], - input=input_text, capture_output=True, text=True, timeout=30, env=environment, + [sys.executable, "-X", "faulthandler", "-m", "chronos_core.cli", *arguments], + env=environment, input=input_text, capture_output=True, text=True, timeout=30, ) assert result.returncode == (0 if ok else 1), result.stderr + assert "Traceback" not in result.stderr if not ok: assert result.stdout == "" - assert "Traceback" not in result.stderr return json.loads(result.stdout) if result.stdout.strip() else None +def _invoke_sql(database: DatabaseConfig, *args: str, + ok: bool = True, input_text: str | None = None) -> Any: + arguments = ["--database-url", database.database_url] + if database.metadata_url is not None: + arguments.extend(["--metadata-url", database.metadata_url]) + environment = dict(os.environ) + environment.pop("CHRONOS_METADATA_URL", None) + return _run_cli([*arguments, *args], environment=environment, ok=ok, input_text=input_text) + + +def _invoke_qdrant(target: QdrantConfig, *args: str, ok: bool = True, input_text: str | None = None): + env = {key: value for key, value in os.environ.items() if not key.startswith("CHRONOS_")} + env["CHRONOS_METADATA_URL"] = target.metadata_url + env["CHRONOS_QDRANT_COLLECTION_PREFIX"] = target.collection_prefix + if target.path is not None: + env["CHRONOS_QDRANT_PATH"] = str(target.path) + else: + env["CHRONOS_QDRANT_URL"] = target.url + if target.api_key is not None: + env["CHRONOS_QDRANT_API_KEY"] = target.api_key + return _run_cli(["qdrant", *args], environment=env, ok=ok, input_text=input_text) + + @pytest.fixture(scope="session") def postgres_cli_database_url() -> Iterator[str]: """Create one owned database for the run, without resetting the admin database.""" @@ -83,97 +103,224 @@ def cli_database(request: pytest.FixtureRequest, tmp_path: Path) -> DatabaseConf ) if kind == "duckdb-postgres" else DatabaseConfig(url) -def test_relational_workflow(cli_database: DatabaseConfig) -> None: - invoke = partial(_invoke_cli, cli_database) - invoke("table", "create", "CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)") - assert invoke("table", "register", "items", "--primary-key", "id") is None - assert invoke("branch", "create", "trial") is None - invoke("branch", "create", "trial", ok=False) - assert {branch["branch_id"] for branch in invoke("branch", "list")} == {"main", "trial"} - assert invoke("execute", "INSERT INTO items (id, value) VALUES (:id, :value)", "--branch", "trial", - "--params", '{"id": 1, "value": "hello"}') == {"rowcount": 1} - assert invoke("query", "SELECT * FROM items", "--branch", "trial") == [{"id": 1, "value": "hello"}] - assert invoke("query", "SELECT * FROM items") == [] - assert invoke("checkpoint", "create", "saved", "--branch", "trial")["checkpoint_id"] == "saved" - assert len(invoke("checkpoint", "list", "--branch", "trial")) == 1 - assert len(invoke("diff", "main", "trial")["changes"]) == 1 - assert len(invoke("diff", "main", "trial", "--table", "items")) == 1 - assert invoke("merge", "preview", "trial", "main")["conflicts"] == [] - assert invoke("merge", "apply", "trial", "main")["applied"] == 1 - assert invoke("query", "SELECT * FROM items") == [{"id": 1, "value": "hello"}] - invoke("branch", "create", "conflicting") - invoke("execute", "UPDATE items SET value = 'source' WHERE id = 1", "--branch", "conflicting") - invoke("execute", "UPDATE items SET value = 'target' WHERE id = 1") - assert len(invoke("merge", "preview", "conflicting", "main")["conflicts"]) == 1 - invoke("merge", "apply", "conflicting", "main", ok=False) - invoke("merge", "apply", "conflicting", "main", "--policy", "snapshot_isolation", ok=False) - assert invoke("query", "SELECT value FROM items") == [{"value": "target"}] - invoke("branch", "delete", "conflicting") - invoke("query", "SELECT * FROM items", "--branch", "missing", ok=False) - invoke("execute", "DROP TABLE items", ok=False) - assert invoke("branch", "delete", "trial") is None - assert [branch["branch_id"] for branch in invoke("branch", "list")] == ["main"] - - -def test_cli_only_setup(cli_database: DatabaseConfig) -> None: - invoke = partial(_invoke_cli, cli_database) - - assert invoke("table", "create", - "CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)") is None - invoke("table", "register", "items", "--primary-key", "id") - invoke("branch", "create", "experiment") - assert invoke("execute", "INSERT INTO items (id, value) VALUES (1, 'hello')", - "--branch", "experiment") == {"rowcount": 1} - assert invoke("query", "SELECT * FROM items", "--branch", "experiment") == [{"id": 1, "value": "hello"}] - assert invoke("query", "SELECT * FROM items") == [] - assert len(invoke("diff", "main", "experiment")["changes"]) == 1 - assert invoke("merge", "apply", "experiment", "main")["applied"] == 1 - assert invoke("query", "SELECT * FROM items") == [{"id": 1, "value": "hello"}] - - -def test_relational_sql_inputs_and_rollback(cli_database: DatabaseConfig, tmp_path: Path) -> None: - invoke = partial(_invoke_cli, cli_database) - statement = tmp_path / "statement.sql" - statement.write_text( - "CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)", encoding="utf-8", - ) - invoke("table", "create", "--file", str(statement)) - invoke("table", "register", "items", "--primary-key", "id") - statement.write_text( - "INSERT INTO items (id, value) VALUES (:id, :value)", encoding="utf-8", +@pytest.fixture(params=["local-sqlite", "remote-sqlite", "local-postgres", "remote-postgres"]) +def qdrant_target(request: pytest.FixtureRequest): + client_type = pytest.importorskip("qdrant_client").QdrantClient + from qdrant_client.http.exceptions import ResponseHandlingException, UnexpectedResponse + + kind = request.param + remote_url = os.environ.get("CHRONOS_CLI_TEST_QDRANT_URL") + api_key = os.environ.get("CHRONOS_CLI_TEST_QDRANT_API_KEY") + if kind.startswith("remote") and not remote_url: + pytest.skip("Set CHRONOS_CLI_TEST_QDRANT_URL to test the Qdrant server") + client = None + prefix = f"cli_test_{uuid4().hex}_" + if kind.startswith("remote"): + client = client_type(url=remote_url, api_key=api_key, timeout=2) + deadline = time.monotonic() + 30 + while True: + try: + client.get_collections() + break + except (ResponseHandlingException, UnexpectedResponse): + if time.monotonic() >= deadline: + client.close() + pytest.fail("Configured Qdrant test server is not ready after 30 seconds") + time.sleep(0.5) + try: + with TemporaryDirectory(prefix="chronos-cli-qdrant-") as directory: + root = Path(directory) + metadata = f"sqlite:///{root / 'metadata.sqlite'}" + if kind.endswith("postgres"): + metadata = request.getfixturevalue("postgres_cli_database_url") + # Only reset the database that our session fixture owns. + with psycopg.connect(metadata, autocommit=True) as admin: + admin.execute("DROP SCHEMA public CASCADE") + admin.execute("CREATE SCHEMA public") + yield QdrantConfig( + metadata_url=metadata, + path=None if kind.startswith("remote") else root / "vectors", + url=remote_url if kind.startswith("remote") else None, + api_key=api_key if kind.startswith("remote") else None, + collection_prefix=prefix, + ) + finally: + if client is not None: + try: + for collection in client.get_collections().collections: + if collection.name.startswith(prefix): + client.delete_collection(collection.name) + finally: + client.close() + + +def _open_qdrant_api(target: QdrantConfig): + from chronos_core.workspace.qdrant import ChronosQdrantStore + + if target.path is not None: + return ChronosQdrantStore.local( + target.metadata_url, path=target.path, collection_prefix=target.collection_prefix, + ) + return ChronosQdrantStore.remote( + target.metadata_url, url=target.url, api_key=target.api_key, + collection_prefix=target.collection_prefix, ) - assert invoke("execute", "--file", str(statement), "--params", - '{"id": 1, "value": "100% done"}') == {"rowcount": 1} - assert invoke("query", "--stdin", "--params", '{"pattern": "100%"}', - input_text="SELECT * FROM items WHERE value LIKE :pattern") == [ - {"id": 1, "value": "100% done"}, - ] - assert invoke("execute", "--stdin", "--params", '{"id": 1, "value": "updated"}', - input_text="UPDATE items SET value = :value WHERE id = :id") == {"rowcount": 1} - statement.write_text("SELECT * FROM items", encoding="utf-8") - assert invoke("query", "--file", str(statement)) == [{"id": 1, "value": "updated"}] - # A failed statement must not leave its earlier rows committed. - invoke("execute", "INSERT INTO items (id, value) VALUES (2, 'partial'), (1, 'duplicate')", ok=False) - assert invoke("query", input_text="SELECT * FROM items ORDER BY id") == [ - {"id": 1, "value": "updated"}, - ] - - - -def test_relational_aggregation(cli_database: DatabaseConfig) -> None: - invoke = partial(_invoke_cli, cli_database) - invoke("table", "create", "CREATE TABLE sales (id INTEGER PRIMARY KEY, category TEXT, amount BIGINT)") - invoke("table", "register", "sales", "--primary-key", "id") - invoke("execute", "INSERT INTO sales (id, category, amount) VALUES (1, 'books', 10), (2, 'books', 20), (3, 'games', 50)") - invoke("branch", "create", "forecast") - invoke("execute", "UPDATE sales SET amount = amount * :factor WHERE category = :category", - "--branch", "forecast", "--params", '{"factor": 2, "category": "books"}') - query = "SELECT category, SUM(amount) AS total FROM sales GROUP BY category ORDER BY category" - # PostgreSQL SUM(BIGINT) returns NUMERIC, serialized as a precision-preserving string. - total = str if urlsplit(cli_database.database_url).scheme in {"postgres", "postgresql"} else int - assert invoke("query", query) == [ - {"category": "books", "total": total(30)}, {"category": "games", "total": total(50)}, - ] - assert invoke("query", query, "--branch", "forecast") == [ - {"category": "books", "total": total(60)}, {"category": "games", "total": total(50)}, - ] + + +class TestRelationalWorkflows: + def test_relational_workflow(self, cli_database: DatabaseConfig) -> None: + invoke = partial(_invoke_sql, cli_database) + invoke("table", "create", "CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)") + assert invoke("table", "register", "items", "--primary-key", "id") is None + assert invoke("branch", "create", "trial") is None + invoke("branch", "create", "trial", ok=False) + assert {branch["branch_id"] for branch in invoke("branch", "list")} == {"main", "trial"} + assert invoke("execute", "INSERT INTO items (id, value) VALUES (:id, :value)", "--branch", "trial", + "--params", '{"id": 1, "value": "hello"}') == {"rowcount": 1} + assert invoke("query", "SELECT * FROM items", "--branch", "trial") == [{"id": 1, "value": "hello"}] + assert invoke("query", "SELECT * FROM items") == [] + assert invoke("checkpoint", "create", "saved", "--branch", "trial")["checkpoint_id"] == "saved" + assert len(invoke("checkpoint", "list", "--branch", "trial")) == 1 + assert len(invoke("diff", "main", "trial")["changes"]) == 1 + assert len(invoke("diff", "main", "trial", "--table", "items")) == 1 + assert invoke("merge", "preview", "trial", "main")["conflicts"] == [] + assert invoke("merge", "apply", "trial", "main")["applied"] == 1 + assert invoke("query", "SELECT * FROM items") == [{"id": 1, "value": "hello"}] + invoke("branch", "create", "conflicting") + invoke("execute", "UPDATE items SET value = 'source' WHERE id = 1", "--branch", "conflicting") + invoke("execute", "UPDATE items SET value = 'target' WHERE id = 1") + assert len(invoke("merge", "preview", "conflicting", "main")["conflicts"]) == 1 + invoke("merge", "apply", "conflicting", "main", ok=False) + invoke("merge", "apply", "conflicting", "main", "--policy", "snapshot_isolation", ok=False) + assert invoke("query", "SELECT value FROM items") == [{"value": "target"}] + invoke("branch", "delete", "conflicting") + invoke("query", "SELECT * FROM items", "--branch", "missing", ok=False) + invoke("execute", "DROP TABLE items", ok=False) + assert invoke("branch", "delete", "trial") is None + assert [branch["branch_id"] for branch in invoke("branch", "list")] == ["main"] + + def test_relational_sql_inputs_and_rollback(self, cli_database: DatabaseConfig, tmp_path: Path) -> None: + invoke = partial(_invoke_sql, cli_database) + statement = tmp_path / "statement.sql" + statement.write_text( + "CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)", encoding="utf-8", + ) + invoke("table", "create", "--file", str(statement)) + invoke("table", "register", "items", "--primary-key", "id") + statement.write_text( + "INSERT INTO items (id, value) VALUES (:id, :value)", encoding="utf-8", + ) + assert invoke("execute", "--file", str(statement), "--params", + '{"id": 1, "value": "100% done"}') == {"rowcount": 1} + assert invoke("query", "--stdin", "--params", '{"pattern": "100%"}', + input_text="SELECT * FROM items WHERE value LIKE :pattern") == [ + {"id": 1, "value": "100% done"}, + ] + assert invoke("execute", "--stdin", "--params", '{"id": 1, "value": "updated"}', + input_text="UPDATE items SET value = :value WHERE id = :id") == {"rowcount": 1} + statement.write_text("SELECT * FROM items", encoding="utf-8") + assert invoke("query", "--file", str(statement)) == [{"id": 1, "value": "updated"}] + # A failed statement must not leave its earlier rows committed. + invoke("execute", "INSERT INTO items (id, value) VALUES (2, 'partial'), (1, 'duplicate')", ok=False) + assert invoke("query", input_text="SELECT * FROM items ORDER BY id") == [ + {"id": 1, "value": "updated"}, + ] + + def test_relational_aggregation(self, cli_database: DatabaseConfig) -> None: + invoke = partial(_invoke_sql, cli_database) + invoke("table", "create", "CREATE TABLE sales (id INTEGER PRIMARY KEY, category TEXT, amount BIGINT)") + invoke("table", "register", "sales", "--primary-key", "id") + invoke("execute", "INSERT INTO sales (id, category, amount) VALUES (1, 'books', 10), (2, 'books', 20), (3, 'games', 50)") + invoke("branch", "create", "forecast") + invoke("execute", "UPDATE sales SET amount = amount * :factor WHERE category = :category", + "--branch", "forecast", "--params", '{"factor": 2, "category": "books"}') + query = "SELECT category, SUM(amount) AS total FROM sales GROUP BY category ORDER BY category" + # PostgreSQL SUM(BIGINT) returns NUMERIC, serialized as a precision-preserving string. + total = str if urlsplit(cli_database.database_url).scheme in {"postgres", "postgresql"} else int + assert invoke("query", query) == [ + {"category": "books", "total": total(30)}, {"category": "games", "total": total(50)}, + ] + assert invoke("query", query, "--branch", "forecast") == [ + {"category": "books", "total": total(60)}, {"category": "games", "total": total(50)}, + ] + + +class TestQdrantWorkflows: + def test_qdrant_cli_workflow(self, qdrant_target: QdrantConfig) -> None: + target = qdrant_target + info = _invoke_qdrant(target, "collection", "register", "documents", "--dimensions", "3") + assert info["name"] == "documents" + assert info["dimensions"] == 3 + assert _invoke_qdrant(target, "collection", "info", "documents") == info + assert _invoke_qdrant(target, "collection", "list") == [info] + first = {"vector": [1, 0, 0], "payload": {"title": "Runtime"}} + second = {"vector": [0, 1, 0], "payload": {"title": "Billing"}} + point = _invoke_qdrant(target, "point", "upsert", "documents", "runtime", json.dumps(first)) + _invoke_qdrant(target, "point", "upsert", "documents", "billing", json.dumps(second)) + assert _invoke_qdrant(target, "point", "get", "documents", "runtime") == point + assert {item["id"] for item in _invoke_qdrant(target, "point", "list", "documents")} == {"runtime", "billing"} + assert _invoke_qdrant(target, "point", "get", "documents", "missing") is None + results = _invoke_qdrant(target, "search", "documents", "[1,0,0]", "--limit", "2") + assert [item["id"] for item in results] == ["runtime", "billing"] + assert results[0]["score"] == pytest.approx(1.0, abs=1e-6) + assert results[1]["score"] == pytest.approx(0.0, abs=1e-6) + assert len(_invoke_qdrant(target, "search", "documents", "[1,0,0]", "--score-threshold", "0.9")) == 1 + + # Independent expected data plus a direct public-API comparison. + store = _open_qdrant_api(target) + try: + with store.context.checkout("main") as control: + session = store.checkout_control(control) + assert point == asdict(session.get("documents", "runtime")) + assert results == [asdict(item) for item in session.search("documents", [1, 0, 0], limit=2)] + finally: + store.close() + + _invoke_qdrant(target, "branch", "create", "experiment") + assert {item["branch_id"] for item in _invoke_qdrant(target, "branch", "list")} == {"main", "experiment"} + changed = {"vector": [0, 0, 1], "payload": {"title": "Experiment"}} + _invoke_qdrant(target, "point", "upsert", "documents", "runtime", json.dumps(changed), "--branch", "experiment") + assert _invoke_qdrant(target, "point", "delete", "documents", "billing", "--branch", "experiment") is True + assert _invoke_qdrant(target, "point", "get", "documents", "billing", "--branch", "experiment") is None + assert _invoke_qdrant(target, "point", "get", "documents", "billing")["payload"] == second["payload"] + assert _invoke_qdrant(target, "point", "get", "documents", "runtime")["payload"] == first["payload"] + assert _invoke_qdrant(target, "checkpoint", "create", "saved", "--branch", "experiment")["checkpoint_id"] == "saved" + assert len(_invoke_qdrant(target, "checkpoint", "list", "--branch", "experiment")) == 1 + changes = _invoke_qdrant(target, "diff", "main", "experiment")["changes"] + assert {(item["key"]["id"], item["change"]) for item in changes} == {("runtime", "modified"), ("billing", "deleted")} + assert _invoke_qdrant(target, "merge", "preview", "experiment", "main")["conflicts"] == [] + assert _invoke_qdrant(target, "merge", "apply", "experiment", "main")["applied"] == 2 + assert _invoke_qdrant(target, "point", "get", "documents", "runtime")["payload"] == changed["payload"] + assert _invoke_qdrant(target, "point", "get", "documents", "billing") is None + assert _invoke_qdrant(target, "point", "delete", "documents", "missing") is False + _invoke_qdrant(target, "branch", "delete", "experiment") + + def test_qdrant_cli_conflicts_and_failed_operations(self, qdrant_target: QdrantConfig) -> None: + target = qdrant_target + _invoke_qdrant(target, "collection", "register", "docs", "--dimensions", "3") + _invoke_qdrant(target, "point", "upsert", "docs", "one", '{"vector":[1,0,0],"payload":{"value":"base"}}') + _invoke_qdrant(target, "branch", "create", "trial") + _invoke_qdrant(target, "branch", "create", "trial", ok=False) + _invoke_qdrant(target, "point", "upsert", "docs", "one", '{"vector":[0,1,0],"payload":{"value":"trial"}}', "--branch", "trial") + _invoke_qdrant(target, "point", "upsert", "docs", "one", '{"vector":[0,0,1],"payload":{"value":"main"}}') + assert len(_invoke_qdrant(target, "merge", "preview", "trial", "main")["conflicts"]) == 1 + _invoke_qdrant(target, "merge", "apply", "trial", "main", ok=False) + _invoke_qdrant(target, "merge", "apply", "trial", "main", "--policy", "snapshot_isolation", ok=False) + _invoke_qdrant(target, "point", "upsert", "docs", "bad", '{"vector":[1,0]}', ok=False) + assert _invoke_qdrant(target, "point", "get", "docs", "bad") is None + assert _invoke_qdrant(target, "point", "get", "docs", "one")["payload"] == {"value": "main"} + _invoke_qdrant(target, "point", "get", "docs", "one", "--branch", "missing", ok=False) + _invoke_qdrant(target, "collection", "info", "missing", ok=False) + + def test_qdrant_cli_json_files_and_stdin(self, qdrant_target: QdrantConfig, tmp_path: Path) -> None: + target = qdrant_target + _invoke_qdrant(target, "collection", "register", "docs", "--dimensions", "3") + data = tmp_path / "point.json" + data.write_text('{"vector":[1,0,0],"payload":{"source":"file"}}', encoding="utf-8") + result = _invoke_qdrant(target, "point", "upsert", "docs", "one", "--file", str(data), "--revision", "initial") + assert result["payload"] == {"source": "file"} + assert _invoke_qdrant(target, "search", "docs", "--stdin", input_text="[1,0,0]")[0]["id"] == "one" + _invoke_qdrant(target, "point", "upsert", "docs", "two", "--stdin", input_text='{"vector":[0,1,0]}') + data.write_text("[0,1,0]", encoding="utf-8") + assert _invoke_qdrant(target, "search", "docs", "--file", str(data))[0]["id"] == "two" + assert _invoke_qdrant(target, "search", "docs", input_text="[1,0,0]")[0]["id"] == "one" diff --git a/tests/test_cli_qdrant.py b/tests/test_cli_qdrant.py deleted file mode 100644 index a698c9e..0000000 --- a/tests/test_cli_qdrant.py +++ /dev/null @@ -1,218 +0,0 @@ -"""Command parsing, API forwarding, errors, and resource ownership for Qdrant.""" -from __future__ import annotations - -import json -import os -import subprocess -import sys -from pathlib import Path -from unittest.mock import MagicMock - -import pytest -from typer.testing import CliRunner - -from chronos_core.cli import _qdrant_support as support -from chronos_core.cli.main import app - -runner = CliRunner() - - -@pytest.fixture -def factory(monkeypatch: pytest.MonkeyPatch) -> MagicMock: - pytest.importorskip("qdrant_client") - for name in ("CHRONOS_METADATA_URL", "CHRONOS_QDRANT_PATH", "CHRONOS_QDRANT_URL", - "CHRONOS_QDRANT_API_KEY", "CHRONOS_QDRANT_COLLECTION_PREFIX"): - monkeypatch.delenv(name, raising=False) - factory = MagicMock() - factory.remote.return_value = factory.local.return_value - monkeypatch.setattr(support, "_store_class", lambda: factory) - return factory - - -def invoke(*args: str, **kwargs): - return runner.invoke(app, ["qdrant", "--path", "vectors", "--metadata-url", "sqlite:///metadata.sqlite", *args], **kwargs) - - -def test_help_does_not_connect(factory: MagicMock) -> None: - for args in (["--help"], ["qdrant", "--help"], ["qdrant", "point", "--help"], - ["qdrant", "search", "--help"]): - assert runner.invoke(app, args).exit_code == 0 - factory.local.assert_not_called() - factory.remote.assert_not_called() - - -@pytest.mark.parametrize(("args", "method", "positional", "keywords"), [ - (["collection", "register", "docs", "--dimensions", "3"], "register_collection", ("docs", 3), {"distance": "cosine"}), - (["collection", "register", "docs", "--dimensions", "3", "--distance", "dot"], "register_collection", ("docs", 3), {"distance": "dot"}), - (["collection", "list"], "list_collections", (), {}), - (["collection", "info", "docs"], "collection_info", ("docs",), {}), - (["branch", "create", "trial", "--from", "parent"], "create_branch", ("trial",), {"from_branch": "parent"}), - (["branch", "delete", "trial"], "delete_branch", ("trial",), {}), - (["checkpoint", "create", "saved", "--branch", "trial"], "create_checkpoint", ("saved",), {"branch": "trial"}), - (["diff", "main", "trial"], "diff", ("main", "trial"), {}), - (["merge", "preview", "trial", "main"], "merge_preview", ("trial", "main"), {"policy": "abort_on_conflict"}), - (["merge", "apply", "trial", "main", "--policy", "snapshot_isolation"], "merge_apply", ("trial", "main"), {"policy": "snapshot_isolation"}), -]) -def test_store_api_forwarding(factory: MagicMock, args, method, positional, keywords) -> None: - store = factory.local.return_value - getattr(store, method).return_value = {"result": "from API"} - result = invoke(*args) - assert result.exit_code == 0, result.output - getattr(store, method).assert_called_once_with(*positional, **keywords) - assert json.loads(result.stdout) == {"result": "from API"} - store.close.assert_called_once() - - -@pytest.mark.parametrize(("args", "method", "keywords"), [ - (["branch", "list"], "list_branches", {}), - (["checkpoint", "list", "--branch", "trial"], "list_checkpoints", {"branch": "trial"}), -]) -def test_metadata_context_calls(factory: MagicMock, args, method, keywords) -> None: - store = factory.local.return_value - getattr(store.context, method).return_value = [] - result = invoke(*args) - assert result.exit_code == 0, result.output - getattr(store.context, method).assert_called_once_with(**keywords) - assert json.loads(result.stdout) == [] - store.close.assert_called_once() - - -@pytest.mark.parametrize(("command", "method", "positional"), [ - (["point", "get", "docs", "one"], "get", ("docs", "one")), - (["point", "list", "docs"], "list_points", ("docs",)), - (["point", "delete", "docs", "one"], "delete", ("docs", "one")), -]) -def test_point_session_calls(factory: MagicMock, command, method, positional) -> None: - store = factory.local.return_value - session = store.checkout_control.return_value - getattr(session, method).return_value = False if method == "delete" else None - result = invoke(*command, "--branch", "trial") - assert result.exit_code == 0, result.output - store.context.checkout.assert_called_once_with("trial") - store.checkout_control.assert_called_once_with(store.context.checkout.return_value.__enter__.return_value) - getattr(session, method).assert_called_once_with(*positional) - assert result.stdout == ("false\n" if method == "delete" else "") - store.context.checkout.return_value.__exit__.assert_called_once() - store.close.assert_called_once() - - -@pytest.mark.parametrize("source", ["text", "file", "stdin", "pipe"]) -@pytest.mark.parametrize("command", ["upsert", "search"]) -def test_json_sources(factory: MagicMock, tmp_path: Path, source: str, command: str) -> None: - text = '{"vector": [1, 0, 0], "payload": {"title": "hello"}}' if command == "upsert" else "[1, 0, 0]" - args = ["point", "upsert", "docs", "one", "--revision", "r1"] if command == "upsert" else ["search", "docs", "--limit", "2", "--score-threshold", "0.5"] - input_text = None - if source == "text": - args.append(text) - elif source == "file": - path = tmp_path / "input.json" - path.write_text(text, encoding="utf-8") - args.extend(["--file", str(path)]) - else: - input_text = text - if source == "stdin": - args.append("--stdin") - store = factory.local.return_value - session = store.checkout_control.return_value - getattr(session, command).return_value = [] - result = invoke(*args, input=input_text) - assert result.exit_code == 0, result.output - if command == "upsert": - session.upsert.assert_called_once_with("docs", "one", [1, 0, 0], {"title": "hello"}, revision="r1") - else: - session.search.assert_called_once_with("docs", [1, 0, 0], limit=2, score_threshold=0.5) - assert json.loads(result.stdout) == [] - store.close.assert_called_once() - - -def test_connection_flags_and_environment(factory: MagicMock) -> None: - store = factory.local.return_value - store.list_collections.return_value = [] - env = {"CHRONOS_METADATA_URL": "sqlite:///env.sqlite", "CHRONOS_QDRANT_URL": "http://localhost:6333", - "CHRONOS_QDRANT_API_KEY": "test-secret"} - result = runner.invoke(app, ["qdrant", "--metadata-url", "sqlite:///flag.sqlite", "--collection-prefix", "demo_", - "--timeout", "12", "collection", "list"], env=env) - assert result.exit_code == 0, result.output - factory.remote.assert_called_once_with("sqlite:///flag.sqlite", url="http://localhost:6333", - api_key="test-secret", collection_prefix="demo_", timeout=12.0) - factory.local.assert_not_called() - store.close.assert_called_once() - - -@pytest.mark.parametrize("args", [ - ["collection", "list"], - ["--metadata-url", "sqlite:///m.sqlite", "collection", "list"], - ["--metadata-url", "sqlite:///m.sqlite", "--path", "vectors", "--url", "http://localhost:6333", "collection", "list"], - ["--metadata-url", "sqlite:///:memory:", "--path", "vectors", "collection", "list"], - ["--metadata-url", "sqlite:///m.sqlite", "--path", ":memory:", "collection", "list"], - ["--metadata-url", "sqlite:///m.sqlite", "--path", "vectors", "--api-key", "secret", "collection", "list"], -]) -def test_invalid_configuration_does_not_connect(factory: MagicMock, args) -> None: - assert runner.invoke(app, ["qdrant", *args]).exit_code == 2 - factory.local.assert_not_called() - factory.remote.assert_not_called() - - -@pytest.mark.parametrize("args", [ - ["search", "docs"], ["search", "docs", "not JSON"], ["search", "docs", "[NaN]"], - ["search", "docs", "[true]"], ["search", "docs", '{}'], ["search", "docs", "[1]", "--stdin"], - ["search", "docs", "--file", "missing.json"], - ["point", "upsert", "docs", "one", '{"vector": [1], "payload": []}'], - ["point", "upsert", "docs", "one", '{"vector": [1], "unexpected": 1}'], - ["collection", "register", "docs", "--dimensions", "0"], -]) -def test_invalid_input_does_not_connect(factory: MagicMock, args) -> None: - result = invoke(*args) - assert result.exit_code == 2, result.output - factory.local.assert_not_called() - - -def test_failure_closes_session_before_store_and_redacts_secrets(factory: MagicMock) -> None: - store = factory.local.return_value - session = store.checkout_control.return_value - events = [] - store.context.checkout.return_value.__exit__.side_effect = lambda *args: events.append("session") - store.close.side_effect = lambda: events.append("store") - session.search.side_effect = RuntimeError("Failed with api-key test-secret at postgresql://user:pw@host/meta") - result = runner.invoke(app, ["qdrant", "--url", "http://localhost:6333", "--api-key", "test-secret", - "--metadata-url", "postgresql://user:pw@host/meta", "search", "docs", "[1,0,0]"]) - assert result.exit_code == 1 - assert result.stdout == "" - assert "test-secret" not in result.stderr - assert "user:pw" not in result.stderr - assert events == ["session", "store"] - - -def test_factory_failure_does_not_close_unassigned_store(factory: MagicMock) -> None: - factory.local.side_effect = ValueError("Cannot open storage") - result = invoke("collection", "list") - assert result.exit_code == 1 - factory.local.return_value.close.assert_not_called() - - -def test_optional_dependency_is_not_needed_for_help(tmp_path: Path) -> None: - code = ''' -import importlib.abc -import sys -class Block(importlib.abc.MetaPathFinder): - def find_spec(self, fullname, path=None, target=None): - if fullname == "qdrant_client" or fullname.startswith("qdrant_client."): - raise ModuleNotFoundError("simulated missing qdrant-client") -sys.meta_path.insert(0, Block()) -from chronos_core.cli.main import app -sys.argv = ["chronos", *sys.argv[1:]] -app() -''' - env = {key: value for key, value in os.environ.items() if not key.startswith("CHRONOS_")} - for args in (["--help"], ["qdrant", "--help"]): - result = subprocess.run([sys.executable, "-c", code, *args], capture_output=True, text=True, env=env) - assert result.returncode == 0, result.stderr - result = subprocess.run([sys.executable, "-c", code, "qdrant", "--metadata-url", f"sqlite:///{tmp_path / 'metadata.sqlite'}", - "--path", str(tmp_path / "vectors"), "collection", "list"], capture_output=True, text=True, env=env) - assert result.returncode == 1 - assert "chronos-core[qdrant]" in result.stderr - assert list(tmp_path.iterdir()) == [] - result = subprocess.run([sys.executable, "-c", code, "--database-url", f"sqlite:///{tmp_path / 'sql.sqlite'}", - "branch", "list"], capture_output=True, text=True, env=env) - assert result.returncode == 0, result.stderr - assert [branch["branch_id"] for branch in json.loads(result.stdout)] == ["main"] diff --git a/tests/test_cli_qdrant_integration.py b/tests/test_cli_qdrant_integration.py deleted file mode 100644 index 4d01a6a..0000000 --- a/tests/test_cli_qdrant_integration.py +++ /dev/null @@ -1,186 +0,0 @@ -"""Persistent local and server-backed Qdrant through separate CLI processes.""" -from __future__ import annotations - -import json -import os -import subprocess -import sys -import time -from dataclasses import asdict -from pathlib import Path -from tempfile import TemporaryDirectory -from uuid import uuid4 - -import psycopg -import pytest - -pytest.importorskip("qdrant_client") -from qdrant_client import QdrantClient -from qdrant_client.http.exceptions import ResponseHandlingException, UnexpectedResponse - -from chronos_core.cli._qdrant_support import QdrantConfig -from chronos_core.workspace.qdrant import ChronosQdrantStore -# Reuse the existing isolated, session-owned PostgreSQL database fixture. -from tests.test_cli_integration import postgres_cli_database_url # noqa: F401 - - -@pytest.fixture(params=["local-sqlite", "remote-sqlite", "local-postgres", "remote-postgres"]) -def qdrant_target(request: pytest.FixtureRequest): - kind = request.param - remote_url = os.environ.get("CHRONOS_CLI_TEST_QDRANT_URL") - api_key = os.environ.get("CHRONOS_CLI_TEST_QDRANT_API_KEY") - if kind.startswith("remote") and not remote_url: - pytest.skip("Set CHRONOS_CLI_TEST_QDRANT_URL to test the Qdrant server") - client = None - prefix = f"cli_test_{uuid4().hex}_" - if kind.startswith("remote"): - client = QdrantClient(url=remote_url, api_key=api_key, timeout=2) - deadline = time.monotonic() + 30 - while True: - try: - client.get_collections() - break - except (ResponseHandlingException, UnexpectedResponse): - if time.monotonic() >= deadline: - client.close() - pytest.fail("Configured Qdrant test server is not ready after 30 seconds") - time.sleep(0.5) - try: - with TemporaryDirectory(prefix="chronos-cli-qdrant-") as directory: - root = Path(directory) - metadata = f"sqlite:///{root / 'metadata.sqlite'}" - if kind.endswith("postgres"): - metadata = request.getfixturevalue("postgres_cli_database_url") - # Only reset the database that our session fixture owns. - with psycopg.connect(metadata, autocommit=True) as admin: - admin.execute("DROP SCHEMA public CASCADE") - admin.execute("CREATE SCHEMA public") - yield QdrantConfig( - metadata_url=metadata, - path=None if kind.startswith("remote") else root / "vectors", - url=remote_url if kind.startswith("remote") else None, - api_key=api_key if kind.startswith("remote") else None, - collection_prefix=prefix, - ) - finally: - if client is not None: - try: - for collection in client.get_collections().collections: - if collection.name.startswith(prefix): - client.delete_collection(collection.name) - finally: - client.close() - - -def invoke(target: QdrantConfig, *args: str, ok: bool = True, input_text: str | None = None): - env = {key: value for key, value in os.environ.items() if not key.startswith("CHRONOS_")} - env["CHRONOS_METADATA_URL"] = target.metadata_url - env["CHRONOS_QDRANT_COLLECTION_PREFIX"] = target.collection_prefix - if target.path is not None: - env["CHRONOS_QDRANT_PATH"] = str(target.path) - else: - env["CHRONOS_QDRANT_URL"] = target.url - if target.api_key is not None: - env["CHRONOS_QDRANT_API_KEY"] = target.api_key - result = subprocess.run( - [sys.executable, "-m", "chronos_core.cli", "qdrant", *args], - env=env, input=input_text, capture_output=True, text=True, timeout=30, - ) - assert result.returncode == (0 if ok else 1), result.stderr - assert "Traceback" not in result.stderr - if not ok: - assert result.stdout == "" - return json.loads(result.stdout) if result.stdout.strip() else None - - -def open_api(target: QdrantConfig): - if target.path is not None: - return ChronosQdrantStore.local( - target.metadata_url, path=target.path, collection_prefix=target.collection_prefix, - ) - return ChronosQdrantStore.remote( - target.metadata_url, url=target.url, api_key=target.api_key, - collection_prefix=target.collection_prefix, - ) - - -def test_qdrant_cli_workflow(qdrant_target: QdrantConfig) -> None: - target = qdrant_target - info = invoke(target, "collection", "register", "documents", "--dimensions", "3") - assert info["name"] == "documents" - assert info["dimensions"] == 3 - assert invoke(target, "collection", "info", "documents") == info - assert invoke(target, "collection", "list") == [info] - first = {"vector": [1, 0, 0], "payload": {"title": "Runtime"}} - second = {"vector": [0, 1, 0], "payload": {"title": "Billing"}} - point = invoke(target, "point", "upsert", "documents", "runtime", json.dumps(first)) - invoke(target, "point", "upsert", "documents", "billing", json.dumps(second)) - assert invoke(target, "point", "get", "documents", "runtime") == point - assert {item["id"] for item in invoke(target, "point", "list", "documents")} == {"runtime", "billing"} - assert invoke(target, "point", "get", "documents", "missing") is None - results = invoke(target, "search", "documents", "[1,0,0]", "--limit", "2") - assert [item["id"] for item in results] == ["runtime", "billing"] - assert results[0]["score"] == pytest.approx(1.0, abs=1e-6) - assert results[1]["score"] == pytest.approx(0.0, abs=1e-6) - assert len(invoke(target, "search", "documents", "[1,0,0]", "--score-threshold", "0.9")) == 1 - - # Independent expected data plus a direct public-API comparison. - store = open_api(target) - try: - with store.context.checkout("main") as control: - session = store.checkout_control(control) - assert point == asdict(session.get("documents", "runtime")) - assert results == [asdict(item) for item in session.search("documents", [1, 0, 0], limit=2)] - finally: - store.close() - - invoke(target, "branch", "create", "experiment") - assert {item["branch_id"] for item in invoke(target, "branch", "list")} == {"main", "experiment"} - changed = {"vector": [0, 0, 1], "payload": {"title": "Experiment"}} - invoke(target, "point", "upsert", "documents", "runtime", json.dumps(changed), "--branch", "experiment") - assert invoke(target, "point", "delete", "documents", "billing", "--branch", "experiment") is True - assert invoke(target, "point", "get", "documents", "billing", "--branch", "experiment") is None - assert invoke(target, "point", "get", "documents", "billing")["payload"] == second["payload"] - assert invoke(target, "point", "get", "documents", "runtime")["payload"] == first["payload"] - assert invoke(target, "checkpoint", "create", "saved", "--branch", "experiment")["checkpoint_id"] == "saved" - assert len(invoke(target, "checkpoint", "list", "--branch", "experiment")) == 1 - changes = invoke(target, "diff", "main", "experiment")["changes"] - assert {(item["key"]["id"], item["change"]) for item in changes} == {("runtime", "modified"), ("billing", "deleted")} - assert invoke(target, "merge", "preview", "experiment", "main")["conflicts"] == [] - assert invoke(target, "merge", "apply", "experiment", "main")["applied"] == 2 - assert invoke(target, "point", "get", "documents", "runtime")["payload"] == changed["payload"] - assert invoke(target, "point", "get", "documents", "billing") is None - assert invoke(target, "point", "delete", "documents", "missing") is False - invoke(target, "branch", "delete", "experiment") - - -def test_qdrant_cli_conflicts_and_failed_operations(qdrant_target: QdrantConfig) -> None: - target = qdrant_target - invoke(target, "collection", "register", "docs", "--dimensions", "3") - invoke(target, "point", "upsert", "docs", "one", '{"vector":[1,0,0],"payload":{"value":"base"}}') - invoke(target, "branch", "create", "trial") - invoke(target, "branch", "create", "trial", ok=False) - invoke(target, "point", "upsert", "docs", "one", '{"vector":[0,1,0],"payload":{"value":"trial"}}', "--branch", "trial") - invoke(target, "point", "upsert", "docs", "one", '{"vector":[0,0,1],"payload":{"value":"main"}}') - assert len(invoke(target, "merge", "preview", "trial", "main")["conflicts"]) == 1 - invoke(target, "merge", "apply", "trial", "main", ok=False) - invoke(target, "merge", "apply", "trial", "main", "--policy", "snapshot_isolation", ok=False) - invoke(target, "point", "upsert", "docs", "bad", '{"vector":[1,0]}', ok=False) - assert invoke(target, "point", "get", "docs", "bad") is None - assert invoke(target, "point", "get", "docs", "one")["payload"] == {"value": "main"} - invoke(target, "point", "get", "docs", "one", "--branch", "missing", ok=False) - invoke(target, "collection", "info", "missing", ok=False) - - -def test_qdrant_cli_json_files_and_stdin(qdrant_target: QdrantConfig, tmp_path: Path) -> None: - target = qdrant_target - invoke(target, "collection", "register", "docs", "--dimensions", "3") - data = tmp_path / "point.json" - data.write_text('{"vector":[1,0,0],"payload":{"source":"file"}}', encoding="utf-8") - result = invoke(target, "point", "upsert", "docs", "one", "--file", str(data), "--revision", "initial") - assert result["payload"] == {"source": "file"} - assert invoke(target, "search", "docs", "--stdin", input_text="[1,0,0]")[0]["id"] == "one" - invoke(target, "point", "upsert", "docs", "two", "--stdin", input_text='{"vector":[0,1,0]}') - data.write_text("[0,1,0]", encoding="utf-8") - assert invoke(target, "search", "docs", "--file", str(data))[0]["id"] == "two" - assert invoke(target, "search", "docs", input_text="[1,0,0]")[0]["id"] == "one" diff --git a/tests/test_cli_support.py b/tests/test_cli_support.py deleted file mode 100644 index 3aa1c37..0000000 --- a/tests/test_cli_support.py +++ /dev/null @@ -1,41 +0,0 @@ -from __future__ import annotations - -import json -from dataclasses import dataclass -from datetime import date, datetime, time -from decimal import Decimal -from unittest.mock import MagicMock - -import pytest -import typer - -from chronos_core.branching import ChronosBranchContext -from chronos_core.cli._support import DatabaseConfig, connect, print_result - - -def test_json_values(capsys: pytest.CaptureFixture[str]) -> None: - @dataclass - class Result: - values: list - print_result(Result([date(2026, 1, 2), datetime(2026, 1, 2, 3, 4), time(3, 4), Decimal("1.20"), b"hi"])) - assert json.loads(capsys.readouterr().out) == {"values": [ - "2026-01-02", "2026-01-02T03:04:00", "03:04:00", "1.20", {"encoding": "base64", "data": "aGk="} - ]} - print_result(None) - assert capsys.readouterr().out == "" - with pytest.raises(TypeError, match="Unsupported JSON value"): - print_result(object()) - - -@pytest.mark.parametrize("failure", [False, True]) -def test_connection_cleanup(monkeypatch: pytest.MonkeyPatch, failure: bool) -> None: - database = MagicMock() - monkeypatch.setattr(ChronosBranchContext, "connect", MagicMock(return_value=database)) - if failure: - with pytest.raises(typer.Exit): - with connect(DatabaseConfig("sqlite:///:memory:")): - raise ValueError("operation failed") - else: - with connect(DatabaseConfig("sqlite:///:memory:")) as context: - assert context is database - database.close.assert_called_once() From 0fa999de03c5869bffd03b5abaaf003fe4e4979c Mon Sep 17 00:00:00 2001 From: Seth Rojas Date: Mon, 2 Nov 2026 22:38:22 -0700 Subject: [PATCH 06/11] feat(cli): wrap multi-store workspace operations Add TOML-configured workspace commands, named SQL and Qdrant operations, and explicit coordinated and atomic merge routes. Preserve persistent data while owning connections and cleaning up failed invocations. Cover CLI inputs, forwarding, persistence, backend workflows, stale previews, conflicts, and injected merge failures. Add a checked PostgreSQL/Qdrant demo with a SQLite alternative. Validation: 202 targeted tests passed with one expected build-feature skip; both demos passed. --- .github/workflows/relational-package.yml | 14 +- docs/cli.md | 76 +++- examples/cli_workspace_demo.sh | 109 +++++ packages/chronos-core/pyproject.toml | 1 + .../src/chronos_core/cli/_relational.py | 27 +- .../src/chronos_core/cli/_workspace.py | 325 ++++++++++++++ .../src/chronos_core/cli/_workspace_config.py | 252 +++++++++++ .../chronos-core/src/chronos_core/cli/main.py | 2 + tests/test_cli.py | 421 +++++++++++++++++- tests/test_cli_integration.py | 308 +++++++++++++ uv.lock | 2 + 11 files changed, 1519 insertions(+), 18 deletions(-) create mode 100755 examples/cli_workspace_demo.sh create mode 100644 packages/chronos-core/src/chronos_core/cli/_workspace.py create mode 100644 packages/chronos-core/src/chronos_core/cli/_workspace_config.py diff --git a/.github/workflows/relational-package.yml b/.github/workflows/relational-package.yml index 3778aeb..c3bf914 100644 --- a/.github/workflows/relational-package.yml +++ b/.github/workflows/relational-package.yml @@ -106,7 +106,13 @@ jobs: run: | python -I -m pytest tests/test_cli.py::TestRelationalCommands \ tests/test_cli.py::TestTerminalBehavior \ - tests/test_cli_integration.py::TestRelationalWorkflows -q --timeout=60 + tests/test_cli.py::TestWorkspaceConfiguration \ + tests/test_cli.py::TestWorkspaceCommands \ + tests/test_cli_integration.py::TestRelationalWorkflows \ + tests/test_cli_integration.py::TestWorkspaceSQLWorkflows \ + tests/test_cli_integration.py::TestWorkspaceResources::test_sqlite_workspace_persists_after_close \ + tests/test_cli_integration.py::TestWorkspaceResources::test_sqlite_workspace_without_qdrant_dependency \ + -q --timeout=60 - name: Test Qdrant CLI in local and server modes if: matrix.python == '3.12' && matrix.duckdb == 'OFF' env: @@ -118,4 +124,8 @@ jobs: -p 127.0.0.1:6333:6333 qdrant/qdrant:v1.20.0 trap 'docker stop chronos-cli-qdrant-test > /dev/null' EXIT python -I -m pytest tests/test_cli.py::TestQdrantCommands \ - tests/test_cli_integration.py::TestQdrantWorkflows -q --timeout=60 + tests/test_cli.py::TestWorkspaceQdrantCommands \ + tests/test_cli_integration.py::TestQdrantWorkflows \ + tests/test_cli_integration.py::TestWorkspaceResources \ + tests/test_cli_integration.py::TestWorkspaceQdrantWorkflows \ + tests/test_cli_integration.py::TestWorkspaceFailureWorkflows -q --timeout=60 diff --git a/docs/cli.md b/docs/cli.md index 1dc26c1..d26a5ce 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1,9 +1,10 @@ # Chronos CLI -The `chronos` command is a one-off wrapper around the public relational and Qdrant -Python APIs. Each invocation opens the selected store, calls the API, prints its -return value, and closes the connection. It does not remember a current branch -or keep a background connection open. +The `chronos` command wraps the public workspace, relational, and Qdrant Python +APIs. Use a workspace for one or several named stores; direct-store commands +remain available for quick scripts. Each invocation opens its stores, calls the +API, prints its return value, and closes connections. Stored data persists. The +CLI does not remember a current branch or keep background connections open. ## Install and choose a database @@ -35,6 +36,73 @@ macOS's SQLite can crash the process. After a custom build, use `uv run --no-sync --package chronos-core chronos ...` to run the installed build without syncing it again. +## Workspaces + +Configure named stores in `chronos.toml`. For example, one SQLite store: + +```toml +[stores.app] +kind = "sqlite" +database_url = "sqlite:///.chronos/app.sqlite" +``` + +Create `.chronos/` first. Local database URLs and Qdrant paths resolve relative +to the TOML file. Configuration precedence is `--config PATH`, then +`CHRONOS_WORKSPACE_CONFIG`, then `chronos.toml` in the current directory. A +workspace name is not required. Connection strings may reference `${ENV_VAR}`; +unset references are errors. Single-store connection environment variables are +not used for workspace commands, and `.env` files are not loaded automatically. + +```sh +chronos workspace table create --store app \ + 'CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)' +chronos workspace table register --store app items --primary-key id +chronos workspace branch create experiment +chronos workspace query --store app --branch experiment 'SELECT * FROM items' +``` + +SQL participants support `sqlite`, `postgresql`, and `duckdb` kinds, with +`database_url` and optional `metadata_url` (required for DuckDB). Qdrant uses +`kind = "qdrant"`, `metadata_url`, and exactly one of `path` or `url`; remote +connections also accept `api_key` and `timeout`. Select vector operations with +`chronos workspace qdrant --store NAME ...`, using the same collection, point, +and search arguments as the direct Qdrant commands below. + +Branches, checkpoints, diffs, and merges operate across the workspace. With +independent metadata, `merge preview/apply` calls the coordinated API, which +can retain partial changes if a later participant fails. Atomic mode requires +`[workspace].shared_metadata_url` and matching metadata URLs for every store. +Use `merge atomic-preview` and `merge atomic-apply --operation-id ID`; optionally +pass the returned `--preview-token` to reject a changed preview. Atomic mode +does not silently fall back to coordinated merging. Filesystems are not exposed +by the workspace CLI yet. + +### Checked terminal demo + +After installation, run `bash examples/cli_workspace_demo.sh --help` from the +repository. The default demo uses PostgreSQL plus embedded Qdrant; set +`CHRONOS_DEMO_POSTGRES_URL` to a fresh demo database, then run: + +```sh +bash examples/cli_workspace_demo.sh +``` + +For SQLite plus embedded Qdrant without servers: + +```sh +bash examples/cli_workspace_demo.sh --sqlite +``` + +The script performs every Chronos operation through the CLI and uses Python +only to check its JSON output. It prints each check, verifies single-store +startup, adding Qdrant, branch isolation, checkpoint restoration, search, and +atomic merging, then removes its branches. It leaves the checkpoint and data +in place and prints the config path for further exploration. Local files go +into a fresh `.chronos/workspace-demo.*` directory; PostgreSQL data remains in +the supplied database. No servers are started or stopped by the script. + +## Direct-store commands + Set the database URL in your terminal. This example uses a persistent local file: ```sh diff --git a/examples/cli_workspace_demo.sh b/examples/cli_workspace_demo.sh new file mode 100755 index 0000000..0d9e450 --- /dev/null +++ b/examples/cli_workspace_demo.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash +# Run from any directory after installing chronos-core with Qdrant support. +set -euo pipefail + +case "${1:---postgres}" in + --postgres) + : "${CHRONOS_DEMO_POSTGRES_URL:?Set CHRONOS_DEMO_POSTGRES_URL to a fresh PostgreSQL demo database}" + sql_kind="postgresql" + sql_url='${CHRONOS_DEMO_POSTGRES_URL}' + ;; + --sqlite) + sql_kind="sqlite" + sql_url="sqlite:///app.sqlite" + ;; + --help) + printf 'Usage: bash examples/cli_workspace_demo.sh [--postgres|--sqlite]\n' + printf 'Default: PostgreSQL (CHRONOS_DEMO_POSTGRES_URL) + embedded Qdrant.\n' + printf 'Use --sqlite for SQLite + embedded Qdrant with no servers.\n' + exit 0 + ;; + *) printf 'Unknown option: %s\n' "$1" >&2; exit 2 ;; +esac + +repo_root="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")/.." && pwd)" +cd "$repo_root" +mkdir -p .chronos +demo_dir="$(mktemp -d "$repo_root/.chronos/workspace-demo.XXXXXX")" +export CHRONOS_WORKSPACE_CONFIG="$demo_dir/chronos.toml" +cli=(uv run --no-sync --package chronos-core chronos workspace) +python=(uv run --no-sync --package chronos-core python) + +# Python only compares JSON output; every Chronos operation uses the CLI. +check() { + local label="$1" field="$2" expected="$3" actual + shift 3 + printf '\n%s\n' "$label" + actual="$("${cli[@]}" "$@")" + printf '%s\n' "$actual" + "${python[@]}" -c ' +import json, sys +value = json.loads(sys.argv[1] or "null") +for field in filter(None, sys.argv[2].split(".")): + value = value[int(field)] if isinstance(value, list) else value[field] +expected = json.loads(sys.argv[3]) +if value != expected: + raise SystemExit(f"FAIL: expected {expected!r}, got {value!r}") +print("PASS") +' "$actual" "$field" "$expected" +} + +printf 'Persistent demo directory: %s\n' "$demo_dir" +cat > "$CHRONOS_WORKSPACE_CONFIG" <> "$CHRONOS_WORKSPACE_CONFIG" <=3.1", "sqlglot>=25", "typer>=0.25.1", + "tomli>=2.0; python_version < '3.11'", ] [project.optional-dependencies] diff --git a/packages/chronos-core/src/chronos_core/cli/_relational.py b/packages/chronos-core/src/chronos_core/cli/_relational.py index 0f86ed0..e256524 100644 --- a/packages/chronos-core/src/chronos_core/cli/_relational.py +++ b/packages/chronos-core/src/chronos_core/cli/_relational.py @@ -91,6 +91,21 @@ def _parameters(params: str | None) -> dict[str, Any] | None: return value +def _create_table_input(sql: str | None, file: Path | None, stdin: bool) -> str: + statement = _sql_input(sql, file, stdin) + try: + statements = sqlglot.parse(statement) + except sqlglot.errors.ParseError as error: + raise typer.BadParameter("Expected one CREATE TABLE statement.") from error + if ( + len(statements) != 1 + or not isinstance(statements[0], exp.Create) + or statements[0].kind != "TABLE" + ): + raise typer.BadParameter("Expected one CREATE TABLE statement.") + return statement + + branch_app = typer.Typer(no_args_is_help=True) table_app = typer.Typer(no_args_is_help=True) checkpoint_app = typer.Typer(no_args_is_help=True) @@ -135,17 +150,7 @@ def create_table( stdin: bool = typer.Option(False, "--stdin"), ) -> None: """Create an ordinary table for subsequent table register; not a branch-local write.""" - statement = _sql_input(sql, file, stdin) - try: - statements = sqlglot.parse(statement) - except sqlglot.errors.ParseError as error: - raise typer.BadParameter("Expected one CREATE TABLE statement.") from error - if ( - len(statements) != 1 - or not isinstance(statements[0], exp.Create) - or statements[0].kind != "TABLE" - ): - raise typer.BadParameter("Expected one CREATE TABLE statement.") + statement = _create_table_input(sql, file, stdin) with connect(database_config(ctx)) as database: database.db.execute(statement) print_result(database.db.commit()) diff --git a/packages/chronos-core/src/chronos_core/cli/_workspace.py b/packages/chronos-core/src/chronos_core/cli/_workspace.py new file mode 100644 index 0000000..781fef2 --- /dev/null +++ b/packages/chronos-core/src/chronos_core/cli/_workspace.py @@ -0,0 +1,325 @@ +"""One-off commands over the public multi-store workspace API.""" +from __future__ import annotations + +import sqlite3 +import sys +from collections.abc import Generator +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any + +import psycopg +import typer + +from chronos_core.cli._qdrant import QdrantConfig, dense_vector, point_input, read_json_input +from chronos_core.cli._relational import _create_table_input, _parameters, _sql_input +from chronos_core.cli._terminal import MergePolicy, print_result, report_error +from chronos_core.cli._workspace_config import WorkspaceConfigError, load_workspace_config, open_workspace + +if TYPE_CHECKING: + from chronos_core.workspace import ChronosWorkspaceContext + + +@dataclass(frozen=True) +class WorkspaceOptions: + config_path: Path | None + + +@dataclass(frozen=True) +class QdrantSelection: + store: str | None + + +app = typer.Typer(no_args_is_help=True) +branch_app = typer.Typer(no_args_is_help=True) +checkpoint_app = typer.Typer(no_args_is_help=True) +merge_app = typer.Typer(no_args_is_help=True) +table_app = typer.Typer(no_args_is_help=True) +qdrant_app = typer.Typer(no_args_is_help=True) +collection_app = typer.Typer(no_args_is_help=True) +point_app = typer.Typer(no_args_is_help=True) +app.add_typer(branch_app, name="branch", help="Manage branches across the workspace.") +app.add_typer(checkpoint_app, name="checkpoint", help="Checkpoint all participants.") +app.add_typer(merge_app, name="merge", help="Preview and apply coordinated or atomic merges.") +app.add_typer(table_app, name="table", help="Prepare tables in a named relational store.") +app.add_typer(qdrant_app, name="qdrant", help="Use a named Qdrant participant.") +qdrant_app.add_typer(collection_app, name="collection") +qdrant_app.add_typer(point_app, name="point") + + +@app.callback() +def main( + ctx: typer.Context, + config: Path | None = typer.Option( + None, "--config", envvar="CHRONOS_WORKSPACE_CONFIG", + help="Workspace TOML path; defaults to ./chronos.toml.", + ), +) -> None: + """Use configured stores together, including workspaces with just one store.""" + ctx.obj = WorkspaceOptions(config) + + +@contextmanager +def _connect( + ctx: typer.Context, store: str | None = None, *, qdrant: bool = False, +) -> Generator[ChronosWorkspaceContext, None, None]: + options = ctx.find_object(WorkspaceOptions) + try: + config = load_workspace_config(options.config_path if options else None) + except WorkspaceConfigError as error: + raise typer.BadParameter(str(error), param_hint="--config") from error + if store is not None: + try: + config.store(store, qdrant=qdrant) + except WorkspaceConfigError as error: + raise typer.BadParameter(str(error), param_hint="--store") from error + + from chronos_core.branching import BranchingError + + errors = (BranchingError, RuntimeError, ValueError, TypeError, OSError, sqlite3.Error, psycopg.Error) + secrets: list[str | None] = [] + urls: list[str | None] = [] + for entry in config.stores.values(): + target = entry.connection + urls.append(target.metadata_url) + if isinstance(target, QdrantConfig): + secrets.append(target.api_key) + urls.append(target.url) + else: + urls.append(target.database_url) + if any(entry.kind == "qdrant" for entry in config.stores.values()): + try: + import grpc + import httpx + from qdrant_client.http.exceptions import ResponseHandlingException, UnexpectedResponse + except ImportError: + report_error(RuntimeError("Qdrant support requires installing chronos-core[qdrant].")) + errors += (grpc.RpcError, httpx.HTTPError, ResponseHandlingException, UnexpectedResponse) + try: + with open_workspace(config) as workspace: + yield workspace + except errors as error: + report_error(error, secrets=tuple(secrets), urls=tuple(urls)) + + +@contextmanager +def _session(workspace: ChronosWorkspaceContext, branch: str, store: str) -> Generator[Any, None, None]: + checked_out = workspace.checkout(branch) + errors: list[Exception] = [] + try: + yield checked_out.stores[store] + finally: + # SQL sessions expose close(). Qdrant control sessions without a public + # close method are released by their owning context when it closes. + for session in checked_out.stores.values(): + close = getattr(session, "close", None) + if callable(close): + try: + close() + except Exception as error: + errors.append(error) + if errors and sys.exc_info()[0] is None: + raise errors[0] + + +@branch_app.command("list") +def list_branches(ctx: typer.Context) -> None: + with _connect(ctx) as workspace: + print_result(workspace.list_branches()) + + +@branch_app.command("create") +def create_branch(ctx: typer.Context, name: str, from_branch: str = typer.Option("main", "--from")) -> None: + with _connect(ctx) as workspace: + print_result(workspace.create_branch(name, from_branch=from_branch)) + + +@branch_app.command("create-from-checkpoint") +def create_branch_from_checkpoint(ctx: typer.Context, name: str, checkpoint: str) -> None: + with _connect(ctx) as workspace: + print_result(workspace.create_branch_from_checkpoint(name, checkpoint)) + + +@branch_app.command("delete") +def delete_branch(ctx: typer.Context, name: str) -> None: + with _connect(ctx) as workspace: + print_result(workspace.delete_branch(name)) + + +@checkpoint_app.command("create") +def create_checkpoint(ctx: typer.Context, name: str, branch: str = typer.Option("main", "--branch")) -> None: + with _connect(ctx) as workspace: + print_result(workspace.create_checkpoint(name, branch=branch)) + + +@app.command() +def diff(ctx: typer.Context, left: str, right: str) -> None: + with _connect(ctx) as workspace: + print_result(workspace.diff(left, right)) + + +@merge_app.command("preview") +def merge_preview(ctx: typer.Context, source: str, target: str, + policy: MergePolicy = typer.Option(MergePolicy.abort_on_conflict, "--policy")) -> None: + with _connect(ctx) as workspace: + print_result(workspace.merge_preview(source, target, policy=policy.value)) + + +@merge_app.command("apply") +def merge_apply(ctx: typer.Context, source: str, target: str, + policy: MergePolicy = typer.Option(MergePolicy.abort_on_conflict, "--policy")) -> None: + with _connect(ctx) as workspace: + print_result(workspace.merge_apply(source, target, policy=policy.value)) + + +@merge_app.command("atomic-preview") +def merge_atomic_preview(ctx: typer.Context, source: str, target: str, + policy: MergePolicy = typer.Option(MergePolicy.abort_on_conflict, "--policy")) -> None: + with _connect(ctx) as workspace: + print_result(workspace.merge_atomic_preview(source, target, policy=policy.value)) + + +@merge_app.command("atomic-apply") +def merge_atomic( + ctx: typer.Context, source: str, target: str, + operation_id: str = typer.Option(..., "--operation-id"), + preview_token: str | None = typer.Option(None, "--preview-token"), + policy: MergePolicy = typer.Option(MergePolicy.abort_on_conflict, "--policy"), +) -> None: + with _connect(ctx) as workspace: + print_result(workspace.merge_atomic( + source, target, operation_id=operation_id, preview_token=preview_token, policy=policy.value, + )) + + +@table_app.command("create") +def create_table( + ctx: typer.Context, sql: str | None = typer.Argument(None), + store: str = typer.Option(..., "--store"), + file: Path | None = typer.Option(None, "--file"), stdin: bool = typer.Option(False, "--stdin"), +) -> None: + statement = _create_table_input(sql, file, stdin) + with _connect(ctx, store) as workspace: + database = workspace.stores[store] + database.db.execute(statement) + print_result(database.db.commit()) + + +@table_app.command("register") +def register_table(ctx: typer.Context, table: str, store: str = typer.Option(..., "--store"), + primary_key: list[str] = typer.Option(..., "--primary-key")) -> None: + with _connect(ctx, store) as workspace: + print_result(workspace.stores[store].register_table(table, primary_key)) + + +@app.command() +def query( + ctx: typer.Context, sql: str | None = typer.Argument(None), + store: str = typer.Option(..., "--store"), branch: str = typer.Option("main", "--branch"), + params: str | None = typer.Option(None, "--params"), + file: Path | None = typer.Option(None, "--file"), stdin: bool = typer.Option(False, "--stdin"), +) -> None: + statement, parameters = _sql_input(sql, file, stdin), _parameters(params) + with _connect(ctx, store) as workspace: + with _session(workspace, branch, store) as session: + print_result(session.query(statement, parameters)) + + +@app.command() +def execute( + ctx: typer.Context, sql: str | None = typer.Argument(None), + store: str = typer.Option(..., "--store"), branch: str = typer.Option("main", "--branch"), + params: str | None = typer.Option(None, "--params"), + file: Path | None = typer.Option(None, "--file"), stdin: bool = typer.Option(False, "--stdin"), +) -> None: + statement, parameters = _sql_input(sql, file, stdin), _parameters(params) + with _connect(ctx, store) as workspace: + with _session(workspace, branch, store) as session: + print_result(session.execute(statement, parameters)) + + +@qdrant_app.callback() +def select_qdrant(ctx: typer.Context, store: str | None = typer.Option(None, "--store", help="Named Qdrant store; required for operations.")) -> None: + ctx.obj = QdrantSelection(store) + + +def _qdrant_name(ctx: typer.Context) -> str: + selection = ctx.find_object(QdrantSelection) + if selection is None or not selection.store: + raise typer.BadParameter("Select a Qdrant store with --store.", param_hint="--store") + return selection.store + + +@collection_app.command("register") +def register_collection(ctx: typer.Context, name: str, + dimensions: int = typer.Option(..., "--dimensions", min=1), + distance: str = typer.Option("cosine", "--distance")) -> None: + name_in_config = _qdrant_name(ctx) + with _connect(ctx, name_in_config, qdrant=True) as workspace: + print_result(workspace.stores[name_in_config].register_collection(name, dimensions, distance=distance)) + + +@collection_app.command("list") +def list_collections(ctx: typer.Context) -> None: + name = _qdrant_name(ctx) + with _connect(ctx, name, qdrant=True) as workspace: + print_result(workspace.stores[name].list_collections()) + + +@collection_app.command("info") +def collection_info(ctx: typer.Context, name: str) -> None: + target = _qdrant_name(ctx) + with _connect(ctx, target, qdrant=True) as workspace: + print_result(workspace.stores[target].collection_info(name)) + + +@point_app.command("upsert") +def upsert( + ctx: typer.Context, collection: str, point_id: str, data: str | None = typer.Argument(None), + file: Path | None = typer.Option(None, "--file"), stdin: bool = typer.Option(False, "--stdin"), + branch: str = typer.Option("main", "--branch"), revision: str | None = typer.Option(None, "--revision"), +) -> None: + vector, payload = point_input(read_json_input(data, file, stdin)) + name = _qdrant_name(ctx) + with _connect(ctx, name, qdrant=True) as workspace: + with _session(workspace, branch, name) as session: + print_result(session.upsert(collection, point_id, vector, payload, revision=revision)) + + +@point_app.command("get") +def get(ctx: typer.Context, collection: str, point_id: str, branch: str = typer.Option("main", "--branch")) -> None: + name = _qdrant_name(ctx) + with _connect(ctx, name, qdrant=True) as workspace: + with _session(workspace, branch, name) as session: + print_result(session.get(collection, point_id)) + + +@point_app.command("list") +def list_points(ctx: typer.Context, collection: str, branch: str = typer.Option("main", "--branch")) -> None: + name = _qdrant_name(ctx) + with _connect(ctx, name, qdrant=True) as workspace: + with _session(workspace, branch, name) as session: + print_result(session.list_points(collection)) + + +@point_app.command("delete") +def delete_point(ctx: typer.Context, collection: str, point_id: str, branch: str = typer.Option("main", "--branch")) -> None: + name = _qdrant_name(ctx) + with _connect(ctx, name, qdrant=True) as workspace: + with _session(workspace, branch, name) as session: + print_result(session.delete(collection, point_id)) + + +@qdrant_app.command() +def search( + ctx: typer.Context, collection: str, vector: str | None = typer.Argument(None), + file: Path | None = typer.Option(None, "--file"), stdin: bool = typer.Option(False, "--stdin"), + branch: str = typer.Option("main", "--branch"), limit: int = typer.Option(10, "--limit"), + score_threshold: float | None = typer.Option(None, "--score-threshold"), +) -> None: + query_vector = dense_vector(read_json_input(vector, file, stdin)) + name = _qdrant_name(ctx) + with _connect(ctx, name, qdrant=True) as workspace: + with _session(workspace, branch, name) as session: + print_result(session.search(collection, query_vector, limit=limit, score_threshold=score_threshold)) diff --git a/packages/chronos-core/src/chronos_core/cli/_workspace_config.py b/packages/chronos-core/src/chronos_core/cli/_workspace_config.py new file mode 100644 index 0000000..ee99c6a --- /dev/null +++ b/packages/chronos-core/src/chronos_core/cli/_workspace_config.py @@ -0,0 +1,252 @@ +"""Workspace configuration and ownership of public store resources.""" +from __future__ import annotations + +import math +import os +import re +import sys +from collections.abc import Generator +from contextlib import ExitStack, contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import TYPE_CHECKING, Any, Literal +from urllib.parse import urlsplit + +try: + import tomllib +except ModuleNotFoundError: # Python 3.10 + import tomli as tomllib # type: ignore[no-redef] + +from chronos_core.cli._qdrant import QdrantConfig +from chronos_core.cli._relational import DatabaseConfig + +if TYPE_CHECKING: + from chronos_core.workspace import ChronosWorkspaceContext + + +class WorkspaceConfigError(ValueError): + """Invalid workspace configuration, before connecting to any store.""" + + +@dataclass(frozen=True) +class StoreConfig: + kind: Literal["sqlite", "postgresql", "duckdb", "qdrant"] + connection: DatabaseConfig | QdrantConfig + + @property + def metadata_url(self) -> str: + connection = self.connection + if isinstance(connection, QdrantConfig): + return connection.metadata_url + return connection.metadata_url or connection.database_url + + +@dataclass(frozen=True) +class WorkspaceConfig: + config_path: Path + stores: dict[str, StoreConfig] + shared_metadata_url: str | None = None + + def store(self, name: str, *, qdrant: bool = False) -> StoreConfig: + store = self.stores.get(name) + if store is None: + raise WorkspaceConfigError(f"Unknown store {name!r}.") + if (store.kind == "qdrant") != qdrant: + expected = "Qdrant" if qdrant else "relational" + raise WorkspaceConfigError(f"Store {name!r} is not a {expected} store.") + return store + + +def _table(value: Any, field: str, allowed: set[str]) -> dict[str, Any]: + if not isinstance(value, dict): + raise WorkspaceConfigError(f"{field} must be a TOML table.") + if set(value) - allowed: + raise WorkspaceConfigError(f"{field} contains unsupported fields.") + return value + + +def _text(value: Any, field: str) -> str: + if not isinstance(value, str) or not value.strip(): + raise WorkspaceConfigError(f"{field} must be a non-empty string.") + + def substitute(match: re.Match[str]) -> str: + name = match.group(1) + if name not in os.environ: + raise WorkspaceConfigError(f"{field} references unset environment variable {name!r}.") + return os.environ[name] + + result = re.sub(r"\$\{([A-Za-z_][A-Za-z0-9_]*)\}", substitute, value) + if not result.strip(): + raise WorkspaceConfigError(f"{field} must not resolve to an empty value.") + return result + + +def _database_url(value: Any, root: Path, field: str, *, kind: str | None = None) -> str: + url = _text(value, field) + try: + parsed = urlsplit(url) + except ValueError as error: + raise WorkspaceConfigError(f"{field} must be a valid database URL.") from error + scheme = "postgresql" if parsed.scheme == "postgres" else parsed.scheme + expected = {kind} if kind is not None else {"sqlite", "postgresql"} + if scheme not in expected: + raise WorkspaceConfigError(f"{field} has an unsupported database URL scheme.") + if scheme == "postgresql": + return "postgresql:" + url.split(":", 1)[1] + if parsed.netloc or parsed.query or parsed.fragment or not url.startswith(f"{scheme}:///"): + raise WorkspaceConfigError(f"{field} must be a local database file URL.") + local = url[len(f"{scheme}:///"):] + if not local or local == ":memory:": + raise WorkspaceConfigError(f"{field} requires persistent storage, not an in-memory database.") + location = Path(local).expanduser() + if not location.is_absolute(): + location = root / location + return f"{scheme}:///{location.resolve()}" + + +def _store_config(value: Any, root: Path, field: str) -> StoreConfig: + if not isinstance(value, dict): + raise WorkspaceConfigError(f"{field} must be a TOML table.") + kind = value.get("kind") + if kind in ("sqlite", "postgresql", "duckdb"): + data = _table(value, field, {"kind", "database_url", "metadata_url"}) + url = _database_url(data.get("database_url"), root, f"{field}.database_url", kind=kind) + metadata = ( + _database_url(data["metadata_url"], root, f"{field}.metadata_url") + if "metadata_url" in data else None + ) + if kind == "duckdb" and metadata is None: + raise WorkspaceConfigError(f"{field}.metadata_url is required for DuckDB.") + return StoreConfig(kind, DatabaseConfig(url, metadata)) + if kind != "qdrant": + raise WorkspaceConfigError(f"{field}.kind must be sqlite, postgresql, duckdb, or qdrant.") + data = _table(value, field, {"kind", "metadata_url", "path", "url", "api_key", "collection_prefix", "timeout"}) + metadata = _database_url(data.get("metadata_url"), root, f"{field}.metadata_url") + if ("path" in data) == ("url" in data): + raise WorkspaceConfigError(f"{field} requires exactly one of path or url.") + local_path = None + url = None + if "path" in data: + local = _text(data["path"], f"{field}.path") + if local == ":memory:": + raise WorkspaceConfigError(f"{field}.path requires persistent storage.") + local_path = Path(local).expanduser() + if not local_path.is_absolute(): + local_path = root / local_path + local_path = local_path.resolve() + if "api_key" in data or "timeout" in data: + raise WorkspaceConfigError(f"{field}: api_key and timeout require url.") + else: + url = _text(data["url"], f"{field}.url") + try: + remote = urlsplit(url) + except ValueError as error: + raise WorkspaceConfigError(f"{field}.url must be a valid HTTP or HTTPS URL.") from error + if remote.scheme not in {"http", "https"} or not remote.hostname: + raise WorkspaceConfigError(f"{field}.url must use HTTP or HTTPS.") + api_key = _text(data["api_key"], f"{field}.api_key") if "api_key" in data else None + prefix = _text(data.get("collection_prefix", "chronos_"), f"{field}.collection_prefix") + timeout = data.get("timeout") + if timeout is not None and ( + isinstance(timeout, bool) or not isinstance(timeout, (int, float)) + or not math.isfinite(timeout) or timeout < 0.001 + ): + raise WorkspaceConfigError(f"{field}.timeout must be a finite number of at least 0.001 seconds.") + return StoreConfig("qdrant", QdrantConfig(metadata, local_path, url, api_key, prefix, timeout)) + + +def load_workspace_config(path: Path | None = None) -> WorkspaceConfig: + """Read explicit path, environment path, or ./chronos.toml without opening stores.""" + selected = path if path is not None else Path(os.environ.get("CHRONOS_WORKSPACE_CONFIG", "chronos.toml")) + config_path = selected.expanduser().resolve() + try: + raw = tomllib.loads(config_path.read_text(encoding="utf-8")) + except (OSError, UnicodeError) as error: + raise WorkspaceConfigError(f"Cannot read workspace configuration: {config_path}") from error + except tomllib.TOMLDecodeError as error: + # Decoder messages can contain configuration values, including secrets. + raise WorkspaceConfigError("Invalid TOML in workspace configuration.") from error + data = _table(raw, "configuration", {"workspace", "stores"}) + workspace = _table(data.get("workspace", {}), "workspace", {"shared_metadata_url"}) + shared = ( + _database_url(workspace["shared_metadata_url"], config_path.parent, "workspace.shared_metadata_url") + if "shared_metadata_url" in workspace else None + ) + entries = data.get("stores") + if not isinstance(entries, dict) or not entries: + raise WorkspaceConfigError("Configure at least one named store in [stores.NAME].") + # These names cannot be used by WorkspaceBranchSession, even via its stores mapping. + from chronos_core.workspace.runtime import WorkspaceBranchSession + + reserved = {"branch_id", "fs", "stores", "transaction", *dir(WorkspaceBranchSession)} + stores = {} + for name, value in entries.items(): + if not name.strip() or name in reserved: + raise WorkspaceConfigError(f"Invalid or reserved workspace store name {name!r}.") + stores[name] = _store_config(value, config_path.parent, f"stores.{name}") + metadata_urls = [store.metadata_url for store in stores.values()] + if shared is not None and any(url != shared for url in metadata_urls): + raise WorkspaceConfigError("Every store must use workspace.shared_metadata_url for atomic operations.") + if shared is None and len(set(metadata_urls)) != len(metadata_urls): + raise WorkspaceConfigError("Stores sharing metadata require workspace.shared_metadata_url.") + return WorkspaceConfig(config_path, stores, shared) + + +@contextmanager +def open_workspace(config: WorkspaceConfig) -> Generator[ChronosWorkspaceContext, None, None]: + """Close every owned resource; never remove persistent data or replace an API error.""" + from chronos_core.branching import ChronosBranchContext + from chronos_core.workspace import ChronosWorkspaceContext + + errors: list[Exception] = [] + + def close(resource: Any) -> None: + try: + resource.close() + except Exception as error: + errors.append(error) + + try: + with ExitStack() as resources: + stores: dict[str, Any] = {} + contexts: dict[str, Any] = {} + # Open SQL first so Qdrant may borrow an existing row-store context, + # regardless of the order in which users declare their stores. + for name, entry in config.stores.items(): + database = entry.connection + if not isinstance(database, DatabaseConfig): + continue + if database.metadata_url is None or database.metadata_url == database.database_url: + context = ChronosBranchContext.connect(database.database_url) + contexts[database.database_url] = context + else: + context = ChronosBranchContext.connect_split(database.database_url, database.metadata_url) + resources.callback(close, context) + stores[name] = context + for name, entry in config.stores.items(): + target = entry.connection + if not isinstance(target, QdrantConfig): + continue + from chronos_core.cli._qdrant import _store_class + + factory = _store_class() + options = {"collection_prefix": target.collection_prefix, "context": contexts.get(target.metadata_url)} + if target.path is not None: + store = factory.local(target.metadata_url, path=target.path, **options) + else: + store = factory.remote( + target.metadata_url, url=target.url, api_key=target.api_key, + timeout=target.timeout, **options, + ) + resources.callback(close, store) + stores[name] = store + workspace = ChronosWorkspaceContext( + stores={name: stores[name] for name in config.stores}, + shared_metadata_url=config.shared_metadata_url, + ) + # ExitStack owns the participants directly because workspace.close() + # stops at the first close failure instead of attempting all stores. + yield workspace + finally: + if errors and sys.exc_info()[0] is None: + raise errors[0] diff --git a/packages/chronos-core/src/chronos_core/cli/main.py b/packages/chronos-core/src/chronos_core/cli/main.py index 33a715f..7c5ffec 100644 --- a/packages/chronos-core/src/chronos_core/cli/main.py +++ b/packages/chronos-core/src/chronos_core/cli/main.py @@ -5,10 +5,12 @@ from chronos_core.cli._qdrant import app as qdrant_app from chronos_core.cli._relational import DatabaseConfig, register_commands +from chronos_core.cli._workspace import app as workspace_app app = typer.Typer(no_args_is_help=True, pretty_exceptions_enable=False) register_commands(app) app.add_typer(qdrant_app, name="qdrant", help="Branch-aware vector storage and search.") +app.add_typer(workspace_app, name="workspace", help="Coordinate configured stores, or use just one.") @app.callback() diff --git a/tests/test_cli.py b/tests/test_cli.py index 2a4c3db..148c7cd 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -11,20 +11,56 @@ from decimal import Decimal from pathlib import Path from unittest.mock import MagicMock +from contextlib import contextmanager import pytest import typer from typer.testing import CliRunner from chronos_core.branching import BranchingError, ChronosBranchContext -from chronos_core.cli import _qdrant +from chronos_core.cli import _qdrant, _workspace from chronos_core.cli._relational import DatabaseConfig, connect from chronos_core.cli._terminal import print_result +from chronos_core.cli._workspace_config import ( + WorkspaceConfigError, load_workspace_config, open_workspace, +) from chronos_core.cli.main import app runner = CliRunner() +@pytest.fixture +def workspace_connection(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> MagicMock: + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("CHRONOS_WORKSPACE_CONFIG", raising=False) + (tmp_path / "chronos.toml").write_text( + '[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"\n' + '[stores.other]\nkind="sqlite"\ndatabase_url="sqlite:///other.sqlite"\n' + ) + workspace = MagicMock() + for method in ("create_branch", "create_branch_from_checkpoint", "delete_branch", "create_checkpoint", + "diff", "merge_preview", "merge_apply", "merge_atomic_preview", "merge_atomic"): + getattr(workspace, method).return_value = None + workspace.list_branches.return_value = ["main"] + workspace.stores = {"app": MagicMock(), "other": MagicMock()} + workspace.stores["app"].register_table.return_value = None + workspace.stores["app"].db.commit.return_value = None + session = MagicMock() + session.query.return_value = [{"id": 1}] + session.execute.return_value = None + workspace.checkout.return_value.stores = {"app": session, "other": MagicMock()} + + @contextmanager + def opened(config): + try: + yield workspace + finally: + workspace.close() + + monkeypatch.setattr(_workspace, "open_workspace", MagicMock(side_effect=opened)) + return workspace + + @pytest.fixture def connection(monkeypatch: pytest.MonkeyPatch) -> MagicMock: database = MagicMock() @@ -447,3 +483,386 @@ class Result: assert capsys.readouterr().out == "" with pytest.raises(TypeError, match="Unsupported JSON value"): print_result(object()) + + +class TestWorkspaceConfiguration: + def test_path_precedence_and_relative_storage(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.chdir(tmp_path) + monkeypatch.delenv("CHRONOS_WORKSPACE_CONFIG", raising=False) + default = tmp_path / "chronos.toml" + default.write_text('[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///.chronos/app.sqlite"\n') + assert load_workspace_config().stores["app"].metadata_url == f"sqlite:///{tmp_path / '.chronos/app.sqlite'}" + other = tmp_path / "project" + other.mkdir() + configured = other / "workspace.toml" + configured.write_text('[stores.analytics]\nkind="duckdb"\ndatabase_url="duckdb:///data.duckdb"\nmetadata_url="sqlite:///meta.sqlite"\n') + monkeypatch.setenv("CHRONOS_WORKSPACE_CONFIG", str(configured)) + selected = load_workspace_config() + assert selected.config_path == configured + assert selected.stores["analytics"].connection == DatabaseConfig( + f"duckdb:///{other / 'data.duckdb'}", f"sqlite:///{other / 'meta.sqlite'}", + ) + assert load_workspace_config(default).config_path == default + assert not (tmp_path / ".chronos").exists() + assert sorted(item.name for item in other.iterdir()) == ["workspace.toml"] + + def test_environment_values_and_atomic_metadata(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("WORKSPACE_DATABASE", "postgres://user:secret@localhost/test") + monkeypatch.setenv("VECTOR_KEY", "vector-secret") + config = tmp_path / "chronos.toml" + config.write_text(''' +[workspace] +shared_metadata_url = "${WORKSPACE_DATABASE}" +[stores.app] +kind = "postgresql" +database_url = "${WORKSPACE_DATABASE}" +[stores.vectors] +kind = "qdrant" +metadata_url = "${WORKSPACE_DATABASE}" +url = "http://localhost:6333" +api_key = "${VECTOR_KEY}" +timeout = 2.5 +''') + loaded = load_workspace_config(config) + assert loaded.shared_metadata_url == "postgresql://user:secret@localhost/test" + assert loaded.stores["vectors"].connection == _qdrant.QdrantConfig( + loaded.shared_metadata_url, url="http://localhost:6333", api_key="vector-secret", timeout=2.5, + ) + monkeypatch.delenv("VECTOR_KEY") + with pytest.raises(WorkspaceConfigError, match="unset environment variable 'VECTOR_KEY'") as failure: + load_workspace_config(config) + assert "secret" not in str(failure.value) + + @pytest.mark.parametrize("source", [ + '', '[stores.app]\nkind="other"', + '[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///:memory:"', + '[stores.app]\nkind="sqlite"\ndatabase_url="postgresql://user:secret@host/test"', + '[stores.app]\nkind="duckdb"\ndatabase_url="duckdb:///app.duckdb"', + '[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"\ntypo="secret"', + '[stores.fs]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"', + '[stores.vectors]\nkind="qdrant"\nmetadata_url="sqlite:///meta.sqlite"', + '[stores.vectors]\nkind="qdrant"\nmetadata_url="sqlite:///meta.sqlite"\npath=":memory:"', + '[stores.vectors]\nkind="qdrant"\nmetadata_url="sqlite:///meta.sqlite"\npath="data"\nurl="http://localhost:6333"', + '[stores.vectors]\nkind="qdrant"\nmetadata_url="sqlite:///meta.sqlite"\npath="data"\napi_key="secret"', + '[stores.vectors]\nkind="qdrant"\nmetadata_url="sqlite:///meta.sqlite"\nurl="http://localhost:6333"\ntimeout=nan', + '[stores.vectors]\nkind="qdrant"\nmetadata_url="sqlite:///meta.sqlite"\nurl="http://localhost:6333"\ntimeout=true', + '[workspace]\nshared_metadata_url="sqlite:///shared.sqlite"\n[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///other.sqlite"', + '[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///same.sqlite"\n[stores.vectors]\nkind="qdrant"\nmetadata_url="sqlite:///same.sqlite"\npath="data"', + '[stores.app]\nkind="sqlite"\ndatabase_url="secret', + ]) + def test_invalid_configuration_does_not_create_storage(self, source: str, tmp_path: Path) -> None: + config = tmp_path / "chronos.toml" + config.write_text(source) + with pytest.raises(WorkspaceConfigError) as failure: + load_workspace_config(config) + assert "secret" not in str(failure.value) + assert list(tmp_path.iterdir()) == [config] + + def test_store_selection(self, tmp_path: Path) -> None: + config = tmp_path / "chronos.toml" + config.write_text('[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"') + loaded = load_workspace_config(config) + assert loaded.store("app").kind == "sqlite" + with pytest.raises(WorkspaceConfigError, match="Unknown store"): + loaded.store("missing") + with pytest.raises(WorkspaceConfigError, match="not a Qdrant store"): + loaded.store("app", qdrant=True) + + @pytest.mark.parametrize("failure_stage", ["none", "open", "workspace", "operation", "close"]) + def test_resource_ownership(self, failure_stage: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + import chronos_core.workspace as workspace_api + + config_path = tmp_path / "chronos.toml" + # Declaration order must not affect shared-context construction or cleanup. + config_path.write_text(''' +[workspace] +shared_metadata_url="sqlite:///app.sqlite" +[stores.vectors] +kind="qdrant" +metadata_url="sqlite:///app.sqlite" +path="vectors" +[stores.app] +kind="sqlite" +database_url="sqlite:///app.sqlite" +''') + loaded = load_workspace_config(config_path) + database = MagicMock() + vectors = MagicMock() + factory = MagicMock() + factory.local.return_value = vectors + workspace_factory = MagicMock() + monkeypatch.setattr(ChronosBranchContext, "connect", MagicMock(return_value=database)) + monkeypatch.setattr(_qdrant, "_store_class", lambda: factory) + monkeypatch.setattr(workspace_api, "ChronosWorkspaceContext", workspace_factory) + events = [] + database.close.side_effect = lambda: events.append("database") + + def close_vectors() -> None: + events.append("vectors") + if failure_stage in {"operation", "close"}: + raise RuntimeError("close failed") + + vectors.close.side_effect = close_vectors + if failure_stage == "open": + factory.local.side_effect = RuntimeError("open failed") + if failure_stage == "workspace": + workspace_factory.side_effect = RuntimeError("workspace failed") + if failure_stage == "none": + with open_workspace(loaded) as workspace: + assert workspace is workspace_factory.return_value + else: + expected = "operation failed" if failure_stage == "operation" else f"{failure_stage} failed" + with pytest.raises(RuntimeError, match=expected): + with open_workspace(loaded): + if failure_stage == "operation": + raise RuntimeError("operation failed") + assert events == (["database"] if failure_stage == "open" else ["vectors", "database"]) + if failure_stage != "open": + factory.local.assert_called_once_with( + loaded.shared_metadata_url, path=tmp_path / "vectors", collection_prefix="chronos_", context=database, + ) + workspace_factory.assert_called_once_with( + stores={"vectors": vectors, "app": database}, shared_metadata_url=loaded.shared_metadata_url, + ) + workspace_factory.return_value.close.assert_not_called() + + def test_split_sql_and_remote_qdrant(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + import chronos_core.workspace as workspace_api + + config_file = tmp_path / "chronos.toml" + config_file.write_text(''' +[workspace] +shared_metadata_url="postgresql://localhost/metadata" +[stores.analytics] +kind="duckdb" +database_url="duckdb:///analytics.duckdb" +metadata_url="postgresql://localhost/metadata" +[stores.vectors] +kind="qdrant" +metadata_url="postgresql://localhost/metadata" +url="http://localhost:6333" +api_key="test-key" +collection_prefix="project_" +timeout=3 +''') + config = load_workspace_config(config_file) + split = MagicMock() + monkeypatch.setattr(ChronosBranchContext, "connect_split", split) + factory = MagicMock() + monkeypatch.setattr(_qdrant, "_store_class", lambda: factory) + workspace_factory = MagicMock() + monkeypatch.setattr(workspace_api, "ChronosWorkspaceContext", workspace_factory) + with open_workspace(config): + pass + split.assert_called_once_with(f"duckdb:///{tmp_path / 'analytics.duckdb'}", "postgresql://localhost/metadata") + factory.remote.assert_called_once_with( + "postgresql://localhost/metadata", url="http://localhost:6333", api_key="test-key", + collection_prefix="project_", timeout=3, context=None, + ) + split.return_value.close.assert_called_once_with() + factory.remote.return_value.close.assert_called_once_with() + + def test_missing_file(self, tmp_path: Path) -> None: + with pytest.raises(WorkspaceConfigError, match="Cannot read workspace configuration"): + load_workspace_config(tmp_path / "missing.toml") + + +class TestWorkspaceCommands: + @pytest.mark.parametrize(("args", "method", "expected_args", "expected_kwargs"), [ + (["branch", "list"], "list_branches", (), {}), + (["branch", "create", "trial", "--from", "parent"], "create_branch", ("trial",), {"from_branch": "parent"}), + (["branch", "create-from-checkpoint", "trial", "saved"], "create_branch_from_checkpoint", ("trial", "saved"), {}), + (["branch", "delete", "trial"], "delete_branch", ("trial",), {}), + (["checkpoint", "create", "saved", "--branch", "trial"], "create_checkpoint", ("saved",), {"branch": "trial"}), + (["diff", "main", "trial"], "diff", ("main", "trial"), {}), + (["merge", "preview", "trial", "main"], "merge_preview", ("trial", "main"), {"policy": "abort_on_conflict"}), + (["merge", "apply", "trial", "main", "--policy", "snapshot_isolation"], "merge_apply", ("trial", "main"), {"policy": "snapshot_isolation"}), + (["merge", "atomic-preview", "trial", "main"], "merge_atomic_preview", ("trial", "main"), {"policy": "abort_on_conflict"}), + (["merge", "atomic-apply", "trial", "main", "--operation-id", "merge-1", "--preview-token", "reviewed"], + "merge_atomic", ("trial", "main"), {"policy": "abort_on_conflict", "operation_id": "merge-1", "preview_token": "reviewed"}), + ]) + def test_workspace_forwarding(self, workspace_connection: MagicMock, args, method, expected_args, expected_kwargs) -> None: + result = runner.invoke(app, ["workspace", *args]) + assert result.exit_code == 0, result.output + getattr(workspace_connection, method).assert_called_once_with(*expected_args, **expected_kwargs) + workspace_connection.close.assert_called_once_with() + if method == "list_branches": + assert json.loads(result.stdout) == ["main"] + else: + assert result.stdout == "" + + def test_named_relational_commands(self, workspace_connection: MagicMock) -> None: + workspace = workspace_connection + result = runner.invoke(app, ["workspace", "table", "create", "--store", "app", "CREATE TABLE items (id INTEGER)"]) + assert result.exit_code == 0, result.output + workspace.stores["app"].db.execute.assert_called_once_with("CREATE TABLE items (id INTEGER)") + workspace.stores["app"].db.commit.assert_called_once_with() + result = runner.invoke(app, ["workspace", "table", "register", "items", "--store", "app", "--primary-key", "id", "--primary-key", "tenant"]) + assert result.exit_code == 0, result.output + workspace.stores["app"].register_table.assert_called_once_with("items", ["id", "tenant"]) + result = runner.invoke(app, ["workspace", "query", "SELECT :id", "--store", "app", "--branch", "trial", "--params", '{"id":1}']) + assert result.exit_code == 0, result.output + assert json.loads(result.stdout) == [{"id": 1}] + session = workspace.checkout.return_value.stores["app"] + session.query.assert_called_once_with("SELECT :id", {"id": 1}) + workspace.checkout.assert_called_once_with("trial") + for checked_out in workspace.checkout.return_value.stores.values(): + checked_out.close.assert_called_once_with() + result = runner.invoke(app, ["workspace", "execute", "UPDATE items SET id = :id", "--store", "app", "--params", '{"id":2}']) + assert result.exit_code == 0, result.output + session.execute.assert_called_once_with("UPDATE items SET id = :id", {"id": 2}) + + def test_sql_input_modes(self, workspace_connection: MagicMock, tmp_path: Path) -> None: + sql_file = tmp_path / "query.sql" + sql_file.write_text("SELECT :id") + session = workspace_connection.checkout.return_value.stores["app"] + for args, input_text in [(["--file", str(sql_file)], None), (["--stdin"], "SELECT :id"), ([], "SELECT :id")]: + result = runner.invoke(app, ["workspace", "query", "--store", "app", "--params", '{"id":3}', *args], input=input_text) + assert result.exit_code == 0, result.output + assert session.query.call_args.args == ("SELECT :id", {"id": 3}) + + @pytest.mark.parametrize("args", [ + ["query", "SELECT 1", "--store", "missing"], + ["query", "SELECT 1", "--store", "app", "--params", "[]"], + ["query", "SELECT 1", "--store", "app", "--stdin"], + ["query", "--store", "app", "--stdin"], + ["table", "create", "--store", "app", "DELETE FROM items"], + ["qdrant", "--store", "app", "collection", "list"], + ["qdrant", "collection", "list"], + ["merge", "atomic-apply", "trial", "main"], + ]) + def test_invalid_arguments_do_not_open_stores(self, workspace_connection: MagicMock, args) -> None: + result = runner.invoke(app, ["workspace", *args], input="") + assert result.exit_code == 2, result.output + _workspace.open_workspace.assert_not_called() + + def test_help_and_configuration_precedence(self, workspace_connection: MagicMock, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + for args in ([], ["branch"], ["query"], ["qdrant"], ["qdrant", "point"], ["qdrant", "point", "upsert"], ["merge", "atomic-apply"]): + result = runner.invoke(app, ["workspace", "--config", "missing.toml", *args, "--help"]) + assert result.exit_code == 0, result.output + _workspace.open_workspace.assert_not_called() + monkeypatch.setenv("CHRONOS_WORKSPACE_CONFIG", str(tmp_path / "missing.toml")) + result = runner.invoke(app, ["workspace", "branch", "list"]) + assert result.exit_code == 2 + _workspace.open_workspace.assert_not_called() + result = runner.invoke(app, ["workspace", "--config", str(tmp_path / "chronos.toml"), "branch", "list"]) + assert result.exit_code == 0, result.output + assert _workspace.open_workspace.call_args.args[0].config_path == tmp_path / "chronos.toml" + + def test_api_error_and_session_cleanup(self, workspace_connection: MagicMock) -> None: + workspace = workspace_connection + session = workspace.checkout.return_value.stores["app"] + session.query.side_effect = BranchingError("query failed") + session.close.side_effect = RuntimeError("close failed") + result = runner.invoke(app, ["workspace", "query", "SELECT 1", "--store", "app"]) + assert result.exit_code == 1 + assert result.stdout == "" + assert "query failed" in result.stderr + assert "close failed" not in result.stderr + session.close.assert_called_once_with() + workspace.checkout.return_value.stores["other"].close.assert_called_once_with() + workspace.close.assert_called_once_with() + + def test_api_result_field_names_are_preserved(self, workspace_connection: MagicMock) -> None: + from chronos_core.workspace import AtomicMergeResult, WorkspaceBranchToken + token = WorkspaceBranchToken("main", 1) + result = AtomicMergeResult("merge-1", "trial", "main", "committed", token, token, token, 2, 0, {"app": 1, "other": 1}) + workspace_connection.merge_atomic.return_value = result + invoked = runner.invoke(app, ["workspace", "merge", "atomic-apply", "trial", "main", "--operation-id", "merge-1"]) + assert invoked.exit_code == 0, invoked.output + assert json.loads(invoked.stdout) == { + "operation_id": "merge-1", "source": "trial", "target": "main", "status": "committed", + "source_token": {"branch_id": "main", "current_segment_id": 1}, + "old_target_token": {"branch_id": "main", "current_segment_id": 1}, + "new_target_token": {"branch_id": "main", "current_segment_id": 1}, + "selected": 2, "skipped": 0, "stores": {"app": 1, "other": 1}, + } + + +class TestWorkspaceQdrantCommands: + @pytest.fixture + def workspace(self, workspace_connection: MagicMock, tmp_path: Path) -> MagicMock: + pytest.importorskip("qdrant_client") + (tmp_path / "chronos.toml").write_text(''' +[stores.app] +kind="sqlite" +database_url="sqlite:///app.sqlite" +[stores.vectors] +kind="qdrant" +metadata_url="sqlite:///vectors.sqlite" +path="vectors" +''') + vector_store = MagicMock() + for method in ("register_collection", "list_collections", "collection_info"): + getattr(vector_store, method).return_value = None + vector_session = MagicMock() + for method in ("upsert", "get", "list_points", "delete", "search"): + getattr(vector_session, method).return_value = None + workspace_connection.stores["vectors"] = vector_store + workspace_connection.checkout.return_value.stores["vectors"] = vector_session + return workspace_connection + + @pytest.mark.parametrize(("args", "method", "positional", "keywords", "session"), [ + (["collection", "register", "docs", "--dimensions", "3", "--distance", "dot"], "register_collection", ("docs", 3), {"distance": "dot"}, False), + (["collection", "list"], "list_collections", (), {}, False), + (["collection", "info", "docs"], "collection_info", ("docs",), {}, False), + (["point", "upsert", "docs", "one", '{"vector":[1,0,0],"payload":{"value":"one"}}', "--revision", "v1"], + "upsert", ("docs", "one", [1, 0, 0], {"value": "one"}), {"revision": "v1"}, True), + (["point", "get", "docs", "one"], "get", ("docs", "one"), {}, True), + (["point", "list", "docs"], "list_points", ("docs",), {}, True), + (["point", "delete", "docs", "one"], "delete", ("docs", "one"), {}, True), + (["search", "docs", "[1,0,0]", "--limit", "3", "--score-threshold", "0.5"], "search", ("docs", [1, 0, 0]), {"limit": 3, "score_threshold": 0.5}, True), + ]) + def test_forwarding(self, workspace: MagicMock, args, method, positional, keywords, session: bool) -> None: + result = runner.invoke(app, ["workspace", "qdrant", "--store", "vectors", *args, *(["--branch", "trial"] if session else [])]) + assert result.exit_code == 0, result.output + assert result.stdout == "" + target = workspace.checkout.return_value.stores["vectors"] if session else workspace.stores["vectors"] + getattr(target, method).assert_called_once_with(*positional, **keywords) + if session: + workspace.checkout.assert_called_once_with("trial") + workspace.close.assert_called_once_with() + + @pytest.mark.parametrize("data", ['{"vector":[true]}', '{"vector":[NaN]}', '{"vector":[1],"payload":[]}']) + def test_invalid_point_does_not_connect(self, workspace: MagicMock, data: str) -> None: + result = runner.invoke(app, ["workspace", "qdrant", "--store", "vectors", "point", "upsert", "docs", "one", data]) + assert result.exit_code == 2, result.output + _workspace.open_workspace.assert_not_called() + + def test_transport_error_redacts_credentials(self, workspace: MagicMock, tmp_path: Path) -> None: + import httpx + (tmp_path / "chronos.toml").write_text(''' +[stores.app] +kind="postgresql" +database_url="postgresql://user:sql-secret@localhost/app" +[stores.vectors] +kind="qdrant" +metadata_url="sqlite:///vectors.sqlite" +url="http://localhost:6333" +api_key="vector-secret" +''') + workspace.checkout.return_value.stores["vectors"].search.side_effect = httpx.ConnectError( + "Failed using vector-secret and postgresql://user:sql-secret@localhost/app" + ) + result = runner.invoke(app, ["workspace", "qdrant", "--store", "vectors", "search", "docs", "[1,0,0]"]) + assert result.exit_code == 1 + assert result.stdout == "" + assert "secret" not in result.stderr + assert "" in result.stderr + workspace.close.assert_called_once_with() + + def test_json_file_and_stdin(self, workspace: MagicMock, tmp_path: Path) -> None: + data = tmp_path / "point.json" + data.write_text('{"vector":[1,0,0],"payload":{"source":"file"}}') + result = runner.invoke(app, ["workspace", "qdrant", "--store", "vectors", "point", "upsert", "docs", "one", "--file", str(data)]) + assert result.exit_code == 0, result.output + session = workspace.checkout.return_value.stores["vectors"] + session.upsert.assert_called_once_with("docs", "one", [1, 0, 0], {"source": "file"}, revision=None) + for args in (["--stdin"], []): + result = runner.invoke(app, ["workspace", "qdrant", "--store", "vectors", "search", "docs", *args], input="[0,1,0]") + assert result.exit_code == 0, result.output + assert session.search.call_args.args == ("docs", [0, 1, 0]) + data.write_text("[1,0,0]") + result = runner.invoke(app, ["workspace", "qdrant", "--store", "vectors", "search", "docs", "--file", str(data)]) + assert result.exit_code == 0, result.output + assert session.search.call_args.args == ("docs", [1, 0, 0]) diff --git a/tests/test_cli_integration.py b/tests/test_cli_integration.py index 2167d06..d114282 100644 --- a/tests/test_cli_integration.py +++ b/tests/test_cli_integration.py @@ -5,6 +5,7 @@ import os import subprocess import sys +import textwrap import time from dataclasses import asdict from functools import partial @@ -16,11 +17,13 @@ import psycopg import pytest +from typer.testing import CliRunner from psycopg import sql as postgres_sql from chronos_core import _native_interval from chronos_core.cli._qdrant import QdrantConfig from chronos_core.cli._relational import DatabaseConfig +from chronos_core.cli._workspace_config import load_workspace_config, open_workspace def _run_cli(arguments: list[str], *, environment: dict[str, str], @@ -324,3 +327,308 @@ def test_qdrant_cli_json_files_and_stdin(self, qdrant_target: QdrantConfig, tmp_ data.write_text("[0,1,0]", encoding="utf-8") assert _invoke_qdrant(target, "search", "docs", "--file", str(data))[0]["id"] == "two" assert _invoke_qdrant(target, "search", "docs", input_text="[1,0,0]")[0]["id"] == "one" + + +class TestWorkspaceResources: + def test_sqlite_workspace_without_qdrant_dependency(self, tmp_path: Path) -> None: + config_file = tmp_path / "chronos.toml" + config_file.write_text('[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"\n') + code = ''' + import importlib.abc + import json + import sys + from pathlib import Path + + class BlockQdrant(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == "qdrant_client" or fullname.startswith("qdrant_client."): + raise ModuleNotFoundError("simulated missing qdrant-client") + + sys.meta_path.insert(0, BlockQdrant()) + from chronos_core.cli._workspace_config import load_workspace_config, open_workspace + with open_workspace(load_workspace_config(Path(sys.argv[1]))) as workspace: + print(json.dumps(workspace.list_branches())) + ''' + result = subprocess.run( + [sys.executable, "-X", "faulthandler", "-c", textwrap.dedent(code), str(config_file)], + capture_output=True, text=True, timeout=30, + ) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout) == ["main"] + + def test_sqlite_workspace_persists_after_close(self, tmp_path: Path) -> None: + config_file = tmp_path / "chronos.toml" + config_file.write_text('[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"\n') + config = load_workspace_config(config_file) + with open_workspace(config) as workspace: + database = workspace.stores["app"] + database.db.execute("CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)") + database.db.commit() + database.register_table("items", ["id"]) + session = workspace.checkout().stores["app"] + session.execute("INSERT INTO items (id, value) VALUES (1, 'persisted')") + assert (tmp_path / "app.sqlite").is_file() + with open_workspace(config) as workspace: + assert workspace.checkout().stores["app"].query("SELECT * FROM items") == [ + {"id": 1, "value": "persisted"}, + ] + + def test_add_qdrant_to_existing_sqlite_workspace(self, tmp_path: Path) -> None: + pytest.importorskip("qdrant_client") + config_file = tmp_path / "chronos.toml" + sqlite_config = '[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"\n' + config_file.write_text(sqlite_config) + with open_workspace(load_workspace_config(config_file)) as workspace: + database = workspace.stores["app"] + database.db.execute("CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)") + database.db.commit() + database.register_table("items", ["id"]) + workspace.checkout().stores["app"].execute("INSERT INTO items (id, value) VALUES (1, 'base')") + workspace.create_branch("experiment") + workspace.checkout("experiment").stores["app"].execute("UPDATE items SET value = 'candidate' WHERE id = 1") + config_file.write_text(sqlite_config + ''' +[workspace] +shared_metadata_url="sqlite:///app.sqlite" +[stores.vectors] +kind="qdrant" +metadata_url="sqlite:///app.sqlite" +path="vectors" +''') + config = load_workspace_config(config_file) + with open_workspace(config) as workspace: + assert set(workspace.list_branches()) == {"main", "experiment"} + workspace.stores["vectors"].register_collection("documents", 2) + main = workspace.checkout("main") + experiment = workspace.checkout("experiment") + assert main.stores["app"].query("SELECT * FROM items") == [{"id": 1, "value": "base"}] + assert experiment.stores["app"].query("SELECT * FROM items") == [{"id": 1, "value": "candidate"}] + experiment.stores["vectors"].upsert("documents", "one", [1.0, 0.0], {"value": "candidate"}) + assert main.stores["vectors"].get("documents", "one") is None + result = workspace.merge_atomic( + "experiment", "main", operation_id="added-qdrant", policy="abort_on_conflict", + ) + assert result.status == "committed" + assert result.stores == {"app": 1, "vectors": 1} + with open_workspace(config) as workspace: + main = workspace.checkout("main") + assert main.stores["app"].query("SELECT * FROM items") == [{"id": 1, "value": "candidate"}] + assert main.stores["vectors"].get("documents", "one").payload == {"value": "candidate"} + + @pytest.mark.parametrize("atomic", [False, True]) + def test_sqlite_qdrant_workspace_reopens(self, tmp_path: Path, atomic: bool) -> None: + pytest.importorskip("qdrant_client") + config_file = tmp_path / "chronos.toml" + header = '[workspace]\nshared_metadata_url="sqlite:///app.sqlite"\n' if atomic else "" + metadata = "app.sqlite" if atomic else "vectors.sqlite" + config_file.write_text(header + f''' +[stores.app] +kind="sqlite" +database_url="sqlite:///app.sqlite" +[stores.vectors] +kind="qdrant" +metadata_url="sqlite:///{metadata}" +path="vectors" +''') + config = load_workspace_config(config_file) + with open_workspace(config) as workspace: + database = workspace.stores["app"] + database.db.execute("CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)") + database.db.commit() + database.register_table("items", ["id"]) + workspace.stores["vectors"].register_collection("documents", 2) + workspace.create_branch("experiment") + session = workspace.checkout("experiment") + session.stores["app"].execute("INSERT INTO items (id, value) VALUES (1, 'candidate')") + session.stores["vectors"].upsert("documents", "one", [1.0, 0.0], {"value": "candidate"}) + workspace.create_checkpoint("saved", branch="experiment") + assert (tmp_path / "app.sqlite").is_file() + assert (tmp_path / "vectors").is_dir() + with open_workspace(config) as workspace: + main = workspace.checkout("main") + assert main.stores["app"].query("SELECT * FROM items") == [] + assert main.stores["vectors"].get("documents", "one") is None + assert {name: len(diff.changes) for name, diff in workspace.diff("main", "experiment").items()} == { + "app": 1, "vectors": 1, + } + if atomic: + preview = workspace.merge_atomic_preview("experiment", "main", policy="abort_on_conflict") + result = workspace.merge_atomic( + "experiment", "main", policy="abort_on_conflict", + operation_id="workspace-persistence", preview_token=preview.preview_token, + ) + assert result.status == "committed" + assert result.stores == {"app": 1, "vectors": 1} + else: + assert set(workspace.merge_apply("experiment", "main", policy="abort_on_conflict")) == {"app", "vectors"} + with open_workspace(config) as workspace: + main = workspace.checkout("main") + assert main.stores["app"].query("SELECT * FROM items") == [{"id": 1, "value": "candidate"}] + assert main.stores["vectors"].get("documents", "one").payload == {"value": "candidate"} + workspace.delete_branch("experiment") + + +def _workspace_config_file(root: Path, database: DatabaseConfig, vectors: QdrantConfig | None = None, *, atomic: bool = False) -> Path: + lines = [] + if atomic: + lines.extend(["[workspace]", f"shared_metadata_url={json.dumps(database.metadata_url or database.database_url)}"]) + lines.extend(["[stores.app]", f"kind={json.dumps(urlsplit(database.database_url).scheme)}", + f"database_url={json.dumps(database.database_url)}"]) + if database.metadata_url: + lines.append(f"metadata_url={json.dumps(database.metadata_url)}") + if vectors is not None: + lines.extend(["[stores.vectors]", 'kind="qdrant"', f"metadata_url={json.dumps(vectors.metadata_url)}", + f"collection_prefix={json.dumps(vectors.collection_prefix)}"]) + if vectors.path is not None: + lines.append(f"path={json.dumps(str(vectors.path))}") + else: + lines.append(f"url={json.dumps(vectors.url)}") + if vectors.api_key: + lines.append(f"api_key={json.dumps(vectors.api_key)}") + config = root / "chronos.toml" + config.write_text("\n".join(lines) + "\n") + return config + + +def _invoke_workspace(config: Path, *args: str, ok: bool = True, input_text: str | None = None) -> Any: + environment = {key: value for key, value in os.environ.items() if not key.startswith("CHRONOS_")} + environment["CHRONOS_WORKSPACE_CONFIG"] = str(config) + return _run_cli(["workspace", *args], environment=environment, ok=ok, input_text=input_text) + + +class TestWorkspaceSQLWorkflows: + def test_relational_workspace(self, cli_database: DatabaseConfig, tmp_path: Path) -> None: + config = _workspace_config_file(tmp_path, cli_database) + _invoke_workspace(config, "table", "create", "CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)", "--store", "app") + _invoke_workspace(config, "table", "register", "items", "--primary-key", "id", "--store", "app") + _invoke_workspace(config, "execute", "INSERT INTO items (id, value) VALUES (:id, :value)", "--store", "app", "--params", '{"id":1,"value":"base"}') + # A second independently backed SQL store needs no special adapter in the CLI. + audit_url = f"sqlite:///{tmp_path / 'audit.sqlite'}" + with config.open("a") as stream: + stream.write(f'\n[stores.audit]\nkind="sqlite"\ndatabase_url={json.dumps(audit_url)}\n') + _invoke_workspace(config, "table", "create", "CREATE TABLE audit (id INTEGER PRIMARY KEY, value TEXT)", "--store", "audit") + _invoke_workspace(config, "table", "register", "audit", "--primary-key", "id", "--store", "audit") + _invoke_workspace(config, "branch", "create", "experiment") + _invoke_workspace(config, "execute", "UPDATE items SET value = 'candidate' WHERE id = 1", "--store", "app", "--branch", "experiment") + _invoke_workspace(config, "execute", "INSERT INTO audit (id, value) VALUES (1, 'candidate')", "--store", "audit", "--branch", "experiment") + assert _invoke_workspace(config, "query", "SELECT value FROM items", "--store", "app") == [{"value": "base"}] + assert _invoke_workspace(config, "query", "SELECT * FROM audit", "--store", "audit") == [] + checkpoint = _invoke_workspace(config, "checkpoint", "create", "saved", "--branch", "experiment") + assert set(checkpoint) == {"app", "audit"} + _invoke_workspace(config, "branch", "create-from-checkpoint", "restored", "saved") + assert _invoke_workspace(config, "query", "SELECT value FROM items", "--store", "app", "--branch", "restored") == [{"value": "candidate"}] + difference = _invoke_workspace(config, "diff", "main", "experiment") + assert {name: len(result["changes"]) for name, result in difference.items()} == {"app": 1, "audit": 1} + assert set(_invoke_workspace(config, "merge", "preview", "experiment", "main")) == {"app", "audit"} + _invoke_workspace(config, "merge", "atomic-preview", "experiment", "main", ok=False) + assert set(_invoke_workspace(config, "merge", "apply", "experiment", "main")) == {"app", "audit"} + assert _invoke_workspace(config, "query", "SELECT value FROM items", "--store", "app") == [{"value": "candidate"}] + assert _invoke_workspace(config, "query", "SELECT value FROM audit", "--store", "audit") == [{"value": "candidate"}] + _invoke_workspace(config, "branch", "delete", "restored") + _invoke_workspace(config, "branch", "delete", "experiment") + assert _invoke_workspace(config, "branch", "list") == ["main"] + + +class TestWorkspaceQdrantWorkflows: + @pytest.mark.parametrize("atomic", [False, True]) + def test_two_store_workflow(self, qdrant_target: QdrantConfig, tmp_path: Path, atomic: bool) -> None: + target = qdrant_target + database = DatabaseConfig(target.metadata_url if atomic else f"sqlite:///{tmp_path / 'app.sqlite'}") + # Begin with SQLite/PostgreSQL alone, then add Qdrant to existing data. + config = _workspace_config_file(tmp_path, database) + _invoke_workspace(config, "table", "create", "CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)", "--store", "app") + _invoke_workspace(config, "table", "register", "items", "--primary-key", "id", "--store", "app") + _invoke_workspace(config, "execute", "INSERT INTO items (id, value) VALUES (1, 'base')", "--store", "app") + config = _workspace_config_file(tmp_path, database, target, atomic=atomic) + _invoke_workspace(config, "qdrant", "--store", "vectors", "collection", "register", "documents", "--dimensions", "3") + assert _invoke_workspace(config, "qdrant", "--store", "vectors", "collection", "info", "documents")["dimensions"] == 3 + _invoke_workspace(config, "branch", "create", "experiment") + _invoke_workspace(config, "execute", "UPDATE items SET value = 'candidate' WHERE id = 1", "--store", "app", "--branch", "experiment") + _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "upsert", "documents", "one", "--stdin", "--branch", "experiment", input_text='{"vector":[1,0,0],"payload":{"value":"candidate"}}') + assert _invoke_workspace(config, "query", "--store", "app", "--stdin", input_text="SELECT value FROM items") == [{"value": "base"}] + assert _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "get", "documents", "one") is None + assert _invoke_workspace(config, "qdrant", "--store", "vectors", "search", "documents", "[1,0,0]", "--branch", "experiment")[0]["id"] == "one" + _invoke_workspace(config, "checkpoint", "create", "saved", "--branch", "experiment") + _invoke_workspace(config, "branch", "create-from-checkpoint", "restored", "saved") + assert _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "get", "documents", "one", "--branch", "restored")["payload"] == {"value": "candidate"} + assert {name: len(result["changes"]) for name, result in _invoke_workspace(config, "diff", "main", "experiment").items()} == {"app": 1, "vectors": 1} + if atomic: + preview = _invoke_workspace(config, "merge", "atomic-preview", "experiment", "main") + _invoke_workspace(config, "merge", "apply", "experiment", "main", ok=False) + result = _invoke_workspace(config, "merge", "atomic-apply", "experiment", "main", "--operation-id", "workspace-demo", "--preview-token", preview["preview_token"]) + assert result["status"] == "committed" + assert result["stores"] == {"app": 1, "vectors": 1} + else: + assert set(_invoke_workspace(config, "merge", "preview", "experiment", "main")) == {"app", "vectors"} + _invoke_workspace(config, "merge", "apply", "experiment", "main") + assert _invoke_workspace(config, "query", "SELECT value FROM items", "--store", "app") == [{"value": "candidate"}] + assert _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "get", "documents", "one")["payload"] == {"value": "candidate"} + _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "delete", "documents", "one") + assert _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "list", "documents") == [] + _invoke_workspace(config, "branch", "delete", "restored") + _invoke_workspace(config, "branch", "delete", "experiment") + assert _invoke_workspace(config, "branch", "list") == ["main"] + + +def _prepare_workspace_merge(root: Path, *, atomic: bool) -> Path: + pytest.importorskip("qdrant_client") + database = DatabaseConfig(f"sqlite:///{root / 'app.sqlite'}") + target = QdrantConfig( + metadata_url=database.database_url if atomic else f"sqlite:///{root / 'vectors.sqlite'}", + path=root / "vectors", + ) + config = _workspace_config_file(root, database, target, atomic=atomic) + _invoke_workspace(config, "table", "create", "CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)", "--store", "app") + _invoke_workspace(config, "table", "register", "items", "--primary-key", "id", "--store", "app") + _invoke_workspace(config, "execute", "INSERT INTO items (id, value) VALUES (1, 'base')", "--store", "app") + _invoke_workspace(config, "qdrant", "--store", "vectors", "collection", "register", "documents", "--dimensions", "3") + _invoke_workspace(config, "branch", "create", "experiment") + _invoke_workspace(config, "execute", "UPDATE items SET value = 'candidate' WHERE id = 1", "--store", "app", "--branch", "experiment") + _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "upsert", "documents", "one", '{"vector":[1,0,0]}', "--branch", "experiment") + return config + + +class TestWorkspaceFailureWorkflows: + def test_atomic_stale_preview_is_rejected(self, tmp_path: Path) -> None: + config = _prepare_workspace_merge(tmp_path, atomic=True) + preview = _invoke_workspace(config, "merge", "atomic-preview", "experiment", "main") + _invoke_workspace(config, "execute", "UPDATE items SET value = 'changed after preview' WHERE id = 1", "--store", "app", "--branch", "experiment") + _invoke_workspace(config, "merge", "atomic-apply", "experiment", "main", "--operation-id", "stale", "--preview-token", preview["preview_token"], ok=False) + assert _invoke_workspace(config, "query", "SELECT value FROM items", "--store", "app") == [{"value": "base"}] + assert _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "get", "documents", "one") is None + + @pytest.mark.parametrize("atomic", [False, True]) + def test_merge_failure_retains_api_guarantees(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, atomic: bool) -> None: + config = _prepare_workspace_merge(tmp_path, atomic=atomic) + from chronos_core.cli.main import app + from chronos_core.workspace.qdrant import ChronosQdrantStore + + def fail_vector_store(*args, **kwargs): + raise RuntimeError("injected vector failure") + + # Fail the second participant after SQL has applied/staged its changes. + method = "stage_branch_transaction_changes" if atomic else "merge_apply" + monkeypatch.setattr(ChronosQdrantStore, method, fail_vector_store) + command = "atomic-apply" if atomic else "apply" + arguments = ["workspace", "--config", str(config), "merge", command, "experiment", "main"] + if atomic: + arguments.extend(["--operation-id", "failed-merge"]) + result = CliRunner().invoke(app, arguments) + assert result.exit_code == 1, result.output + assert result.stdout == "" + assert "injected vector failure" in result.stderr + assert "Traceback" not in result.stderr + # Atomic mode never publishes the staged SQL change; coordinated mode + # preserves the API's partial apply instead of pretending to roll it back. + expected = "base" if atomic else "candidate" + assert _invoke_workspace(config, "query", "SELECT value FROM items", "--store", "app") == [{"value": expected}] + assert _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "get", "documents", "one") is None + assert _invoke_workspace(config, "query", "SELECT value FROM items", "--store", "app", "--branch", "experiment") == [{"value": "candidate"}] + + def test_conflict_blocks_atomic_merge(self, tmp_path: Path) -> None: + config = _prepare_workspace_merge(tmp_path, atomic=True) + _invoke_workspace(config, "execute", "UPDATE items SET value = 'main change' WHERE id = 1", "--store", "app") + preview = _invoke_workspace(config, "merge", "atomic-preview", "experiment", "main") + assert len(preview["stores"]["app"]["conflicts"]) == 1 + _invoke_workspace(config, "merge", "atomic-apply", "experiment", "main", "--operation-id", "conflicting", ok=False) + assert _invoke_workspace(config, "query", "SELECT value FROM items", "--store", "app") == [{"value": "main change"}] + assert _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "get", "documents", "one") is None diff --git a/uv.lock b/uv.lock index 16d097e..3efc8bf 100644 --- a/uv.lock +++ b/uv.lock @@ -190,6 +190,7 @@ source = { editable = "packages/chronos-core" } dependencies = [ { name = "psycopg", extra = ["binary"] }, { name = "sqlglot" }, + { name = "tomli", marker = "python_full_version < '3.11'" }, { name = "typer" }, ] @@ -227,6 +228,7 @@ requires-dist = [ { name = "qdrant-client", marker = "extra == 'qdrant'", specifier = ">=1.18,<2" }, { name = "qdrant-client", marker = "extra == 'test'", specifier = ">=1.18,<2" }, { name = "sqlglot", specifier = ">=25" }, + { name = "tomli", marker = "python_full_version < '3.11'", specifier = ">=2.0" }, { name = "tomli", marker = "python_full_version < '3.11' and extra == 'mcp'", specifier = ">=2.0" }, { name = "tomli", marker = "python_full_version < '3.11' and extra == 'test'", specifier = ">=2.0" }, { name = "typer", specifier = ">=0.25.1" }, From b7b849abaa2302cbfd4f9d19f910161d8c926415 Mon Sep 17 00:00:00 2001 From: Seth Rojas Date: Tue, 3 Nov 2026 19:43:42 -0700 Subject: [PATCH 07/11] refactor(cli): consolidate commands around workspace API --- .github/workflows/relational-package.yml | 15 +- docs/README.md | 2 +- docs/cli.md | 345 ++--------- examples/cli_workspace_demo.sh | 2 +- packages/chronos-core/README.md | 2 +- .../cli/{_workspace_config.py => _config.py} | 73 ++- .../src/chronos_core/cli/_qdrant.py | 239 ++----- .../src/chronos_core/cli/_relational.py | 183 +----- .../src/chronos_core/cli/_terminal.py | 20 +- .../src/chronos_core/cli/_workspace.py | 195 +----- .../chronos-core/src/chronos_core/cli/main.py | 26 +- tests/test_cli.py | 586 +++++------------- tests/test_cli_integration.py | 404 ++++++------ 13 files changed, 575 insertions(+), 1517 deletions(-) rename packages/chronos-core/src/chronos_core/cli/{_workspace_config.py => _config.py} (83%) diff --git a/.github/workflows/relational-package.yml b/.github/workflows/relational-package.yml index c3bf914..6e8812e 100644 --- a/.github/workflows/relational-package.yml +++ b/.github/workflows/relational-package.yml @@ -104,15 +104,20 @@ jobs: CHRONOS_CLI_TEST_POSTGRES_ADMIN_URL: postgresql://postgres:postgres@127.0.0.1:5432/postgres CHRONOS_CLI_REQUIRE_DUCKDB: ${{ matrix.duckdb }} run: | - python -I -m pytest tests/test_cli.py::TestRelationalCommands \ - tests/test_cli.py::TestTerminalBehavior \ + python -I -m pytest tests/test_cli.py::TestTerminalBehavior \ tests/test_cli.py::TestWorkspaceConfiguration \ tests/test_cli.py::TestWorkspaceCommands \ - tests/test_cli_integration.py::TestRelationalWorkflows \ tests/test_cli_integration.py::TestWorkspaceSQLWorkflows \ tests/test_cli_integration.py::TestWorkspaceResources::test_sqlite_workspace_persists_after_close \ tests/test_cli_integration.py::TestWorkspaceResources::test_sqlite_workspace_without_qdrant_dependency \ -q --timeout=60 + - name: Test workspace DuckDB and Qdrant together + if: matrix.duckdb == 'ON' + env: + CHRONOS_CLI_REQUIRE_DUCKDB: 'ON' + run: | + python -m pip install 'qdrant-client==1.20.0' + python -I -m pytest tests/test_cli_integration.py::TestWorkspaceQdrantWorkflows::test_duckdb_and_qdrant -q --timeout=60 - name: Test Qdrant CLI in local and server modes if: matrix.python == '3.12' && matrix.duckdb == 'OFF' env: @@ -123,9 +128,7 @@ jobs: docker run --detach --rm --name chronos-cli-qdrant-test \ -p 127.0.0.1:6333:6333 qdrant/qdrant:v1.20.0 trap 'docker stop chronos-cli-qdrant-test > /dev/null' EXIT - python -I -m pytest tests/test_cli.py::TestQdrantCommands \ - tests/test_cli.py::TestWorkspaceQdrantCommands \ - tests/test_cli_integration.py::TestQdrantWorkflows \ + python -I -m pytest tests/test_cli.py::TestWorkspaceQdrantCommands \ tests/test_cli_integration.py::TestWorkspaceResources \ tests/test_cli_integration.py::TestWorkspaceQdrantWorkflows \ tests/test_cli_integration.py::TestWorkspaceFailureWorkflows -q --timeout=60 diff --git a/docs/README.md b/docs/README.md index a770c25..6a10f79 100644 --- a/docs/README.md +++ b/docs/README.md @@ -13,7 +13,7 @@ page says otherwise, commands and APIs apply to Chronos 0.2.0a1. optional DuckDB, filesystem, and S3 features. - [Integration](integration.md) explains how to adopt existing tables and route application access through Chronos. -- [CLI](cli.md) wraps the relational and Qdrant APIs with JSON output for +- [CLI](cli.md) wraps one or several workspace stores with JSON output for pipes and files. - [Compatibility and limits](compatibility.md) states what is supported and where direct database access can bypass branch isolation. diff --git a/docs/cli.md b/docs/cli.md index d26a5ce..04b76e4 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -1,12 +1,12 @@ # Chronos CLI -The `chronos` command wraps the public workspace, relational, and Qdrant Python -APIs. Use a workspace for one or several named stores; direct-store commands -remain available for quick scripts. Each invocation opens its stores, calls the -API, prints its return value, and closes connections. Stored data persists. The -CLI does not remember a current branch or keep background connections open. +The `chronos` command is a one-off wrapper around `ChronosWorkspaceContext` and +its named store APIs. A workspace can contain one store or several. Each command +opens its configured stores, calls the API, prints the returned data, and closes +connections. Stored data persists; the CLI keeps no background connection or +current-branch state. -## Install and choose a database +## Install The CLI ships with `chronos-core`. Follow [installation](installation.md) for native build dependencies. From the repository root, install the workspace package: @@ -50,34 +50,36 @@ Create `.chronos/` first. Local database URLs and Qdrant paths resolve relative to the TOML file. Configuration precedence is `--config PATH`, then `CHRONOS_WORKSPACE_CONFIG`, then `chronos.toml` in the current directory. A workspace name is not required. Connection strings may reference `${ENV_VAR}`; -unset references are errors. Single-store connection environment variables are -not used for workspace commands, and `.env` files are not loaded automatically. +unset references are errors. `.env` files are not loaded automatically. ```sh -chronos workspace table create --store app \ +chronos table create --store app \ 'CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)' -chronos workspace table register --store app items --primary-key id -chronos workspace branch create experiment -chronos workspace query --store app --branch experiment 'SELECT * FROM items' +chronos table register --store app items --primary-key id +chronos branch create experiment +chronos query --store app --branch experiment 'SELECT * FROM items' ``` SQL participants support `sqlite`, `postgresql`, and `duckdb` kinds, with `database_url` and optional `metadata_url` (required for DuckDB). Qdrant uses `kind = "qdrant"`, `metadata_url`, and exactly one of `path` or `url`; remote connections also accept `api_key` and `timeout`. Select vector operations with -`chronos workspace qdrant --store NAME ...`, using the same collection, point, -and search arguments as the direct Qdrant commands below. +`chronos qdrant --store NAME ...`. Branches, checkpoints, diffs, and merges operate across the workspace. With independent metadata, `merge preview/apply` calls the coordinated API, which can retain partial changes if a later participant fails. Atomic mode requires `[workspace].shared_metadata_url` and matching metadata URLs for every store. +This CLI version supports one SQL participant in atomic mode; multiple SQL +participants can use independent metadata. Shared-metadata SQL participants +need per-table merge scopes, and the current API's diff does not honor those +scopes, so the CLI rejects that topology rather than implementing custom behavior. Use `merge atomic-preview` and `merge atomic-apply --operation-id ID`; optionally pass the returned `--preview-token` to reject a changed preview. Atomic mode does not silently fall back to coordinated merging. Filesystems are not exposed by the workspace CLI yet. -### Checked terminal demo +## Checked terminal demo After installation, run `bash examples/cli_workspace_demo.sh --help` from the repository. The default demo uses PostgreSQL plus embedded Qdrant; set @@ -101,297 +103,44 @@ in place and prints the config path for further exploration. Local files go into a fresh `.chronos/workspace-demo.*` directory; PostgreSQL data remains in the supplied database. No servers are started or stopped by the script. -## Direct-store commands +## Store operations and output -Set the database URL in your terminal. This example uses a persistent local file: +Prepare existing SQL tables with `table register --store NAME TABLE --primary-key KEY`; +repeat `--primary-key` for composite keys. `table create` runs one ordinary CREATE +TABLE statement before registration; it is not a branch-local schema change. +After registration, use the branch-session `query` and `execute` commands for +managed reads and writes. See [integration](integration.md) before adopting +application data; direct access to the original table bypasses Chronos. -```sh -mkdir -p .chronos -export CHRONOS_DATABASE_URL="sqlite:///$PWD/.chronos/demo.sqlite" -``` - -This keeps the demo in `.chronos/` under your current directory. The absolute URL -continues to select that file if you change directories. Files ending in `.sqlite` -and their SQLite journal files are ignored by this repository's Git configuration. -SQLite creates a missing database file; its parent directory must already exist. -PostgreSQL URLs can also be supplied, for example through an environment variable -populated by your secret-management tooling. No PostgreSQL server is started by -the CLI. A `.env` file is not loaded automatically. - -Override the environment variable by placing the shared flag before the command: +SQL accepts text, `--file PATH`, or `--stdin`. Piped input is read automatically +when no source is supplied. Sources are mutually exclusive; SQL is passed to the +API without splitting scripts. `--params` accepts a JSON object of named values. +Branch defaults are `main`. ```sh -uv run --package chronos-core chronos \ - --database-url "sqlite:///$PWD/.chronos/other.sqlite" branch list -``` - -Help does not require a database URL and does not open a connection. - -## Create and register a table - -For a fresh database, create an ordinary table directly through the CLI, then -register it with Chronos: - -```sh -uv run --package chronos-core chronos table create \ - 'CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)' -uv run --package chronos-core chronos table register items --primary-key id -``` - -`table create` wraps the public database adapter's `execute()` and `commit()` -methods for initial setup. It accepts one CREATE TABLE statement directly, through -`--file`, or through stdin. This creates an ordinary database table; it is not a -branch-local schema change. Run it once before registration and managed access. -The branch-session `execute` command continues to use the API's default behavior. - -If your database already contains the application table, skip `table create` and -register it directly. For composite keys, repeat `--primary-key`. Register each -table once. Registration copies existing data into Chronos-managed storage; -subsequent managed reads and writes must use branch sessions. Direct access to -the original table bypasses Chronos. See [integration](integration.md) before -adopting application data. - -## Branch, query, checkpoint, and merge - -Run this walkthrough against the newly prepared database: - -```sh -uv run --package chronos-core chronos branch list -uv run --package chronos-core chronos branch create experiment --from main - -uv run --package chronos-core chronos execute \ +chronos execute --store app --branch experiment \ 'INSERT INTO items (id, value) VALUES (:id, :value)' \ - --branch experiment --params '{"id": 1, "value": "hello"}' - -uv run --package chronos-core chronos query \ - 'SELECT * FROM items WHERE id = :id' \ - --branch experiment --params '{"id": 1}' - -uv run --package chronos-core chronos checkpoint create saved --branch experiment -uv run --package chronos-core chronos checkpoint list --branch experiment -uv run --package chronos-core chronos diff main experiment -uv run --package chronos-core chronos diff main experiment --table items -uv run --package chronos-core chronos merge preview experiment main -uv run --package chronos-core chronos merge apply experiment main -uv run --package chronos-core chronos query 'SELECT * FROM items' -uv run --package chronos-core chronos branch delete experiment + --params '{"id": 1, "value": "candidate"}' +chronos query --store app --file query.sql +printf 'SELECT * FROM items' | chronos query --store app ``` -Queries, writes, and checkpoint creation default to branch `main`. Branch creation -defaults to `--from main`. Checkpoint listing without `--branch` lists all -checkpoints. Both merge commands accept `--policy abort_on_conflict` (the default) -or `--policy snapshot_isolation`. These are API policies; snapshot isolation does -not mean force-overwriting conflicts. API validation and conflict behavior apply -unchanged. - -## SQL input - -Both `query` and `execute` accept one SQL source: - -```sh -# Direct text, including multiline text inside shell quotes. -uv run --package chronos-core chronos query 'SELECT * FROM items' - -# A UTF-8 file. -uv run --package chronos-core chronos query --file query.sql - -# Piped input, automatically detected when no other source is supplied. -printf 'SELECT * FROM items\n' | uv run --package chronos-core chronos query - -# Explicit stdin; type SQL and finish with Ctrl-D in an interactive terminal. -uv run --package chronos-core chronos query --stdin -``` - -Empty SQL and conflicting explicit sources are rejected. With no SQL source in an -interactive terminal, the CLI shows an input error rather than opening an editor. -`--params` must be a JSON object of named parameter values. SQL and parameters are -passed directly to `session.query()` or `session.execute()`; files are not split -into a script of multiple statements. Supported SQL is determined by the API. - -## Results and errors - -Non-`None` API return values are printed as JSON to stdout: - -- Queries produce an array of row objects; no rows produces `[]`. -- Writes produce the API's result fields, such as `{"rowcount": 1}`. -- Lists, checkpoints, diffs, and merges preserve the API result field names. -- Methods returning `None`, including successful branch creation/deletion and - table registration, produce no output. - -Dates and times use ISO strings, decimals use strings to preserve precision, and -bytes use `{"encoding": "base64", "data": "..."}` objects. Unsupported values -produce an error instead of silently changing their representation. No query -results are cached or saved unless you redirect stdout: - -```sh -uv run --package chronos-core chronos query 'SELECT * FROM items' > rows.json -uv run --package chronos-core chronos query 'SELECT * FROM items' | python -m json.tool -``` - -Exit code 0 means the API call and output completed successfully. Expected -operational failures are reported on stderr with exit code 1; invalid command-line -input uses exit code 2. A merge preview may successfully return conflicts as data; -merge application follows the API's rejection behavior. Errors do not emit a JSON -success result. - -The CLI uses the default interval backend with one relational data store per -invocation. DuckDB uses a separate metadata store as described below. Workspace -coordination, a persistent interactive shell, advanced connection tuning, and -custom Python merge callbacks are not exposed. - -## Qdrant - -The `qdrant` command group wraps `ChronosQdrantStore` and its branch sessions. -Install the optional Qdrant dependency; for an existing checkout environment: - -```sh -uv pip install 'qdrant-client>=1.18,<2' -mkdir -p .chronos/qdrant-demo -export CHRONOS_QDRANT_PATH="$PWD/.chronos/qdrant-demo/vectors" -export CHRONOS_METADATA_URL="sqlite:///$PWD/.chronos/qdrant-demo/metadata.sqlite" - -uv run --no-sync --package chronos-core chronos qdrant collection register documents --dimensions 3 -uv run --no-sync --package chronos-core chronos qdrant point upsert documents example \ - '{"vector":[1,0,0],"payload":{"title":"Example"}}' -uv run --no-sync --package chronos-core chronos qdrant search documents '[1,0,0]' -``` - -Choose persistent local storage with `--path` / `CHRONOS_QDRANT_PATH`, or a server -with `--url` / `CHRONOS_QDRANT_URL`, not both. Server authentication uses -`--api-key` / `CHRONOS_QDRANT_API_KEY`. Metadata requires SQLite or PostgreSQL via -`--metadata-url` / `CHRONOS_METADATA_URL`. Explicit Qdrant options go after -`qdrant` and before the subcommand; flags override their environment variables. - -Point upsert and search also accept JSON from `--file`, `--stdin`, or a pipe. -Use `--branch` to select a branch; `main` is the default. The group also exposes -branch, checkpoint, diff, and merge operations through the public store APIs. -Run `chronos qdrant --help` or subcommand help for available options. Results use -the JSON conventions above; search consumes vectors rather than generating them. - -The example stores everything under `.chronos/qdrant-demo/`. After closing clients, -remove that folder to delete both its vectors and metadata. Unset the demo -environment variables before switching stores. No storage is created by help. - -## DuckDB data with separate metadata - -DuckDB is an in-process SQL database designed for analytical queries such as -aggregations and joins. The CLI opens a local DuckDB file; no DuckDB server or -port is required. Chronos stores branch metadata separately in SQLite or -PostgreSQL, using the existing `ChronosBranchContext.connect_split()` API. - -The native extension must be built with DuckDB enabled. Installing the Python -`duckdb` package alone does not enable Chronos's native driver. On Linux x86-64, -the build downloads the pinned DuckDB library: - -```sh -uv sync --inexact --package chronos-core --reinstall-package chronos-core \ - -Ccmake.define.CHRONOS_WITH_DUCKDB=ON -``` - -On macOS or other architectures, supply `-Ccmake.define.CHRONOS_DUCKDB_ROOT=PATH` -with a directory containing `duckdb.h` and the compatible `libduckdb` library. -The repository pins DuckDB 1.5.4 for its automatic Linux build. For example, after -extracting the [official release's macOS universal library](https://github.com/duckdb/duckdb/releases/tag/v1.5.4) into -`packages/chronos-core/build/duckdb-sdk`, rebuild with: - -```sh -CMAKE_PREFIX_PATH="$(brew --prefix libpq)" \ -uv sync --inexact --package chronos-core --reinstall-package chronos-core \ - -C "cmake.define.SQLite3_INCLUDE_DIR=$(brew --prefix sqlite)/include" \ - -C "cmake.define.SQLite3_LIBRARY=$(brew --prefix sqlite)/lib/libsqlite3.dylib" \ - -Ccmake.define.CHRONOS_WITH_DUCKDB=ON \ - -C "cmake.define.CHRONOS_DUCKDB_ROOT=$PWD/packages/chronos-core/build/duckdb-sdk" -``` - -Use the same table, branch, query, checkpoint, diff, and merge commands with two -connection URLs. Both databases must persist between one-off commands: - -```sh -mkdir -p .chronos -export CHRONOS_DATABASE_URL="duckdb:///$PWD/.chronos/analytics.duckdb" -export CHRONOS_METADATA_URL="sqlite:///$PWD/.chronos/analytics-metadata.sqlite" - -uv run --no-sync --package chronos-core chronos table create \ - 'CREATE TABLE sales (id INTEGER PRIMARY KEY, amount BIGINT)' -uv run --no-sync --package chronos-core chronos table register sales --primary-key id -uv run --no-sync --package chronos-core chronos execute \ - 'INSERT INTO sales (id, amount) VALUES (1, 10), (2, 20)' -uv run --no-sync --package chronos-core chronos branch create forecast -uv run --no-sync --package chronos-core chronos execute \ - 'UPDATE sales SET amount = amount * 2' --branch forecast -uv run --no-sync --package chronos-core chronos query \ - 'SELECT SUM(amount) AS total FROM sales' --branch forecast -``` - -The forecast totals 60 while `main` still totals 30. You can substitute a -PostgreSQL URL for `CHRONOS_METADATA_URL`; no multi-store workspace is required. -The explicit `--metadata-url` flag precedes the command name and overrides its -environment variable, just like `--database-url`. Missing DuckDB metadata produces -a usage error before connecting. An unavailable native driver produces the API's -build-option error. - -Each invocation must use the same data/metadata pair to access the same branches. -Unset `CHRONOS_METADATA_URL` when returning to the default single-database SQLite -or PostgreSQL mode. This wrapper preserves the API's split-store semantics; it -does not add cross-database transaction guarantees. DuckDB's normal file locking -also applies, so use sequential invocations for this one-off CLI workflow. - -## Test PostgreSQL locally - -The same relational commands work with SQLite, PostgreSQL, and DuckDB. To run the -CLI integration tests, start one disposable PostgreSQL server using Docker: - -```sh -docker run --detach --rm --name chronos-cli-postgres-test \ - -e POSTGRES_PASSWORD=chronos_test \ - -p 127.0.0.1:55434:5432 \ - --health-cmd 'pg_isready -U postgres' \ - --health-interval 1s --health-timeout 3s --health-retries 30 \ - postgres:16-alpine - -docker inspect --format '{{.State.Health.Status}}' chronos-cli-postgres-test -``` - -Wait until the health status is `healthy`. Port `55434` on your computer forwards -to PostgreSQL's port `5432` inside the container; it is bound only to localhost. -This port must be free before starting the container. Reuse the existing -container if you already started it; do not start one per test. - -With the package and pytest installed in your environment, run from the -repository root: - -```sh -CHRONOS_CLI_TEST_POSTGRES_ADMIN_URL=postgresql://postgres:chronos_test@127.0.0.1:55434/postgres \ - uv run --no-sync --package chronos-core python -m pytest \ - tests/test_cli.py tests/test_cli_integration.py -v -``` - -The PostgreSQL account must have permission to create and drop databases. The -test session creates one uniquely named test database and reuses it, clearing its -`public` schema between cases. It drops that database on completion and never -resets the database named in the admin URL. No additional servers are started by -pytest. Without the environment variable, PostgreSQL cases are reported as skips; -an explicitly configured server that cannot be reached causes a test failure. - -A DuckDB-enabled build also runs the workflows with SQLite and PostgreSQL -metadata independently. Without its native driver those cases are skipped. Set -`CHRONOS_CLI_REQUIRE_DUCKDB=ON` to fail rather than skip if the driver is missing. - -The tests cover CLI-only setup, branch isolation, parameterized writes, SQL files -and stdin, checkpoints, diffs, successful merges, conflicts, and rollback after a -failed write. CI is configured to test the installed wheel on Python 3.10 and 3.12, with -an additional DuckDB-enabled Python 3.12 build. - -`test_cli.py` groups relational, Qdrant, and shared terminal checks, using fake -connections where appropriate. `test_cli_integration.py` groups real relational -and Qdrant workflows. Optional stores and services are skipped when unavailable; -SQL tests do not require the Qdrant dependency. - -Stop the disposable server when finished: +Qdrant requires `chronos-core[qdrant]`. Register collections, then use point and +search commands on a selected store. Dense vectors and payloads are JSON; point +upserts and searches also accept `--file` and `--stdin`. ```sh -docker stop chronos-cli-postgres-test +chronos qdrant --store vectors collection register documents --dimensions 3 +chronos qdrant --store vectors point upsert documents one \ + '{"vector": [1, 0, 0], "payload": {"title": "Example"}}' --branch experiment +chronos qdrant --store vectors search documents '[1, 0, 0]' --branch experiment ``` -Because the container uses `--rm`, stopping it removes the container and its -temporary database storage. No Docker volume is retained for this test server. +Results are the API's JSON values, including workspace store names and result +field names. `None` prints nothing. Dataclasses become objects, dates use ISO +strings, decimals become strings, and bytes become tagged base64 objects. +Expected operational errors go to stderr with exit code 1; invalid CLI arguments +use exit code 2. Help opens no connections. JSON output can be piped to `jq` or +redirected to a file. +Non-finite floats returned by the API fail clearly because JSON cannot represent +them; query execution and the API's returned values are not modified. diff --git a/examples/cli_workspace_demo.sh b/examples/cli_workspace_demo.sh index 0d9e450..33906ae 100755 --- a/examples/cli_workspace_demo.sh +++ b/examples/cli_workspace_demo.sh @@ -26,7 +26,7 @@ cd "$repo_root" mkdir -p .chronos demo_dir="$(mktemp -d "$repo_root/.chronos/workspace-demo.XXXXXX")" export CHRONOS_WORKSPACE_CONFIG="$demo_dir/chronos.toml" -cli=(uv run --no-sync --package chronos-core chronos workspace) +cli=(uv run --no-sync --package chronos-core chronos) python=(uv run --no-sync --package chronos-core python) # Python only compares JSON output; every Chronos operation uses the CLI. diff --git a/packages/chronos-core/README.md b/packages/chronos-core/README.md index 0ec4c70..668f2f7 100644 --- a/packages/chronos-core/README.md +++ b/packages/chronos-core/README.md @@ -27,7 +27,7 @@ Optional builds use `-Ccmake.define.CHRONOS_WITH_FILESYSTEM=ON`, Python extras do not change compiled capabilities. See the repository's docs/installation.md, docs/integration.md, and docs/tutorials. -The installed `chronos` command wraps the relational and Qdrant APIs; run +The installed `chronos` command wraps a workspace of named relational and Qdrant stores; run `chronos --help` and see [the CLI guide](../../docs/cli.md) for setup and usage. This is a Python library, not a PostgreSQL-wire proxy. Managed access must pass through branch sessions. Version 0.2.0a1 is an experimental release candidate. diff --git a/packages/chronos-core/src/chronos_core/cli/_workspace_config.py b/packages/chronos-core/src/chronos_core/cli/_config.py similarity index 83% rename from packages/chronos-core/src/chronos_core/cli/_workspace_config.py rename to packages/chronos-core/src/chronos_core/cli/_config.py index ee99c6a..6a0b083 100644 --- a/packages/chronos-core/src/chronos_core/cli/_workspace_config.py +++ b/packages/chronos-core/src/chronos_core/cli/_config.py @@ -10,20 +10,33 @@ from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Any, Literal -from urllib.parse import urlsplit +from urllib.parse import unquote, urlsplit try: import tomllib except ModuleNotFoundError: # Python 3.10 import tomli as tomllib # type: ignore[no-redef] -from chronos_core.cli._qdrant import QdrantConfig -from chronos_core.cli._relational import DatabaseConfig - if TYPE_CHECKING: from chronos_core.workspace import ChronosWorkspaceContext +@dataclass(frozen=True) +class DatabaseConfig: + database_url: str + metadata_url: str | None = None + + +@dataclass(frozen=True) +class QdrantConfig: + metadata_url: str + path: Path | None = None + url: str | None = None + api_key: str | None = None + collection_prefix: str = "chronos_" + timeout: float | None = None + + class WorkspaceConfigError(ValueError): """Invalid workspace configuration, before connecting to any store.""" @@ -81,6 +94,22 @@ def substitute(match: re.Match[str]) -> str: return result +def _local_path(value: str | Path, root: Path, field: str) -> Path: + try: + location = Path(value).expanduser() + return (location if location.is_absolute() else root / location).resolve() + except (OSError, RuntimeError, ValueError) as error: + raise WorkspaceConfigError(f"{field} must be a valid local path.") from error + + +def _metadata_identity(url: str, root: Path) -> str: + if url.startswith("sqlite:///"): + # Match the adapter's decoding for comparisons without rewriting the + # URL passed to the API (literal percent signs may themselves be encoded). + return f"sqlite:///{_local_path(unquote(url[len('sqlite:///'):]), root, 'metadata_url')}" + return url + + def _database_url(value: Any, root: Path, field: str, *, kind: str | None = None) -> str: url = _text(value, field) try: @@ -98,10 +127,7 @@ def _database_url(value: Any, root: Path, field: str, *, kind: str | None = None local = url[len(f"{scheme}:///"):] if not local or local == ":memory:": raise WorkspaceConfigError(f"{field} requires persistent storage, not an in-memory database.") - location = Path(local).expanduser() - if not location.is_absolute(): - location = root / location - return f"{scheme}:///{location.resolve()}" + return f"{scheme}:///{_local_path(local, root, field)}" def _store_config(value: Any, root: Path, field: str) -> StoreConfig: @@ -130,10 +156,7 @@ def _store_config(value: Any, root: Path, field: str) -> StoreConfig: local = _text(data["path"], f"{field}.path") if local == ":memory:": raise WorkspaceConfigError(f"{field}.path requires persistent storage.") - local_path = Path(local).expanduser() - if not local_path.is_absolute(): - local_path = root / local_path - local_path = local_path.resolve() + local_path = _local_path(local, root, f"{field}.path") if "api_key" in data or "timeout" in data: raise WorkspaceConfigError(f"{field}: api_key and timeout require url.") else: @@ -147,10 +170,14 @@ def _store_config(value: Any, root: Path, field: str) -> StoreConfig: api_key = _text(data["api_key"], f"{field}.api_key") if "api_key" in data else None prefix = _text(data.get("collection_prefix", "chronos_"), f"{field}.collection_prefix") timeout = data.get("timeout") - if timeout is not None and ( - isinstance(timeout, bool) or not isinstance(timeout, (int, float)) - or not math.isfinite(timeout) or timeout < 0.001 - ): + try: + valid_timeout = timeout is None or ( + not isinstance(timeout, bool) and isinstance(timeout, (int, float)) + and math.isfinite(timeout) and timeout >= 0.001 + ) + except OverflowError: + valid_timeout = False + if not valid_timeout: raise WorkspaceConfigError(f"{field}.timeout must be a finite number of at least 0.001 seconds.") return StoreConfig("qdrant", QdrantConfig(metadata, local_path, url, api_key, prefix, timeout)) @@ -158,7 +185,7 @@ def _store_config(value: Any, root: Path, field: str) -> StoreConfig: def load_workspace_config(path: Path | None = None) -> WorkspaceConfig: """Read explicit path, environment path, or ./chronos.toml without opening stores.""" selected = path if path is not None else Path(os.environ.get("CHRONOS_WORKSPACE_CONFIG", "chronos.toml")) - config_path = selected.expanduser().resolve() + config_path = _local_path(selected, Path.cwd(), "--config") try: raw = tomllib.loads(config_path.read_text(encoding="utf-8")) except (OSError, UnicodeError) as error: @@ -187,7 +214,13 @@ def load_workspace_config(path: Path | None = None) -> WorkspaceConfig: metadata_urls = [store.metadata_url for store in stores.values()] if shared is not None and any(url != shared for url in metadata_urls): raise WorkspaceConfigError("Every store must use workspace.shared_metadata_url for atomic operations.") - if shared is None and len(set(metadata_urls)) != len(metadata_urls): + if shared is not None and sum(store.kind != "qdrant" for store in stores.values()) > 1: + raise WorkspaceConfigError( + "This CLI does not yet support multiple SQL participants sharing metadata. " + "Use independent metadata, or one SQL participant with Qdrant." + ) + identities = [_metadata_identity(url, config_path.parent) for url in metadata_urls] + if shared is None and len(set(identities)) != len(identities): raise WorkspaceConfigError("Stores sharing metadata require workspace.shared_metadata_url.") return WorkspaceConfig(config_path, stores, shared) @@ -227,9 +260,9 @@ def close(resource: Any) -> None: target = entry.connection if not isinstance(target, QdrantConfig): continue - from chronos_core.cli._qdrant import _store_class + from chronos_core.workspace.qdrant import ChronosQdrantStore - factory = _store_class() + factory = ChronosQdrantStore options = {"collection_prefix": target.collection_prefix, "context": contexts.get(target.metadata_url)} if target.path is not None: store = factory.local(target.metadata_url, path=target.path, **options) diff --git a/packages/chronos-core/src/chronos_core/cli/_qdrant.py b/packages/chronos-core/src/chronos_core/cli/_qdrant.py index 55819e2..0705e66 100644 --- a/packages/chronos-core/src/chronos_core/cli/_qdrant.py +++ b/packages/chronos-core/src/chronos_core/cli/_qdrant.py @@ -1,94 +1,28 @@ -"""Qdrant commands, JSON input, and ownership of public API resources.""" +"""JSON input and commands for named Qdrant workspace participants.""" from __future__ import annotations import json import math -import sqlite3 import sys -from collections.abc import Generator -from contextlib import contextmanager from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any +from typing import Any -import psycopg import typer -from chronos_core.cli._terminal import MergePolicy, print_result, report_error - -if TYPE_CHECKING: - from chronos_core.workspace.qdrant import ChronosQdrantStore, QdrantBranchSession - +from chronos_core.cli._terminal import print_result +from chronos_core.cli._workspace import connect_workspace, store_session @dataclass(frozen=True) -class QdrantConfig: - metadata_url: str - path: Path | None = None - url: str | None = None - api_key: str | None = None - collection_prefix: str = "chronos_" - timeout: float | None = None - - -def qdrant_config(ctx: typer.Context) -> QdrantConfig: - config = ctx.find_object(QdrantConfig) - if config is None or not config.metadata_url: - raise typer.BadParameter("Provide --metadata-url or CHRONOS_METADATA_URL.", param_hint="--metadata-url") - if (config.path is None) == (config.url is None): - raise typer.BadParameter("Choose exactly one of --path or --url.") - if config.metadata_url in {":memory:", "sqlite:///:memory:"} or str(config.path) == ":memory:": - raise typer.BadParameter("One-off commands require persistent storage, not :memory:.") - if config.path is not None and (config.api_key is not None or config.timeout is not None): - raise typer.BadParameter("--api-key and --timeout require --url.") - return config - - -def _store_class() -> type[ChronosQdrantStore]: - # Optional dependencies must not be imported when showing help or using SQL. - from chronos_core.workspace.qdrant import ChronosQdrantStore - return ChronosQdrantStore - - -@contextmanager -def open_qdrant(config: QdrantConfig) -> Generator[ChronosQdrantStore, None, None]: - try: - import grpc - import httpx - from qdrant_client.http.exceptions import ResponseHandlingException, UnexpectedResponse - from chronos_core.branching import BranchingError - except ImportError: - report_error(RuntimeError("Qdrant support requires installing chronos-core[qdrant].")) +class QdrantSelection: + store: str | None - try: - factory = _store_class() - if config.path is not None: - store = factory.local( - config.metadata_url, path=config.path, collection_prefix=config.collection_prefix, - ) - elif config.url is not None: - store = factory.remote( - config.metadata_url, url=config.url, api_key=config.api_key, - collection_prefix=config.collection_prefix, timeout=config.timeout, - ) - else: - raise ValueError("Choose exactly one of --path or --url.") - try: - yield store - finally: - store.close() - except ( - BranchingError, RuntimeError, ValueError, TypeError, OSError, sqlite3.Error, - psycopg.Error, httpx.HTTPError, grpc.RpcError, ResponseHandlingException, UnexpectedResponse, - ) as error: - report_error(error, secrets=(config.api_key,), urls=(config.metadata_url, config.url)) - - -@contextmanager -def qdrant_session(store: ChronosQdrantStore, branch: str) -> Generator[QdrantBranchSession, None, None]: - # Qdrant sessions wrap a relational control session but have no close method. - # Both of these are public APIs; keep control-session ownership explicit. - with store.context.checkout(branch) as control: - yield store.checkout_control(control) + +app = typer.Typer(no_args_is_help=True) +collection_app = typer.Typer(no_args_is_help=True) +point_app = typer.Typer(no_args_is_help=True) +app.add_typer(collection_app, name="collection", help="Register and inspect collections.") +app.add_typer(point_app, name="point", help="Write, retrieve, and delete branch-visible points.") def read_json_input(text: str | None, file: Path | None, stdin: bool) -> Any: @@ -135,152 +69,87 @@ def point_input(value: Any) -> tuple[list[float], dict[str, Any] | None]: return dense_vector(value["vector"]), payload -app = typer.Typer(no_args_is_help=True) -collection_app = typer.Typer(no_args_is_help=True) -point_app = typer.Typer(no_args_is_help=True) -branch_app = typer.Typer(no_args_is_help=True) -checkpoint_app = typer.Typer(no_args_is_help=True) -merge_app = typer.Typer(no_args_is_help=True) -app.add_typer(collection_app, name="collection", help="Register and inspect collections.") -app.add_typer(point_app, name="point", help="Write, retrieve, and delete branch-visible points.") -app.add_typer(branch_app, name="branch", help="Manage Qdrant branches.") -app.add_typer(checkpoint_app, name="checkpoint", help="Create and list checkpoints.") -app.add_typer(merge_app, name="merge", help="Preview and apply Qdrant merges.") +@app.callback() +def select_qdrant(ctx: typer.Context, store: str | None = typer.Option(None, "--store", help="Named Qdrant store; required for operations.")) -> None: + ctx.obj = QdrantSelection(store) -@app.callback() -def main( - ctx: typer.Context, - metadata_url: str | None = typer.Option(None, "--metadata-url", envvar="CHRONOS_METADATA_URL"), - path: Path | None = typer.Option(None, "--path", envvar="CHRONOS_QDRANT_PATH", file_okay=False), - url: str | None = typer.Option(None, "--url", envvar="CHRONOS_QDRANT_URL"), - api_key: str | None = typer.Option(None, "--api-key", envvar="CHRONOS_QDRANT_API_KEY", show_default=False), - collection_prefix: str = typer.Option("chronos_", "--collection-prefix", envvar="CHRONOS_QDRANT_COLLECTION_PREFIX"), - timeout: float | None = typer.Option(None, "--timeout", min=0.001, help="Remote transport timeout in seconds."), -) -> None: - """Use persistent local Qdrant (--path) or a server (--url) with Chronos metadata.""" - ctx.obj = QdrantConfig(metadata_url or "", path, url, api_key, collection_prefix, timeout) +def _qdrant_name(ctx: typer.Context) -> str: + selection = ctx.find_object(QdrantSelection) + if selection is None or not selection.store: + raise typer.BadParameter("Select a Qdrant store with --store.", param_hint="--store") + return selection.store @collection_app.command("register") -def register_collection( - ctx: typer.Context, name: str, - dimensions: int = typer.Option(..., "--dimensions", min=1), - distance: str = typer.Option("cosine", "--distance", help="cosine, dot, euclid, or manhattan."), -) -> None: - with open_qdrant(qdrant_config(ctx)) as store: - print_result(store.register_collection(name, dimensions, distance=distance)) +def register_collection(ctx: typer.Context, name: str, + dimensions: int = typer.Option(..., "--dimensions", min=1), + distance: str = typer.Option("cosine", "--distance")) -> None: + name_in_config = _qdrant_name(ctx) + with connect_workspace(ctx, name_in_config, qdrant=True) as workspace: + print_result(workspace.stores[name_in_config].register_collection(name, dimensions, distance=distance)) @collection_app.command("list") def list_collections(ctx: typer.Context) -> None: - with open_qdrant(qdrant_config(ctx)) as store: - print_result(store.list_collections()) + name = _qdrant_name(ctx) + with connect_workspace(ctx, name, qdrant=True) as workspace: + print_result(workspace.stores[name].list_collections()) @collection_app.command("info") def collection_info(ctx: typer.Context, name: str) -> None: - with open_qdrant(qdrant_config(ctx)) as store: - print_result(store.collection_info(name)) + target = _qdrant_name(ctx) + with connect_workspace(ctx, target, qdrant=True) as workspace: + print_result(workspace.stores[target].collection_info(name)) @point_app.command("upsert") def upsert( - ctx: typer.Context, collection: str, point_id: str, - data: str | None = typer.Argument(None, help='JSON: {"vector": [...], "payload": {...}}'), - file: Path | None = typer.Option(None, "--file"), - stdin: bool = typer.Option(False, "--stdin"), - branch: str = typer.Option("main", "--branch"), - revision: str | None = typer.Option(None, "--revision"), + ctx: typer.Context, collection: str, point_id: str, data: str | None = typer.Argument(None), + file: Path | None = typer.Option(None, "--file"), stdin: bool = typer.Option(False, "--stdin"), + branch: str = typer.Option("main", "--branch"), revision: str | None = typer.Option(None, "--revision"), ) -> None: vector, payload = point_input(read_json_input(data, file, stdin)) - with open_qdrant(qdrant_config(ctx)) as store: - with qdrant_session(store, branch) as session: + name = _qdrant_name(ctx) + with connect_workspace(ctx, name, qdrant=True) as workspace: + with store_session(workspace, branch, name) as session: print_result(session.upsert(collection, point_id, vector, payload, revision=revision)) @point_app.command("get") def get(ctx: typer.Context, collection: str, point_id: str, branch: str = typer.Option("main", "--branch")) -> None: - with open_qdrant(qdrant_config(ctx)) as store: - with qdrant_session(store, branch) as session: + name = _qdrant_name(ctx) + with connect_workspace(ctx, name, qdrant=True) as workspace: + with store_session(workspace, branch, name) as session: print_result(session.get(collection, point_id)) @point_app.command("list") def list_points(ctx: typer.Context, collection: str, branch: str = typer.Option("main", "--branch")) -> None: - with open_qdrant(qdrant_config(ctx)) as store: - with qdrant_session(store, branch) as session: + name = _qdrant_name(ctx) + with connect_workspace(ctx, name, qdrant=True) as workspace: + with store_session(workspace, branch, name) as session: print_result(session.list_points(collection)) @point_app.command("delete") -def delete(ctx: typer.Context, collection: str, point_id: str, branch: str = typer.Option("main", "--branch")) -> None: - with open_qdrant(qdrant_config(ctx)) as store: - with qdrant_session(store, branch) as session: +def delete_point(ctx: typer.Context, collection: str, point_id: str, branch: str = typer.Option("main", "--branch")) -> None: + name = _qdrant_name(ctx) + with connect_workspace(ctx, name, qdrant=True) as workspace: + with store_session(workspace, branch, name) as session: print_result(session.delete(collection, point_id)) @app.command() def search( - ctx: typer.Context, collection: str, - vector: str | None = typer.Argument(None, help="Query vector as a JSON array."), - file: Path | None = typer.Option(None, "--file"), - stdin: bool = typer.Option(False, "--stdin"), - branch: str = typer.Option("main", "--branch"), - limit: int = typer.Option(10, "--limit"), + ctx: typer.Context, collection: str, vector: str | None = typer.Argument(None), + file: Path | None = typer.Option(None, "--file"), stdin: bool = typer.Option(False, "--stdin"), + branch: str = typer.Option("main", "--branch"), limit: int = typer.Option(10, "--limit"), score_threshold: float | None = typer.Option(None, "--score-threshold"), ) -> None: query_vector = dense_vector(read_json_input(vector, file, stdin)) - with open_qdrant(qdrant_config(ctx)) as store: - with qdrant_session(store, branch) as session: + name = _qdrant_name(ctx) + with connect_workspace(ctx, name, qdrant=True) as workspace: + with store_session(workspace, branch, name) as session: print_result(session.search(collection, query_vector, limit=limit, score_threshold=score_threshold)) - - -@branch_app.command("list") -def list_branches(ctx: typer.Context) -> None: - with open_qdrant(qdrant_config(ctx)) as store: - print_result(store.context.list_branches()) - - -@branch_app.command("create") -def create_branch(ctx: typer.Context, name: str, from_branch: str = typer.Option("main", "--from")) -> None: - with open_qdrant(qdrant_config(ctx)) as store: - print_result(store.create_branch(name, from_branch=from_branch)) - - -@branch_app.command("delete") -def delete_branch(ctx: typer.Context, name: str) -> None: - with open_qdrant(qdrant_config(ctx)) as store: - print_result(store.delete_branch(name)) - - -@checkpoint_app.command("create") -def create_checkpoint(ctx: typer.Context, name: str, branch: str = typer.Option("main", "--branch")) -> None: - with open_qdrant(qdrant_config(ctx)) as store: - print_result(store.create_checkpoint(name, branch=branch)) - - -@checkpoint_app.command("list") -def list_checkpoints(ctx: typer.Context, branch: str | None = typer.Option(None, "--branch")) -> None: - with open_qdrant(qdrant_config(ctx)) as store: - print_result(store.context.list_checkpoints(branch=branch)) - - -@app.command() -def diff(ctx: typer.Context, left: str, right: str) -> None: - with open_qdrant(qdrant_config(ctx)) as store: - print_result(store.diff(left, right)) - - -@merge_app.command("preview") -def merge_preview(ctx: typer.Context, source: str, target: str, - policy: MergePolicy = typer.Option(MergePolicy.abort_on_conflict, "--policy")) -> None: - with open_qdrant(qdrant_config(ctx)) as store: - print_result(store.merge_preview(source, target, policy=policy.value)) - - -@merge_app.command("apply") -def merge_apply(ctx: typer.Context, source: str, target: str, - policy: MergePolicy = typer.Option(MergePolicy.abort_on_conflict, "--policy")) -> None: - with open_qdrant(qdrant_config(ctx)) as store: - print_result(store.merge_apply(source, target, policy=policy.value)) diff --git a/packages/chronos-core/src/chronos_core/cli/_relational.py b/packages/chronos-core/src/chronos_core/cli/_relational.py index e256524..1baf6df 100644 --- a/packages/chronos-core/src/chronos_core/cli/_relational.py +++ b/packages/chronos-core/src/chronos_core/cli/_relational.py @@ -1,63 +1,20 @@ -"""Relational commands, SQL input, and connection ownership.""" +"""SQL input and commands for named relational workspace participants.""" from __future__ import annotations import json -import sqlite3 import sys -from collections.abc import Generator -from contextlib import contextmanager -from dataclasses import dataclass from pathlib import Path -from typing import TYPE_CHECKING, Any -from urllib.parse import urlsplit +from typing import Any -import psycopg import sqlglot import typer from sqlglot import exp -from chronos_core.cli._terminal import MergePolicy, print_result, report_error +from chronos_core.cli._terminal import print_result +from chronos_core.cli._workspace import connect_workspace, store_session -if TYPE_CHECKING: - from chronos_core.branching import ChronosBranchContext - -@dataclass(frozen=True) -class DatabaseConfig: - database_url: str - metadata_url: str | None = None - - -def database_config(ctx: typer.Context) -> DatabaseConfig: - database = ctx.find_object(DatabaseConfig) - if database is None or not database.database_url: - raise typer.BadParameter( - "Provide --database-url or set CHRONOS_DATABASE_URL.", - param_hint="--database-url", - ) - if urlsplit(database.database_url).scheme == "duckdb" and not database.metadata_url: - raise typer.BadParameter( - "DuckDB requires --metadata-url or CHRONOS_METADATA_URL for SQLite or PostgreSQL metadata.", - param_hint="--metadata-url", - ) - return database - - -@contextmanager -def connect(database: DatabaseConfig) -> Generator[ChronosBranchContext, None, None]: - from chronos_core.branching import BranchingError, ChronosBranchContext - - try: - if database.metadata_url is None: - context = ChronosBranchContext.connect(database.database_url) - else: - context = ChronosBranchContext.connect_split(database.database_url, database.metadata_url) - try: - yield context - finally: - context.close() - except (BranchingError, RuntimeError, sqlite3.Error, psycopg.Error, OSError, ValueError, TypeError) as error: - report_error(error, urls=(database.database_url, database.metadata_url)) +table_app = typer.Typer(no_args_is_help=True) def _sql_input(sql: str | None, file: Path | None, stdin: bool) -> str: @@ -106,135 +63,51 @@ def _create_table_input(sql: str | None, file: Path | None, stdin: bool) -> str: return statement -branch_app = typer.Typer(no_args_is_help=True) -table_app = typer.Typer(no_args_is_help=True) -checkpoint_app = typer.Typer(no_args_is_help=True) -merge_app = typer.Typer(no_args_is_help=True) - - -@branch_app.command("list") -def list_branches(ctx: typer.Context) -> None: - with connect(database_config(ctx)) as database: - print_result(database.list_branches()) - - -@branch_app.command("create") -def create_branch( - ctx: typer.Context, name: str, - from_branch: str = typer.Option("main", "--from"), -) -> None: - with connect(database_config(ctx)) as database: - print_result(database.create_branch(name, from_branch=from_branch)) - - -@branch_app.command("delete") -def delete_branch(ctx: typer.Context, name: str) -> None: - with connect(database_config(ctx)) as database: - print_result(database.delete_branch(name)) - - -@table_app.command("register") -def register_table( - ctx: typer.Context, table: str, - primary_key: list[str] = typer.Option(..., "--primary-key"), -) -> None: - with connect(database_config(ctx)) as database: - print_result(database.register_table(table, primary_key)) - - @table_app.command("create") def create_table( - ctx: typer.Context, - sql: str | None = typer.Argument(None), - file: Path | None = typer.Option(None, "--file"), - stdin: bool = typer.Option(False, "--stdin"), + ctx: typer.Context, sql: str | None = typer.Argument(None), + store: str = typer.Option(..., "--store"), + file: Path | None = typer.Option(None, "--file"), stdin: bool = typer.Option(False, "--stdin"), ) -> None: - """Create an ordinary table for subsequent table register; not a branch-local write.""" statement = _create_table_input(sql, file, stdin) - with connect(database_config(ctx)) as database: + with connect_workspace(ctx, store) as workspace: + database = workspace.stores[store] database.db.execute(statement) print_result(database.db.commit()) +@table_app.command("register") +def register_table(ctx: typer.Context, table: str, store: str = typer.Option(..., "--store"), + primary_key: list[str] = typer.Option(..., "--primary-key")) -> None: + with connect_workspace(ctx, store) as workspace: + print_result(workspace.stores[store].register_table(table, primary_key)) + + def query( - ctx: typer.Context, - sql: str | None = typer.Argument(None), - branch: str = typer.Option("main", "--branch"), + ctx: typer.Context, sql: str | None = typer.Argument(None), + store: str = typer.Option(..., "--store"), branch: str = typer.Option("main", "--branch"), params: str | None = typer.Option(None, "--params"), - file: Path | None = typer.Option(None, "--file"), - stdin: bool = typer.Option(False, "--stdin"), + file: Path | None = typer.Option(None, "--file"), stdin: bool = typer.Option(False, "--stdin"), ) -> None: - """Call the branch session's query API with SQL and optional named parameters.""" statement, parameters = _sql_input(sql, file, stdin), _parameters(params) - with connect(database_config(ctx)) as database: - with database.checkout(branch) as session: + with connect_workspace(ctx, store) as workspace: + with store_session(workspace, branch, store) as session: print_result(session.query(statement, parameters)) def execute( - ctx: typer.Context, - sql: str | None = typer.Argument(None), - branch: str = typer.Option("main", "--branch"), + ctx: typer.Context, sql: str | None = typer.Argument(None), + store: str = typer.Option(..., "--store"), branch: str = typer.Option("main", "--branch"), params: str | None = typer.Option(None, "--params"), - file: Path | None = typer.Option(None, "--file"), - stdin: bool = typer.Option(False, "--stdin"), + file: Path | None = typer.Option(None, "--file"), stdin: bool = typer.Option(False, "--stdin"), ) -> None: - """Call the branch session's execute API; SQL is not split into a script.""" statement, parameters = _sql_input(sql, file, stdin), _parameters(params) - with connect(database_config(ctx)) as database: - with database.checkout(branch) as session: + with connect_workspace(ctx, store) as workspace: + with store_session(workspace, branch, store) as session: print_result(session.execute(statement, parameters)) -@checkpoint_app.command("create") -def create_checkpoint( - ctx: typer.Context, name: str, - branch: str = typer.Option("main", "--branch"), -) -> None: - with connect(database_config(ctx)) as database: - print_result(database.create_checkpoint(name, branch=branch)) - - -@checkpoint_app.command("list") -def list_checkpoints( - ctx: typer.Context, branch: str | None = typer.Option(None, "--branch"), -) -> None: - with connect(database_config(ctx)) as database: - print_result(database.list_checkpoints(branch=branch)) - - -def diff( - ctx: typer.Context, left: str, right: str, - table: str | None = typer.Option(None, "--table"), -) -> None: - with connect(database_config(ctx)) as database: - print_result(database.diff(left, right) if table is None else database.diff_rows(left, right, table)) - - -@merge_app.command("preview") -def merge_preview( - ctx: typer.Context, source: str, target: str, - policy: MergePolicy = typer.Option(MergePolicy.abort_on_conflict, "--policy"), -) -> None: - with connect(database_config(ctx)) as database: - print_result(database.merge_preview(source, target, policy=policy.value)) - - -@merge_app.command("apply") -def merge_apply( - ctx: typer.Context, source: str, target: str, - policy: MergePolicy = typer.Option(MergePolicy.abort_on_conflict, "--policy"), -) -> None: - with connect(database_config(ctx)) as database: - print_result(database.merge_apply(source, target, policy=policy.value)) - - def register_commands(app: typer.Typer) -> None: - """Attach relational commands to the root without introducing a SQL subcommand.""" app.command()(query) app.command()(execute) - app.command()(diff) - app.add_typer(branch_app, name="branch", help="Manage branches.") - app.add_typer(table_app, name="table", help="Create ordinary tables and register them with Chronos.") - app.add_typer(checkpoint_app, name="checkpoint", help="Create and list checkpoints.") - app.add_typer(merge_app, name="merge", help="Preview and apply merges.") + app.add_typer(table_app, name="table", help="Prepare tables in a named relational store.") diff --git a/packages/chronos-core/src/chronos_core/cli/_terminal.py b/packages/chronos-core/src/chronos_core/cli/_terminal.py index adea51c..90e9814 100644 --- a/packages/chronos-core/src/chronos_core/cli/_terminal.py +++ b/packages/chronos-core/src/chronos_core/cli/_terminal.py @@ -9,6 +9,7 @@ from decimal import Decimal from enum import Enum from typing import Any, NoReturn +from urllib.parse import parse_qsl, unquote, urlsplit import typer @@ -32,7 +33,7 @@ def _json_default(value: Any) -> Any: def print_result(result: Any) -> None: if result is not None: - typer.echo(json.dumps(result, default=_json_default, ensure_ascii=False, indent=2)) + typer.echo(json.dumps(result, default=_json_default, ensure_ascii=False, indent=2, allow_nan=False)) def report_error( @@ -40,12 +41,23 @@ def report_error( ) -> NoReturn: """Report an operational failure without exposing configured credentials.""" message = str(error) + sensitive = [secret for secret in secrets if secret] for url in urls: if url: message = message.replace(url, "") - for secret in secrets: - if secret: - message = message.replace(secret, "") + try: + parsed = urlsplit(url) + except ValueError: + continue + if parsed.password: + sensitive.extend((parsed.password, unquote(parsed.password))) + sensitive.extend(value for key, value in parse_qsl(parsed.query) if key == "password" and value) + for item in parsed.query.split("&"): + key, separator, value = item.partition("=") + if separator and unquote(key) == "password" and value: + sensitive.append(value) + for secret in sorted(set(sensitive), key=len, reverse=True): + message = message.replace(secret, "") message = re.sub(r"([a-zA-Z][\w+.-]*://)[^\s/@]+@", r"\1@", message) message = re.sub(r"(?i)(password\s*=\s*)(?:'[^']*'|\"[^\"]*\"|[^\s]+)", r"\1", message) typer.echo(f"Error: {message}", err=True) diff --git a/packages/chronos-core/src/chronos_core/cli/_workspace.py b/packages/chronos-core/src/chronos_core/cli/_workspace.py index 781fef2..60ece27 100644 --- a/packages/chronos-core/src/chronos_core/cli/_workspace.py +++ b/packages/chronos-core/src/chronos_core/cli/_workspace.py @@ -1,4 +1,4 @@ -"""One-off commands over the public multi-store workspace API.""" +"""Workspace lifecycle commands and invocation-owned connections and sessions.""" from __future__ import annotations import sqlite3 @@ -12,10 +12,8 @@ import psycopg import typer -from chronos_core.cli._qdrant import QdrantConfig, dense_vector, point_input, read_json_input -from chronos_core.cli._relational import _create_table_input, _parameters, _sql_input +from chronos_core.cli._config import QdrantConfig, WorkspaceConfigError, load_workspace_config, open_workspace from chronos_core.cli._terminal import MergePolicy, print_result, report_error -from chronos_core.cli._workspace_config import WorkspaceConfigError, load_workspace_config, open_workspace if TYPE_CHECKING: from chronos_core.workspace import ChronosWorkspaceContext @@ -26,42 +24,13 @@ class WorkspaceOptions: config_path: Path | None -@dataclass(frozen=True) -class QdrantSelection: - store: str | None - - -app = typer.Typer(no_args_is_help=True) branch_app = typer.Typer(no_args_is_help=True) checkpoint_app = typer.Typer(no_args_is_help=True) merge_app = typer.Typer(no_args_is_help=True) -table_app = typer.Typer(no_args_is_help=True) -qdrant_app = typer.Typer(no_args_is_help=True) -collection_app = typer.Typer(no_args_is_help=True) -point_app = typer.Typer(no_args_is_help=True) -app.add_typer(branch_app, name="branch", help="Manage branches across the workspace.") -app.add_typer(checkpoint_app, name="checkpoint", help="Checkpoint all participants.") -app.add_typer(merge_app, name="merge", help="Preview and apply coordinated or atomic merges.") -app.add_typer(table_app, name="table", help="Prepare tables in a named relational store.") -app.add_typer(qdrant_app, name="qdrant", help="Use a named Qdrant participant.") -qdrant_app.add_typer(collection_app, name="collection") -qdrant_app.add_typer(point_app, name="point") - - -@app.callback() -def main( - ctx: typer.Context, - config: Path | None = typer.Option( - None, "--config", envvar="CHRONOS_WORKSPACE_CONFIG", - help="Workspace TOML path; defaults to ./chronos.toml.", - ), -) -> None: - """Use configured stores together, including workspaces with just one store.""" - ctx.obj = WorkspaceOptions(config) @contextmanager -def _connect( +def connect_workspace( ctx: typer.Context, store: str | None = None, *, qdrant: bool = False, ) -> Generator[ChronosWorkspaceContext, None, None]: options = ctx.find_object(WorkspaceOptions) @@ -104,7 +73,7 @@ def _connect( @contextmanager -def _session(workspace: ChronosWorkspaceContext, branch: str, store: str) -> Generator[Any, None, None]: +def store_session(workspace: ChronosWorkspaceContext, branch: str, store: str) -> Generator[Any, None, None]: checked_out = workspace.checkout(branch) errors: list[Exception] = [] try: @@ -125,58 +94,57 @@ def _session(workspace: ChronosWorkspaceContext, branch: str, store: str) -> Gen @branch_app.command("list") def list_branches(ctx: typer.Context) -> None: - with _connect(ctx) as workspace: + with connect_workspace(ctx) as workspace: print_result(workspace.list_branches()) @branch_app.command("create") def create_branch(ctx: typer.Context, name: str, from_branch: str = typer.Option("main", "--from")) -> None: - with _connect(ctx) as workspace: + with connect_workspace(ctx) as workspace: print_result(workspace.create_branch(name, from_branch=from_branch)) @branch_app.command("create-from-checkpoint") def create_branch_from_checkpoint(ctx: typer.Context, name: str, checkpoint: str) -> None: - with _connect(ctx) as workspace: + with connect_workspace(ctx) as workspace: print_result(workspace.create_branch_from_checkpoint(name, checkpoint)) @branch_app.command("delete") def delete_branch(ctx: typer.Context, name: str) -> None: - with _connect(ctx) as workspace: + with connect_workspace(ctx) as workspace: print_result(workspace.delete_branch(name)) @checkpoint_app.command("create") def create_checkpoint(ctx: typer.Context, name: str, branch: str = typer.Option("main", "--branch")) -> None: - with _connect(ctx) as workspace: + with connect_workspace(ctx) as workspace: print_result(workspace.create_checkpoint(name, branch=branch)) -@app.command() def diff(ctx: typer.Context, left: str, right: str) -> None: - with _connect(ctx) as workspace: + with connect_workspace(ctx) as workspace: print_result(workspace.diff(left, right)) @merge_app.command("preview") def merge_preview(ctx: typer.Context, source: str, target: str, policy: MergePolicy = typer.Option(MergePolicy.abort_on_conflict, "--policy")) -> None: - with _connect(ctx) as workspace: + with connect_workspace(ctx) as workspace: print_result(workspace.merge_preview(source, target, policy=policy.value)) @merge_app.command("apply") def merge_apply(ctx: typer.Context, source: str, target: str, policy: MergePolicy = typer.Option(MergePolicy.abort_on_conflict, "--policy")) -> None: - with _connect(ctx) as workspace: + with connect_workspace(ctx) as workspace: print_result(workspace.merge_apply(source, target, policy=policy.value)) @merge_app.command("atomic-preview") def merge_atomic_preview(ctx: typer.Context, source: str, target: str, policy: MergePolicy = typer.Option(MergePolicy.abort_on_conflict, "--policy")) -> None: - with _connect(ctx) as workspace: + with connect_workspace(ctx) as workspace: print_result(workspace.merge_atomic_preview(source, target, policy=policy.value)) @@ -187,139 +155,14 @@ def merge_atomic( preview_token: str | None = typer.Option(None, "--preview-token"), policy: MergePolicy = typer.Option(MergePolicy.abort_on_conflict, "--policy"), ) -> None: - with _connect(ctx) as workspace: + with connect_workspace(ctx) as workspace: print_result(workspace.merge_atomic( source, target, operation_id=operation_id, preview_token=preview_token, policy=policy.value, )) -@table_app.command("create") -def create_table( - ctx: typer.Context, sql: str | None = typer.Argument(None), - store: str = typer.Option(..., "--store"), - file: Path | None = typer.Option(None, "--file"), stdin: bool = typer.Option(False, "--stdin"), -) -> None: - statement = _create_table_input(sql, file, stdin) - with _connect(ctx, store) as workspace: - database = workspace.stores[store] - database.db.execute(statement) - print_result(database.db.commit()) - - -@table_app.command("register") -def register_table(ctx: typer.Context, table: str, store: str = typer.Option(..., "--store"), - primary_key: list[str] = typer.Option(..., "--primary-key")) -> None: - with _connect(ctx, store) as workspace: - print_result(workspace.stores[store].register_table(table, primary_key)) - - -@app.command() -def query( - ctx: typer.Context, sql: str | None = typer.Argument(None), - store: str = typer.Option(..., "--store"), branch: str = typer.Option("main", "--branch"), - params: str | None = typer.Option(None, "--params"), - file: Path | None = typer.Option(None, "--file"), stdin: bool = typer.Option(False, "--stdin"), -) -> None: - statement, parameters = _sql_input(sql, file, stdin), _parameters(params) - with _connect(ctx, store) as workspace: - with _session(workspace, branch, store) as session: - print_result(session.query(statement, parameters)) - - -@app.command() -def execute( - ctx: typer.Context, sql: str | None = typer.Argument(None), - store: str = typer.Option(..., "--store"), branch: str = typer.Option("main", "--branch"), - params: str | None = typer.Option(None, "--params"), - file: Path | None = typer.Option(None, "--file"), stdin: bool = typer.Option(False, "--stdin"), -) -> None: - statement, parameters = _sql_input(sql, file, stdin), _parameters(params) - with _connect(ctx, store) as workspace: - with _session(workspace, branch, store) as session: - print_result(session.execute(statement, parameters)) - - -@qdrant_app.callback() -def select_qdrant(ctx: typer.Context, store: str | None = typer.Option(None, "--store", help="Named Qdrant store; required for operations.")) -> None: - ctx.obj = QdrantSelection(store) - - -def _qdrant_name(ctx: typer.Context) -> str: - selection = ctx.find_object(QdrantSelection) - if selection is None or not selection.store: - raise typer.BadParameter("Select a Qdrant store with --store.", param_hint="--store") - return selection.store - - -@collection_app.command("register") -def register_collection(ctx: typer.Context, name: str, - dimensions: int = typer.Option(..., "--dimensions", min=1), - distance: str = typer.Option("cosine", "--distance")) -> None: - name_in_config = _qdrant_name(ctx) - with _connect(ctx, name_in_config, qdrant=True) as workspace: - print_result(workspace.stores[name_in_config].register_collection(name, dimensions, distance=distance)) - - -@collection_app.command("list") -def list_collections(ctx: typer.Context) -> None: - name = _qdrant_name(ctx) - with _connect(ctx, name, qdrant=True) as workspace: - print_result(workspace.stores[name].list_collections()) - - -@collection_app.command("info") -def collection_info(ctx: typer.Context, name: str) -> None: - target = _qdrant_name(ctx) - with _connect(ctx, target, qdrant=True) as workspace: - print_result(workspace.stores[target].collection_info(name)) - - -@point_app.command("upsert") -def upsert( - ctx: typer.Context, collection: str, point_id: str, data: str | None = typer.Argument(None), - file: Path | None = typer.Option(None, "--file"), stdin: bool = typer.Option(False, "--stdin"), - branch: str = typer.Option("main", "--branch"), revision: str | None = typer.Option(None, "--revision"), -) -> None: - vector, payload = point_input(read_json_input(data, file, stdin)) - name = _qdrant_name(ctx) - with _connect(ctx, name, qdrant=True) as workspace: - with _session(workspace, branch, name) as session: - print_result(session.upsert(collection, point_id, vector, payload, revision=revision)) - - -@point_app.command("get") -def get(ctx: typer.Context, collection: str, point_id: str, branch: str = typer.Option("main", "--branch")) -> None: - name = _qdrant_name(ctx) - with _connect(ctx, name, qdrant=True) as workspace: - with _session(workspace, branch, name) as session: - print_result(session.get(collection, point_id)) - - -@point_app.command("list") -def list_points(ctx: typer.Context, collection: str, branch: str = typer.Option("main", "--branch")) -> None: - name = _qdrant_name(ctx) - with _connect(ctx, name, qdrant=True) as workspace: - with _session(workspace, branch, name) as session: - print_result(session.list_points(collection)) - - -@point_app.command("delete") -def delete_point(ctx: typer.Context, collection: str, point_id: str, branch: str = typer.Option("main", "--branch")) -> None: - name = _qdrant_name(ctx) - with _connect(ctx, name, qdrant=True) as workspace: - with _session(workspace, branch, name) as session: - print_result(session.delete(collection, point_id)) - - -@qdrant_app.command() -def search( - ctx: typer.Context, collection: str, vector: str | None = typer.Argument(None), - file: Path | None = typer.Option(None, "--file"), stdin: bool = typer.Option(False, "--stdin"), - branch: str = typer.Option("main", "--branch"), limit: int = typer.Option(10, "--limit"), - score_threshold: float | None = typer.Option(None, "--score-threshold"), -) -> None: - query_vector = dense_vector(read_json_input(vector, file, stdin)) - name = _qdrant_name(ctx) - with _connect(ctx, name, qdrant=True) as workspace: - with _session(workspace, branch, name) as session: - print_result(session.search(collection, query_vector, limit=limit, score_threshold=score_threshold)) +def register_commands(app: typer.Typer) -> None: + app.command()(diff) + app.add_typer(branch_app, name="branch", help="Manage branches across the workspace.") + app.add_typer(checkpoint_app, name="checkpoint", help="Checkpoint all participants.") + app.add_typer(merge_app, name="merge", help="Preview and apply coordinated or atomic merges.") diff --git a/packages/chronos-core/src/chronos_core/cli/main.py b/packages/chronos-core/src/chronos_core/cli/main.py index 7c5ffec..52de987 100644 --- a/packages/chronos-core/src/chronos_core/cli/main.py +++ b/packages/chronos-core/src/chronos_core/cli/main.py @@ -1,30 +1,30 @@ -"""Assemble the public Chronos CLI and its root connection options.""" +"""Assemble the workspace CLI and its root configuration option.""" from __future__ import annotations +from pathlib import Path + import typer from chronos_core.cli._qdrant import app as qdrant_app -from chronos_core.cli._relational import DatabaseConfig, register_commands -from chronos_core.cli._workspace import app as workspace_app +from chronos_core.cli._relational import register_commands as register_relational_commands +from chronos_core.cli._workspace import WorkspaceOptions, register_commands as register_workspace_commands app = typer.Typer(no_args_is_help=True, pretty_exceptions_enable=False) -register_commands(app) -app.add_typer(qdrant_app, name="qdrant", help="Branch-aware vector storage and search.") -app.add_typer(workspace_app, name="workspace", help="Coordinate configured stores, or use just one.") +register_workspace_commands(app) +register_relational_commands(app) +app.add_typer(qdrant_app, name="qdrant", help="Use a named Qdrant participant.") @app.callback() def main( ctx: typer.Context, - database_url: str | None = typer.Option( - None, "--database-url", envvar="CHRONOS_DATABASE_URL", help="Database connection URL." - ), - metadata_url: str | None = typer.Option( - None, "--metadata-url", envvar="CHRONOS_METADATA_URL", - help="Separate SQLite or PostgreSQL metadata URL; required for DuckDB.", + config: Path | None = typer.Option( + None, "--config", envvar="CHRONOS_WORKSPACE_CONFIG", + help="Workspace TOML path; defaults to ./chronos.toml.", ), ) -> None: - ctx.obj = DatabaseConfig(database_url, metadata_url) if database_url else None + """Use configured Chronos stores together, including workspaces with just one store.""" + ctx.obj = WorkspaceOptions(config) if __name__ == "__main__": diff --git a/tests/test_cli.py b/tests/test_cli.py index 148c7cd..da7cd2c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -18,11 +18,10 @@ from typer.testing import CliRunner from chronos_core.branching import BranchingError, ChronosBranchContext -from chronos_core.cli import _qdrant, _workspace -from chronos_core.cli._relational import DatabaseConfig, connect +from chronos_core.cli import _workspace from chronos_core.cli._terminal import print_result -from chronos_core.cli._workspace_config import ( - WorkspaceConfigError, load_workspace_config, open_workspace, +from chronos_core.cli._config import ( + DatabaseConfig, QdrantConfig, WorkspaceConfigError, load_workspace_config, open_workspace, ) from chronos_core.cli.main import app @@ -61,415 +60,6 @@ def opened(config): return workspace -@pytest.fixture -def connection(monkeypatch: pytest.MonkeyPatch) -> MagicMock: - database = MagicMock() - database.list_branches.return_value = [] - database.checkout.return_value.__enter__.return_value.query.return_value = [] - monkeypatch.setattr(ChronosBranchContext, "connect", MagicMock(return_value=database)) - monkeypatch.delenv("CHRONOS_DATABASE_URL", raising=False) - monkeypatch.delenv("CHRONOS_METADATA_URL", raising=False) - return database - - -@pytest.fixture -def factory(monkeypatch: pytest.MonkeyPatch) -> MagicMock: - pytest.importorskip("qdrant_client") - for name in ("CHRONOS_METADATA_URL", "CHRONOS_QDRANT_PATH", "CHRONOS_QDRANT_URL", - "CHRONOS_QDRANT_API_KEY", "CHRONOS_QDRANT_COLLECTION_PREFIX"): - monkeypatch.delenv(name, raising=False) - factory = MagicMock() - factory.remote.return_value = factory.local.return_value - monkeypatch.setattr(_qdrant, "_store_class", lambda: factory) - return factory - - -def _invoke_qdrant(*args: str, **kwargs): - return runner.invoke(app, ["qdrant", "--path", "vectors", "--metadata-url", "sqlite:///metadata.sqlite", *args], **kwargs) - - -class TestRelationalCommands: - def test_configuration_and_help(self, connection: MagicMock) -> None: - for args in (["--help"], ["branch", "--help"], ["query", "--help"]): - assert runner.invoke(app, args).exit_code == 0 - ChronosBranchContext.connect.assert_not_called() - result = runner.invoke(app, ["branch", "list"]) - assert result.exit_code == 2 - ChronosBranchContext.connect.assert_not_called() - result = runner.invoke(app, ["branch", "list"], env={"CHRONOS_DATABASE_URL": "sqlite:///env.db"}) - assert result.exit_code == 0 - ChronosBranchContext.connect.assert_called_with("sqlite:///env.db") - result = runner.invoke(app, ["--database-url", "sqlite:///flag.db", "branch", "list"], - env={"CHRONOS_DATABASE_URL": "sqlite:///env.db"}) - assert result.exit_code == 0 - ChronosBranchContext.connect.assert_called_with("sqlite:///flag.db") - assert json.loads(result.stdout) == [] - assert connection.close.call_count == 2 - - @pytest.mark.parametrize(("args", "method", "expected_args", "expected_kwargs"), [ - (["branch", "list"], "list_branches", (), {}), - (["branch", "create", "trial"], "create_branch", ("trial",), {"from_branch": "main"}), - (["branch", "create", "trial", "--from", "parent"], "create_branch", ("trial",), {"from_branch": "parent"}), - (["branch", "delete", "trial"], "delete_branch", ("trial",), {}), - (["table", "register", "items", "--primary-key", "id", "--primary-key", "tenant"], "register_table", ("items", ["id", "tenant"]), {}), - (["checkpoint", "create", "saved", "--branch", "trial"], "create_checkpoint", ("saved",), {"branch": "trial"}), - (["checkpoint", "list"], "list_checkpoints", (), {"branch": None}), - (["checkpoint", "list", "--branch", "trial"], "list_checkpoints", (), {"branch": "trial"}), - (["diff", "main", "trial"], "diff", ("main", "trial"), {}), - (["diff", "main", "trial", "--table", "items"], "diff_rows", ("main", "trial", "items"), {}), - (["merge", "preview", "trial", "main"], "merge_preview", ("trial", "main"), {"policy": "abort_on_conflict"}), - (["merge", "apply", "trial", "main", "--policy", "snapshot_isolation"], "merge_apply", ("trial", "main"), {"policy": "snapshot_isolation"}), - ]) - def test_api_forwarding(self, connection: MagicMock, args: list[str], method: str, - expected_args: tuple, expected_kwargs: dict) -> None: - getattr(connection, method).return_value = None - result = runner.invoke(app, ["--database-url", "sqlite:///test.db", *args]) - assert result.exit_code == 0, result.output - getattr(connection, method).assert_called_once_with(*expected_args, **expected_kwargs) - assert result.stdout == "" - connection.close.assert_called_once() - - @pytest.mark.parametrize("command", ["query", "execute"]) - @pytest.mark.parametrize("source", ["string", "file", "stdin", "pipe"]) - def test_sql_sources(self, connection: MagicMock, tmp_path: Path, command: str, source: str) -> None: - sql = "SELECT *\nFROM items WHERE id = :id\n" - args = ["--database-url", "sqlite:///test.db", command, "--branch", "trial", "--params", '{"id": 3}'] - input_text = None - if source == "string": - args.append(sql) - elif source == "file": - path = tmp_path / "query.sql" - path.write_text(sql, encoding="utf-8") - args.extend(["--file", str(path)]) - else: - input_text = sql - if source == "stdin": - args.append("--stdin") - session = connection.checkout.return_value.__enter__.return_value - getattr(session, command).return_value = [{"id": 3}] if command == "query" else {"rowcount": 1} - result = runner.invoke(app, args, input=input_text) - assert result.exit_code == 0, result.output - connection.checkout.assert_called_once_with("trial") - getattr(session, command).assert_called_once_with(sql, {"id": 3}) - assert json.loads(result.stdout) == getattr(session, command).return_value - connection.checkout.return_value.__exit__.assert_called_once() - connection.close.assert_called_once() - - @pytest.mark.parametrize("args", [ - ["query"], ["query", " "], ["query", "SELECT 1", "--stdin"], - ["query", "SELECT 1", "--file", "missing.sql"], - ["query", "--file", "missing.sql"], - ["query", "SELECT 1", "--params", "[]"], - ["execute", "SELECT 1", "--params", "broken"], - ["merge", "apply", "a", "b", "--policy", "unknown"], - ["table", "register", "items"], - ]) - def test_bad_inputs_do_not_connect(self, connection: MagicMock, args: list[str]) -> None: - result = runner.invoke(app, ["--database-url", "sqlite:///test.db", *args]) - assert result.exit_code == 2, result.output - ChronosBranchContext.connect.assert_not_called() - - def test_interactive_missing_sql(self, connection: MagicMock, monkeypatch: pytest.MonkeyPatch) -> None: - monkeypatch.setattr(sys.stdin, "isatty", lambda: True) - from chronos_core.cli._relational import _sql_input - import typer - with pytest.raises(typer.BadParameter, match="Provide SQL"): - _sql_input(None, None, False) - ChronosBranchContext.connect.assert_not_called() - - def test_cleanup_and_redaction(self, connection: MagicMock) -> None: - url = "postgresql://user:secret@localhost/db" - session = connection.checkout.return_value.__enter__.return_value - session.query.side_effect = BranchingError(f"Cannot use {url}; password=secret") - result = runner.invoke(app, ["--database-url", url, "query", "SELECT 1"]) - assert result.exit_code == 1 - assert "secret" not in result.output - assert "Error:" in result.stderr - connection.checkout.return_value.__exit__.assert_called_once() - connection.close.assert_called_once() - - def test_connection_failure_does_not_close_unassigned_context(self, connection: MagicMock) -> None: - ChronosBranchContext.connect.side_effect = ValueError("unsupported database") - result = runner.invoke(app, ["--database-url", "bad", "branch", "list"]) - assert result.exit_code == 1 - assert "unsupported database" in result.stderr - connection.close.assert_not_called() - - def test_installed_entrypoints(self) -> None: - executable = Path(sys.executable).parent / "chronos" - for args in ([str(executable), "--help"], [sys.executable, "-m", "chronos_core.cli", "--help"]): - env = dict(os.environ) - env.pop("CHRONOS_DATABASE_URL", None) - env.pop("CHRONOS_METADATA_URL", None) - result = subprocess.run(args, capture_output=True, text=True, env=env, timeout=30) - assert result.returncode == 0, result.stderr - assert "--database-url" in result.stdout - - def test_create_table_forwarding(self, connection: MagicMock) -> None: - connection.db.commit.return_value = None - statement = "CREATE TABLE items (id INTEGER PRIMARY KEY)" - result = runner.invoke(app, ["--database-url", "sqlite:///test.db", "table", "create", statement]) - assert result.exit_code == 0, result.output - connection.db.execute.assert_called_once_with(statement) - connection.db.commit.assert_called_once() - connection.checkout.assert_not_called() - assert result.stdout == "" - - @pytest.mark.parametrize("statement", [ - "DELETE FROM items", "CREATE INDEX idx ON items(id)", - "CREATE TABLE items (id INTEGER); DROP TABLE items", "not sql", - ]) - def test_create_table_rejects_other_sql(self, connection: MagicMock, statement: str) -> None: - result = runner.invoke(app, ["--database-url", "sqlite:///test.db", "table", "create", statement]) - assert result.exit_code == 2, result.output - ChronosBranchContext.connect.assert_not_called() - - def test_native_error_is_reported_without_traceback(self, connection: MagicMock) -> None: - session = connection.checkout.return_value.__enter__.return_value - session.execute.side_effect = RuntimeError("INSERT must specify a column list") - result = runner.invoke(app, [ - "--database-url", "sqlite:///test.db", "execute", "INSERT INTO items VALUES (1)", - ]) - assert result.exit_code == 1 - assert result.stdout == "" - assert result.stderr.strip() == "Error: INSERT must specify a column list" - connection.checkout.return_value.__exit__.assert_called_once() - connection.close.assert_called_once() - - @pytest.mark.parametrize("source", ["flag", "environment", "override"]) - def test_split_connection_configuration(self, connection: MagicMock, monkeypatch: pytest.MonkeyPatch, source: str) -> None: - split = MagicMock(return_value=connection) - monkeypatch.setattr(ChronosBranchContext, "connect_split", split) - args = ["--database-url", "duckdb:////tmp/data.duckdb"] - environment = {} - if source != "flag": - environment["CHRONOS_METADATA_URL"] = "sqlite:////tmp/env.sqlite" - if source != "environment": - args.extend(["--metadata-url", "sqlite:////tmp/flag.sqlite"]) - result = runner.invoke(app, [*args, "branch", "list"], env=environment) - assert result.exit_code == 0, result.output - expected_metadata = "sqlite:////tmp/env.sqlite" if source == "environment" else "sqlite:////tmp/flag.sqlite" - split.assert_called_once_with("duckdb:////tmp/data.duckdb", expected_metadata) - ChronosBranchContext.connect.assert_not_called() - connection.close.assert_called_once() - - def test_duckdb_requires_metadata_before_connecting(self, connection: MagicMock) -> None: - result = runner.invoke(app, ["--database-url", "duckdb:////tmp/data.duckdb", "branch", "list"]) - assert result.exit_code == 2 - assert "--metadata-url" in result.output - ChronosBranchContext.connect.assert_not_called() - - def test_split_connection_redacts_metadata_credentials(self, connection: MagicMock, monkeypatch: pytest.MonkeyPatch) -> None: - metadata = "postgresql://user:secret@localhost/metadata" - split = MagicMock(side_effect=ValueError(f"Cannot connect to {metadata}")) - monkeypatch.setattr(ChronosBranchContext, "connect_split", split) - result = runner.invoke(app, ["--database-url", "duckdb:////tmp/data.duckdb", - "--metadata-url", metadata, "branch", "list"]) - assert result.exit_code == 1 - assert "secret" not in result.output - assert "" in result.stderr - connection.close.assert_not_called() - - def test_unavailable_duckdb_driver_reports_api_error(self, connection: MagicMock, monkeypatch: pytest.MonkeyPatch) -> None: - split = MagicMock(side_effect=RuntimeError("DuckDB support requires CHRONOS_WITH_DUCKDB=ON")) - monkeypatch.setattr(ChronosBranchContext, "connect_split", split) - result = runner.invoke(app, ["--database-url", "duckdb:////tmp/data.duckdb", - "--metadata-url", "sqlite:////tmp/metadata.sqlite", "branch", "list"]) - assert result.exit_code == 1 - assert result.stdout == "" - assert "CHRONOS_WITH_DUCKDB=ON" in result.stderr - connection.close.assert_not_called() - - @pytest.mark.parametrize("failure", [False, True]) - def test_connection_cleanup(self, monkeypatch: pytest.MonkeyPatch, failure: bool) -> None: - database = MagicMock() - monkeypatch.setattr(ChronosBranchContext, "connect", MagicMock(return_value=database)) - if failure: - with pytest.raises(typer.Exit): - with connect(DatabaseConfig("sqlite:///:memory:")): - raise ValueError("operation failed") - else: - with connect(DatabaseConfig("sqlite:///:memory:")) as context: - assert context is database - database.close.assert_called_once() - - -class TestQdrantCommands: - def test_help_does_not_connect(self, factory: MagicMock) -> None: - for args in (["--help"], ["qdrant", "--help"], ["qdrant", "point", "--help"], - ["qdrant", "search", "--help"]): - assert runner.invoke(app, args).exit_code == 0 - factory.local.assert_not_called() - factory.remote.assert_not_called() - - @pytest.mark.parametrize(("args", "method", "positional", "keywords"), [ - (["collection", "register", "docs", "--dimensions", "3"], "register_collection", ("docs", 3), {"distance": "cosine"}), - (["collection", "register", "docs", "--dimensions", "3", "--distance", "dot"], "register_collection", ("docs", 3), {"distance": "dot"}), - (["collection", "list"], "list_collections", (), {}), - (["collection", "info", "docs"], "collection_info", ("docs",), {}), - (["branch", "create", "trial", "--from", "parent"], "create_branch", ("trial",), {"from_branch": "parent"}), - (["branch", "delete", "trial"], "delete_branch", ("trial",), {}), - (["checkpoint", "create", "saved", "--branch", "trial"], "create_checkpoint", ("saved",), {"branch": "trial"}), - (["diff", "main", "trial"], "diff", ("main", "trial"), {}), - (["merge", "preview", "trial", "main"], "merge_preview", ("trial", "main"), {"policy": "abort_on_conflict"}), - (["merge", "apply", "trial", "main", "--policy", "snapshot_isolation"], "merge_apply", ("trial", "main"), {"policy": "snapshot_isolation"}), - ]) - def test_store_api_forwarding(self, factory: MagicMock, args, method, positional, keywords) -> None: - store = factory.local.return_value - getattr(store, method).return_value = {"result": "from API"} - result = _invoke_qdrant(*args) - assert result.exit_code == 0, result.output - getattr(store, method).assert_called_once_with(*positional, **keywords) - assert json.loads(result.stdout) == {"result": "from API"} - store.close.assert_called_once() - - @pytest.mark.parametrize(("args", "method", "keywords"), [ - (["branch", "list"], "list_branches", {}), - (["checkpoint", "list", "--branch", "trial"], "list_checkpoints", {"branch": "trial"}), - ]) - def test_metadata_context_calls(self, factory: MagicMock, args, method, keywords) -> None: - store = factory.local.return_value - getattr(store.context, method).return_value = [] - result = _invoke_qdrant(*args) - assert result.exit_code == 0, result.output - getattr(store.context, method).assert_called_once_with(**keywords) - assert json.loads(result.stdout) == [] - store.close.assert_called_once() - - @pytest.mark.parametrize(("command", "method", "positional"), [ - (["point", "get", "docs", "one"], "get", ("docs", "one")), - (["point", "list", "docs"], "list_points", ("docs",)), - (["point", "delete", "docs", "one"], "delete", ("docs", "one")), - ]) - def test_point_session_calls(self, factory: MagicMock, command, method, positional) -> None: - store = factory.local.return_value - session = store.checkout_control.return_value - getattr(session, method).return_value = False if method == "delete" else None - result = _invoke_qdrant(*command, "--branch", "trial") - assert result.exit_code == 0, result.output - store.context.checkout.assert_called_once_with("trial") - store.checkout_control.assert_called_once_with(store.context.checkout.return_value.__enter__.return_value) - getattr(session, method).assert_called_once_with(*positional) - assert result.stdout == ("false\n" if method == "delete" else "") - store.context.checkout.return_value.__exit__.assert_called_once() - store.close.assert_called_once() - - @pytest.mark.parametrize("source", ["text", "file", "stdin", "pipe"]) - @pytest.mark.parametrize("command", ["upsert", "search"]) - def test_json_sources(self, factory: MagicMock, tmp_path: Path, source: str, command: str) -> None: - text = '{"vector": [1, 0, 0], "payload": {"title": "hello"}}' if command == "upsert" else "[1, 0, 0]" - args = ["point", "upsert", "docs", "one", "--revision", "r1"] if command == "upsert" else ["search", "docs", "--limit", "2", "--score-threshold", "0.5"] - input_text = None - if source == "text": - args.append(text) - elif source == "file": - path = tmp_path / "input.json" - path.write_text(text, encoding="utf-8") - args.extend(["--file", str(path)]) - else: - input_text = text - if source == "stdin": - args.append("--stdin") - store = factory.local.return_value - session = store.checkout_control.return_value - getattr(session, command).return_value = [] - result = _invoke_qdrant(*args, input=input_text) - assert result.exit_code == 0, result.output - if command == "upsert": - session.upsert.assert_called_once_with("docs", "one", [1, 0, 0], {"title": "hello"}, revision="r1") - else: - session.search.assert_called_once_with("docs", [1, 0, 0], limit=2, score_threshold=0.5) - assert json.loads(result.stdout) == [] - store.close.assert_called_once() - - def test_connection_flags_and_environment(self, factory: MagicMock) -> None: - store = factory.local.return_value - store.list_collections.return_value = [] - env = {"CHRONOS_METADATA_URL": "sqlite:///env.sqlite", "CHRONOS_QDRANT_URL": "http://localhost:6333", - "CHRONOS_QDRANT_API_KEY": "test-secret"} - result = runner.invoke(app, ["qdrant", "--metadata-url", "sqlite:///flag.sqlite", "--collection-prefix", "demo_", - "--timeout", "12", "collection", "list"], env=env) - assert result.exit_code == 0, result.output - factory.remote.assert_called_once_with("sqlite:///flag.sqlite", url="http://localhost:6333", - api_key="test-secret", collection_prefix="demo_", timeout=12.0) - factory.local.assert_not_called() - store.close.assert_called_once() - - @pytest.mark.parametrize("args", [ - ["collection", "list"], - ["--metadata-url", "sqlite:///m.sqlite", "collection", "list"], - ["--metadata-url", "sqlite:///m.sqlite", "--path", "vectors", "--url", "http://localhost:6333", "collection", "list"], - ["--metadata-url", "sqlite:///:memory:", "--path", "vectors", "collection", "list"], - ["--metadata-url", "sqlite:///m.sqlite", "--path", ":memory:", "collection", "list"], - ["--metadata-url", "sqlite:///m.sqlite", "--path", "vectors", "--api-key", "secret", "collection", "list"], - ]) - def test_invalid_configuration_does_not_connect(self, factory: MagicMock, args) -> None: - assert runner.invoke(app, ["qdrant", *args]).exit_code == 2 - factory.local.assert_not_called() - factory.remote.assert_not_called() - - @pytest.mark.parametrize("args", [ - ["search", "docs"], ["search", "docs", "not JSON"], ["search", "docs", "[NaN]"], - ["search", "docs", "[true]"], ["search", "docs", '{}'], ["search", "docs", "[1]", "--stdin"], - ["search", "docs", "--file", "missing.json"], - ["point", "upsert", "docs", "one", '{"vector": [1], "payload": []}'], - ["point", "upsert", "docs", "one", '{"vector": [1], "unexpected": 1}'], - ["collection", "register", "docs", "--dimensions", "0"], - ]) - def test_invalid_input_does_not_connect(self, factory: MagicMock, args) -> None: - result = _invoke_qdrant(*args) - assert result.exit_code == 2, result.output - factory.local.assert_not_called() - - def test_failure_closes_session_before_store_and_redacts_secrets(self, factory: MagicMock) -> None: - store = factory.local.return_value - session = store.checkout_control.return_value - events = [] - store.context.checkout.return_value.__exit__.side_effect = lambda *args: events.append("session") - store.close.side_effect = lambda: events.append("store") - session.search.side_effect = RuntimeError("Failed with api-key test-secret at postgresql://user:pw@host/meta") - result = runner.invoke(app, ["qdrant", "--url", "http://localhost:6333", "--api-key", "test-secret", - "--metadata-url", "postgresql://user:pw@host/meta", "search", "docs", "[1,0,0]"]) - assert result.exit_code == 1 - assert result.stdout == "" - assert "test-secret" not in result.stderr - assert "user:pw" not in result.stderr - assert events == ["session", "store"] - - def test_factory_failure_does_not_close_unassigned_store(self, factory: MagicMock) -> None: - factory.local.side_effect = ValueError("Cannot open storage") - result = _invoke_qdrant("collection", "list") - assert result.exit_code == 1 - factory.local.return_value.close.assert_not_called() - - def test_optional_dependency_is_not_needed_for_help(self, tmp_path: Path) -> None: - code = ''' - import importlib.abc - import sys - class Block(importlib.abc.MetaPathFinder): - def find_spec(self, fullname, path=None, target=None): - if fullname == "qdrant_client" or fullname.startswith("qdrant_client."): - raise ModuleNotFoundError("simulated missing qdrant-client") - sys.meta_path.insert(0, Block()) - from chronos_core.cli.main import app - sys.argv = ["chronos", *sys.argv[1:]] - app() - ''' - env = {key: value for key, value in os.environ.items() if not key.startswith("CHRONOS_")} - for args in (["--help"], ["qdrant", "--help"]): - result = subprocess.run([sys.executable, "-c", textwrap.dedent(code), *args], capture_output=True, text=True, env=env) - assert result.returncode == 0, result.stderr - result = subprocess.run([sys.executable, "-c", textwrap.dedent(code), "qdrant", "--metadata-url", f"sqlite:///{tmp_path / 'metadata.sqlite'}", - "--path", str(tmp_path / "vectors"), "collection", "list"], capture_output=True, text=True, env=env) - assert result.returncode == 1 - assert "chronos-core[qdrant]" in result.stderr - assert list(tmp_path.iterdir()) == [] - result = subprocess.run([sys.executable, "-c", textwrap.dedent(code), "--database-url", f"sqlite:///{tmp_path / 'sql.sqlite'}", - "branch", "list"], capture_output=True, text=True, env=env) - assert result.returncode == 0, result.stderr - assert [branch["branch_id"] for branch in json.loads(result.stdout)] == ["main"] - - class TestTerminalBehavior: def test_json_values(self, capsys: pytest.CaptureFixture[str]) -> None: @dataclass @@ -486,6 +76,35 @@ class Result: class TestWorkspaceConfiguration: + @pytest.mark.parametrize("source", [ + '[stores.vectors]\nkind="qdrant"\nmetadata_url="sqlite:///meta.sqlite"\nurl="http://localhost:6333"\ntimeout=' + '9' * 400, + '[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///~chronos_no_such_user_987654/app.sqlite"', + '[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///shared.sqlite"\n[stores.vectors]\nkind="qdrant"\nmetadata_url="sqlite:///shared%2Esqlite"\npath="vectors"', + '[workspace]\nshared_metadata_url="sqlite:///shared.sqlite"\n[stores.first]\nkind="sqlite"\ndatabase_url="sqlite:///shared.sqlite"\n[stores.second]\nkind="sqlite"\ndatabase_url="sqlite:///shared.sqlite"', + '[workspace]\nshared_metadata_url="sqlite:///shared.sqlite"\n[stores.first]\nkind="duckdb"\ndatabase_url="duckdb:///first.duckdb"\nmetadata_url="sqlite:///shared.sqlite"\n[stores.second]\nkind="duckdb"\ndatabase_url="duckdb:///second.duckdb"\nmetadata_url="sqlite:///shared.sqlite"', + ]) + def test_reviewed_config_errors_fail_before_connecting(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, source: str) -> None: + config = tmp_path / "chronos.toml" + config.write_text(source) + opened = MagicMock() + monkeypatch.setattr(_workspace, "open_workspace", opened) + result = runner.invoke(app, ["--config", str(config), "branch", "list"]) + assert result.exit_code == 2, result.output + assert "Invalid value" in result.stderr + opened.assert_not_called() + assert list(tmp_path.iterdir()) == [config] + + def test_encoded_filename_preserves_api_url(self, tmp_path: Path) -> None: + config = tmp_path / "chronos.toml" + config.write_text('[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///literal%252Ename.sqlite"') + loaded = load_workspace_config(config) + assert loaded.stores["app"].connection.database_url == f"sqlite:///{tmp_path / 'literal%252Ename.sqlite'}" + + def test_invalid_config_path_is_a_usage_error(self) -> None: + result = runner.invoke(app, ["--config", "~chronos_no_such_user_987654/chronos.toml", "branch", "list"]) + assert result.exit_code == 2, result.output + assert "valid local path" in result.stderr + def test_path_precedence_and_relative_storage(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.chdir(tmp_path) monkeypatch.delenv("CHRONOS_WORKSPACE_CONFIG", raising=False) @@ -525,7 +144,7 @@ def test_environment_values_and_atomic_metadata(self, tmp_path: Path, monkeypatc ''') loaded = load_workspace_config(config) assert loaded.shared_metadata_url == "postgresql://user:secret@localhost/test" - assert loaded.stores["vectors"].connection == _qdrant.QdrantConfig( + assert loaded.stores["vectors"].connection == QdrantConfig( loaded.shared_metadata_url, url="http://localhost:6333", api_key="vector-secret", timeout=2.5, ) monkeypatch.delenv("VECTOR_KEY") @@ -571,6 +190,7 @@ def test_store_selection(self, tmp_path: Path) -> None: @pytest.mark.parametrize("failure_stage", ["none", "open", "workspace", "operation", "close"]) def test_resource_ownership(self, failure_stage: str, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: import chronos_core.workspace as workspace_api + from chronos_core.workspace.qdrant import ChronosQdrantStore config_path = tmp_path / "chronos.toml" # Declaration order must not affect shared-context construction or cleanup. @@ -592,7 +212,8 @@ def test_resource_ownership(self, failure_stage: str, tmp_path: Path, monkeypatc factory.local.return_value = vectors workspace_factory = MagicMock() monkeypatch.setattr(ChronosBranchContext, "connect", MagicMock(return_value=database)) - monkeypatch.setattr(_qdrant, "_store_class", lambda: factory) + monkeypatch.setattr(ChronosQdrantStore, "local", factory.local) + monkeypatch.setattr(ChronosQdrantStore, "remote", factory.remote) monkeypatch.setattr(workspace_api, "ChronosWorkspaceContext", workspace_factory) events = [] database.close.side_effect = lambda: events.append("database") @@ -628,6 +249,7 @@ def close_vectors() -> None: def test_split_sql_and_remote_qdrant(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: import chronos_core.workspace as workspace_api + from chronos_core.workspace.qdrant import ChronosQdrantStore config_file = tmp_path / "chronos.toml" config_file.write_text(''' @@ -649,7 +271,8 @@ def test_split_sql_and_remote_qdrant(self, tmp_path: Path, monkeypatch: pytest.M split = MagicMock() monkeypatch.setattr(ChronosBranchContext, "connect_split", split) factory = MagicMock() - monkeypatch.setattr(_qdrant, "_store_class", lambda: factory) + monkeypatch.setattr(ChronosQdrantStore, "local", factory.local) + monkeypatch.setattr(ChronosQdrantStore, "remote", factory.remote) workspace_factory = MagicMock() monkeypatch.setattr(workspace_api, "ChronosWorkspaceContext", workspace_factory) with open_workspace(config): @@ -668,6 +291,93 @@ def test_missing_file(self, tmp_path: Path) -> None: class TestWorkspaceCommands: + @pytest.mark.parametrize("value", [float("inf"), float("-inf"), float("nan")]) + def test_nonfinite_query_result_fails_clearly(self, workspace_connection: MagicMock, value: float) -> None: + session = workspace_connection.checkout.return_value.stores["app"] + session.query.return_value = [{"value": value}] + result = runner.invoke(app, ["query", "--store", "app", "SELECT 1"]) + assert result.exit_code == 1, result.output + assert result.stdout == "" + assert "JSON compliant" in result.stderr + session.close.assert_called_once_with() + workspace_connection.close.assert_called_once_with() + + @pytest.mark.parametrize("credentials", [ + 'review_user:review_secret%zz@127.0.0.1/db', + 'review_user:review%40secret%zz@127.0.0.1/db', + 'review_user@127.0.0.1/db?password=review%40secret%zz', + ]) + def test_postgres_parser_error_hides_password(self, tmp_path: Path, credentials: str) -> None: + config = tmp_path / "chronos.toml" + config.write_text(f'[stores.app]\nkind="postgresql"\ndatabase_url="postgresql://{credentials}"') + # Invalid percent escapes fail in the real driver's parser, before any network call. + result = runner.invoke(app, ["--config", str(config), "branch", "list"]) + assert result.exit_code == 1, result.output + assert result.stdout == "" + assert "invalid percent-encoded token" in result.stderr + assert "secret" not in result.stderr + assert "" in result.stderr + + def test_installed_entrypoints(self) -> None: + executable = Path(sys.executable).parent / "chronos" + for args in ([str(executable), "--help"], [sys.executable, "-m", "chronos_core.cli", "--help"]): + environment = {key: value for key, value in os.environ.items() if not key.startswith("CHRONOS_")} + result = subprocess.run(args, capture_output=True, text=True, env=environment, timeout=30) + assert result.returncode == 0, result.stderr + assert "--config" in result.stdout + assert "--database-url" not in result.stdout + + def test_help_and_sql_without_qdrant_dependency(self, tmp_path: Path) -> None: + code = ''' + import importlib.abc + import sys + class Block(importlib.abc.MetaPathFinder): + def find_spec(self, fullname, path=None, target=None): + if fullname == "qdrant_client" or fullname.startswith("qdrant_client."): + raise ModuleNotFoundError("simulated missing qdrant-client") + sys.meta_path.insert(0, Block()) + from chronos_core.cli.main import app + sys.argv = ["chronos", *sys.argv[1:]] + app() + ''' + environment = {key: value for key, value in os.environ.items() if not key.startswith("CHRONOS_")} + for args in (["--help"], ["qdrant", "--help"], ["qdrant", "point", "upsert", "--help"]): + result = subprocess.run([sys.executable, "-c", textwrap.dedent(code), *args], capture_output=True, text=True, env=environment) + assert result.returncode == 0, result.stderr + config = tmp_path / "chronos.toml" + config.write_text('[stores.vectors]\nkind="qdrant"\nmetadata_url="sqlite:///meta.sqlite"\npath="vectors"\n') + result = subprocess.run( + [sys.executable, "-c", textwrap.dedent(code), "--config", str(config), "qdrant", "--store", "vectors", "collection", "list"], + capture_output=True, text=True, env=environment, + ) + assert result.returncode == 1 + assert "chronos-core[qdrant]" in result.stderr + assert list(tmp_path.iterdir()) == [config] + config.write_text('[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"\n') + result = subprocess.run( + [sys.executable, "-c", textwrap.dedent(code), "--config", str(config), "branch", "list"], + capture_output=True, text=True, env=environment, + ) + assert result.returncode == 0, result.stderr + assert json.loads(result.stdout) == ["main"] + + def test_open_failure_reports_credentials_safely(self, workspace_connection: MagicMock) -> None: + _workspace.open_workspace.side_effect = RuntimeError("Backend unavailable at postgresql://user:secret@host/meta") + result = runner.invoke(app, ["branch", "list"]) + assert result.exit_code == 1 + assert result.stdout == "" + assert "secret" not in result.stderr + assert "Backend unavailable" in result.stderr + workspace_connection.close.assert_not_called() + + def test_unsupported_result_closes_workspace(self, workspace_connection: MagicMock) -> None: + workspace_connection.diff.return_value = object() + result = runner.invoke(app, ["diff", "main", "trial"]) + assert result.exit_code == 1 + assert result.stdout == "" + assert "Unsupported JSON value type" in result.stderr + workspace_connection.close.assert_called_once_with() + @pytest.mark.parametrize(("args", "method", "expected_args", "expected_kwargs"), [ (["branch", "list"], "list_branches", (), {}), (["branch", "create", "trial", "--from", "parent"], "create_branch", ("trial",), {"from_branch": "parent"}), @@ -682,7 +392,7 @@ class TestWorkspaceCommands: "merge_atomic", ("trial", "main"), {"policy": "abort_on_conflict", "operation_id": "merge-1", "preview_token": "reviewed"}), ]) def test_workspace_forwarding(self, workspace_connection: MagicMock, args, method, expected_args, expected_kwargs) -> None: - result = runner.invoke(app, ["workspace", *args]) + result = runner.invoke(app, [*args]) assert result.exit_code == 0, result.output getattr(workspace_connection, method).assert_called_once_with(*expected_args, **expected_kwargs) workspace_connection.close.assert_called_once_with() @@ -693,14 +403,14 @@ def test_workspace_forwarding(self, workspace_connection: MagicMock, args, metho def test_named_relational_commands(self, workspace_connection: MagicMock) -> None: workspace = workspace_connection - result = runner.invoke(app, ["workspace", "table", "create", "--store", "app", "CREATE TABLE items (id INTEGER)"]) + result = runner.invoke(app, ["table", "create", "--store", "app", "CREATE TABLE items (id INTEGER)"]) assert result.exit_code == 0, result.output workspace.stores["app"].db.execute.assert_called_once_with("CREATE TABLE items (id INTEGER)") workspace.stores["app"].db.commit.assert_called_once_with() - result = runner.invoke(app, ["workspace", "table", "register", "items", "--store", "app", "--primary-key", "id", "--primary-key", "tenant"]) + result = runner.invoke(app, ["table", "register", "items", "--store", "app", "--primary-key", "id", "--primary-key", "tenant"]) assert result.exit_code == 0, result.output workspace.stores["app"].register_table.assert_called_once_with("items", ["id", "tenant"]) - result = runner.invoke(app, ["workspace", "query", "SELECT :id", "--store", "app", "--branch", "trial", "--params", '{"id":1}']) + result = runner.invoke(app, ["query", "SELECT :id", "--store", "app", "--branch", "trial", "--params", '{"id":1}']) assert result.exit_code == 0, result.output assert json.loads(result.stdout) == [{"id": 1}] session = workspace.checkout.return_value.stores["app"] @@ -708,7 +418,7 @@ def test_named_relational_commands(self, workspace_connection: MagicMock) -> Non workspace.checkout.assert_called_once_with("trial") for checked_out in workspace.checkout.return_value.stores.values(): checked_out.close.assert_called_once_with() - result = runner.invoke(app, ["workspace", "execute", "UPDATE items SET id = :id", "--store", "app", "--params", '{"id":2}']) + result = runner.invoke(app, ["execute", "UPDATE items SET id = :id", "--store", "app", "--params", '{"id":2}']) assert result.exit_code == 0, result.output session.execute.assert_called_once_with("UPDATE items SET id = :id", {"id": 2}) @@ -717,35 +427,43 @@ def test_sql_input_modes(self, workspace_connection: MagicMock, tmp_path: Path) sql_file.write_text("SELECT :id") session = workspace_connection.checkout.return_value.stores["app"] for args, input_text in [(["--file", str(sql_file)], None), (["--stdin"], "SELECT :id"), ([], "SELECT :id")]: - result = runner.invoke(app, ["workspace", "query", "--store", "app", "--params", '{"id":3}', *args], input=input_text) + result = runner.invoke(app, ["query", "--store", "app", "--params", '{"id":3}', *args], input=input_text) assert result.exit_code == 0, result.output assert session.query.call_args.args == ("SELECT :id", {"id": 3}) @pytest.mark.parametrize("args", [ ["query", "SELECT 1", "--store", "missing"], ["query", "SELECT 1", "--store", "app", "--params", "[]"], + ["execute", "SELECT 1", "--store", "app", "--params", "broken"], + ["query", "SELECT 1", "--store", "app", "--file", "missing.sql"], + ["query", "--store", "app", "--file", "missing.sql"], ["query", "SELECT 1", "--store", "app", "--stdin"], ["query", "--store", "app", "--stdin"], ["table", "create", "--store", "app", "DELETE FROM items"], + ["table", "create", "--store", "app", "CREATE INDEX idx ON items(id)"], + ["table", "create", "--store", "app", "CREATE TABLE items (id INTEGER); DROP TABLE items"], + ["table", "create", "--store", "app", "not sql"], + ["table", "register", "--store", "app", "items"], ["qdrant", "--store", "app", "collection", "list"], ["qdrant", "collection", "list"], ["merge", "atomic-apply", "trial", "main"], + ["merge", "apply", "trial", "main", "--policy", "unknown"], ]) def test_invalid_arguments_do_not_open_stores(self, workspace_connection: MagicMock, args) -> None: - result = runner.invoke(app, ["workspace", *args], input="") + result = runner.invoke(app, [*args], input="") assert result.exit_code == 2, result.output _workspace.open_workspace.assert_not_called() def test_help_and_configuration_precedence(self, workspace_connection: MagicMock, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: for args in ([], ["branch"], ["query"], ["qdrant"], ["qdrant", "point"], ["qdrant", "point", "upsert"], ["merge", "atomic-apply"]): - result = runner.invoke(app, ["workspace", "--config", "missing.toml", *args, "--help"]) + result = runner.invoke(app, ["--config", "missing.toml", *args, "--help"]) assert result.exit_code == 0, result.output _workspace.open_workspace.assert_not_called() monkeypatch.setenv("CHRONOS_WORKSPACE_CONFIG", str(tmp_path / "missing.toml")) - result = runner.invoke(app, ["workspace", "branch", "list"]) + result = runner.invoke(app, ["branch", "list"]) assert result.exit_code == 2 _workspace.open_workspace.assert_not_called() - result = runner.invoke(app, ["workspace", "--config", str(tmp_path / "chronos.toml"), "branch", "list"]) + result = runner.invoke(app, ["--config", str(tmp_path / "chronos.toml"), "branch", "list"]) assert result.exit_code == 0, result.output assert _workspace.open_workspace.call_args.args[0].config_path == tmp_path / "chronos.toml" @@ -754,7 +472,7 @@ def test_api_error_and_session_cleanup(self, workspace_connection: MagicMock) -> session = workspace.checkout.return_value.stores["app"] session.query.side_effect = BranchingError("query failed") session.close.side_effect = RuntimeError("close failed") - result = runner.invoke(app, ["workspace", "query", "SELECT 1", "--store", "app"]) + result = runner.invoke(app, ["query", "SELECT 1", "--store", "app"]) assert result.exit_code == 1 assert result.stdout == "" assert "query failed" in result.stderr @@ -768,7 +486,7 @@ def test_api_result_field_names_are_preserved(self, workspace_connection: MagicM token = WorkspaceBranchToken("main", 1) result = AtomicMergeResult("merge-1", "trial", "main", "committed", token, token, token, 2, 0, {"app": 1, "other": 1}) workspace_connection.merge_atomic.return_value = result - invoked = runner.invoke(app, ["workspace", "merge", "atomic-apply", "trial", "main", "--operation-id", "merge-1"]) + invoked = runner.invoke(app, ["merge", "atomic-apply", "trial", "main", "--operation-id", "merge-1"]) assert invoked.exit_code == 0, invoked.output assert json.loads(invoked.stdout) == { "operation_id": "merge-1", "source": "trial", "target": "main", "status": "committed", @@ -814,7 +532,7 @@ def workspace(self, workspace_connection: MagicMock, tmp_path: Path) -> MagicMoc (["search", "docs", "[1,0,0]", "--limit", "3", "--score-threshold", "0.5"], "search", ("docs", [1, 0, 0]), {"limit": 3, "score_threshold": 0.5}, True), ]) def test_forwarding(self, workspace: MagicMock, args, method, positional, keywords, session: bool) -> None: - result = runner.invoke(app, ["workspace", "qdrant", "--store", "vectors", *args, *(["--branch", "trial"] if session else [])]) + result = runner.invoke(app, ["qdrant", "--store", "vectors", *args, *(["--branch", "trial"] if session else [])]) assert result.exit_code == 0, result.output assert result.stdout == "" target = workspace.checkout.return_value.stores["vectors"] if session else workspace.stores["vectors"] @@ -825,7 +543,7 @@ def test_forwarding(self, workspace: MagicMock, args, method, positional, keywor @pytest.mark.parametrize("data", ['{"vector":[true]}', '{"vector":[NaN]}', '{"vector":[1],"payload":[]}']) def test_invalid_point_does_not_connect(self, workspace: MagicMock, data: str) -> None: - result = runner.invoke(app, ["workspace", "qdrant", "--store", "vectors", "point", "upsert", "docs", "one", data]) + result = runner.invoke(app, ["qdrant", "--store", "vectors", "point", "upsert", "docs", "one", data]) assert result.exit_code == 2, result.output _workspace.open_workspace.assert_not_called() @@ -844,7 +562,7 @@ def test_transport_error_redacts_credentials(self, workspace: MagicMock, tmp_pat workspace.checkout.return_value.stores["vectors"].search.side_effect = httpx.ConnectError( "Failed using vector-secret and postgresql://user:sql-secret@localhost/app" ) - result = runner.invoke(app, ["workspace", "qdrant", "--store", "vectors", "search", "docs", "[1,0,0]"]) + result = runner.invoke(app, ["qdrant", "--store", "vectors", "search", "docs", "[1,0,0]"]) assert result.exit_code == 1 assert result.stdout == "" assert "secret" not in result.stderr @@ -854,15 +572,15 @@ def test_transport_error_redacts_credentials(self, workspace: MagicMock, tmp_pat def test_json_file_and_stdin(self, workspace: MagicMock, tmp_path: Path) -> None: data = tmp_path / "point.json" data.write_text('{"vector":[1,0,0],"payload":{"source":"file"}}') - result = runner.invoke(app, ["workspace", "qdrant", "--store", "vectors", "point", "upsert", "docs", "one", "--file", str(data)]) + result = runner.invoke(app, ["qdrant", "--store", "vectors", "point", "upsert", "docs", "one", "--file", str(data)]) assert result.exit_code == 0, result.output session = workspace.checkout.return_value.stores["vectors"] session.upsert.assert_called_once_with("docs", "one", [1, 0, 0], {"source": "file"}, revision=None) for args in (["--stdin"], []): - result = runner.invoke(app, ["workspace", "qdrant", "--store", "vectors", "search", "docs", *args], input="[0,1,0]") + result = runner.invoke(app, ["qdrant", "--store", "vectors", "search", "docs", *args], input="[0,1,0]") assert result.exit_code == 0, result.output assert session.search.call_args.args == ("docs", [0, 1, 0]) data.write_text("[1,0,0]") - result = runner.invoke(app, ["workspace", "qdrant", "--store", "vectors", "search", "docs", "--file", str(data)]) + result = runner.invoke(app, ["qdrant", "--store", "vectors", "search", "docs", "--file", str(data)]) assert result.exit_code == 0, result.output assert session.search.call_args.args == ("docs", [1, 0, 0]) diff --git a/tests/test_cli_integration.py b/tests/test_cli_integration.py index d114282..45ab965 100644 --- a/tests/test_cli_integration.py +++ b/tests/test_cli_integration.py @@ -21,9 +21,7 @@ from psycopg import sql as postgres_sql from chronos_core import _native_interval -from chronos_core.cli._qdrant import QdrantConfig -from chronos_core.cli._relational import DatabaseConfig -from chronos_core.cli._workspace_config import load_workspace_config, open_workspace +from chronos_core.cli._config import DatabaseConfig, QdrantConfig, load_workspace_config, open_workspace def _run_cli(arguments: list[str], *, environment: dict[str, str], @@ -39,29 +37,6 @@ def _run_cli(arguments: list[str], *, environment: dict[str, str], return json.loads(result.stdout) if result.stdout.strip() else None -def _invoke_sql(database: DatabaseConfig, *args: str, - ok: bool = True, input_text: str | None = None) -> Any: - arguments = ["--database-url", database.database_url] - if database.metadata_url is not None: - arguments.extend(["--metadata-url", database.metadata_url]) - environment = dict(os.environ) - environment.pop("CHRONOS_METADATA_URL", None) - return _run_cli([*arguments, *args], environment=environment, ok=ok, input_text=input_text) - - -def _invoke_qdrant(target: QdrantConfig, *args: str, ok: bool = True, input_text: str | None = None): - env = {key: value for key, value in os.environ.items() if not key.startswith("CHRONOS_")} - env["CHRONOS_METADATA_URL"] = target.metadata_url - env["CHRONOS_QDRANT_COLLECTION_PREFIX"] = target.collection_prefix - if target.path is not None: - env["CHRONOS_QDRANT_PATH"] = str(target.path) - else: - env["CHRONOS_QDRANT_URL"] = target.url - if target.api_key is not None: - env["CHRONOS_QDRANT_API_KEY"] = target.api_key - return _run_cli(["qdrant", *args], environment=env, ok=ok, input_text=input_text) - - @pytest.fixture(scope="session") def postgres_cli_database_url() -> Iterator[str]: """Create one owned database for the run, without resetting the admin database.""" @@ -170,163 +145,58 @@ def _open_qdrant_api(target: QdrantConfig): ) -class TestRelationalWorkflows: - def test_relational_workflow(self, cli_database: DatabaseConfig) -> None: - invoke = partial(_invoke_sql, cli_database) - invoke("table", "create", "CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)") - assert invoke("table", "register", "items", "--primary-key", "id") is None - assert invoke("branch", "create", "trial") is None - invoke("branch", "create", "trial", ok=False) - assert {branch["branch_id"] for branch in invoke("branch", "list")} == {"main", "trial"} - assert invoke("execute", "INSERT INTO items (id, value) VALUES (:id, :value)", "--branch", "trial", - "--params", '{"id": 1, "value": "hello"}') == {"rowcount": 1} - assert invoke("query", "SELECT * FROM items", "--branch", "trial") == [{"id": 1, "value": "hello"}] - assert invoke("query", "SELECT * FROM items") == [] - assert invoke("checkpoint", "create", "saved", "--branch", "trial")["checkpoint_id"] == "saved" - assert len(invoke("checkpoint", "list", "--branch", "trial")) == 1 - assert len(invoke("diff", "main", "trial")["changes"]) == 1 - assert len(invoke("diff", "main", "trial", "--table", "items")) == 1 - assert invoke("merge", "preview", "trial", "main")["conflicts"] == [] - assert invoke("merge", "apply", "trial", "main")["applied"] == 1 - assert invoke("query", "SELECT * FROM items") == [{"id": 1, "value": "hello"}] - invoke("branch", "create", "conflicting") - invoke("execute", "UPDATE items SET value = 'source' WHERE id = 1", "--branch", "conflicting") - invoke("execute", "UPDATE items SET value = 'target' WHERE id = 1") - assert len(invoke("merge", "preview", "conflicting", "main")["conflicts"]) == 1 - invoke("merge", "apply", "conflicting", "main", ok=False) - invoke("merge", "apply", "conflicting", "main", "--policy", "snapshot_isolation", ok=False) - assert invoke("query", "SELECT value FROM items") == [{"value": "target"}] - invoke("branch", "delete", "conflicting") - invoke("query", "SELECT * FROM items", "--branch", "missing", ok=False) - invoke("execute", "DROP TABLE items", ok=False) - assert invoke("branch", "delete", "trial") is None - assert [branch["branch_id"] for branch in invoke("branch", "list")] == ["main"] +def _workspace_config_file(root: Path, database: DatabaseConfig, vectors: QdrantConfig | None = None, *, atomic: bool = False) -> Path: + lines = [] + if atomic: + lines.extend(["[workspace]", f"shared_metadata_url={json.dumps(database.metadata_url or database.database_url)}"]) + lines.extend(["[stores.app]", f"kind={json.dumps(urlsplit(database.database_url).scheme)}", + f"database_url={json.dumps(database.database_url)}"]) + if database.metadata_url: + lines.append(f"metadata_url={json.dumps(database.metadata_url)}") + if vectors is not None: + lines.extend(["[stores.vectors]", 'kind="qdrant"', f"metadata_url={json.dumps(vectors.metadata_url)}", + f"collection_prefix={json.dumps(vectors.collection_prefix)}"]) + if vectors.path is not None: + lines.append(f"path={json.dumps(str(vectors.path))}") + else: + lines.append(f"url={json.dumps(vectors.url)}") + if vectors.api_key: + lines.append(f"api_key={json.dumps(vectors.api_key)}") + config = root / "chronos.toml" + config.write_text("\n".join(lines) + "\n") + return config - def test_relational_sql_inputs_and_rollback(self, cli_database: DatabaseConfig, tmp_path: Path) -> None: - invoke = partial(_invoke_sql, cli_database) - statement = tmp_path / "statement.sql" - statement.write_text( - "CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)", encoding="utf-8", - ) - invoke("table", "create", "--file", str(statement)) - invoke("table", "register", "items", "--primary-key", "id") - statement.write_text( - "INSERT INTO items (id, value) VALUES (:id, :value)", encoding="utf-8", - ) - assert invoke("execute", "--file", str(statement), "--params", - '{"id": 1, "value": "100% done"}') == {"rowcount": 1} - assert invoke("query", "--stdin", "--params", '{"pattern": "100%"}', - input_text="SELECT * FROM items WHERE value LIKE :pattern") == [ - {"id": 1, "value": "100% done"}, - ] - assert invoke("execute", "--stdin", "--params", '{"id": 1, "value": "updated"}', - input_text="UPDATE items SET value = :value WHERE id = :id") == {"rowcount": 1} - statement.write_text("SELECT * FROM items", encoding="utf-8") - assert invoke("query", "--file", str(statement)) == [{"id": 1, "value": "updated"}] - # A failed statement must not leave its earlier rows committed. - invoke("execute", "INSERT INTO items (id, value) VALUES (2, 'partial'), (1, 'duplicate')", ok=False) - assert invoke("query", input_text="SELECT * FROM items ORDER BY id") == [ - {"id": 1, "value": "updated"}, - ] - def test_relational_aggregation(self, cli_database: DatabaseConfig) -> None: - invoke = partial(_invoke_sql, cli_database) - invoke("table", "create", "CREATE TABLE sales (id INTEGER PRIMARY KEY, category TEXT, amount BIGINT)") - invoke("table", "register", "sales", "--primary-key", "id") - invoke("execute", "INSERT INTO sales (id, category, amount) VALUES (1, 'books', 10), (2, 'books', 20), (3, 'games', 50)") - invoke("branch", "create", "forecast") - invoke("execute", "UPDATE sales SET amount = amount * :factor WHERE category = :category", - "--branch", "forecast", "--params", '{"factor": 2, "category": "books"}') - query = "SELECT category, SUM(amount) AS total FROM sales GROUP BY category ORDER BY category" - # PostgreSQL SUM(BIGINT) returns NUMERIC, serialized as a precision-preserving string. - total = str if urlsplit(cli_database.database_url).scheme in {"postgres", "postgresql"} else int - assert invoke("query", query) == [ - {"category": "books", "total": total(30)}, {"category": "games", "total": total(50)}, - ] - assert invoke("query", query, "--branch", "forecast") == [ - {"category": "books", "total": total(60)}, {"category": "games", "total": total(50)}, - ] +def _invoke_workspace(config: Path, *args: str, ok: bool = True, input_text: str | None = None) -> Any: + environment = {key: value for key, value in os.environ.items() if not key.startswith("CHRONOS_")} + environment["CHRONOS_WORKSPACE_CONFIG"] = str(config) + return _run_cli([*args], environment=environment, ok=ok, input_text=input_text) -class TestQdrantWorkflows: - def test_qdrant_cli_workflow(self, qdrant_target: QdrantConfig) -> None: - target = qdrant_target - info = _invoke_qdrant(target, "collection", "register", "documents", "--dimensions", "3") - assert info["name"] == "documents" - assert info["dimensions"] == 3 - assert _invoke_qdrant(target, "collection", "info", "documents") == info - assert _invoke_qdrant(target, "collection", "list") == [info] - first = {"vector": [1, 0, 0], "payload": {"title": "Runtime"}} - second = {"vector": [0, 1, 0], "payload": {"title": "Billing"}} - point = _invoke_qdrant(target, "point", "upsert", "documents", "runtime", json.dumps(first)) - _invoke_qdrant(target, "point", "upsert", "documents", "billing", json.dumps(second)) - assert _invoke_qdrant(target, "point", "get", "documents", "runtime") == point - assert {item["id"] for item in _invoke_qdrant(target, "point", "list", "documents")} == {"runtime", "billing"} - assert _invoke_qdrant(target, "point", "get", "documents", "missing") is None - results = _invoke_qdrant(target, "search", "documents", "[1,0,0]", "--limit", "2") - assert [item["id"] for item in results] == ["runtime", "billing"] - assert results[0]["score"] == pytest.approx(1.0, abs=1e-6) - assert results[1]["score"] == pytest.approx(0.0, abs=1e-6) - assert len(_invoke_qdrant(target, "search", "documents", "[1,0,0]", "--score-threshold", "0.9")) == 1 - - # Independent expected data plus a direct public-API comparison. - store = _open_qdrant_api(target) - try: - with store.context.checkout("main") as control: - session = store.checkout_control(control) - assert point == asdict(session.get("documents", "runtime")) - assert results == [asdict(item) for item in session.search("documents", [1, 0, 0], limit=2)] - finally: - store.close() +def _prepare_workspace_merge(root: Path, *, atomic: bool) -> Path: + pytest.importorskip("qdrant_client") + database = DatabaseConfig(f"sqlite:///{root / 'app.sqlite'}") + target = QdrantConfig( + metadata_url=database.database_url if atomic else f"sqlite:///{root / 'vectors.sqlite'}", + path=root / "vectors", + ) + config = _workspace_config_file(root, database, target, atomic=atomic) + _invoke_workspace(config, "table", "create", "CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)", "--store", "app") + _invoke_workspace(config, "table", "register", "items", "--primary-key", "id", "--store", "app") + _invoke_workspace(config, "execute", "INSERT INTO items (id, value) VALUES (1, 'base')", "--store", "app") + _invoke_workspace(config, "qdrant", "--store", "vectors", "collection", "register", "documents", "--dimensions", "3") + _invoke_workspace(config, "branch", "create", "experiment") + _invoke_workspace(config, "execute", "UPDATE items SET value = 'candidate' WHERE id = 1", "--store", "app", "--branch", "experiment") + _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "upsert", "documents", "one", '{"vector":[1,0,0]}', "--branch", "experiment") + return config - _invoke_qdrant(target, "branch", "create", "experiment") - assert {item["branch_id"] for item in _invoke_qdrant(target, "branch", "list")} == {"main", "experiment"} - changed = {"vector": [0, 0, 1], "payload": {"title": "Experiment"}} - _invoke_qdrant(target, "point", "upsert", "documents", "runtime", json.dumps(changed), "--branch", "experiment") - assert _invoke_qdrant(target, "point", "delete", "documents", "billing", "--branch", "experiment") is True - assert _invoke_qdrant(target, "point", "get", "documents", "billing", "--branch", "experiment") is None - assert _invoke_qdrant(target, "point", "get", "documents", "billing")["payload"] == second["payload"] - assert _invoke_qdrant(target, "point", "get", "documents", "runtime")["payload"] == first["payload"] - assert _invoke_qdrant(target, "checkpoint", "create", "saved", "--branch", "experiment")["checkpoint_id"] == "saved" - assert len(_invoke_qdrant(target, "checkpoint", "list", "--branch", "experiment")) == 1 - changes = _invoke_qdrant(target, "diff", "main", "experiment")["changes"] - assert {(item["key"]["id"], item["change"]) for item in changes} == {("runtime", "modified"), ("billing", "deleted")} - assert _invoke_qdrant(target, "merge", "preview", "experiment", "main")["conflicts"] == [] - assert _invoke_qdrant(target, "merge", "apply", "experiment", "main")["applied"] == 2 - assert _invoke_qdrant(target, "point", "get", "documents", "runtime")["payload"] == changed["payload"] - assert _invoke_qdrant(target, "point", "get", "documents", "billing") is None - assert _invoke_qdrant(target, "point", "delete", "documents", "missing") is False - _invoke_qdrant(target, "branch", "delete", "experiment") - - def test_qdrant_cli_conflicts_and_failed_operations(self, qdrant_target: QdrantConfig) -> None: - target = qdrant_target - _invoke_qdrant(target, "collection", "register", "docs", "--dimensions", "3") - _invoke_qdrant(target, "point", "upsert", "docs", "one", '{"vector":[1,0,0],"payload":{"value":"base"}}') - _invoke_qdrant(target, "branch", "create", "trial") - _invoke_qdrant(target, "branch", "create", "trial", ok=False) - _invoke_qdrant(target, "point", "upsert", "docs", "one", '{"vector":[0,1,0],"payload":{"value":"trial"}}', "--branch", "trial") - _invoke_qdrant(target, "point", "upsert", "docs", "one", '{"vector":[0,0,1],"payload":{"value":"main"}}') - assert len(_invoke_qdrant(target, "merge", "preview", "trial", "main")["conflicts"]) == 1 - _invoke_qdrant(target, "merge", "apply", "trial", "main", ok=False) - _invoke_qdrant(target, "merge", "apply", "trial", "main", "--policy", "snapshot_isolation", ok=False) - _invoke_qdrant(target, "point", "upsert", "docs", "bad", '{"vector":[1,0]}', ok=False) - assert _invoke_qdrant(target, "point", "get", "docs", "bad") is None - assert _invoke_qdrant(target, "point", "get", "docs", "one")["payload"] == {"value": "main"} - _invoke_qdrant(target, "point", "get", "docs", "one", "--branch", "missing", ok=False) - _invoke_qdrant(target, "collection", "info", "missing", ok=False) - def test_qdrant_cli_json_files_and_stdin(self, qdrant_target: QdrantConfig, tmp_path: Path) -> None: - target = qdrant_target - _invoke_qdrant(target, "collection", "register", "docs", "--dimensions", "3") - data = tmp_path / "point.json" - data.write_text('{"vector":[1,0,0],"payload":{"source":"file"}}', encoding="utf-8") - result = _invoke_qdrant(target, "point", "upsert", "docs", "one", "--file", str(data), "--revision", "initial") - assert result["payload"] == {"source": "file"} - assert _invoke_qdrant(target, "search", "docs", "--stdin", input_text="[1,0,0]")[0]["id"] == "one" - _invoke_qdrant(target, "point", "upsert", "docs", "two", "--stdin", input_text='{"vector":[0,1,0]}') - data.write_text("[0,1,0]", encoding="utf-8") - assert _invoke_qdrant(target, "search", "docs", "--file", str(data))[0]["id"] == "two" - assert _invoke_qdrant(target, "search", "docs", input_text="[1,0,0]")[0]["id"] == "one" +def _qdrant_workspace_file(root: Path, target: QdrantConfig) -> Path: + config = _workspace_config_file(root, DatabaseConfig(target.metadata_url), target) + # A Qdrant-only workspace still has its own relational metadata context. + text = config.read_text() + config.write_text(text[text.index("[stores.vectors]"):]) + return config class TestWorkspaceResources: @@ -345,7 +215,7 @@ def find_spec(self, fullname, path=None, target=None): raise ModuleNotFoundError("simulated missing qdrant-client") sys.meta_path.insert(0, BlockQdrant()) - from chronos_core.cli._workspace_config import load_workspace_config, open_workspace + from chronos_core.cli._config import load_workspace_config, open_workspace with open_workspace(load_workspace_config(Path(sys.argv[1]))) as workspace: print(json.dumps(workspace.list_branches())) ''' @@ -467,35 +337,15 @@ def test_sqlite_qdrant_workspace_reopens(self, tmp_path: Path, atomic: bool) -> workspace.delete_branch("experiment") -def _workspace_config_file(root: Path, database: DatabaseConfig, vectors: QdrantConfig | None = None, *, atomic: bool = False) -> Path: - lines = [] - if atomic: - lines.extend(["[workspace]", f"shared_metadata_url={json.dumps(database.metadata_url or database.database_url)}"]) - lines.extend(["[stores.app]", f"kind={json.dumps(urlsplit(database.database_url).scheme)}", - f"database_url={json.dumps(database.database_url)}"]) - if database.metadata_url: - lines.append(f"metadata_url={json.dumps(database.metadata_url)}") - if vectors is not None: - lines.extend(["[stores.vectors]", 'kind="qdrant"', f"metadata_url={json.dumps(vectors.metadata_url)}", - f"collection_prefix={json.dumps(vectors.collection_prefix)}"]) - if vectors.path is not None: - lines.append(f"path={json.dumps(str(vectors.path))}") - else: - lines.append(f"url={json.dumps(vectors.url)}") - if vectors.api_key: - lines.append(f"api_key={json.dumps(vectors.api_key)}") - config = root / "chronos.toml" - config.write_text("\n".join(lines) + "\n") - return config - - -def _invoke_workspace(config: Path, *args: str, ok: bool = True, input_text: str | None = None) -> Any: - environment = {key: value for key, value in os.environ.items() if not key.startswith("CHRONOS_")} - environment["CHRONOS_WORKSPACE_CONFIG"] = str(config) - return _run_cli(["workspace", *args], environment=environment, ok=ok, input_text=input_text) - - class TestWorkspaceSQLWorkflows: + def test_nonfinite_api_result_is_not_printed_as_invalid_json(self, tmp_path: Path) -> None: + config = _workspace_config_file(tmp_path, DatabaseConfig(f"sqlite:///{tmp_path / 'app.sqlite'}")) + with open_workspace(load_workspace_config(config)) as workspace: + with workspace.stores["app"].checkout("main") as session: + assert session.query("SELECT 1e999 AS value") == [{"value": float("inf")}] + _invoke_workspace(config, "query", "SELECT 1e999 AS value", "--store", "app", ok=False) + assert _invoke_workspace(config, "query", "SELECT 1 AS value", "--store", "app") == [{"value": 1}] + def test_relational_workspace(self, cli_database: DatabaseConfig, tmp_path: Path) -> None: config = _workspace_config_file(tmp_path, cli_database) _invoke_workspace(config, "table", "create", "CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)", "--store", "app") @@ -523,12 +373,95 @@ def test_relational_workspace(self, cli_database: DatabaseConfig, tmp_path: Path assert set(_invoke_workspace(config, "merge", "apply", "experiment", "main")) == {"app", "audit"} assert _invoke_workspace(config, "query", "SELECT value FROM items", "--store", "app") == [{"value": "candidate"}] assert _invoke_workspace(config, "query", "SELECT value FROM audit", "--store", "audit") == [{"value": "candidate"}] + _invoke_workspace(config, "branch", "create", "conflicting") + _invoke_workspace(config, "execute", "UPDATE items SET value = 'source' WHERE id = 1", "--store", "app", "--branch", "conflicting") + _invoke_workspace(config, "execute", "UPDATE items SET value = 'target' WHERE id = 1", "--store", "app") + assert len(_invoke_workspace(config, "merge", "preview", "conflicting", "main")["app"]["conflicts"]) == 1 + for policy in ("abort_on_conflict", "snapshot_isolation"): + _invoke_workspace(config, "merge", "apply", "conflicting", "main", "--policy", policy, ok=False) + assert _invoke_workspace(config, "query", "SELECT value FROM items", "--store", "app") == [{"value": "target"}] + _invoke_workspace(config, "branch", "delete", "conflicting") + _invoke_workspace(config, "query", "SELECT * FROM items", "--store", "app", "--branch", "missing", ok=False) + _invoke_workspace(config, "execute", "DROP TABLE items", "--store", "app", ok=False) _invoke_workspace(config, "branch", "delete", "restored") _invoke_workspace(config, "branch", "delete", "experiment") assert _invoke_workspace(config, "branch", "list") == ["main"] + def test_relational_sql_inputs_and_rollback(self, cli_database: DatabaseConfig, tmp_path: Path) -> None: + config = _workspace_config_file(tmp_path, cli_database) + invoke = partial(_invoke_workspace, config) + statement = tmp_path / "statement.sql" + statement.write_text( + "CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)", encoding="utf-8", + ) + invoke("table", "create", "--store", "app", "--file", str(statement)) + invoke("table", "register", "--store", "app", "items", "--primary-key", "id") + statement.write_text( + "INSERT INTO items (id, value) VALUES (:id, :value)", encoding="utf-8", + ) + assert invoke("execute", "--store", "app", "--file", str(statement), "--params", + '{"id": 1, "value": "100% done"}') == {"rowcount": 1} + assert invoke("query", "--store", "app", "--stdin", "--params", '{"pattern": "100%"}', + input_text="SELECT * FROM items WHERE value LIKE :pattern") == [ + {"id": 1, "value": "100% done"}, + ] + assert invoke("execute", "--store", "app", "--stdin", "--params", '{"id": 1, "value": "updated"}', + input_text="UPDATE items SET value = :value WHERE id = :id") == {"rowcount": 1} + statement.write_text("SELECT * FROM items", encoding="utf-8") + assert invoke("query", "--store", "app", "--file", str(statement)) == [{"id": 1, "value": "updated"}] + # A failed statement must not leave its earlier rows committed. + invoke("execute", "--store", "app", "INSERT INTO items (id, value) VALUES (2, 'partial'), (1, 'duplicate')", ok=False) + assert invoke("query", "--store", "app", input_text="SELECT * FROM items ORDER BY id") == [ + {"id": 1, "value": "updated"}, + ] + + def test_relational_aggregation(self, cli_database: DatabaseConfig, tmp_path: Path) -> None: + config = _workspace_config_file(tmp_path, cli_database) + invoke = partial(_invoke_workspace, config) + invoke("table", "create", "--store", "app", "CREATE TABLE sales (id INTEGER PRIMARY KEY, category TEXT, amount BIGINT)") + invoke("table", "register", "--store", "app", "sales", "--primary-key", "id") + invoke("execute", "--store", "app", "INSERT INTO sales (id, category, amount) VALUES (1, 'books', 10), (2, 'books', 20), (3, 'games', 50)") + invoke("branch", "create", "forecast") + invoke("execute", "--store", "app", "UPDATE sales SET amount = amount * :factor WHERE category = :category", + "--branch", "forecast", "--params", '{"factor": 2, "category": "books"}') + query = "SELECT category, SUM(amount) AS total FROM sales GROUP BY category ORDER BY category" + # PostgreSQL SUM(BIGINT) returns NUMERIC, serialized as a precision-preserving string. + total = str if urlsplit(cli_database.database_url).scheme in {"postgres", "postgresql"} else int + assert invoke("query", "--store", "app", query) == [ + {"category": "books", "total": total(30)}, {"category": "games", "total": total(50)}, + ] + assert invoke("query", "--store", "app", query, "--branch", "forecast") == [ + {"category": "books", "total": total(60)}, {"category": "games", "total": total(50)}, + ] + class TestWorkspaceQdrantWorkflows: + def test_duckdb_and_qdrant(self, tmp_path: Path) -> None: + if not _native_interval.has_duckdb: + if os.environ.get("CHRONOS_CLI_REQUIRE_DUCKDB") == "ON": + pytest.fail("The CLI test matrix requires a DuckDB-enabled native build") + pytest.skip("DuckDB disabled in this native build") + pytest.importorskip("qdrant_client") + metadata = f"sqlite:///{tmp_path / 'metadata.sqlite'}" + database = DatabaseConfig(f"duckdb:///{tmp_path / 'data.duckdb'}", metadata) + vectors = QdrantConfig(metadata, path=tmp_path / "vectors") + config = _workspace_config_file(tmp_path, database, vectors, atomic=True) + _invoke_workspace(config, "table", "create", "CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)", "--store", "app") + _invoke_workspace(config, "table", "register", "items", "--primary-key", "id", "--store", "app") + _invoke_workspace(config, "qdrant", "--store", "vectors", "collection", "register", "documents", "--dimensions", "3") + _invoke_workspace(config, "branch", "create", "experiment") + _invoke_workspace(config, "execute", "INSERT INTO items (id, value) VALUES (1, 'candidate')", "--store", "app", "--branch", "experiment") + _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "upsert", "documents", "one", '{"vector":[1,0,0]}', "--branch", "experiment") + assert _invoke_workspace(config, "query", "SELECT * FROM items", "--store", "app") == [] + assert _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "get", "documents", "one") is None + preview = _invoke_workspace(config, "merge", "atomic-preview", "experiment", "main") + assert {name: len(result["changes"]) for name, result in preview["stores"].items()} == {"app": 1, "vectors": 1} + result = _invoke_workspace(config, "merge", "atomic-apply", "experiment", "main", "--operation-id", "duckdb-qdrant", "--preview-token", preview["preview_token"]) + assert result["status"] == "committed" + assert result["stores"] == {"app": 1, "vectors": 1} + assert _invoke_workspace(config, "query", "SELECT * FROM items", "--store", "app") == [{"id": 1, "value": "candidate"}] + assert _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "get", "documents", "one")["vector"] == [1, 0, 0] + @pytest.mark.parametrize("atomic", [False, True]) def test_two_store_workflow(self, qdrant_target: QdrantConfig, tmp_path: Path, atomic: bool) -> None: target = qdrant_target @@ -568,23 +501,48 @@ def test_two_store_workflow(self, qdrant_target: QdrantConfig, tmp_path: Path, a _invoke_workspace(config, "branch", "delete", "experiment") assert _invoke_workspace(config, "branch", "list") == ["main"] + def test_qdrant_cli_conflicts_and_failed_operations(self, qdrant_target: QdrantConfig, tmp_path: Path) -> None: + config = _qdrant_workspace_file(tmp_path, qdrant_target) + vectors = partial(_invoke_workspace, config, "qdrant", "--store", "vectors") + workspace = partial(_invoke_workspace, config) + vectors("collection", "register", "docs", "--dimensions", "3") + vectors("point", "upsert", "docs", "one", '{"vector":[1,0,0],"payload":{"value":"base"}}') + workspace("branch", "create", "trial") + workspace("branch", "create", "trial", ok=False) + vectors("point", "upsert", "docs", "one", '{"vector":[0,1,0],"payload":{"value":"trial"}}', "--branch", "trial") + vectors("point", "upsert", "docs", "one", '{"vector":[0,0,1],"payload":{"value":"main"}}') + assert len(workspace("merge", "preview", "trial", "main")["vectors"]["conflicts"]) == 1 + workspace("merge", "apply", "trial", "main", ok=False) + workspace("merge", "apply", "trial", "main", "--policy", "snapshot_isolation", ok=False) + vectors("point", "upsert", "docs", "bad", '{"vector":[1,0]}', ok=False) + assert vectors("point", "get", "docs", "bad") is None + assert vectors("point", "get", "docs", "one")["payload"] == {"value": "main"} + vectors("point", "get", "docs", "one", "--branch", "missing", ok=False) + vectors("collection", "info", "missing", ok=False) -def _prepare_workspace_merge(root: Path, *, atomic: bool) -> Path: - pytest.importorskip("qdrant_client") - database = DatabaseConfig(f"sqlite:///{root / 'app.sqlite'}") - target = QdrantConfig( - metadata_url=database.database_url if atomic else f"sqlite:///{root / 'vectors.sqlite'}", - path=root / "vectors", - ) - config = _workspace_config_file(root, database, target, atomic=atomic) - _invoke_workspace(config, "table", "create", "CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)", "--store", "app") - _invoke_workspace(config, "table", "register", "items", "--primary-key", "id", "--store", "app") - _invoke_workspace(config, "execute", "INSERT INTO items (id, value) VALUES (1, 'base')", "--store", "app") - _invoke_workspace(config, "qdrant", "--store", "vectors", "collection", "register", "documents", "--dimensions", "3") - _invoke_workspace(config, "branch", "create", "experiment") - _invoke_workspace(config, "execute", "UPDATE items SET value = 'candidate' WHERE id = 1", "--store", "app", "--branch", "experiment") - _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "upsert", "documents", "one", '{"vector":[1,0,0]}', "--branch", "experiment") - return config + def test_qdrant_cli_json_files_and_stdin(self, qdrant_target: QdrantConfig, tmp_path: Path) -> None: + config = _qdrant_workspace_file(tmp_path, qdrant_target) + vectors = partial(_invoke_workspace, config, "qdrant", "--store", "vectors") + workspace = partial(_invoke_workspace, config) + vectors("collection", "register", "docs", "--dimensions", "3") + data = tmp_path / "point.json" + data.write_text('{"vector":[1,0,0],"payload":{"source":"file"}}', encoding="utf-8") + result = vectors("point", "upsert", "docs", "one", "--file", str(data), "--revision", "initial") + assert result["payload"] == {"source": "file"} + store = _open_qdrant_api(qdrant_target) + try: + with store.context.checkout("main") as control: + session = store.checkout_control(control) + assert result == asdict(session.get("docs", "one")) + expected_search = [asdict(item) for item in session.search("docs", [1, 0, 0], limit=1)] + finally: + store.close() + assert vectors("search", "docs", "[1,0,0]", "--limit", "1") == expected_search + assert vectors("search", "docs", "--stdin", input_text="[1,0,0]")[0]["id"] == "one" + vectors("point", "upsert", "docs", "two", "--stdin", input_text='{"vector":[0,1,0]}') + data.write_text("[0,1,0]", encoding="utf-8") + assert vectors("search", "docs", "--file", str(data))[0]["id"] == "two" + assert vectors("search", "docs", input_text="[1,0,0]")[0]["id"] == "one" class TestWorkspaceFailureWorkflows: @@ -609,7 +567,7 @@ def fail_vector_store(*args, **kwargs): method = "stage_branch_transaction_changes" if atomic else "merge_apply" monkeypatch.setattr(ChronosQdrantStore, method, fail_vector_store) command = "atomic-apply" if atomic else "apply" - arguments = ["workspace", "--config", str(config), "merge", command, "experiment", "main"] + arguments = ["--config", str(config), "merge", command, "experiment", "main"] if atomic: arguments.extend(["--operation-id", "failed-merge"]) result = CliRunner().invoke(app, arguments) From 271b85b917a72d1424a3b8f61a6b952c109ea19f Mon Sep 17 00:00:00 2001 From: Seth Rojas Date: Fri, 6 Nov 2026 08:20:46 -0700 Subject: [PATCH 08/11] feat(cli): add OverlayFS workspace support --- .github/workflows/relational-package.yml | 9 ++ docs/cli.md | 71 ++++++++++- .../src/chronos_core/cli/_config.py | 36 +++++- .../src/chronos_core/cli/_filesystem.py | 55 ++++++++ .../src/chronos_core/cli/_terminal.py | 5 + .../src/chronos_core/cli/_workspace.py | 22 ++-- .../chronos-core/src/chronos_core/cli/main.py | 2 + tests/test_cli.py | 117 ++++++++++++++++++ tests/test_cli_integration.py | 71 +++++++++++ 9 files changed, 374 insertions(+), 14 deletions(-) create mode 100644 packages/chronos-core/src/chronos_core/cli/_filesystem.py diff --git a/.github/workflows/relational-package.yml b/.github/workflows/relational-package.yml index 6e8812e..e96a7b1 100644 --- a/.github/workflows/relational-package.yml +++ b/.github/workflows/relational-package.yml @@ -107,10 +107,19 @@ jobs: python -I -m pytest tests/test_cli.py::TestTerminalBehavior \ tests/test_cli.py::TestWorkspaceConfiguration \ tests/test_cli.py::TestWorkspaceCommands \ + tests/test_cli.py::TestWorkspaceFilesystemCommands \ tests/test_cli_integration.py::TestWorkspaceSQLWorkflows \ tests/test_cli_integration.py::TestWorkspaceResources::test_sqlite_workspace_persists_after_close \ tests/test_cli_integration.py::TestWorkspaceResources::test_sqlite_workspace_without_qdrant_dependency \ -q --timeout=60 + - name: Test workspace filesystem on Linux + if: matrix.python == '3.12' && matrix.duckdb == 'OFF' + env: + CHRONOS_CLI_REQUIRE_FILESYSTEM: 'ON' + run: | + sudo apt-get install -y fuse-overlayfs fuse3 + sudo modprobe fuse + python -I -m pytest tests/test_cli_integration.py::TestWorkspaceFilesystemWorkflows -q --timeout=120 - name: Test workspace DuckDB and Qdrant together if: matrix.duckdb == 'ON' env: diff --git a/docs/cli.md b/docs/cli.md index 04b76e4..6b8d4c3 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -136,7 +136,8 @@ chronos qdrant --store vectors point upsert documents one \ chronos qdrant --store vectors search documents '[1, 0, 0]' --branch experiment ``` -Results are the API's JSON values, including workspace store names and result +Except for `fs run`'s default command-output mode described below, results are +the API's JSON values, including workspace store names and result field names. `None` prints nothing. Dataclasses become objects, dates use ISO strings, decimals become strings, and bytes become tagged base64 objects. Expected operational errors go to stderr with exit code 1; invalid CLI arguments @@ -144,3 +145,71 @@ use exit code 2. Help opens no connections. JSON output can be piped to `jq` or redirected to a file. Non-finite floats returned by the API fail clearly because JSON cannot represent them; query execution and the API's returned values are not modified. + +## Filesystem workspace + +The filesystem participant wraps `ChronosFilesystemStore` and requires Linux, +`fuse-overlayfs`, `fusermount3` (or `fusermount`), and permission to use `/dev/fuse`. +It can be the only participant or accompany named SQL/Qdrant stores. Configure +an existing base directory and an explicit persistent state directory, preferably +outside the base directory. Relative paths resolve against the TOML file: + +```toml +[filesystem] +root = "base" +state_dir = ".chronos/filesystem" +``` + +Branch, checkpoint, and diff commands include this participant automatically. +Run ordinary programs against a branch using the existing session `run()` API: + +```sh +chronos branch create experiment +chronos fs run --branch experiment -- sh -c 'printf "hello\n" > report.txt' +chronos fs run --branch experiment -- cat report.txt +chronos checkpoint create saved --branch experiment +chronos diff main experiment +chronos merge preview experiment main --policy api-default +chronos merge apply experiment main --policy api-default +``` + +`fs run` defaults to branch `main` and working directory `.`. Options must precede +`-- COMMAND...`. `--cwd` selects a directory within the branch; `--timeout` passes +a timeout in seconds; `--env '{"NAME":"value"}'` replaces the child environment +(the API inherits the current environment when this option is omitted). Commands +are passed as argument lists; shell syntax requires explicitly running a shell. +The mounted working directory is not a security sandbox for the child process. + +Output is captured by the API and printed to stdout/stderr after completion. +The CLI exits with the child's exit code (signals use the shell convention +`128 + signal`). `--json` instead prints the API result fields `args`, `returncode`, +`stdout`, and `stderr`, while retaining that exit status. These child-generated +outputs are passed through as-is. Startup failures and timeouts exit with code 1. + +`--policy api-default` passes `None` to the workspace API. This is required for +the current filesystem merge implementation, which applies file-level changes +without three-way conflict detection. The existing `abort_on_conflict` default +and explicit policies remain unchanged; the API rejects them for this filesystem. +The CLI's OverlayFS backend does not support shared-metadata atomic workspaces. +These configurations are rejected before opening stores. The separate native +ChronosFS backend is not exposed by this CLI. Coordinated merges do not promise all-store rollback. + +Each invocation releases its mounts while preserving state and versioned files. +Use short-lived commands; this interface does not keep a mount alive for later +external use or background processes. For a disposable demo, keep the base, +configuration, and state inside one dedicated directory, and remove that directory +only after commands finish and its mounts are released. + +Filesystem integration tests live in `TestWorkspaceFilesystemWorkflows` in +`tests/test_cli_integration.py`. Run them in Linux with the dependencies above: + +```sh +CHRONOS_CLI_REQUIRE_FILESYSTEM=ON python -m pytest \ + tests/test_cli_integration.py::TestWorkspaceFilesystemWorkflows -q +``` + +For Docker, the test container needs `/dev/fuse`, `SYS_ADMIN`, and a mount-permitting +security profile (for example `--device /dev/fuse --cap-add SYS_ADMIN +--security-opt apparmor=unconfined`). Keep test data on the container's Linux +filesystem. The required-test flag prevents missing dependencies from counting +as a successful skipped run. diff --git a/packages/chronos-core/src/chronos_core/cli/_config.py b/packages/chronos-core/src/chronos_core/cli/_config.py index 6a0b083..bc106ea 100644 --- a/packages/chronos-core/src/chronos_core/cli/_config.py +++ b/packages/chronos-core/src/chronos_core/cli/_config.py @@ -54,11 +54,18 @@ def metadata_url(self) -> str: return connection.metadata_url or connection.database_url +@dataclass(frozen=True) +class FilesystemConfig: + root: Path + state_dir: Path + + @dataclass(frozen=True) class WorkspaceConfig: config_path: Path stores: dict[str, StoreConfig] shared_metadata_url: str | None = None + filesystem: FilesystemConfig | None = None def store(self, name: str, *, qdrant: bool = False) -> StoreConfig: store = self.stores.get(name) @@ -193,22 +200,31 @@ def load_workspace_config(path: Path | None = None) -> WorkspaceConfig: except tomllib.TOMLDecodeError as error: # Decoder messages can contain configuration values, including secrets. raise WorkspaceConfigError("Invalid TOML in workspace configuration.") from error - data = _table(raw, "configuration", {"workspace", "stores"}) + data = _table(raw, "configuration", {"workspace", "stores", "filesystem"}) workspace = _table(data.get("workspace", {}), "workspace", {"shared_metadata_url"}) shared = ( _database_url(workspace["shared_metadata_url"], config_path.parent, "workspace.shared_metadata_url") if "shared_metadata_url" in workspace else None ) - entries = data.get("stores") - if not isinstance(entries, dict) or not entries: - raise WorkspaceConfigError("Configure at least one named store in [stores.NAME].") + filesystem = None + if "filesystem" in data: + fields = _table(data["filesystem"], "filesystem", {"root", "state_dir"}) + filesystem = FilesystemConfig(*( + _local_path(_text(fields.get(key), f"filesystem.{key}"), config_path.parent, f"filesystem.{key}") + for key in ("root", "state_dir") + )) + if shared is not None: + raise WorkspaceConfigError("The OverlayFS backend does not support shared-metadata atomic workspaces.") + entries = data.get("stores", {}) + if not isinstance(entries, dict) or (not entries and filesystem is None): + raise WorkspaceConfigError("Configure at least one named store in [stores.NAME] or [filesystem].") # These names cannot be used by WorkspaceBranchSession, even via its stores mapping. from chronos_core.workspace.runtime import WorkspaceBranchSession reserved = {"branch_id", "fs", "stores", "transaction", *dir(WorkspaceBranchSession)} stores = {} for name, value in entries.items(): - if not name.strip() or name in reserved: + if not name.strip() or name in reserved or (filesystem is not None and name == "filesystem"): raise WorkspaceConfigError(f"Invalid or reserved workspace store name {name!r}.") stores[name] = _store_config(value, config_path.parent, f"stores.{name}") metadata_urls = [store.metadata_url for store in stores.values()] @@ -222,7 +238,7 @@ def load_workspace_config(path: Path | None = None) -> WorkspaceConfig: identities = [_metadata_identity(url, config_path.parent) for url in metadata_urls] if shared is None and len(set(identities)) != len(identities): raise WorkspaceConfigError("Stores sharing metadata require workspace.shared_metadata_url.") - return WorkspaceConfig(config_path, stores, shared) + return WorkspaceConfig(config_path, stores, shared, filesystem) @contextmanager @@ -273,9 +289,17 @@ def close(resource: Any) -> None: ) resources.callback(close, store) stores[name] = store + filesystem_options: dict[str, Any] = {} + if config.filesystem is not None: + from chronos_core.workspace import ChronosFilesystemStore + + filesystem = ChronosFilesystemStore(config.filesystem.root, state_dir=config.filesystem.state_dir) + resources.callback(close, filesystem) + filesystem_options["filesystem"] = filesystem workspace = ChronosWorkspaceContext( stores={name: stores[name] for name in config.stores}, shared_metadata_url=config.shared_metadata_url, + **filesystem_options, ) # ExitStack owns the participants directly because workspace.close() # stops at the first close failure instead of attempting all stores. diff --git a/packages/chronos-core/src/chronos_core/cli/_filesystem.py b/packages/chronos-core/src/chronos_core/cli/_filesystem.py new file mode 100644 index 0000000..7e41e0c --- /dev/null +++ b/packages/chronos-core/src/chronos_core/cli/_filesystem.py @@ -0,0 +1,55 @@ +"""Terminal access to the workspace's public filesystem session API.""" +from __future__ import annotations + +import json +import math +import subprocess + +import typer + +from chronos_core.cli._terminal import print_result +from chronos_core.cli._workspace import connect_workspace, workspace_session + +app = typer.Typer(no_args_is_help=True) + + +@app.command("run", context_settings={"allow_interspersed_args": False}) +def run( + ctx: typer.Context, + argv: list[str] = typer.Argument(..., help="Command and arguments; use -- before the command."), + branch: str = typer.Option("main", "--branch"), + cwd: str = typer.Option(".", "--cwd", help="Working directory relative to the branch mount."), + env: str | None = typer.Option(None, "--env", help="JSON object replacing the child environment; omitted inherits it."), + timeout: float | None = typer.Option(None, "--timeout", min=0.001), + as_json: bool = typer.Option(False, "--json", help="Print args, returncode, stdout, and stderr as JSON."), +) -> None: + """Run a command, print captured output after completion, then release mounts.""" + if timeout is not None and not math.isfinite(timeout): + raise typer.BadParameter("Expected a finite timeout.", param_hint="--timeout") + environment = None + if env is not None: + try: + environment = json.loads(env) + except ValueError as error: + raise typer.BadParameter("Expected a JSON object of string values.", param_hint="--env") from error + if not isinstance(environment, dict) or not all(isinstance(value, str) for value in environment.values()): + raise typer.BadParameter("Expected a JSON object of string values.", param_hint="--env") + with connect_workspace(ctx) as workspace: + if workspace.filesystem is None: + raise typer.BadParameter("Configure [filesystem] to run filesystem commands.") + try: + with workspace_session(workspace, branch) as checked_out: + result = checked_out.fs.run(argv, cwd=cwd, env=environment, timeout=timeout) + except subprocess.TimeoutExpired: + # Avoid echoing argv: it can contain credentials supplied to the child. + typer.echo("Error: filesystem command timed out.", err=True) + raise typer.Exit(1) from None + if as_json: + print_result({"args": result.args, "returncode": result.returncode, + "stdout": result.stdout, "stderr": result.stderr}) + else: + typer.echo(result.stdout, nl=False, color=True) + typer.echo(result.stderr, err=True, nl=False, color=True) + # subprocess uses negative return codes for signals; shells use 128+signal. + status = result.returncode if result.returncode >= 0 else 128 - result.returncode + raise typer.Exit(status) diff --git a/packages/chronos-core/src/chronos_core/cli/_terminal.py b/packages/chronos-core/src/chronos_core/cli/_terminal.py index 90e9814..e6b1e07 100644 --- a/packages/chronos-core/src/chronos_core/cli/_terminal.py +++ b/packages/chronos-core/src/chronos_core/cli/_terminal.py @@ -15,9 +15,14 @@ class MergePolicy(str, Enum): + api_default = "api-default" abort_on_conflict = "abort_on_conflict" snapshot_isolation = "snapshot_isolation" + @property + def api_value(self) -> str | None: + return None if self is MergePolicy.api_default else self.value + def _json_default(value: Any) -> Any: if is_dataclass(value) and not isinstance(value, type): diff --git a/packages/chronos-core/src/chronos_core/cli/_workspace.py b/packages/chronos-core/src/chronos_core/cli/_workspace.py index 60ece27..9f113ee 100644 --- a/packages/chronos-core/src/chronos_core/cli/_workspace.py +++ b/packages/chronos-core/src/chronos_core/cli/_workspace.py @@ -17,6 +17,7 @@ if TYPE_CHECKING: from chronos_core.workspace import ChronosWorkspaceContext + from chronos_core.workspace.runtime import WorkspaceBranchSession @dataclass(frozen=True) @@ -45,8 +46,9 @@ def connect_workspace( raise typer.BadParameter(str(error), param_hint="--store") from error from chronos_core.branching import BranchingError + from chronos_core.workspace.filesystem import FilesystemStoreError - errors = (BranchingError, RuntimeError, ValueError, TypeError, OSError, sqlite3.Error, psycopg.Error) + errors = (FilesystemStoreError, BranchingError, RuntimeError, ValueError, TypeError, OSError, sqlite3.Error, psycopg.Error) secrets: list[str | None] = [] urls: list[str | None] = [] for entry in config.stores.values(): @@ -73,11 +75,11 @@ def connect_workspace( @contextmanager -def store_session(workspace: ChronosWorkspaceContext, branch: str, store: str) -> Generator[Any, None, None]: +def workspace_session(workspace: ChronosWorkspaceContext, branch: str) -> Generator[WorkspaceBranchSession, None, None]: checked_out = workspace.checkout(branch) errors: list[Exception] = [] try: - yield checked_out.stores[store] + yield checked_out finally: # SQL sessions expose close(). Qdrant control sessions without a public # close method are released by their owning context when it closes. @@ -92,6 +94,12 @@ def store_session(workspace: ChronosWorkspaceContext, branch: str, store: str) - raise errors[0] +@contextmanager +def store_session(workspace: ChronosWorkspaceContext, branch: str, store: str) -> Generator[Any, None, None]: + with workspace_session(workspace, branch) as checked_out: + yield checked_out.stores[store] + + @branch_app.command("list") def list_branches(ctx: typer.Context) -> None: with connect_workspace(ctx) as workspace: @@ -131,21 +139,21 @@ def diff(ctx: typer.Context, left: str, right: str) -> None: def merge_preview(ctx: typer.Context, source: str, target: str, policy: MergePolicy = typer.Option(MergePolicy.abort_on_conflict, "--policy")) -> None: with connect_workspace(ctx) as workspace: - print_result(workspace.merge_preview(source, target, policy=policy.value)) + print_result(workspace.merge_preview(source, target, policy=policy.api_value)) @merge_app.command("apply") def merge_apply(ctx: typer.Context, source: str, target: str, policy: MergePolicy = typer.Option(MergePolicy.abort_on_conflict, "--policy")) -> None: with connect_workspace(ctx) as workspace: - print_result(workspace.merge_apply(source, target, policy=policy.value)) + print_result(workspace.merge_apply(source, target, policy=policy.api_value)) @merge_app.command("atomic-preview") def merge_atomic_preview(ctx: typer.Context, source: str, target: str, policy: MergePolicy = typer.Option(MergePolicy.abort_on_conflict, "--policy")) -> None: with connect_workspace(ctx) as workspace: - print_result(workspace.merge_atomic_preview(source, target, policy=policy.value)) + print_result(workspace.merge_atomic_preview(source, target, policy=policy.api_value)) @merge_app.command("atomic-apply") @@ -157,7 +165,7 @@ def merge_atomic( ) -> None: with connect_workspace(ctx) as workspace: print_result(workspace.merge_atomic( - source, target, operation_id=operation_id, preview_token=preview_token, policy=policy.value, + source, target, operation_id=operation_id, preview_token=preview_token, policy=policy.api_value, )) diff --git a/packages/chronos-core/src/chronos_core/cli/main.py b/packages/chronos-core/src/chronos_core/cli/main.py index 52de987..48778a4 100644 --- a/packages/chronos-core/src/chronos_core/cli/main.py +++ b/packages/chronos-core/src/chronos_core/cli/main.py @@ -5,11 +5,13 @@ import typer +from chronos_core.cli._filesystem import app as filesystem_app from chronos_core.cli._qdrant import app as qdrant_app from chronos_core.cli._relational import register_commands as register_relational_commands from chronos_core.cli._workspace import WorkspaceOptions, register_commands as register_workspace_commands app = typer.Typer(no_args_is_help=True, pretty_exceptions_enable=False) +app.add_typer(filesystem_app, name="fs", help="Run commands in the workspace filesystem.") register_workspace_commands(app) register_relational_commands(app) app.add_typer(qdrant_app, name="qdrant", help="Use a named Qdrant participant.") diff --git a/tests/test_cli.py b/tests/test_cli.py index da7cd2c..61d76e2 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -584,3 +584,120 @@ def test_json_file_and_stdin(self, workspace: MagicMock, tmp_path: Path) -> None result = runner.invoke(app, ["qdrant", "--store", "vectors", "search", "docs", "--file", str(data)]) assert result.exit_code == 0, result.output assert session.search.call_args.args == ("docs", [1, 0, 0]) + + +class TestWorkspaceFilesystemCommands: + def test_config_and_resource_ownership(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + from chronos_core import workspace as api + + config_file = tmp_path / 'chronos.toml' + config_file.write_text('[filesystem]\nroot="base"\nstate_dir="state"\n') + config = load_workspace_config(config_file) + assert config.stores == {} + assert config.filesystem.root == tmp_path / 'base' + assert config.filesystem.state_dir == tmp_path / 'state' + factory = MagicMock() + monkeypatch.setattr(api, 'ChronosFilesystemStore', factory) + with open_workspace(config) as workspace: + assert workspace.filesystem is factory.return_value + factory.assert_called_once_with(tmp_path / 'base', state_dir=tmp_path / 'state') + factory.return_value.close.assert_called_once_with() + factory.return_value.cleanup.assert_not_called() + + @pytest.mark.parametrize('extra', [ + '[workspace]\nshared_metadata_url="sqlite:///meta.sqlite"\n', + '[stores.filesystem]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"\n', + ]) + def test_unsupported_config(self, tmp_path: Path, extra: str) -> None: + config_file = tmp_path / 'chronos.toml' + config_file.write_text('[filesystem]\nroot="base"\nstate_dir="state"\n' + extra) + with pytest.raises(WorkspaceConfigError): + load_workspace_config(config_file) + + def test_state_directory_is_required(self, tmp_path: Path) -> None: + config_file = tmp_path / 'chronos.toml' + config_file.write_text('[filesystem]\nroot="base"\n') + with pytest.raises(WorkspaceConfigError, match='state_dir'): + load_workspace_config(config_file) + + @pytest.mark.parametrize('as_json', [False, True]) + def test_run_forwards_arguments_and_output(self, workspace_connection: MagicMock, as_json: bool) -> None: + workspace = workspace_connection + command = ['example', '--child-option', 'two words'] + filesystem = workspace.checkout.return_value.fs + filesystem.run.return_value = subprocess.CompletedProcess(command, 7, '\x1b[32moutput\x1b[0m\n', '\x1b[31merror\x1b[0m\n') + args = ['fs', 'run', '--branch', 'feature', '--cwd', 'nested', '--env', '{"MODE":"test"}', '--timeout', '2'] + if as_json: + args.append('--json') + result = runner.invoke(app, [*args, '--', *command]) + assert result.exit_code == 7, result.output + filesystem.run.assert_called_once_with(command, cwd='nested', env={'MODE': 'test'}, timeout=2.0) + workspace.checkout.assert_called_once_with('feature') + if as_json: + assert json.loads(result.stdout) == dict(args=command, returncode=7, stdout='\x1b[32moutput\x1b[0m\n', stderr='\x1b[31merror\x1b[0m\n') + assert result.stderr == '' + else: + assert result.stdout == '\x1b[32moutput\x1b[0m\n' + assert result.stderr == '\x1b[31merror\x1b[0m\n' + for session in workspace.checkout.return_value.stores.values(): + session.close.assert_called_once_with() + workspace.close.assert_called_once_with() + + @pytest.mark.parametrize('args', [[], ['--env', '[]', '--', 'echo'], ['--env', '{"X":1}', '--', 'echo'], + ['--timeout', 'nan', '--', 'echo']]) + def test_invalid_input_before_connection(self, workspace_connection: MagicMock, args: list[str]) -> None: + result = runner.invoke(app, ['fs', 'run', *args]) + assert result.exit_code == 2 + _workspace.open_workspace.assert_not_called() + + @pytest.mark.parametrize('error', [FileNotFoundError('missing command'), subprocess.TimeoutExpired(['secret'], 1)]) + def test_run_failure_closes_resources(self, workspace_connection: MagicMock, error: Exception) -> None: + workspace_connection.checkout.return_value.fs.run.side_effect = error + result = runner.invoke(app, ['fs', 'run', '--', 'secret']) + assert result.exit_code == 1 + assert 'secret' not in result.stderr + workspace_connection.close.assert_called_once_with() + for session in workspace_connection.checkout.return_value.stores.values(): + session.close.assert_called_once_with() + + def test_no_filesystem(self, workspace_connection: MagicMock) -> None: + workspace_connection.filesystem = None + result = runner.invoke(app, ['fs', 'run', '--', 'echo']) + assert result.exit_code == 2 + workspace_connection.checkout.assert_not_called() + workspace_connection.close.assert_called_once_with() + + @pytest.mark.parametrize('operation', ['preview', 'apply', 'atomic-preview', 'atomic-apply']) + def test_api_default_merge_policy(self, workspace_connection: MagicMock, operation: str) -> None: + args = ['merge', operation, 'feature', 'main', '--policy', 'api-default'] + if operation == 'atomic-apply': + args.extend(['--operation-id', 'operation']) + result = runner.invoke(app, args) + assert result.exit_code == 0, result.output + name = {'preview': 'merge_preview', 'apply': 'merge_apply', + 'atomic-preview': 'merge_atomic_preview', 'atomic-apply': 'merge_atomic'}[operation] + assert getattr(workspace_connection, name).call_args.kwargs['policy'] is None + + def test_signal_exit_and_api_defaults(self, workspace_connection: MagicMock) -> None: + filesystem = workspace_connection.checkout.return_value.fs + filesystem.run.return_value = subprocess.CompletedProcess(['command'], -15, '', '') + result = runner.invoke(app, ['fs', 'run', '--', 'command']) + assert result.exit_code == 143 + filesystem.run.assert_called_once_with(['command'], cwd='.', env=None, timeout=None) + workspace_connection.checkout.assert_called_once_with('main') + + def test_help_needs_no_config(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.chdir(tmp_path) + monkeypatch.delenv('CHRONOS_WORKSPACE_CONFIG', raising=False) + result = runner.invoke(app, ['fs', 'run', '--help']) + assert result.exit_code == 0 + assert '--json' in result.stdout + + def test_filesystem_error_is_reported(self, workspace_connection: MagicMock) -> None: + from chronos_core.workspace.filesystem import FilesystemStoreError + + workspace_connection.checkout.side_effect = FilesystemStoreError('failed to mount branch') + result = runner.invoke(app, ['fs', 'run', '--', 'command']) + assert result.exit_code == 1 + assert 'failed to mount branch' in result.stderr + workspace_connection.close.assert_called_once_with() diff --git a/tests/test_cli_integration.py b/tests/test_cli_integration.py index 45ab965..6632ce0 100644 --- a/tests/test_cli_integration.py +++ b/tests/test_cli_integration.py @@ -590,3 +590,74 @@ def test_conflict_blocks_atomic_merge(self, tmp_path: Path) -> None: _invoke_workspace(config, "merge", "atomic-apply", "experiment", "main", "--operation-id", "conflicting", ok=False) assert _invoke_workspace(config, "query", "SELECT value FROM items", "--store", "app") == [{"value": "main change"}] assert _invoke_workspace(config, "qdrant", "--store", "vectors", "point", "get", "documents", "one") is None + + +class TestWorkspaceFilesystemWorkflows: + @pytest.mark.parametrize('with_sql', [False, True]) + def test_persistent_workspace(self, tmp_path: Path, with_sql: bool) -> None: + import shutil + + available = (sys.platform == 'linux' and shutil.which('fuse-overlayfs') + and (shutil.which('fusermount3') or shutil.which('fusermount')) + and Path('/dev/fuse').exists()) + if not available: + if os.environ.get('CHRONOS_CLI_REQUIRE_FILESYSTEM') == 'ON': + pytest.fail('Filesystem integration requires Linux, /dev/fuse, and fuse-overlayfs') + pytest.skip('Filesystem integration requires Linux FUSE') + base = tmp_path / 'base' + base.mkdir() + (base / 'report.txt').write_text('original') + config = tmp_path / 'chronos.toml' + config.write_text('[filesystem]\nroot="base"\nstate_dir="state"\n' + ( + '[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"\n' if with_sql else '' + )) + + def run(branch: str, code: str) -> dict[str, Any]: + return _invoke_workspace(config, 'fs', 'run', '--branch', branch, '--json', '--', sys.executable, '-c', code) + + assert _invoke_workspace(config, 'branch', 'list') == ['main'] + if with_sql: + _invoke_workspace(config, 'table', 'create', '--store', 'app', 'CREATE TABLE docs (id INTEGER PRIMARY KEY)') + _invoke_workspace(config, 'table', 'register', '--store', 'app', 'docs', '--primary-key', 'id') + _invoke_workspace(config, 'branch', 'create', 'feature') + run('feature', 'from pathlib import Path; Path("report.txt").write_text("changed")') + assert run('main', 'print(open("report.txt").read())')['stdout'] == 'original\n' + assert run('feature', 'print(open("report.txt").read())')['stdout'] == 'changed\n' + if with_sql: + _invoke_workspace(config, 'execute', '--store', 'app', '--branch', 'feature', 'INSERT INTO docs (id) VALUES (1)') + checkpoint = _invoke_workspace(config, 'checkpoint', 'create', 'saved', '--branch', 'feature') + assert 'filesystem' in checkpoint + _invoke_workspace(config, 'branch', 'create-from-checkpoint', 'restored', 'saved') + assert run('restored', 'print(open("report.txt").read())')['stdout'] == 'changed\n' + diff = _invoke_workspace(config, 'diff', 'main', 'feature') + with open_workspace(load_workspace_config(config)) as workspace: + assert diff == json.loads(json.dumps({key: asdict(value) for key, value in workspace.diff('main', 'feature').items()})) + assert diff['filesystem']['changes'][0]['path'] == 'report.txt' + # Preserve the API's refusal of policy-aware filesystem merging. + _invoke_workspace(config, 'merge', 'preview', 'feature', 'main', ok=False) + _invoke_workspace(config, 'merge', 'apply', 'feature', 'main', ok=False) + preview = _invoke_workspace(config, 'merge', 'preview', 'feature', 'main', '--policy', 'api-default') + assert preview['filesystem'] == diff['filesystem'] + merged = _invoke_workspace(config, 'merge', 'apply', 'feature', 'main', '--policy', 'api-default') + assert merged['filesystem']['applied'] == 1 + assert run('main', 'print(open("report.txt").read())')['stdout'] == 'changed\n' + if with_sql: + assert _invoke_workspace(config, 'query', '--store', 'app', 'SELECT * FROM docs') == [{'id': 1}] + _invoke_workspace(config, 'branch', 'delete', 'restored') + _invoke_workspace(config, 'branch', 'delete', 'feature') + assert _invoke_workspace(config, 'branch', 'list') == ['main'] + assert (base / 'report.txt').read_text() == 'original' + assert (tmp_path / 'state' / 'metadata.json').exists() + assert str(tmp_path / 'state' / 'mounts') not in Path('/proc/self/mountinfo').read_text() + + environment = {key: value for key, value in os.environ.items() if not key.startswith('CHRONOS_')} + command = [sys.executable, '-m', 'chronos_core.cli', '--config', str(config), 'fs', 'run'] + failed = subprocess.run([*command, '--', sys.executable, '-c', + 'import sys; print("out"); print("err", file=sys.stderr); sys.exit(7)'], + capture_output=True, text=True, env=environment, timeout=30) + assert (failed.returncode, failed.stdout, failed.stderr) == (7, 'out\n', 'err\n') + for args in [ ['--cwd', '..', '--', 'true'], ['--timeout', '0.01', '--', sys.executable, '-c', 'import time; time.sleep(1)'] ]: + failed = subprocess.run([*command, *args], capture_output=True, text=True, env=environment, timeout=30) + assert failed.returncode == 1 + assert 'Traceback' not in failed.stderr + assert str(tmp_path / 'state' / 'mounts') not in Path('/proc/self/mountinfo').read_text() From db87206840689dd9341845f09027648d7dc74317 Mon Sep 17 00:00:00 2001 From: Seth Rojas Date: Fri, 6 Nov 2026 09:27:38 -0700 Subject: [PATCH 09/11] feat(cli): expose native ChronosFS and scoped atomic merges --- .github/workflows/relational-package.yml | 14 +- docs/cli.md | 148 ++++++++++++++++-- .../src/chronos_core/cli/_config.py | 87 ++++++++-- .../src/chronos_core/cli/_filesystem.py | 85 ++++++++++ .../src/chronos_core/cli/_workspace.py | 11 +- .../chronos-core/src/chronos_core/cli/main.py | 2 +- tests/test_cli.py | 96 ++++++++++++ tests/test_cli_integration.py | 147 +++++++++++++++++ 8 files changed, 551 insertions(+), 39 deletions(-) diff --git a/.github/workflows/relational-package.yml b/.github/workflows/relational-package.yml index e96a7b1..1cc5d91 100644 --- a/.github/workflows/relational-package.yml +++ b/.github/workflows/relational-package.yml @@ -80,20 +80,23 @@ jobs: include: - python: '3.10' duckdb: 'OFF' + filesystem: 'OFF' - python: '3.12' duckdb: 'OFF' + filesystem: 'ON' - python: '3.12' duckdb: 'ON' + filesystem: 'OFF' steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python }} - - run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev libpq-dev libboost-dev + - run: sudo apt-get update && sudo apt-get install -y libsqlite3-dev libpq-dev libboost-dev libfuse3-dev - run: python -m pip install build pytest pytest-asyncio pytest-timeout - name: Build from source distribution and install the wheel run: | - python -m build packages/chronos-core --outdir /tmp/chronos-wheel -Cbuild-dir=/tmp/chronos-build -Cbuild.tool-args=-j2 -Ccmake.define.CHRONOS_WITH_DUCKDB=${{ matrix.duckdb }} + python -m build packages/chronos-core --outdir /tmp/chronos-wheel -Cbuild-dir=/tmp/chronos-build -Cbuild.tool-args=-j2 -Ccmake.define.CHRONOS_WITH_DUCKDB=${{ matrix.duckdb }} -Ccmake.define.CHRONOS_WITH_FILESYSTEM=${{ matrix.filesystem }} python -m pip install /tmp/chronos-wheel/*.whl - name: Test the installed relational artifact run: | @@ -120,6 +123,13 @@ jobs: sudo apt-get install -y fuse-overlayfs fuse3 sudo modprobe fuse python -I -m pytest tests/test_cli_integration.py::TestWorkspaceFilesystemWorkflows -q --timeout=120 + - name: Test native ChronosFS CLI + if: matrix.filesystem == 'ON' + env: + CHRONOS_CLI_REQUIRE_CHRONOSFS: 'ON' + run: | + python -I -m pytest tests/test_cli.py -k native -q --timeout=60 + python -I -m pytest tests/test_cli_integration.py::TestNativeChronosFSWorkflows -q --timeout=120 - name: Test workspace DuckDB and Qdrant together if: matrix.duckdb == 'ON' env: diff --git a/docs/cli.md b/docs/cli.md index 6b8d4c3..d33b2fc 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -46,18 +46,22 @@ kind = "sqlite" database_url = "sqlite:///.chronos/app.sqlite" ``` -Create `.chronos/` first. Local database URLs and Qdrant paths resolve relative +Save the example as `chronos.toml` and create `.chronos/` first. Local database URLs and Qdrant paths resolve relative to the TOML file. Configuration precedence is `--config PATH`, then `CHRONOS_WORKSPACE_CONFIG`, then `chronos.toml` in the current directory. A workspace name is not required. Connection strings may reference `${ENV_VAR}`; unset references are errors. `.env` files are not loaded automatically. +From the repository root, run the commands below. If `chronos` is already on +your PATH, you can use it directly in place of the `uv run` invocation. + ```sh -chronos table create --store app \ +mkdir -p .chronos +uv run --package chronos-core chronos table create --store app \ 'CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)' -chronos table register --store app items --primary-key id -chronos branch create experiment -chronos query --store app --branch experiment 'SELECT * FROM items' +uv run --package chronos-core chronos table register --store app items --primary-key id +uv run --package chronos-core chronos branch create experiment +uv run --package chronos-core chronos query --store app --branch experiment 'SELECT * FROM items' ``` SQL participants support `sqlite`, `postgresql`, and `duckdb` kinds, with @@ -70,14 +74,13 @@ Branches, checkpoints, diffs, and merges operate across the workspace. With independent metadata, `merge preview/apply` calls the coordinated API, which can retain partial changes if a later participant fails. Atomic mode requires `[workspace].shared_metadata_url` and matching metadata URLs for every store. -This CLI version supports one SQL participant in atomic mode; multiple SQL -participants can use independent metadata. Shared-metadata SQL participants -need per-table merge scopes, and the current API's diff does not honor those -scopes, so the CLI rejects that topology rather than implementing custom behavior. -Use `merge atomic-preview` and `merge atomic-apply --operation-id ID`; optionally -pass the returned `--preview-token` to reject a changed preview. Atomic mode -does not silently fall back to coordinated merging. Filesystems are not exposed -by the workspace CLI yet. +Multiple SQL participants sharing metadata require a nonempty `merge_tables` +scope for each store. See the atomic SQL example below. Use +`merge atomic-preview` to inspect the candidate changes and +`merge atomic-apply --operation-id ID` to publish them. Pass the returned +`--preview-token` to reject a changed preview. Ordinary `diff` can fail in +split-data workspaces because the Python API's diff does not use merge scopes. +See the filesystem sections below for OverlayFS and native ChronosFS setup. ## Checked terminal demo @@ -146,9 +149,9 @@ redirected to a file. Non-finite floats returned by the API fail clearly because JSON cannot represent them; query execution and the API's returned values are not modified. -## Filesystem workspace +## OverlayFS workspace -The filesystem participant wraps `ChronosFilesystemStore` and requires Linux, +The OverlayFS participant wraps `ChronosFilesystemStore` and requires Linux, `fuse-overlayfs`, `fusermount3` (or `fusermount`), and permission to use `/dev/fuse`. It can be the only participant or accompany named SQL/Qdrant stores. Configure an existing base directory and an explicit persistent state directory, preferably @@ -191,8 +194,7 @@ the current filesystem merge implementation, which applies file-level changes without three-way conflict detection. The existing `abort_on_conflict` default and explicit policies remain unchanged; the API rejects them for this filesystem. The CLI's OverlayFS backend does not support shared-metadata atomic workspaces. -These configurations are rejected before opening stores. The separate native -ChronosFS backend is not exposed by this CLI. Coordinated merges do not promise all-store rollback. +These configurations are rejected before opening stores. Coordinated merges do not promise all-store rollback. Each invocation releases its mounts while preserving state and versioned files. Use short-lived commands; this interface does not keep a mount alive for later @@ -213,3 +215,115 @@ security profile (for example `--device /dev/fuse --cap-add SYS_ADMIN --security-opt apparmor=unconfined`). Keep test data on the container's Linux filesystem. The required-test flag prevents missing dependencies from counting as a successful skipped run. + +## Native ChronosFS workspace + +Native ChronosFS stores versioned files in Chronos. Use a Linux package built +with `CHRONOS_WITH_FILESYSTEM=ON`. File commands use the Python API directly +and do not need a FUSE mount. Select one filesystem backend per workspace. +Omitting `kind` keeps the existing OverlayFS configuration behavior. + +```toml +[filesystem] +kind = "chronosfs" +database_url = "sqlite:///files.sqlite" +``` + +```sh +chronos branch create experiment +printf 'hello\n' | chronos fs write /notes/report.txt --branch experiment --parents --stdin +chronos fs ls /notes --branch experiment +chronos fs read /notes/report.txt --branch experiment +``` + +`fs read` writes raw bytes to stdout. `fs write` accepts either `--file PATH` +or `--stdin` and prints the API's result as JSON. `fs run` is available with +the OverlayFS backend. + +For an atomic workspace with SQLite, configure the SQL store and native +ChronosFS to use the same database: + +```toml +[workspace] +shared_metadata_url = "sqlite:///app.sqlite" + +[stores.app] +kind = "sqlite" +database_url = "sqlite:///app.sqlite" + +[filesystem] +kind = "chronosfs" +database_url = "sqlite:///app.sqlite" +``` + +Use the existing `merge atomic-preview` and `merge atomic-apply` commands to +publish SQL and filesystem changes together. The API preview can show the +filesystem changes both as interval rows under the SQL participant and as +file changes under the filesystem participant. + +To keep SQL rows and native file data in separate databases, put the SQL +participant's table scope in the workspace TOML: + +```toml +[workspace] +shared_metadata_url = "sqlite:///app.sqlite" + +[stores.app] +kind = "sqlite" +database_url = "sqlite:///app.sqlite" +merge_tables = ["items"] + +[filesystem] +kind = "chronosfs" +database_url = "sqlite:///files.sqlite" +metadata_url = "sqlite:///app.sqlite" +``` + +Omitting `merge_tables` or setting it to `[]` keeps the API's all-tables +default. Split atomic configurations that require explicit table ownership +need a nonempty list. The CLI passes that list to `set_merge_table_scope()` +when it opens the SQL store. Register tables separately with `chronos table register`. +Update the scope when you want another registered table included in merges. +This setting controls merge scope and does not change workspace `diff`. + +On Linux with FUSE access, you can also mount a native branch into an existing +directory: + +```sh +mkdir -p ./chronos-mount +chronos fs mount ./chronos-mount --branch experiment +``` + +The command stays in the foreground. Use the directory from another terminal +and unmount it there with `fusermount3 -u ./chronos-mount` when finished. +Repeat `--option` (or `-o`) to pass individual FUSE options to the Python API. + +## Atomic merge across SQL stores + +Declare each SQL store in the workspace TOML and point each one at the shared +metadata database. Give each store an explicit `merge_tables` list containing +the tables it should handle during an atomic merge: + +```toml +[workspace] +shared_metadata_url = "sqlite:///metadata.sqlite" + +[stores.app] +kind = "sqlite" +database_url = "sqlite:///app.sqlite" +metadata_url = "sqlite:///metadata.sqlite" +merge_tables = ["items"] + +[stores.audit] +kind = "sqlite" +database_url = "sqlite:///audit.sqlite" +metadata_url = "sqlite:///metadata.sqlite" +merge_tables = ["events"] +``` + +Register each table in its owning store. Use `merge atomic-preview` to inspect +the changes per store and `merge atomic-apply` to publish them together. +`merge_tables` is forwarded to the Python API's merge scope. To compare a +candidate for publication in this split setup, use atomic preview. The +ordinary workspace `diff` API does not use merge scopes and can fail when it +looks for a table owned by a different data database. diff --git a/packages/chronos-core/src/chronos_core/cli/_config.py b/packages/chronos-core/src/chronos_core/cli/_config.py index bc106ea..7ff42a4 100644 --- a/packages/chronos-core/src/chronos_core/cli/_config.py +++ b/packages/chronos-core/src/chronos_core/cli/_config.py @@ -45,6 +45,7 @@ class WorkspaceConfigError(ValueError): class StoreConfig: kind: Literal["sqlite", "postgresql", "duckdb", "qdrant"] connection: DatabaseConfig | QdrantConfig + merge_tables: tuple[str, ...] | None = None @property def metadata_url(self) -> str: @@ -65,7 +66,7 @@ class WorkspaceConfig: config_path: Path stores: dict[str, StoreConfig] shared_metadata_url: str | None = None - filesystem: FilesystemConfig | None = None + filesystem: FilesystemConfig | DatabaseConfig | None = None def store(self, name: str, *, qdrant: bool = False) -> StoreConfig: store = self.stores.get(name) @@ -142,7 +143,7 @@ def _store_config(value: Any, root: Path, field: str) -> StoreConfig: raise WorkspaceConfigError(f"{field} must be a TOML table.") kind = value.get("kind") if kind in ("sqlite", "postgresql", "duckdb"): - data = _table(value, field, {"kind", "database_url", "metadata_url"}) + data = _table(value, field, {"kind", "database_url", "metadata_url", "merge_tables"}) url = _database_url(data.get("database_url"), root, f"{field}.database_url", kind=kind) metadata = ( _database_url(data["metadata_url"], root, f"{field}.metadata_url") @@ -150,7 +151,15 @@ def _store_config(value: Any, root: Path, field: str) -> StoreConfig: ) if kind == "duckdb" and metadata is None: raise WorkspaceConfigError(f"{field}.metadata_url is required for DuckDB.") - return StoreConfig(kind, DatabaseConfig(url, metadata)) + merge_tables = None + if "merge_tables" in data: + tables = data["merge_tables"] + if not isinstance(tables, list) or any( + not isinstance(table, str) or not table.strip() for table in tables + ): + raise WorkspaceConfigError(f"{field}.merge_tables must be an array of non-empty table names.") + merge_tables = tuple(tables) or None + return StoreConfig(kind, DatabaseConfig(url, metadata), merge_tables) if kind != "qdrant": raise WorkspaceConfigError(f"{field}.kind must be sqlite, postgresql, duckdb, or qdrant.") data = _table(value, field, {"kind", "metadata_url", "path", "url", "api_key", "collection_prefix", "timeout"}) @@ -208,13 +217,26 @@ def load_workspace_config(path: Path | None = None) -> WorkspaceConfig: ) filesystem = None if "filesystem" in data: - fields = _table(data["filesystem"], "filesystem", {"root", "state_dir"}) - filesystem = FilesystemConfig(*( - _local_path(_text(fields.get(key), f"filesystem.{key}"), config_path.parent, f"filesystem.{key}") - for key in ("root", "state_dir") - )) - if shared is not None: - raise WorkspaceConfigError("The OverlayFS backend does not support shared-metadata atomic workspaces.") + fields = _table(data["filesystem"], "filesystem", {"kind", "root", "state_dir", "database_url", "metadata_url"}) + kind = fields.get("kind", "overlayfs") + if kind == "overlayfs": + _table(fields, "filesystem", {"kind", "root", "state_dir"}) + filesystem = FilesystemConfig(*( + _local_path(_text(fields.get(key), f"filesystem.{key}"), config_path.parent, f"filesystem.{key}") + for key in ("root", "state_dir") + )) + if shared is not None: + raise WorkspaceConfigError("The OverlayFS backend does not support shared-metadata atomic workspaces.") + elif kind == "chronosfs": + _table(fields, "filesystem", {"kind", "database_url", "metadata_url"}) + url = _database_url(fields.get("database_url"), config_path.parent, "filesystem.database_url") + metadata = ( + _database_url(fields["metadata_url"], config_path.parent, "filesystem.metadata_url") + if "metadata_url" in fields else None + ) + filesystem = DatabaseConfig(url, metadata) + else: + raise WorkspaceConfigError("filesystem.kind must be overlayfs or chronosfs.") entries = data.get("stores", {}) if not isinstance(entries, dict) or (not entries and filesystem is None): raise WorkspaceConfigError("Configure at least one named store in [stores.NAME] or [filesystem].") @@ -228,13 +250,27 @@ def load_workspace_config(path: Path | None = None) -> WorkspaceConfig: raise WorkspaceConfigError(f"Invalid or reserved workspace store name {name!r}.") stores[name] = _store_config(value, config_path.parent, f"stores.{name}") metadata_urls = [store.metadata_url for store in stores.values()] + if isinstance(filesystem, DatabaseConfig): + metadata_urls.append(filesystem.metadata_url or filesystem.database_url) + if shared is not None and any( + isinstance(store.connection, DatabaseConfig) + and store.connection.database_url != filesystem.database_url + and store.merge_tables is None + for store in stores.values() + ): + raise WorkspaceConfigError( + "SQL stores with separate data from native ChronosFS require nonempty merge_tables for atomic workspaces." + ) if shared is not None and any(url != shared for url in metadata_urls): raise WorkspaceConfigError("Every store must use workspace.shared_metadata_url for atomic operations.") - if shared is not None and sum(store.kind != "qdrant" for store in stores.values()) > 1: - raise WorkspaceConfigError( - "This CLI does not yet support multiple SQL participants sharing metadata. " - "Use independent metadata, or one SQL participant with Qdrant." - ) + sql_stores = {name: store for name, store in stores.items() if store.kind != "qdrant"} + if shared is not None and len(sql_stores) > 1: + unscoped = [name for name, store in sql_stores.items() if store.merge_tables is None] + if unscoped: + raise WorkspaceConfigError( + "Multiple SQL stores sharing metadata require nonempty merge_tables for each store. " + f"Missing for: {', '.join(unscoped)}." + ) identities = [_metadata_identity(url, config_path.parent) for url in metadata_urls] if shared is None and len(set(identities)) != len(identities): raise WorkspaceConfigError("Stores sharing metadata require workspace.shared_metadata_url.") @@ -271,6 +307,8 @@ def close(resource: Any) -> None: else: context = ChronosBranchContext.connect_split(database.database_url, database.metadata_url) resources.callback(close, context) + if entry.merge_tables is not None: + context.set_merge_table_scope(list(entry.merge_tables)) stores[name] = context for name, entry in config.stores.items(): target = entry.connection @@ -290,12 +328,29 @@ def close(resource: Any) -> None: resources.callback(close, store) stores[name] = store filesystem_options: dict[str, Any] = {} - if config.filesystem is not None: + if isinstance(config.filesystem, FilesystemConfig): from chronos_core.workspace import ChronosFilesystemStore filesystem = ChronosFilesystemStore(config.filesystem.root, state_dir=config.filesystem.state_dir) resources.callback(close, filesystem) filesystem_options["filesystem"] = filesystem + elif isinstance(config.filesystem, DatabaseConfig): + from chronos_core.workspace import ChronosFSStore + + target = config.filesystem + context = ( + ChronosBranchContext.connect_split(target.database_url, target.metadata_url) + if target.metadata_url and target.metadata_url != target.database_url + else ChronosBranchContext.connect(target.database_url) + ) + try: + filesystem = ChronosFSStore(context) + except BaseException: + close(context) + raise + resources.callback(close, filesystem) + filesystem.ensure() + filesystem_options["filesystem"] = filesystem workspace = ChronosWorkspaceContext( stores={name: stores[name] for name in config.stores}, shared_metadata_url=config.shared_metadata_url, diff --git a/packages/chronos-core/src/chronos_core/cli/_filesystem.py b/packages/chronos-core/src/chronos_core/cli/_filesystem.py index 7e41e0c..a0b9524 100644 --- a/packages/chronos-core/src/chronos_core/cli/_filesystem.py +++ b/packages/chronos-core/src/chronos_core/cli/_filesystem.py @@ -4,12 +4,19 @@ import json import math import subprocess +from collections.abc import Generator +from contextlib import contextmanager +from pathlib import Path +from typing import TYPE_CHECKING import typer from chronos_core.cli._terminal import print_result from chronos_core.cli._workspace import connect_workspace, workspace_session +if TYPE_CHECKING: + from chronos_core.workspace import ChronosFSBranchSession, ChronosFSStore, ChronosWorkspaceContext + app = typer.Typer(no_args_is_help=True) @@ -34,7 +41,11 @@ def run( raise typer.BadParameter("Expected a JSON object of string values.", param_hint="--env") from error if not isinstance(environment, dict) or not all(isinstance(value, str) for value in environment.values()): raise typer.BadParameter("Expected a JSON object of string values.", param_hint="--env") + from chronos_core.workspace import ChronosFSStore + with connect_workspace(ctx) as workspace: + if isinstance(workspace.filesystem, ChronosFSStore): + raise typer.BadParameter("fs run requires the OverlayFS backend.") if workspace.filesystem is None: raise typer.BadParameter("Configure [filesystem] to run filesystem commands.") try: @@ -53,3 +64,77 @@ def run( # subprocess uses negative return codes for signals; shells use 128+signal. status = result.returncode if result.returncode >= 0 else 128 - result.returncode raise typer.Exit(status) + + +@contextmanager +def _native_store(ctx: typer.Context) -> Generator[tuple[ChronosWorkspaceContext, ChronosFSStore], None, None]: + from chronos_core.workspace import ChronosFSStore + + with connect_workspace(ctx) as workspace: + if not isinstance(workspace.filesystem, ChronosFSStore): + raise typer.BadParameter("This command requires [filesystem] with kind = 'chronosfs'.") + yield workspace, workspace.filesystem + + +@contextmanager +def _native_session(ctx: typer.Context, branch: str) -> Generator[ChronosFSBranchSession, None, None]: + with _native_store(ctx) as (workspace, _): + with workspace_session(workspace, branch) as session: + yield session.fs + + +@app.command("mount") +def mount( + ctx: typer.Context, + mountpoint: Path = typer.Argument(..., exists=True, file_okay=False, resolve_path=True), + branch: str = typer.Option("main", "--branch"), + options: list[str] | None = typer.Option(None, "--option", "-o", help="FUSE option. May be repeated."), +) -> None: + """Mount a native ChronosFS branch in the foreground until unmounted.""" + from chronos_core.workspace import mount_chronosfs + + with _native_store(ctx) as (_, filesystem): + mount_chronosfs(filesystem, mountpoint, branch_id=branch, options=set(options or ())) + + +@app.command("ls") +def list_directory( + ctx: typer.Context, + path: str = typer.Argument("/"), + branch: str = typer.Option("main", "--branch"), +) -> None: + """List entries in a native ChronosFS directory.""" + with _native_session(ctx, branch) as filesystem: + print_result(filesystem.listdir(path)) + + +@app.command("read") +def read_file( + ctx: typer.Context, + path: str, + branch: str = typer.Option("main", "--branch"), +) -> None: + """Write native ChronosFS file bytes to stdout.""" + with _native_session(ctx, branch) as filesystem: + data = filesystem.read_file(path) + typer.get_binary_stream("stdout").write(data) + + +@app.command("write") +def write_file( + ctx: typer.Context, + path: str, + branch: str = typer.Option("main", "--branch"), + file: Path | None = typer.Option(None, "--file", exists=True, dir_okay=False, readable=True), + stdin: bool = typer.Option(False, "--stdin"), + parents: bool = typer.Option(False, "--parents", help="Create missing parent directories."), +) -> None: + """Write bytes to a native ChronosFS file.""" + if (file is not None) == stdin: + raise typer.BadParameter("Choose exactly one of --file or --stdin.") + try: + data = file.read_bytes() if file is not None else typer.get_binary_stream("stdin").read() + except OSError as error: + raise typer.BadParameter("Could not read the input bytes.") from error + with _native_session(ctx, branch) as filesystem: + print_result(filesystem.write_file(path, data, parents=parents)) diff --git a/packages/chronos-core/src/chronos_core/cli/_workspace.py b/packages/chronos-core/src/chronos_core/cli/_workspace.py index 9f113ee..5458850 100644 --- a/packages/chronos-core/src/chronos_core/cli/_workspace.py +++ b/packages/chronos-core/src/chronos_core/cli/_workspace.py @@ -12,7 +12,7 @@ import psycopg import typer -from chronos_core.cli._config import QdrantConfig, WorkspaceConfigError, load_workspace_config, open_workspace +from chronos_core.cli._config import DatabaseConfig, QdrantConfig, WorkspaceConfigError, load_workspace_config, open_workspace from chronos_core.cli._terminal import MergePolicy, print_result, report_error if TYPE_CHECKING: @@ -46,11 +46,16 @@ def connect_workspace( raise typer.BadParameter(str(error), param_hint="--store") from error from chronos_core.branching import BranchingError - from chronos_core.workspace.filesystem import FilesystemStoreError + from chronos_core.workspace import ChronosFSError, ChronosFSMountError, FilesystemStoreError - errors = (FilesystemStoreError, BranchingError, RuntimeError, ValueError, TypeError, OSError, sqlite3.Error, psycopg.Error) + errors = ( + ChronosFSError, ChronosFSMountError, FilesystemStoreError, BranchingError, + RuntimeError, ValueError, TypeError, OSError, sqlite3.Error, psycopg.Error, + ) secrets: list[str | None] = [] urls: list[str | None] = [] + if isinstance(config.filesystem, DatabaseConfig): + urls.extend((config.filesystem.database_url, config.filesystem.metadata_url)) for entry in config.stores.values(): target = entry.connection urls.append(target.metadata_url) diff --git a/packages/chronos-core/src/chronos_core/cli/main.py b/packages/chronos-core/src/chronos_core/cli/main.py index 48778a4..57a0724 100644 --- a/packages/chronos-core/src/chronos_core/cli/main.py +++ b/packages/chronos-core/src/chronos_core/cli/main.py @@ -11,7 +11,7 @@ from chronos_core.cli._workspace import WorkspaceOptions, register_commands as register_workspace_commands app = typer.Typer(no_args_is_help=True, pretty_exceptions_enable=False) -app.add_typer(filesystem_app, name="fs", help="Run commands in the workspace filesystem.") +app.add_typer(filesystem_app, name="fs", help="Use the configured workspace filesystem.") register_workspace_commands(app) register_relational_commands(app) app.add_typer(qdrant_app, name="qdrant", help="Use a named Qdrant participant.") diff --git a/tests/test_cli.py b/tests/test_cli.py index 61d76e2..84ba19c 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -701,3 +701,99 @@ def test_filesystem_error_is_reported(self, workspace_connection: MagicMock) -> assert result.exit_code == 1 assert 'failed to mount branch' in result.stderr workspace_connection.close.assert_called_once_with() + + +def test_native_filesystem_commands(workspace_connection: MagicMock, tmp_path: Path) -> None: + from chronos_core.workspace import ChronosFSStore + + workspace_connection.filesystem = MagicMock(spec=ChronosFSStore) + session = workspace_connection.checkout.return_value.fs + payload = b"\x00\xff\n" + source = tmp_path / "payload" + source.write_bytes(payload) + session.write_file.return_value = 12 + result = runner.invoke(app, ["fs", "write", "/dir/file", "--branch", "draft", "--file", str(source), "--parents"]) + assert result.exit_code == 0, result.output + assert json.loads(result.stdout) == 12 + workspace_connection.checkout.assert_called_with("draft") + session.write_file.assert_called_once_with("/dir/file", payload, parents=True) + + session.read_file.return_value = payload + result = runner.invoke(app, ["fs", "read", "/dir/file"]) + assert result.exit_code == 0, result.output + assert result.stdout_bytes == payload + session.read_file.assert_called_once_with("/dir/file") + + session.listdir.return_value = ["file"] + result = runner.invoke(app, ["fs", "ls", "/dir"]) + assert result.exit_code == 0, result.output + assert json.loads(result.stdout) == ["file"] + session.listdir.assert_called_once_with("/dir") + + result = runner.invoke(app, ["fs", "run", "--", "true"]) + assert result.exit_code == 2 + assert "requires the OverlayFS backend" in result.stderr + + +@pytest.mark.parametrize("args", [[], ["--file", __file__, "--stdin"]]) +def test_native_write_requires_one_input(workspace_connection: MagicMock, args: list[str]) -> None: + result = runner.invoke(app, ["fs", "write", "/file", *args]) + assert result.exit_code == 2 + workspace_connection.checkout.assert_not_called() + + +def test_native_filesystem_requires_matching_atomic_metadata(tmp_path: Path) -> None: + config = tmp_path / "chronos.toml" + config.write_text( + '[workspace]\nshared_metadata_url="sqlite:///shared.sqlite"\n' + '[filesystem]\nkind="chronosfs"\ndatabase_url="sqlite:///files.sqlite"\n' + ) + with pytest.raises(WorkspaceConfigError, match="Every store must use"): + load_workspace_config(config) + + +@pytest.mark.parametrize("scope", ["", "merge_tables=[]\n"]) +def test_native_atomic_split_sql_requires_explicit_scope(tmp_path: Path, scope: str) -> None: + config = tmp_path / "chronos.toml" + config.write_text( + '[workspace]\nshared_metadata_url="sqlite:///app.sqlite"\n' + '[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"\n' + + scope + '[filesystem]\nkind="chronosfs"\ndatabase_url="sqlite:///files.sqlite"\n' + 'metadata_url="sqlite:///app.sqlite"\n' + ) + with pytest.raises(WorkspaceConfigError, match="require nonempty merge_tables"): + load_workspace_config(config) + + +@pytest.mark.parametrize("scope", ["", "merge_tables=[]\n"]) +def test_default_merge_scope_includes_all_tables(tmp_path: Path, scope: str) -> None: + config = tmp_path / "chronos.toml" + config.write_text('[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"\n' + scope) + assert load_workspace_config(config).stores["app"].merge_tables is None + + +@pytest.mark.parametrize("mount_error", [False, True]) +def test_native_mount_delegates_and_closes( + workspace_connection: MagicMock, monkeypatch: pytest.MonkeyPatch, tmp_path: Path, mount_error: bool, +) -> None: + from chronos_core import workspace as api + + filesystem = MagicMock(spec=api.ChronosFSStore) + workspace_connection.filesystem = filesystem + mountpoint = tmp_path / "mount" + mountpoint.mkdir() + failure = api.ChronosFSMountError("mount failed") if mount_error else None + mount = MagicMock(side_effect=failure) + monkeypatch.setattr(api, "mount_chronosfs", mount) + + result = runner.invoke(app, [ + "fs", "mount", str(mountpoint), "--branch", "draft", "-o", "ro", "--option", "default_permissions", + ]) + assert result.exit_code == (1 if mount_error else 0), result.output + if mount_error: + assert "mount failed" in result.stderr + mount.assert_called_once_with( + filesystem, mountpoint.resolve(), branch_id="draft", options={"ro", "default_permissions"}, + ) + workspace_connection.checkout.assert_not_called() + workspace_connection.close.assert_called_once_with() diff --git a/tests/test_cli_integration.py b/tests/test_cli_integration.py index 6632ce0..9ca8a64 100644 --- a/tests/test_cli_integration.py +++ b/tests/test_cli_integration.py @@ -338,6 +338,48 @@ def test_sqlite_qdrant_workspace_reopens(self, tmp_path: Path, atomic: bool) -> class TestWorkspaceSQLWorkflows: + def test_atomic_merge_across_separate_sql_databases(self, tmp_path: Path) -> None: + config = tmp_path / "chronos.toml" + config.write_text(textwrap.dedent(""" + [workspace] + shared_metadata_url = "sqlite:///metadata.sqlite" + + [stores.app] + kind = "sqlite" + database_url = "sqlite:///app.sqlite" + metadata_url = "sqlite:///metadata.sqlite" + merge_tables = ["items"] + + [stores.audit] + kind = "sqlite" + database_url = "sqlite:///audit.sqlite" + metadata_url = "sqlite:///metadata.sqlite" + merge_tables = ["events"] + """)) + invoke = partial(_invoke_workspace, config) + tables = {"app": "items", "audit": "events"} + for store, table in tables.items(): + invoke("table", "create", f"CREATE TABLE {table} (id INTEGER PRIMARY KEY)", "--store", store) + invoke("table", "register", table, "--store", store, "--primary-key", "id") + invoke("branch", "create", "draft") + for store, table in tables.items(): + invoke("execute", f"INSERT INTO {table} (id) VALUES (1)", "--store", store, "--branch", "draft") + assert invoke("query", f"SELECT id FROM {table}", "--store", store) == [] + + preview = invoke("merge", "atomic-preview", "draft", "main") + assert { + store: [change["table"] for change in result["changes"]] + for store, result in preview["stores"].items() + } == {"app": ["items"], "audit": ["events"]} + result = invoke( + "merge", "atomic-apply", "draft", "main", "--operation-id", "separate-sql", + "--preview-token", preview["preview_token"], + ) + assert result["status"] == "committed" + assert result["stores"] == {"app": 1, "audit": 1} + for store, table in tables.items(): + assert invoke("query", f"SELECT id FROM {table}", "--store", store) == [{"id": 1}] + def test_nonfinite_api_result_is_not_printed_as_invalid_json(self, tmp_path: Path) -> None: config = _workspace_config_file(tmp_path, DatabaseConfig(f"sqlite:///{tmp_path / 'app.sqlite'}")) with open_workspace(load_workspace_config(config)) as workspace: @@ -661,3 +703,108 @@ def run(branch: str, code: str) -> dict[str, Any]: assert failed.returncode == 1 assert 'Traceback' not in failed.stderr assert str(tmp_path / 'state' / 'mounts') not in Path('/proc/self/mountinfo').read_text() + + +class TestNativeChronosFSWorkflows: + @pytest.fixture(autouse=True) + def require_native_filesystem(self) -> None: + if not hasattr(_native_interval, "NativeChronosFSStore"): + if os.environ.get("CHRONOS_CLI_REQUIRE_CHRONOSFS") == "ON": + pytest.fail("This test run requires a native ChronosFS build") + pytest.skip("Native ChronosFS is disabled in this build") + + @staticmethod + def file_command(config: Path, *args: str, payload: bytes | None = None) -> subprocess.CompletedProcess[bytes]: + return subprocess.run( + [sys.executable, "-m", "chronos_core.cli", "--config", str(config), "fs", *args], + input=payload, capture_output=True, timeout=30, + ) + + def test_native_only_persists_binary_file(self, tmp_path: Path) -> None: + config = tmp_path / "chronos.toml" + config.write_text('[filesystem]\nkind="chronosfs"\ndatabase_url="sqlite:///files.sqlite"\n') + payload = b"\x00\xff\n" + written = self.file_command(config, "write", "/dir/file", "--stdin", "--parents", payload=payload) + assert written.returncode == 0, written.stderr + assert _invoke_workspace(config, "fs", "ls", "/dir") == ["file"] + read = self.file_command(config, "read", "/dir/file") + assert read.returncode == 0, read.stderr + assert read.stdout == payload + + @pytest.mark.parametrize("split", [False, True], ids=["unified", "split-scoped"]) + def test_sql_and_native_filesystem_atomic_merge(self, tmp_path: Path, split: bool) -> None: + config = tmp_path / "chronos.toml" + sql_config = '[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"\n' + fs_config = '[filesystem]\nkind="chronosfs"\n' + if split: + sql_config += 'merge_tables=["items"]\n' + fs_config += 'database_url="sqlite:///files.sqlite"\nmetadata_url="sqlite:///app.sqlite"\n' + else: + fs_config += 'database_url="sqlite:///app.sqlite"\n' + config.write_text( + '[workspace]\nshared_metadata_url="sqlite:///app.sqlite"\n' + sql_config + fs_config + ) + invoke = partial(_invoke_workspace, config) + invoke("table", "create", "CREATE TABLE items (id INTEGER PRIMARY KEY)", "--store", "app") + invoke("table", "register", "items", "--primary-key", "id", "--store", "app") + invoke("branch", "create", "draft") + invoke("execute", "INSERT INTO items (id) VALUES (1)", "--store", "app", "--branch", "draft") + written = self.file_command(config, "write", "/report", "--branch", "draft", "--stdin", payload=b"draft\n") + assert written.returncode == 0, written.stderr + assert invoke("query", "SELECT * FROM items", "--store", "app") == [] + assert "report" not in invoke("fs", "ls") + preview = invoke("merge", "atomic-preview", "draft", "main") + assert all(preview["stores"][name]["changes"] for name in ("app", "filesystem")) + if split: + assert [change["table"] for change in preview["stores"]["app"]["changes"]] == ["items"] + result = invoke("merge", "atomic-apply", "draft", "main", + "--operation-id", "native-filesystem", "--preview-token", preview["preview_token"]) + assert result["status"] == "committed" + assert invoke("query", "SELECT * FROM items", "--store", "app") == [{"id": 1}] + read = self.file_command(config, "read", "/report") + assert read.returncode == 0, read.stderr + assert read.stdout == b"draft\n" + + def test_mount_writes_to_selected_branch(self, tmp_path: Path) -> None: + import shutil + + unmount = shutil.which("fusermount3") + if unmount is None or not Path("/dev/fuse").exists(): + if os.environ.get("CHRONOS_CLI_REQUIRE_CHRONOSFS") == "ON": + pytest.fail("This test requires /dev/fuse and fusermount3") + pytest.skip("Native mount requires /dev/fuse and fusermount3") + + config = tmp_path / "chronos.toml" + config.write_text('[filesystem]\nkind="chronosfs"\ndatabase_url="sqlite:///files.sqlite"\n') + mountpoint = tmp_path / "mount" + mountpoint.mkdir() + _invoke_workspace(config, "branch", "create", "draft") + command = [ + sys.executable, "-m", "chronos_core.cli", "--config", str(config), + "fs", "mount", str(mountpoint), "--branch", "draft", + ] + with subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE) as process: + try: + deadline = time.monotonic() + 15 + while not mountpoint.is_mount(): + if process.poll() is not None: + _, stderr = process.communicate() + pytest.fail(f"Mount command exited early: {stderr.decode(errors='replace')}") + if time.monotonic() >= deadline: + pytest.fail("Timed out waiting for the native mount") + time.sleep(0.05) + (mountpoint / "report").write_bytes(b"mounted draft\n") + subprocess.run([unmount, "-u", str(mountpoint)], check=True, timeout=10) + _, stderr = process.communicate(timeout=15) + assert process.returncode == 0, stderr + finally: + if mountpoint.is_mount(): + subprocess.run([unmount, "-uz", str(mountpoint)], timeout=10) + if process.poll() is None: + process.kill() + process.wait(timeout=10) + + assert "report" not in _invoke_workspace(config, "fs", "ls") + read = self.file_command(config, "read", "/report", "--branch", "draft") + assert read.returncode == 0, read.stderr + assert read.stdout == b"mounted draft\n" From 8a99b3857afcc6f8aec119957a290e9aa60b0632 Mon Sep 17 00:00:00 2001 From: Seth Rojas Date: Fri, 6 Nov 2026 09:44:44 -0700 Subject: [PATCH 10/11] test(cli): ignore ANSI styling in help assertions --- tests/test_cli.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_cli.py b/tests/test_cli.py index 84ba19c..37a0a1d 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -15,6 +15,7 @@ import pytest import typer +from rich.text import Text from typer.testing import CliRunner from chronos_core.branching import BranchingError, ChronosBranchContext @@ -324,8 +325,9 @@ def test_installed_entrypoints(self) -> None: environment = {key: value for key, value in os.environ.items() if not key.startswith("CHRONOS_")} result = subprocess.run(args, capture_output=True, text=True, env=environment, timeout=30) assert result.returncode == 0, result.stderr - assert "--config" in result.stdout - assert "--database-url" not in result.stdout + help_text = Text.from_ansi(result.stdout).plain + assert "--config" in help_text + assert "--database-url" not in help_text def test_help_and_sql_without_qdrant_dependency(self, tmp_path: Path) -> None: code = ''' @@ -691,7 +693,7 @@ def test_help_needs_no_config(self, tmp_path: Path, monkeypatch: pytest.MonkeyPa monkeypatch.delenv('CHRONOS_WORKSPACE_CONFIG', raising=False) result = runner.invoke(app, ['fs', 'run', '--help']) assert result.exit_code == 0 - assert '--json' in result.stdout + assert "--json" in Text.from_ansi(result.stdout).plain def test_filesystem_error_is_reported(self, workspace_connection: MagicMock) -> None: from chronos_core.workspace.filesystem import FilesystemStoreError From 965bef8da4f224b66dc6a47e836a925362056f15 Mon Sep 17 00:00:00 2001 From: Xinjing Zhou Date: Fri, 6 Nov 2026 17:43:59 -0500 Subject: [PATCH 11/11] Fix local database URL escaping across workspace stores Resolve decoded filenames before encoding absolute database URLs. Decode URL paths exactly once in native SQLite and DuckDB connections, ChronosFS mounts and maintenance, and S3 metadata connections, while preserving raw filesystem paths. Reject DuckDB question-mark paths before opening files because its WAL path handling cannot preserve them. Add workspace-isolation, encoded-filename, native-driver, existing-connection, and FUSE regression coverage, and run the relevant checks in CI. --- .github/workflows/relational-package.yml | 4 ++ docs/cli.md | 6 ++ .../src/chronos_core/cli/_config.py | 12 +++- .../src/chronos_core/native/chronos_s3.cpp | 23 ++++---- .../chronos_core/native/chronosfs_fuse.cpp | 4 +- .../chronos_core/native/database_url_path.hpp | 38 +++++++++++++ .../native/interval_sql_runtime.cpp | 19 +++++-- .../chronos_core/native/native_backends.cpp | 1 + .../workspace/chronosfs/daemon.py | 3 +- .../chronos_core/workspace/chronosfs/fuse.py | 5 +- .../chronos_core/workspace/chronosfs/store.py | 3 +- tests/test_chronosfs.py | 44 ++++++++++++++ tests/test_cli.py | 29 ++++++++++ tests/test_cli_integration.py | 57 +++++++++++++++++-- tests/test_native_interval.py | 54 ++++++++++++++++++ 15 files changed, 276 insertions(+), 26 deletions(-) create mode 100644 packages/chronos-core/src/chronos_core/native/database_url_path.hpp diff --git a/.github/workflows/relational-package.yml b/.github/workflows/relational-package.yml index 7d6927c..6970246 100644 --- a/.github/workflows/relational-package.yml +++ b/.github/workflows/relational-package.yml @@ -112,6 +112,8 @@ jobs: tests/test_cli.py::TestWorkspaceCommands \ tests/test_cli.py::TestWorkspaceFilesystemCommands \ tests/test_cli_integration.py::TestWorkspaceSQLWorkflows \ + tests/test_cli_integration.py::TestWorkspaceResources::test_database_paths_preserve_reserved_characters \ + tests/test_cli_integration.py::TestWorkspaceResources::test_hash_directories_keep_workspace_state_separate \ tests/test_cli_integration.py::TestWorkspaceResources::test_sqlite_workspace_persists_after_close \ tests/test_cli_integration.py::TestWorkspaceResources::test_sqlite_workspace_without_qdrant_dependency \ -q --timeout=60 @@ -130,6 +132,8 @@ jobs: run: | python -I -m pytest tests/test_cli.py -k native -q --timeout=60 python -I -m pytest tests/test_cli_integration.py::TestNativeChronosFSWorkflows -q --timeout=120 + python -I -m pytest tests/test_chronosfs.py::test_shared_daemon_checkpoint_preserves_database_path \ + tests/test_chronosfs.py::test_existing_sqlite_adapter_preserves_database_path -q --timeout=60 - name: Test workspace DuckDB and Qdrant together if: matrix.duckdb == 'ON' env: diff --git a/docs/cli.md b/docs/cli.md index d33b2fc..06cbe92 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -70,6 +70,12 @@ SQL participants support `sqlite`, `postgresql`, and `duckdb` kinds, with connections also accept `api_key` and `timeout`. Select vector operations with `chronos qdrant --store NAME ...`. +Local database URLs use percent-encoded filenames, resolved relative to the +configuration file. For example, `sqlite:///data%23trial.sqlite` names +`data#trial.sqlite`; `%2520` names a literal `%20`. Characters in the config +directory are encoded automatically. DuckDB paths containing `?` are rejected +because the bundled DuckDB version cannot construct their WAL paths correctly. + Branches, checkpoints, diffs, and merges operate across the workspace. With independent metadata, `merge preview/apply` calls the coordinated API, which can retain partial changes if a later participant fails. Atomic mode requires diff --git a/packages/chronos-core/src/chronos_core/cli/_config.py b/packages/chronos-core/src/chronos_core/cli/_config.py index 7ff42a4..cf38b3b 100644 --- a/packages/chronos-core/src/chronos_core/cli/_config.py +++ b/packages/chronos-core/src/chronos_core/cli/_config.py @@ -10,7 +10,7 @@ from dataclasses import dataclass from pathlib import Path from typing import TYPE_CHECKING, Any, Literal -from urllib.parse import unquote, urlsplit +from urllib.parse import quote, unquote, urlsplit try: import tomllib @@ -135,7 +135,15 @@ def _database_url(value: Any, root: Path, field: str, *, kind: str | None = None local = url[len(f"{scheme}:///"):] if not local or local == ":memory:": raise WorkspaceConfigError(f"{field} requires persistent storage, not an in-memory database.") - return f"{scheme}:///{_local_path(local, root, field)}" + # Decode the URL's filename before resolving it, then encode the complete + # path so characters in the config directory cannot become URL syntax. + decoded = unquote(local) + # An encoded leading tilde is a literal filename, not a home-directory + # request. Join it to the root before _local_path expands unescaped '~'. + location = _local_path(decoded if local.startswith("~") else root / decoded, root, field) + if scheme == "duckdb" and "?" in str(location): + raise WorkspaceConfigError(f"{field}: DuckDB database paths cannot contain '?'.") + return f"{scheme}:///{quote(str(location), safe='/')}" def _store_config(value: Any, root: Path, field: str) -> StoreConfig: diff --git a/packages/chronos-core/src/chronos_core/native/chronos_s3.cpp b/packages/chronos-core/src/chronos_core/native/chronos_s3.cpp index 4f5af79..7681cdd 100644 --- a/packages/chronos-core/src/chronos_core/native/chronos_s3.cpp +++ b/packages/chronos-core/src/chronos_core/native/chronos_s3.cpp @@ -1,5 +1,6 @@ #include "chronos_s3.hpp" +#include "database_url_path.hpp" #include "interval_data_plane.hpp" #include @@ -66,16 +67,14 @@ const std::vector kObjectColumns = { }; const std::vector kObjectPrimaryKey = {"bucket", "object_key"}; +std::string url_encode(std::string_view input, bool preserve_slash); +std::string sqlite_file_path(const std::string &path); + std::string sqlite_url(const std::string &path) { std::string url = path; - std::string file_path = path; - const std::string prefix = "sqlite:///"; - if (path.rfind(prefix, 0) == 0) { - file_path = "/" + path.substr(prefix.size()); - } else if (path.rfind("sqlite://", 0) == 0) { - file_path = path.substr(std::strlen("sqlite://")); - } else { - url = "sqlite:///" + path; + const std::string file_path = sqlite_file_path(path); + if (path.rfind("sqlite://", 0) != 0) { + url = "sqlite:///" + url_encode(path, true); } sqlite3 *database = nullptr; if (sqlite3_open_v2( @@ -94,8 +93,12 @@ std::string sqlite_url(const std::string &path) { std::string sqlite_file_path(const std::string &path) { const std::string prefix = "sqlite:///"; - if (path.rfind(prefix, 0) == 0) return "/" + path.substr(prefix.size()); - if (path.rfind("sqlite://", 0) == 0) return path.substr(std::strlen("sqlite://")); + if (path.rfind(prefix, 0) == 0) { + return "/" + decode_database_url_path(path.substr(prefix.size())); + } + if (path.rfind("sqlite://", 0) == 0) { + return decode_database_url_path(path.substr(std::strlen("sqlite://"))); + } return path; } diff --git a/packages/chronos-core/src/chronos_core/native/chronosfs_fuse.cpp b/packages/chronos-core/src/chronos_core/native/chronosfs_fuse.cpp index 4d0b1a9..0663df6 100644 --- a/packages/chronos-core/src/chronos_core/native/chronosfs_fuse.cpp +++ b/packages/chronos-core/src/chronos_core/native/chronosfs_fuse.cpp @@ -2,6 +2,7 @@ #include "chronosfs_fuse.hpp" +#include "database_url_path.hpp" #include "interval_data_plane.hpp" #include @@ -436,7 +437,8 @@ std::string blob_sql_type(const chronos::native::NativeBranchStore &store) { std::string chronosfs_object_dir(const std::string &database_url) { constexpr const char *prefix = "sqlite:///"; if (database_url.rfind(prefix, 0) != 0) return {}; - std::string database_path = database_url.substr(std::strlen(prefix)); + std::string database_path = chronos::native::decode_database_url_path( + database_url.substr(std::strlen(prefix))); if (database_path.empty() || database_path == ":memory:") return {}; return database_path + ".chronosfs-objects"; } diff --git a/packages/chronos-core/src/chronos_core/native/database_url_path.hpp b/packages/chronos-core/src/chronos_core/native/database_url_path.hpp new file mode 100644 index 0000000..369f2c7 --- /dev/null +++ b/packages/chronos-core/src/chronos_core/native/database_url_path.hpp @@ -0,0 +1,38 @@ +#pragma once + +#include +#include +#include + +namespace chronos::native { + +// Database URLs use percent-encoded paths. Decode once, without treating '+' +// as a space; raw filesystem paths must not pass through this helper. +inline std::string decode_database_url_path(std::string_view path) { + const auto hex_digit = [](char ch) -> int { + if (ch >= '0' && ch <= '9') return ch - '0'; + if (ch >= 'a' && ch <= 'f') return ch - 'a' + 10; + if (ch >= 'A' && ch <= 'F') return ch - 'A' + 10; + return -1; + }; + std::string decoded; + decoded.reserve(path.size()); + for (std::size_t i = 0; i < path.size(); ++i) { + char ch = path[i]; + if (ch == '%' && i + 2 < path.size()) { + const int high = hex_digit(path[i + 1]); + const int low = hex_digit(path[i + 2]); + if (high >= 0 && low >= 0) { + ch = static_cast((high << 4) | low); + i += 2; + } + } + if (ch == '\0') { + throw std::invalid_argument("database path must not contain NUL bytes"); + } + decoded.push_back(ch); + } + return decoded; +} + +} // namespace chronos::native diff --git a/packages/chronos-core/src/chronos_core/native/interval_sql_runtime.cpp b/packages/chronos-core/src/chronos_core/native/interval_sql_runtime.cpp index e1cec03..616045b 100644 --- a/packages/chronos-core/src/chronos_core/native/interval_sql_runtime.cpp +++ b/packages/chronos-core/src/chronos_core/native/interval_sql_runtime.cpp @@ -1210,8 +1210,12 @@ py::dict stats_to_py_dict(const BulkUpsertStats &native_stats, std::size_t input std::string sqlite_path_from_url(const std::string &database_url) { const std::string prefix = "sqlite:///"; - if (database_url.rfind(prefix, 0) == 0) return "/" + database_url.substr(prefix.size()); - if (database_url.rfind("sqlite://", 0) == 0) return database_url.substr(std::string("sqlite://").size()); + if (database_url.rfind(prefix, 0) == 0) { + return "/" + decode_database_url_path(database_url.substr(prefix.size())); + } + if (database_url.rfind("sqlite://", 0) == 0) { + return decode_database_url_path(database_url.substr(std::string("sqlite://").size())); + } return database_url; } @@ -1324,16 +1328,16 @@ std::string duckdb_path_from_url(const std::string &database_url) { const std::string url_prefix = "duckdb://"; const std::string scheme_prefix = "duckdb:"; if (database_url.rfind(absolute_prefix, 0) == 0) { - std::string path = database_url.substr(absolute_prefix.size()); + std::string path = decode_database_url_path(database_url.substr(absolute_prefix.size())); if (path == ":memory:") return path; while (!path.empty() && path.front() == '/') path.erase(path.begin()); return "/" + path; } if (database_url.rfind(url_prefix, 0) == 0) { - return database_url.substr(url_prefix.size()); + return decode_database_url_path(database_url.substr(url_prefix.size())); } if (database_url.rfind(scheme_prefix, 0) == 0) { - return database_url.substr(scheme_prefix.size()); + return decode_database_url_path(database_url.substr(scheme_prefix.size())); } return database_url; } @@ -2923,6 +2927,11 @@ class NativeDuckDBDriver final : public NativeSqlDriver { public: explicit NativeDuckDBDriver(const std::string &database_url) : dialect_("duckdb") { path_ = duckdb_path_from_url(database_url); + // DuckDB 1.5.4 treats '?' as a query delimiter when constructing its + // WAL filename, even for local paths. Fail before creating any files. + if (path_.find('?') != std::string::npos) { + throw std::invalid_argument("DuckDB database paths cannot contain '?'"); + } if (path_ != ":memory:" && !path_.empty()) { std::filesystem::path db_path(path_); if (db_path.has_parent_path()) { diff --git a/packages/chronos-core/src/chronos_core/native/native_backends.cpp b/packages/chronos-core/src/chronos_core/native/native_backends.cpp index 5224c20..20802b3 100644 --- a/packages/chronos-core/src/chronos_core/native/native_backends.cpp +++ b/packages/chronos-core/src/chronos_core/native/native_backends.cpp @@ -46,6 +46,7 @@ #include #include "copy_branch_store.hpp" +#include "database_url_path.hpp" #include "interval_data_plane.hpp" namespace py = pybind11; diff --git a/packages/chronos-core/src/chronos_core/workspace/chronosfs/daemon.py b/packages/chronos-core/src/chronos_core/workspace/chronosfs/daemon.py index b494120..ad7fa2a 100644 --- a/packages/chronos-core/src/chronos_core/workspace/chronosfs/daemon.py +++ b/packages/chronos-core/src/chronos_core/workspace/chronosfs/daemon.py @@ -17,6 +17,7 @@ import time from pathlib import Path from typing import Any +from urllib.parse import unquote from chronos_core import _native_interval @@ -133,7 +134,7 @@ def _sqlite_database_path(database_url: str) -> str | None: prefix = "sqlite:///" if not database_url.startswith(prefix): return None - return "/" + database_url[len(prefix) :] + return "/" + unquote(database_url[len(prefix) :]) def _checkpoint_sqlite(database_url: str, state: _DaemonState) -> None: diff --git a/packages/chronos-core/src/chronos_core/workspace/chronosfs/fuse.py b/packages/chronos-core/src/chronos_core/workspace/chronosfs/fuse.py index 874ed24..27a9460 100644 --- a/packages/chronos-core/src/chronos_core/workspace/chronosfs/fuse.py +++ b/packages/chronos-core/src/chronos_core/workspace/chronosfs/fuse.py @@ -12,6 +12,7 @@ import time from contextlib import suppress from pathlib import Path +from urllib.parse import quote from chronos_core import _native_interval from chronos_core.workspace.chronosfs.store import ChronosFSStore @@ -207,7 +208,7 @@ def _database_url_for_mount(store: ChronosFSStore) -> str: if not database_url: database_path = getattr(store.context.db, "database_path", None) if database_path: - database_url = f"sqlite:///{database_path}" + database_url = f"sqlite:///{quote(str(database_path), safe='/')}" if not database_url: raise ChronosFSMountError( "native ChronosFS FUSE requires a file-backed database URL" @@ -221,7 +222,7 @@ def _database_urls_for_mount(store: ChronosFSStore) -> tuple[str, str]: if not metadata_url: metadata_path = getattr(store.context.metadata_db, "database_path", None) if metadata_path: - metadata_url = f"sqlite:///{metadata_path}" + metadata_url = f"sqlite:///{quote(str(metadata_path), safe='/')}" return database_url, str(metadata_url or database_url) diff --git a/packages/chronos-core/src/chronos_core/workspace/chronosfs/store.py b/packages/chronos-core/src/chronos_core/workspace/chronosfs/store.py index d361efa..4654a0b 100644 --- a/packages/chronos-core/src/chronos_core/workspace/chronosfs/store.py +++ b/packages/chronos-core/src/chronos_core/workspace/chronosfs/store.py @@ -9,6 +9,7 @@ from dataclasses import dataclass from pathlib import Path, PurePosixPath from typing import Any, Iterator, Literal +from urllib.parse import quote from chronos_core import _native_interval from chronos_core.branching import ( @@ -1460,7 +1461,7 @@ def _database_url_for_adapter(adapter: Any) -> str: return str(database_url) database_path = getattr(adapter, "database_path", None) if database_path and str(database_path) != ":memory:": - return f"sqlite:///{database_path}" + return f"sqlite:///{quote(str(database_path), safe='/')}" raise ChronosFSError( "native ChronosFS requires a file-backed SQLite or PostgreSQL database" ) diff --git a/tests/test_chronosfs.py b/tests/test_chronosfs.py index baac12b..ad01659 100644 --- a/tests/test_chronosfs.py +++ b/tests/test_chronosfs.py @@ -1292,6 +1292,50 @@ def test_shared_daemon_identity_is_store_scoped() -> None: ] +def test_shared_daemon_checkpoint_preserves_database_path(tmp_path: Path) -> None: + import sqlite3 + from urllib.parse import quote + + from chronos_core.workspace.chronosfs.daemon import _DaemonState, _checkpoint_sqlite + + database = tmp_path / "state #?%20+目录.sqlite" + with sqlite3.connect(database) as connection: + connection.execute("CREATE TABLE marker (id INTEGER PRIMARY KEY)") + connection.execute("INSERT INTO marker VALUES (7)") + state = _DaemonState() + state.request_shutdown() + _checkpoint_sqlite(f"sqlite:///{quote(str(database), safe='/')}", state) + assert sorted(path.name for path in tmp_path.iterdir()) == [database.name] + with sqlite3.connect(database) as connection: + assert connection.execute("SELECT id FROM marker").fetchall() == [(7,)] + + +def test_existing_sqlite_adapter_preserves_database_path(tmp_path: Path) -> None: + import sqlite3 + from urllib.parse import quote + + from chronos_core import _native_interval + from chronos_core.branching.sql_adapters import SQLiteDatabaseAdapter + from chronos_core.workspace.chronosfs.fuse import _database_urls_for_mount + + database = tmp_path / "state #?%20+目录.sqlite" + raw_connection = sqlite3.connect(database) + raw_connection.row_factory = sqlite3.Row + adapter = SQLiteDatabaseAdapter(raw_connection, str(database)) + context = ChronosBranchContext.from_database_adapter(adapter) + store = ChronosFSStore(context) + try: + store.ensure() + store.write_file("main", "/note", b"preserved") + url = f"sqlite:///{quote(str(database), safe='/')}" + assert store._database_url == url + assert _database_urls_for_mount(store) == (url, url) + connection = _native_interval.NativeSqlConnection(url) + assert connection.query_sql("SELECT count(*) FROM _chronosfs_metadata")[0][0] > 0 + finally: + store.close() + + def test_shared_daemon_shutdown_waits_for_mounts() -> None: from chronos_core.workspace.chronosfs.daemon import _DaemonState diff --git a/tests/test_cli.py b/tests/test_cli.py index 37a0a1d..76bc324 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -10,6 +10,7 @@ from datetime import date, datetime, time from decimal import Decimal from pathlib import Path +from urllib.parse import quote from unittest.mock import MagicMock from contextlib import contextmanager @@ -79,6 +80,7 @@ class Result: class TestWorkspaceConfiguration: @pytest.mark.parametrize("source", [ '[stores.vectors]\nkind="qdrant"\nmetadata_url="sqlite:///meta.sqlite"\nurl="http://localhost:6333"\ntimeout=' + '9' * 400, + '[stores.app]\nkind="duckdb"\ndatabase_url="duckdb:///query%3Fdir/data.duckdb"\nmetadata_url="sqlite:///metadata.sqlite"', '[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///~chronos_no_such_user_987654/app.sqlite"', '[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///shared.sqlite"\n[stores.vectors]\nkind="qdrant"\nmetadata_url="sqlite:///shared%2Esqlite"\npath="vectors"', '[workspace]\nshared_metadata_url="sqlite:///shared.sqlite"\n[stores.first]\nkind="sqlite"\ndatabase_url="sqlite:///shared.sqlite"\n[stores.second]\nkind="sqlite"\ndatabase_url="sqlite:///shared.sqlite"', @@ -101,6 +103,33 @@ def test_encoded_filename_preserves_api_url(self, tmp_path: Path) -> None: loaded = load_workspace_config(config) assert loaded.stores["app"].connection.database_url == f"sqlite:///{tmp_path / 'literal%252Ename.sqlite'}" + def test_encoded_tilde_is_a_literal_filename(self, tmp_path: Path) -> None: + config = tmp_path / "chronos.toml" + config.write_text('[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///%7Echronos_no_such_user_987654/app.sqlite"') + loaded = load_workspace_config(config) + assert loaded.stores["app"].connection.database_url == f"sqlite:///{tmp_path / '~chronos_no_such_user_987654/app.sqlite'}" + + @pytest.mark.parametrize("kind", ["sqlite", "duckdb"]) + @pytest.mark.parametrize("directory", ["space dir", "hash#dir", "query?dir", "literal%20dir", "plus+dir", "目录"]) + def test_local_database_urls_encode_config_directory(self, tmp_path: Path, kind: str, directory: str) -> None: + root = tmp_path / directory + root.mkdir() + config = root / "chronos.toml" + filename = "literal%20#.db" if kind == "duckdb" else "literal%20#?.db" + config.write_text( + f'[stores.app]\nkind="{kind}"\ndatabase_url="{kind}:///{quote(filename, safe="")}"\n' + + ('metadata_url="sqlite:///metadata.sqlite"\n' if kind == "duckdb" else '') + ) + if kind == "duckdb" and "?" in directory: + with pytest.raises(WorkspaceConfigError, match="DuckDB database paths"): + load_workspace_config(config) + return + loaded = load_workspace_config(config) + connection = loaded.stores["app"].connection + assert connection.database_url == f"{kind}:///{quote(str(root / filename), safe='/')}" + if kind == "duckdb": + assert connection.metadata_url == f"sqlite:///{quote(str(root / 'metadata.sqlite'), safe='/')}" + def test_invalid_config_path_is_a_usage_error(self) -> None: result = runner.invoke(app, ["--config", "~chronos_no_such_user_987654/chronos.toml", "branch", "list"]) assert result.exit_code == 2, result.output diff --git a/tests/test_cli_integration.py b/tests/test_cli_integration.py index 9ca8a64..670aaff 100644 --- a/tests/test_cli_integration.py +++ b/tests/test_cli_integration.py @@ -12,7 +12,7 @@ from pathlib import Path from tempfile import TemporaryDirectory from typing import Any, Iterator -from urllib.parse import urlsplit, urlunsplit +from urllib.parse import quote, urlsplit, urlunsplit from uuid import uuid4 import psycopg @@ -200,6 +200,50 @@ def _qdrant_workspace_file(root: Path, target: QdrantConfig) -> Path: class TestWorkspaceResources: + @pytest.mark.parametrize("kind", ["sqlite", "sqlite-split", "duckdb-sqlite"]) + def test_database_paths_preserve_reserved_characters(self, tmp_path: Path, kind: str) -> None: + if kind == "duckdb-sqlite" and not _native_interval.has_duckdb: + if os.environ.get("CHRONOS_CLI_REQUIRE_DUCKDB") == "ON": + pytest.fail("This test run requires a DuckDB-enabled native build") + pytest.skip("DuckDB disabled in this native build") + root = tmp_path / ("workspace #%20+目录" if kind == "duckdb-sqlite" else "workspace #?%20+目录") + root.mkdir() + scheme = "duckdb" if kind == "duckdb-sqlite" else "sqlite" + data = root / ("data%20#.duckdb" if scheme == "duckdb" else "data%20#?.sqlite") + metadata = root / "metadata%20#?.sqlite" + config = root / "chronos.toml" + config.write_text( + f'[stores.app]\nkind="{scheme}"\ndatabase_url="{scheme}:///{quote(data.name, safe="")}"\n' + + (f'metadata_url="sqlite:///{quote(metadata.name, safe="")}"\n' if kind != "sqlite" else '') + ) + loaded = load_workspace_config(config) + with open_workspace(loaded) as workspace: + app = workspace.stores["app"] + app.db.execute("CREATE TABLE items (id INTEGER PRIMARY KEY, value TEXT)") + app.db.commit() + app.register_table("items", ["id"]) + with app.checkout("main") as session: + session.execute("INSERT INTO items (id, value) VALUES (1, 'persisted')") + assert data.is_file() + if kind != "sqlite": + assert metadata.is_file() + with open_workspace(loaded) as workspace: + with workspace.stores["app"].checkout("main") as session: + assert session.query("SELECT value FROM items") == [{"value": "persisted"}] + assert sorted(path.name for path in tmp_path.iterdir()) == [root.name] + + def test_hash_directories_keep_workspace_state_separate(self, tmp_path: Path) -> None: + configs = [] + for name in ("workspace#one", "workspace#two"): + root = tmp_path / name + root.mkdir() + config = root / "chronos.toml" + config.write_text('[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"\n') + configs.append(config) + _invoke_workspace(configs[0], "branch", "create", "private-to-one") + assert _invoke_workspace(configs[1], "branch", "list") == ["main"] + assert sorted(path.name for path in tmp_path.iterdir()) == ["workspace#one", "workspace#two"] + def test_sqlite_workspace_without_qdrant_dependency(self, tmp_path: Path) -> None: config_file = tmp_path / "chronos.toml" config_file.write_text('[stores.app]\nkind="sqlite"\ndatabase_url="sqlite:///app.sqlite"\n') @@ -765,7 +809,8 @@ def test_sql_and_native_filesystem_atomic_merge(self, tmp_path: Path, split: boo assert read.returncode == 0, read.stderr assert read.stdout == b"draft\n" - def test_mount_writes_to_selected_branch(self, tmp_path: Path) -> None: + @pytest.mark.parametrize("directory", ["plain", "workspace #?%20+目录"]) + def test_mount_writes_to_selected_branch(self, tmp_path: Path, directory: str) -> None: import shutil unmount = shutil.which("fusermount3") @@ -774,6 +819,8 @@ def test_mount_writes_to_selected_branch(self, tmp_path: Path) -> None: pytest.fail("This test requires /dev/fuse and fusermount3") pytest.skip("Native mount requires /dev/fuse and fusermount3") + tmp_path = tmp_path / directory + tmp_path.mkdir() config = tmp_path / "chronos.toml" config.write_text('[filesystem]\nkind="chronosfs"\ndatabase_url="sqlite:///files.sqlite"\n') mountpoint = tmp_path / "mount" @@ -793,7 +840,8 @@ def test_mount_writes_to_selected_branch(self, tmp_path: Path) -> None: if time.monotonic() >= deadline: pytest.fail("Timed out waiting for the native mount") time.sleep(0.05) - (mountpoint / "report").write_bytes(b"mounted draft\n") + payload = b"mounted draft\n" * 8192 + (mountpoint / "report").write_bytes(payload) subprocess.run([unmount, "-u", str(mountpoint)], check=True, timeout=10) _, stderr = process.communicate(timeout=15) assert process.returncode == 0, stderr @@ -807,4 +855,5 @@ def test_mount_writes_to_selected_branch(self, tmp_path: Path) -> None: assert "report" not in _invoke_workspace(config, "fs", "ls") read = self.file_command(config, "read", "/report", "--branch", "draft") assert read.returncode == 0, read.stderr - assert read.stdout == b"mounted draft\n" + assert read.stdout == payload + assert (tmp_path / "files.sqlite.chronosfs-objects").is_dir() diff --git a/tests/test_native_interval.py b/tests/test_native_interval.py index d9f948a..94b469e 100644 --- a/tests/test_native_interval.py +++ b/tests/test_native_interval.py @@ -1,4 +1,5 @@ import sqlite3 +from urllib.parse import quote import pytest import chronos_core._native_interval as native_interval @@ -59,6 +60,59 @@ def test_native_connectors_smoke(tmp_path): assert native_interval.supports_connection_dialect("postgres") is True +@pytest.mark.parametrize("encoded", [False, True], ids=["raw-path", "url"]) +def test_native_sqlite_connection_preserves_path_characters(tmp_path, encoded): + path = tmp_path / "data #?%20+目录.sqlite" + with sqlite3.connect(path) as db: + db.execute("CREATE TABLE items (id INTEGER PRIMARY KEY)") + db.execute("INSERT INTO items VALUES (7)") + location = f"sqlite:///{quote(str(path), safe='/')}" if encoded else str(path) + conn = native_interval.NativeSqlConnection(location) + assert conn.query_sql("SELECT id FROM items") == [[7]] + + +def test_native_sqlite_connection_rejects_encoded_nul(tmp_path): + path = tmp_path / "data.sqlite" + sqlite3.connect(path).close() + with pytest.raises(ValueError, match="NUL"): + native_interval.NativeSqlConnection(f"sqlite:///{path}%00ignored") + + +@pytest.mark.skipif(not native_interval.has_duckdb, reason="DuckDB driver disabled in this build") +def test_native_duckdb_connection_preserves_path_characters(tmp_path): + path = tmp_path / "data #%20+目录.duckdb" + conn = native_interval.NativeSqlConnection(f"duckdb:///{quote(str(path), safe='/')}") + conn.execute_sql("CREATE TABLE items (id INTEGER PRIMARY KEY)") + conn.execute_sql("INSERT INTO items VALUES (7)") + conn.commit() + assert conn.query_sql("SELECT id FROM items") == [[7]] + assert path.is_file() + + +@pytest.mark.skipif(not native_interval.has_duckdb, reason="DuckDB driver disabled in this build") +@pytest.mark.parametrize("relative", ["data?.duckdb", "query?dir/data.duckdb"]) +def test_native_duckdb_rejects_question_mark_before_creating_files(tmp_path, relative): + path = tmp_path / relative + with pytest.raises(ValueError, match="DuckDB database paths"): + native_interval.NativeSqlConnection(f"duckdb:///{quote(str(path), safe='/')}") + assert list(tmp_path.iterdir()) == [] + + +@pytest.mark.skipif(not hasattr(native_interval, "NativeChronosLakeServer"), reason="S3 support disabled in this build") +@pytest.mark.parametrize("encoded", [False, True], ids=["raw-path", "url"]) +def test_native_lake_metadata_preserves_path_characters(tmp_path, encoded): + path = tmp_path / "metadata #?%20+目录.sqlite" + location = f"sqlite:///{quote(str(path), safe='/')}" if encoded else str(path) + server = native_interval.NativeChronosLakeServer(location, "http://127.0.0.1:9", worker_threads=1) + try: + assert server.branches() == ["main"] + server.create_branch("trial", "main") + assert sorted(server.branches()) == ["main", "trial"] + assert path.is_file() + finally: + server.stop() + + def test_native_sqlite_connection_uses_chronos_wal_checkpoint_default(tmp_path, monkeypatch): monkeypatch.delenv("CHRONOS_NATIVE_SQLITE_WAL_AUTOCHECKPOINT_PAGES", raising=False) db_path = tmp_path / "wal_default.sqlite"