diff --git a/.gitignore b/.gitignore index 3f0c775..fa9a52a 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ wheels/ *.egg # Virtual environments +.venv/ venv/ ENV/ env/ diff --git a/.gts-spec b/.gts-spec index 56ca20d..2a171df 160000 --- a/.gts-spec +++ b/.gts-spec @@ -1 +1 @@ -Subproject commit 56ca20d89df2c2d8e70773da33482e4c748c446d +Subproject commit 2a171dff7810657de147e3b5f06f89537d7a4293 diff --git a/Makefile b/Makefile index 625cbbc..5874cb7 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,30 @@ CI := 1 -.PHONY: help build dev-fmt all check fmt lint mypy test security update-spec e2e coverage +# These recipes rely on POSIX tools (command -v, touch, rm -rf, sleep, kill, +# cat) and POSIX syntax (background jobs, inline env assignments). Require Bash +# explicitly so GNU Make does not fall back to cmd.exe via COMSPEC on Windows. +# On Windows, run these targets from a Bash environment (e.g. Git Bash / MSYS2). +SHELL := /bin/bash + +# Python: PYTHON_BOOTSTRAP is used only to create the virtual environment; +# PYTHON is the venv interpreter used by all other targets. +PYTHON_BOOTSTRAP ?= $(shell command -v python3 2>/dev/null || command -v python 2>/dev/null || echo python3) +PY_ENV_DIR ?= .venv +ifeq ($(OS),Windows_NT) +PYTHON ?= $(PY_ENV_DIR)/Scripts/python +else +PYTHON ?= $(PY_ENV_DIR)/bin/python +endif +PY_ENV_STAMP := $(PY_ENV_DIR)/.stamp +INSTALL_STAMP := $(PY_ENV_DIR)/.install-stamp + +ifneq ($(filter install-local uninstall-local,$(MAKECMDGOALS)),) +ifeq ($(origin PYTHON),file) +$(error PYTHON must be set for local package targets (examples: venv: PYTHON=.venv/bin/python3.13 make install-local; global: PYTHON=python3.13 make install-local)) +endif +endif + +.PHONY: help py-env install build install-local uninstall-local clean dev-fmt all check fmt lint clippy mypy test security update-spec e2e coverage # Default target - show help .DEFAULT_GOAL := help @@ -9,61 +33,103 @@ CI := 1 help: @awk '/^# / { desc=substr($$0, 3) } /^[a-zA-Z0-9_-]+:/ && desc { target=$$1; sub(/:$$/, "", target); printf "%-20s - %s\n", target, desc; desc="" }' Makefile | sort -# Build/install the package in development mode -build: - pip install -e ./gts +# -------- Environment -------- -# Fix formatting issues -dev-fmt: - ruff format gts/src +# Create/update the virtual environment and install dev/test dependencies +py-env: $(PY_ENV_STAMP) -# Run all checks and build -all: check build +$(PY_ENV_STAMP): gts/pyproject.toml .gts-spec/tests/requirements.txt Makefile + @echo "Creating/updating Python virtual environment in $(PY_ENV_DIR)..." + $(PYTHON_BOOTSTRAP) -m venv $(PY_ENV_DIR) + $(PYTHON) -m pip install --upgrade pip + $(PYTHON) -m pip install -r .gts-spec/tests/requirements.txt + $(PYTHON) -m pip install --no-deps 'httprunner>=4,<5' + $(PYTHON) -m pip install ruff mypy + @touch $@ + +# Install gts package into the venv (editable, for development) +install: $(INSTALL_STAMP) + +$(INSTALL_STAMP): $(PY_ENV_STAMP) gts/pyproject.toml + $(PYTHON) -m pip install -e ./gts + @touch $@ + +# Build source and wheel distributions into dist/ +build: py-env + $(PYTHON) -m pip install --upgrade build + $(PYTHON) -m build --outdir dist ./gts + +# Install the locally built wheel, equivalent to installing the published gts package +install-local: build + $(PYTHON) -m pip install --force-reinstall dist/gts-*.whl + +# Uninstall gts from the selected interpreter +uninstall-local: + $(PYTHON) -m pip uninstall --yes gts + @rm -f $(INSTALL_STAMP) + +# Remove venv and build artifacts +clean: + rm -rf $(PY_ENV_DIR) dist/ gts/dist/ gts/*.egg-info + +# -------- Code quality -------- + +# Fix formatting issues +dev-fmt: py-env + $(PYTHON) -m ruff format gts/src # Check code formatting -fmt: - ruff format --check gts/src +fmt: py-env + $(PYTHON) -m ruff format --check gts/src # Run linter (ruff) -lint: - ruff check gts/src +lint: py-env + $(PYTHON) -m ruff check gts/src # Run clippy-equivalent linter with auto-fix -clippy: - ruff check --fix gts/src +clippy: py-env + $(PYTHON) -m ruff check --fix gts/src # Run type checker -mypy: - mypy gts/src/gts --ignore-missing-imports +mypy: py-env + $(PYTHON) -m mypy gts/src/gts --ignore-missing-imports -# Run all tests -test: - pytest tests/ -v +# -------- Tests -------- -# Check dependencies for security vulnerabilities -security: - @command -v pip-audit >/dev/null || (echo "Installing pip-audit..." && pip install pip-audit) - pip-audit +# Run all tests +test: install + $(PYTHON) -m pytest tests/ -v # Measure code coverage -coverage: - pytest tests/ --cov=gts --cov-report=xml --cov-report=term - -# Update gts-spec submodule to latest -update-spec: - git submodule update --remote .gts-spec +coverage: install + $(PYTHON) -m pip install 'pytest-cov>=5,<7' + $(PYTHON) -m pytest tests/ --cov=gts --cov-report=xml --cov-report=term # Run end-to-end tests against gts-spec -e2e: build +e2e: install @echo "Starting server in background..." - @python -m gts server --port 8000 & echo $$! > .server.pid + @$(PYTHON) -m gts server --port 8000 & echo $$! > .server.pid @sleep 2 @echo "Running e2e tests..." - @PYTHONDONTWRITEBYTECODE=1 pytest -p no:cacheprovider --log-file=e2e.log ./.gts-spec/tests || (kill `cat .server.pid` 2>/dev/null; rm -f .server.pid; exit 1) + @PYTHONDONTWRITEBYTECODE=1 $(PYTHON) -m pytest -p no:cacheprovider --log-file=e2e.log ./.gts-spec/tests || (kill `cat .server.pid` 2>/dev/null; rm -f .server.pid; exit 1) @echo "Stopping server..." @kill `cat .server.pid` 2>/dev/null || true @rm -f .server.pid @echo "E2E tests completed successfully" +# -------- Misc -------- + +# Check dependencies for security vulnerabilities +security: py-env + $(PYTHON) -m pip install pip-audit + $(PYTHON) -m pip_audit + +# Update gts-spec submodule to latest +update-spec: + git submodule update --remote .gts-spec + +# Run all checks and build +all: check build + # Run all quality checks check: fmt lint test e2e diff --git a/README.md b/README.md index 8a1a90c..594c9f9 100644 --- a/README.md +++ b/README.md @@ -6,6 +6,8 @@ A minimal, idiomatic Python library for working with **GTS** ([Global Type Syste ## Roadmap +Current supported GTS spec version: 0.13 + Featureset: - [x] **OP#1 - ID Validation**: Verify identifier syntax using regex patterns @@ -22,7 +24,8 @@ Featureset: - [x] **OP#9 - Version Casting**: Transform instances between compatible MINOR versions - [x] **OP#10 - Query Execution**: Filter identifier collections using the GTS query language - [x] **OP#11 - Attribute Access**: Retrieve property values and metadata using the attribute selector (`@`) -- [ ] **OP#12 - Schema Validation**: Validate schema against its precedent schema +- [x] **OP#12 - Type Derivation Validation**: Validate that derived GTS Type Schemas correctly extend their base chain +- [x] **OP#13 - Schema Traits Validation**: Validate schema traits (`x-gts-traits-schema` / `x-gts-traits`). See details in [gts/README.md](gts/README.md) @@ -32,26 +35,46 @@ Other GTS spec [Reference Implementation](https://github.com/globaltypesystem/gt - [x] **CLI** - command-line interface for all GTS operations - [x] **Web server** - a non-production web-server with REST API for the operations processing and testing - [x] **x-gts-ref support** - to support special GTS entity reference annotation in schemas -- [ ] **YAML support** - to support YAML files (*.yml, *.yaml) as input files -- [ ] **TypeSpec support** - add [typespec.io](https://typespec.io/) files (*.tsp) support -- [ ] **UUID for instances** - to support UUID as ID in JSON instances +- [x] **YAML support** - to support YAML files (*.yml, *.yaml) as input files +- [x] **UUID for instances** - to support UUID as ID in JSON instances +- [ ] **TypeSpec support** - direct support for [typespec.io](https://typespec.io/) files (*.tsp) input files Technical Backlog: -- [ ] **Code coverage** - target is 90% -- [ ] **Documentation** - add documentation for all the features -- [ ] **Interface** - export publicly available interface and keep cli and others private -- [ ] **Server API** - finalise the server API -- [ ] **Final code cleanup** - remove unused code, denormalize, add critical comments, etc. +- [x] **Code coverage** - target is 90% +- [x] **Documentation** - add documentation for all the features +- [x] **Interface** - export publicly available interface and keep cli and others private +- [x] **Server API** - finalise the server API +- [x] **Final code cleanup** - remove unused code, denormalize, add critical comments, etc. ## Installation +GTS requires Python 3.9 or later. + +### Local development + +From the repository root, build and install the same wheel artifact that will later be published and installed with `pip install gts`. `install-local` requires an explicit `PYTHON` environment variable so you choose the target interpreter: + +```bash +PYTHON=.venv/bin/python make install-local +``` + +This creates the `.venv` virtual environment when needed, writes source and wheel distributions to `dist/`, and installs the wheel into the interpreter specified by `PYTHON`. Activate it to use the locally installed library and CLI: + ```bash -# install in editable mode -pip install -e ./gts +source .venv/bin/activate +python -c "import gts; print(gts.__file__)" +gts --help +``` + +Use `make build` when you only need the distributable artifacts. Remove the locally installed package with `PYTHON=.venv/bin/python make uninstall-local`. For an editable installation while changing source files, use `make install`. -# install from PyPI, not supported yet -# pip install gts +### Published package + +After `gts` is published to PyPI, install it with: + +```bash +pip install gts ``` ## Usage diff --git a/gts/README.md b/gts/README.md index ba771f8..1520e22 100644 --- a/gts/README.md +++ b/gts/README.md @@ -1,209 +1,333 @@ # GTS Python Library -A minimal, idiomatic Python library for working with **GTS** ([Global Type System](https://github.com/gts-spec/gts-spec)) identifiers and type definitions. +Python helpers and a reference HTTP service for the [Global Type System (GTS)](https://github.com/globaltypesystem/gts-spec). The package supports GTS identifier parsing, JSON Schema-backed validation, schema compatibility and derivation checks, traits, casting, queries, file loading, a CLI, and a FastAPI application. -## File Format Support +The package targets GTS specification v0.13.1 and requires Python 3.9 or later. -GTS Python supports multiple file formats for schemas and instances: +## Installation -### JSON (Native) -Standard JSON format with `.json`, `.jsonc`, and `.gts` extensions. +```bash +python -m pip install gts +``` + +The package installs these runtime dependencies: + +- `jsonschema` for JSON Schema validation; +- `referencing` for standards-aware `$ref` resolution during instance validation; +- `jsonsubschema` for accepted-instance-set inclusion checks; +- `fastapi` and `uvicorn` for the HTTP server; +- `PyYAML` for YAML input. + +## Quick start -### YAML -Full YAML support with `.yaml` and `.yml` extensions. YAML files are automatically parsed and treated identically to JSON. +`GtsOps` is the high-level in-memory API. It is intentionally imported from `gts.ops`; the package root exports the lower-level model, reader, store, and ID classes. ```python -from gts import GtsFileReader +from gts.ops import GtsOps + +ops = GtsOps() +schema_id = "gts.example.demo._.event.v1~" +instance_id = "gts.example.demo._.event.v1~example.demo._.created.v1" + +schema_result = ops.add_entity( + { + "$id": f"gts://{schema_id}", + "$schema": "https://json-schema.org/draft/2020-12/schema", + "type": "object", + "required": ["id", "name"], + "properties": { + "id": {"type": "string"}, + "name": {"type": "string"}, + }, + }, + validate=True, +) +assert schema_result.ok + +instance_result = ops.add_entity( + {"id": instance_id, "name": "created"}, + validate=True, +) +assert instance_result.ok -# Reads both JSON and YAML files -reader = GtsFileReader("path/to/schemas/") -for entity in reader: - print(f"{entity.gts_id.id}: {entity.file.name}") +assert ops.validate_entity(instance_id).ok +print(ops.get_entity(instance_id).to_dict()) ``` -### TypeSpec -TypeSpec (`.tsp`) schemas must be pre-compiled to JSON Schema before use with gts-python. +`add_entity(..., validate=True)` validates a schema fully, including schema-chain, final/abstract, and trait checks. A failed registration is rolled back, including restoration of an entity that was replaced by the candidate. -**Setup:** -```bash -# Install TypeSpec compiler -npm install -g @typespec/compiler @typespec/json-schema +## Public Python API -# Compile TypeSpec to JSON Schema -tsp compile --emit @typespec/json-schema your-schemas/ -``` +### Package-root exports -**Usage:** ```python -from gts import GtsFileReader - -# Point to the generated JSON Schema output directory -reader = GtsFileReader("tsp-output/@typespec/json-schema/") -entities = list(reader) +from gts import ( + DEFAULT_GTS_CONFIG, + GtsConfig, + GtsEntity, + GtsFile, + GtsFileReader, + GtsID, + GtsIdSegment, + GtsPathResolver, + GtsReader, + GtsStore, + GtsWildcard, + JsonEntity, + JsonFile, + JsonPathResolver, + ValidationError, + ValidationResult, +) ``` -See [gts-spec TypeSpec examples](https://github.com/globaltypesystem/gts-spec/tree/main/examples/typespec) for sample TypeSpec definitions. +`JsonEntity`, `JsonFile`, and `JsonPathResolver` are backward-compatible aliases for their `Gts*` counterparts. -## Featureset +### GTS identifiers -GTS specifiaciton reference implementations status: - ---- +```python +from gts import GtsID, GtsWildcard -- [x] **OP#1 - ID Validation**: Verify identifier syntax using regex patterns +schema_id = GtsID("gts.example.demo._.event.v1~") +assert schema_id.is_type +assert schema_id.get_type_id() is None +assert GtsID.is_valid("gts://gts.example.demo._.event.v1~") -```python -from gts import GtsID +instance_id = GtsID( + "gts.example.demo._.event.v1~example.demo._.created.v1" +) +assert instance_id.get_type_id() == "gts.example.demo._.event.v1~" +print(instance_id.to_uuid()) -is_valid = GtsID.is_valid("gts.x.core.events.event.v1~") -print(is_valid) # True or False +pattern = GtsWildcard("gts.example.demo._.event.v1~*") +assert instance_id.wildcard_match(pattern) ``` ---- +`GtsID` accepts either `gts....` or the URI form `gts://gts....`. A type identifier ends in `~`; a well-known instance identifier appends one or more relative segments. `GtsID.to_uuid()` returns a deterministic UUID5, except combined anonymous IDs return their embedded UUID tail. -- [x] **OP#2 - ID Extraction**: Fetch identifiers from JSON objects or JSON Schema documents +`GtsIdSegment` exposes `vendor`, `package`, `namespace`, `type`, `ver_major`, `ver_minor`, `is_type`, and `is_wildcard`. `GtsID.gts_id_segments` contains the parsed segments. `GtsID.split_at_path(value)` separates an optional `@path` selector, and `GtsID.parse_query(expr)` / `GtsID.match_query(obj, gts_field, expr)` provide lower-level query parsing and matching helpers. + +Wildcard patterns may end in `.*` or `~*`. `~*` matches the base type and its descendants for ID matching; OP#10 queries apply an additional depth rule and return only IDs with a suffix at the wildcard position. + +### Entities and configuration ```python -import json -from gts import GtsEntity, DEFAULT_GTS_CONFIG +from gts import DEFAULT_GTS_CONFIG, GtsEntity + +entity = GtsEntity( + content={ + "id": "gts.example.demo._.event.v1~example.demo._.created.v1", + "name": "created", + }, + cfg=DEFAULT_GTS_CONFIG, +) -content = json.load(open("path/to/file.json")) -entity = GtsEntity(content=content, cfg=DEFAULT_GTS_CONFIG) -if entity.gts_id: - print(entity.gts_id.id) +print(entity.raw_id) +print(entity.gts_id) +print(entity.type_id) +print(entity.selected_entity_field) +print(entity.selected_type_id_field) ``` ---- +`GtsEntity` detects schemas from `http://json-schema.org/` and `https://json-schema.org/` `$schema` URLs. It derives a raw ID from `GtsConfig.entity_id_fields` and an instance type ID from `GtsConfig.schema_id_fields`. `DEFAULT_GTS_CONFIG` recognizes common GTS field names including `$id`, `gtsId`, `id`, `gtsType`, and `type`. -- [x] **OP#3 - ID Parsing**: Decompose identifiers into constituent parts (vendor, package, namespace, type, version, etc.) +`GtsPathResolver.resolve(path)` accepts dot paths, slash paths, and array indexes. It returns the resolver with `resolved`, `value`, `error`, and `available_fields` populated; `to_dict()` returns the corresponding serializable result. -```python -from gts import GtsID +Public entity helpers: -gts = GtsID("gts.x.core.events.event.v1~") -print(gts.gts_id_segments) -``` +- `entity.resolve_path(path)` resolves dot, slash, and array-index paths and returns a `GtsPathResolver` result; +- `entity.cast(to_schema, from_schema, resolver=None)` casts an instance to a schema; +- `entity.gts_refs` and `entity.schemaRefs` list discovered GTS IDs and `$ref` values with source paths. ---- +### File loading -- [x] **OP#4 - ID Pattern Matching**: Match identifiers against patterns containing wildcards +`GtsFileReader(path, cfg=None)` accepts one file path or a list of file/directory paths. It recursively loads `.json`, `.jsonc`, `.gts`, `.yaml`, and `.yml` files, skips `node_modules`, `dist`, and `build` directories, and yields entities with valid GTS IDs. JSON-family files use Python's standard JSON parser, so `.jsonc` files must not contain comments. ```python -from gts import GtsID, GtsWildcard +from gts import GtsFileReader, GtsStore -gts = GtsID("gts.x.core.events.event.v1.0~") -pattern = GtsWildcard("gts.x.core.events.event.v1~*") -gts.wildcard_match(pattern) # True - v1~* matches any v1.x~ +reader = GtsFileReader(["schemas", "instances.yaml"]) +store = GtsStore(reader) +for entity_id, entity in store.items(): + print(entity_id, entity.is_schema) ``` ---- +TypeSpec (`.tsp`) inputs must be compiled to JSON Schema before loading. -- [x] **OP#5 - ID to UUID Mapping**: Generate deterministic UUIDs from GTS identifiers +### Store API -```python -from gts import GtsID +`GtsStore` is the low-level registry. Use `GtsStore(reader)` to populate it from a `GtsReader`, or `GtsStore(reader=None)` for an empty in-memory store. -gts = GtsID("gts.x.core.events.event.v1~") -uuid = gts.to_uuid() -print(uuid) -``` +| Method | Purpose | +| --- | --- | +| `register(entity)` / `unregister(entity_id)` | Add or remove an in-memory entity. Instances are keyed by `raw_id`; schemas use their GTS ID. | +| `register_schema(type_id, schema)` | Legacy schema registration helper; `type_id` must end in `~`. | +| `get(entity_id)` | Return `GtsEntity` or `None`. | +| `items()` | Return an iterator over in-memory `(entity_id, entity)` pairs. | +| `get_schema_content(type_id)` | Return a schema dictionary or raise `KeyError`. | +| `validate_schema_basic(type_id)` | Check `$ref` format, `x-gts-ref` declarations, and GTS keyword placement. | +| `validate_schema(type_id)` | Run full JSON Schema, derivation, final/abstract, x-gts-ref, and trait validation. | +| `validate_instance(gts_id)` | Validate a well-known or UUID-addressed instance against its type schema. | +| `is_minor_compatible(old_schema_id, new_schema_id)` | Return compatibility verdicts for two registered schemas. | +| `cast(from_id, target_schema_id)` | Cast a registered **instance** to a target schema. | +| `build_schema_graph(gts_id)` | Build the entity/schema reference graph. | +| `query(expr, limit=100)` | Execute an OP#10 query and return `GtsStoreQueryResult`. | ---- +### High-level operations API -- [x] **OP#6 - Schema Validation**: Validate object instances against their corresponding schemas +Import `GtsOps` and its result dataclasses from `gts.ops`. ```python -from gts import GtsStore, GtsFileReader - -reader = GtsFileReader(path="path/to/gts/files") -store = GtsStore(reader=reader) -try: - store.validate_instance(gts_id="gts.x.core.events.event.v1.0~instance.v1") - print("Validation successful") -except Exception as e: - print(f"Validation failed: {e}") +from gts.ops import GtsOps + +ops = GtsOps(path=["schemas", "instances"]) +ops.reload_from_path("replacement-directory") + +ops.add_entity(content, validate=False) +ops.add_entities([content_a, content_b]) +ops.add_schema(type_id, schema) +ops.extract_id(content) +ops.validate_id(gts_id) +ops.parse_id(gts_id) +ops.match_id_pattern(candidate, pattern) +ops.uuid(gts_id) +ops.validate_instance(gts_id) +ops.validate_schema(type_id) +ops.validate_entity(gts_id) +ops.schema_graph(gts_id) +ops.compatibility(old_schema_id, new_schema_id) +ops.cast(instance_id, target_schema_id) +ops.query(expr, limit=100) +ops.attr("gts.example.demo._.event.v1~@properties.name") +ops.get_entity(gts_id) +ops.get_entities(limit=100) +ops.list(limit=100) ``` ---- +Operation methods return result objects with `.to_dict()`. Validation failures are represented by `ok=False` and an `error` string in the facade API; direct `GtsStore` validation methods raise exceptions. -- [x] **OP#7 - Relationship Resolution**: Load all schemas and instances, resolve inter-dependencies, and detect broken references +## Schema validation and GTS extensions -```python -from gts import GtsStore, GtsFileReader +### JSON Schema dialects and references -reader = GtsFileReader(path="path/to/gts/files") -store = GtsStore(reader=reader) -entity = store.get("gts.x.core.events.event.v1~") -# or build a dependency graph for a specific GTS ID -graph = store.build_schema_graph(gts_id="gts.x.core.events.event.v1~") -``` +The library validates schemas with the dialect named by `$schema`; if absent, it uses Draft 7 for schema meta-validation. GTS permits local JSON Pointers (`#/...`) and GTS references (`gts://gts...`) in `$ref`. Other external `$ref` URI schemes are rejected. ---- +During instance validation, GTS references are resolved with `referencing.Registry`. Draft 2019-09 and Draft 2020-12 `$ref` sibling constraints are preserved, so a sibling such as `minLength` is enforced. -- [x] **OP#8 - Compatibility Checking**: Verify that schemas with different MINOR versions are compatible +### `x-gts-ref` -- [x] **OP#8.1 - Backward compatibility checking** -- [x] **OP#8.2 - Forward compatibility checking** -- [x] **OP#8.3 - Full compatibility checking** +`x-gts-ref` restricts a string value to a GTS ID or pattern. It accepts an absolute `gts.` pattern or a JSON Pointer beginning with `/` that resolves to one. If a store is present, the referenced entity must be registered. -```python -from gts import GtsStore, GtsFileReader +`x-gts-ref` can appear in `oneOf`, `anyOf`, and `allOf`. For x-gts-ref-only combinator branches, GTS evaluates the x-gts-ref constraints as the combinator condition. For ordinary or mixed JSON Schema branches, normal JSON Schema structural matching determines which branch’s x-gts-ref constraints are applied. -reader = GtsFileReader(path="path/to/gts/files") -store = GtsStore(reader=reader) -compatible = store.is_minor_compatible( - "gts.x.core.events.event.v1.0~", - "gts.x.core.events.event.v1.1~" -) -print(compatible.is_backward_compatible) -print(compatible.is_forward_compatible) -print(compatible.is_fully_compatible) -``` +### `x-gts-final` and `x-gts-abstract` + +Both keywords must be top-level booleans and cannot both be `true`: + +- `x-gts-final: true` prevents derived type schemas; +- `x-gts-abstract: true` prevents direct instances and defers required trait completeness to concrete descendants. + +### Traits + +`x-gts-traits-schema` declares the schema of type traits, while `x-gts-traits` supplies values. Effective schemas compose through `allOf`; values merge root-to-leaf according to RFC 7396 JSON Merge Patch. + +Missing trait properties are materialized from the nearest `default`, including `default: null`. A JSON Schema `const` constrains a supplied value but is **not** materialized as a missing trait value. Concrete types must resolve required trait properties; abstract types still validate supplied trait values but defer completeness. + +### Derivation and compatibility + +OP#12 derivation accepts a derived schema only when its declared accepted-instance set is included in its base schema, subject to GTS rules for disabled properties and closed `additionalProperties` branches. + +OP#8 compatibility has three string verdicts: ---- +- `compatible`: inclusion was proved; +- `incompatible`: inclusion was disproved; +- `unknown`: the inclusion engine could not prove the relation. -- [x] **OP#9 - Version Casting**: Transform instances between compatible MINOR versions +`GtsEntityCastResult.to_dict()` returns `backward_compatibility`, `forward_compatibility`, and `full_compatibility` using those strings. Its boolean `is_*_compatible` fields are `True` only for `compatible`; they are `False` for both `incompatible` and `unknown`. + +## HTTP server + +Start a local server: + +```bash +gts --path schemas server --host 127.0.0.1 --port 8000 +``` + +`GtsHttpServer` is an internal implementation detail (module `gts._server`) backing the `gts server` CLI command. It is not part of the public API and may change without notice; embed it at your own risk: ```python -from gts import GtsStore, GtsFileReader +from gts.ops import GtsOps +from gts._server import GtsHttpServer # internal, not a stable API -reader = GtsFileReader(path="path/to/gts/files") -store = GtsStore(reader=reader) -result = store.cast( - from_id="gts.x.core.events.event.v1.0~instance.v1", - target_schema_id="gts.x.core.events.event.v1.1~" -) +app = GtsHttpServer(ops=GtsOps()).app +``` + +| Endpoint | Method | Request | +| --- | --- | --- | +| `/entities` | `GET` | `limit` query parameter, 1–1000; lists registered entities. | +| `/entities/{gts_id}` | `GET` | Retrieves one entity. | +| `/entities` | `POST` | Entity/schema body; optional `validate=true` runs full validation. Failed registration returns 422 and is rolled back. | +| `/entities/bulk` | `POST` | JSON array of entity/schema objects. | +| `/type-schemas` | `POST` | `{"type_id": "...~", "type_schema": {...}}`. | +| `/validate-id` | `GET` | `gts_id` query parameter. | +| `/extract-id` | `POST` | JSON entity/schema object. | +| `/parse-id` | `GET` | `gts_id` query parameter. | +| `/match-id-pattern` | `GET` | `candidate` and `pattern` query parameters. | +| `/uuid` | `GET` | `gts_id` query parameter. | +| `/validate-instance` | `POST` | `{"instance_id": "..."}`. | +| `/validate-type-schema` | `POST` | `{"type_id": "...~"}`. | +| `/validate-entity` | `POST` | `{"entity_id": "..."}` or `{"gts_id": "..."}`. When both are supplied, they must be equal. | +| `/resolve-relationships` | `GET` | `gts_id` query parameter. | +| `/compatibility` | `GET` | `old_type_id` and `new_type_id` query parameters. | +| `/cast` | `POST` | `{"instance_id": "...", "to_type_id": "...~"}`. | +| `/query` | `GET` | `expr` and optional `limit` query parameters, 1–1000. | +| `/attr` | `GET` | `gts_with_path` query parameter containing `@path`. | + +FastAPI exposes interactive OpenAPI documentation when the server is running. Generate the OpenAPI JSON without running the service: + +```bash +gts openapi-spec --out openapi.json ``` ---- +## CLI -- [x] **OP#10 - Query Execution**: Filter identifier collections using the GTS query language +All commands accept optional global `--path`, `--config`, and repeatable `-v` / `--verbose` options. Place global options before the subcommand, for example `gts --path schemas query --expr 'gts.example.*'`. -```python -from gts import GtsStore, GtsFileReader - -reader = GtsFileReader(path="path/to/gts/files") -store = GtsStore(reader=reader) -result = store.query("gts.x.core.events.event.v1~[status=active, user=123]") -print(f"Found {result.count} entities") -for entity in result.results: - print(entity) +```bash +gts validate-id --gts-id 'gts.example.demo._.event.v1~' +gts parse-id --gts-id 'gts.example.demo._.event.v1~' +gts match-id-pattern --candidate 'gts.example.demo._.event.v1~' --pattern 'gts.example.*' +gts uuid --gts-id 'gts.example.demo._.event.v1~' +gts validate-instance --gts-id 'gts.example.demo._.event.v1~example.demo._.created.v1' +gts resolve-relationships --gts-id 'gts.example.demo._.event.v1~' +gts compatibility --old-schema-id 'gts.example.demo._.event.v1.0~' --new-schema-id 'gts.example.demo._.event.v1.1~' +gts cast --from-id 'gts.example.demo._.event.v1~example.demo._.created.v1' --to-schema-id 'gts.example.demo._.event.v1.1~' +gts query --expr 'gts.example.demo.*[status=active]' --limit 10 +gts attr --gts-with-path 'gts.example.demo._.event.v1~@properties.name' +gts list --limit 100 +gts server --port 8000 +gts openapi-spec --out openapi.json ``` ---- +CLI commands print their result as JSON. The CLI provides `validate-instance`, but not a `validate-schema` subcommand; use the Python API or `POST /validate-type-schema` for full type-schema validation. -- [x] **OP#11 - Attribute Access**: Retrieve property values and metadata using the attribute selector (`@`) +## File format support + +### JSON and YAML + +JSON (`.json`, `.jsonc`, `.gts`) and YAML (`.yaml`, `.yml`) files are loaded identically by `GtsFileReader`. + +### TypeSpec + +Compile TypeSpec schemas to JSON Schema before loading them: + +```bash +npm install -g @typespec/compiler @typespec/json-schema +tsp compile --emit @typespec/json-schema your-schemas/ +``` ```python -from gts import GtsStore, GtsFileReader - -reader = GtsFileReader(path="path/to/gts/files") -store = GtsStore(reader=reader) -entity = store.get("gts.x.core.events.event.v1~") -if entity: - res = entity.resolve_path("gtsId") - if res.resolved: - print(res.value) - else: - print(res.error) +from gts import GtsFileReader + +entities = list(GtsFileReader("tsp-output/@typespec/json-schema/")) ``` diff --git a/gts/__main__.py b/gts/__main__.py new file mode 100644 index 0000000..089034e --- /dev/null +++ b/gts/__main__.py @@ -0,0 +1,24 @@ +""" +Trampoline for ``python -m gts`` when run from the repository root. + +The gts/ project directory (src-layout) is picked up by Python as a +namespace package before the installed ``gts`` package, which shadows +the real package. This __main__.py fixes sys.path so the real +src-layout package is found, then delegates to its CLI entry point. +""" + +import os +import sys + +# Put the real src-layout package first on sys.path +_src = os.path.join(os.path.dirname(os.path.abspath(__file__)), "src") +sys.path.insert(0, _src) + +# Drop the namespace-package artefact so the real package is imported +for _key in [k for k in sys.modules if k == "gts" or k.startswith("gts.")]: + del sys.modules[_key] + +from gts._cli import main # noqa: E402 + +if __name__ == "__main__": + main() diff --git a/gts/openapi.json b/gts/openapi.json index bbd443b..df00e6f 100644 --- a/gts/openapi.json +++ b/gts/openapi.json @@ -2,7 +2,7 @@ "openapi": "3.1.0", "info": { "title": "GTS Server", - "version": "0.1.0" + "version": "0.13.0" }, "paths": { "/entities": { diff --git a/gts/pyproject.toml b/gts/pyproject.toml index cb55041..893c599 100644 --- a/gts/pyproject.toml +++ b/gts/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "gts" -version = "0.1.0" +version = "0.13.0" description = "Global Type System (GTS) helpers: identifiers, parsing, validation, and operations" readme = "README.md" authors = [{ name = "GTS Community" }] @@ -12,9 +12,12 @@ license = { text = "Apache-2.0" } requires-python = ">=3.9" dependencies = [ "jsonschema>=4.18,<5", + "referencing>=0.30,<0.37", + "jsonsubschema>=0.0.8,<0.1", "fastapi>=0.110,<1", "uvicorn>=0.23,<1", - "pyyaml>=6.0,<7" + "pyyaml>=6.0,<7", + "eval_type_backport>=0.1,<0.3; python_version < '3.10'" ] [project.urls] @@ -22,7 +25,7 @@ Homepage = "https://github.com/globaltypesystem" Repository = "https://github.com/globaltypesystem/gts-python" [project.scripts] -gts = "gts.cli:main" +gts = "gts._cli:main" [tool.hatch.build.targets.wheel] packages = ["src/gts"] diff --git a/gts/src/gts/__init__.py b/gts/src/gts/__init__.py index 25b4501..87f7530 100644 --- a/gts/src/gts/__init__.py +++ b/gts/src/gts/__init__.py @@ -1,43 +1,43 @@ -from .gts import ( - GtsIdSegment, - GtsID, - GtsWildcard, -) from .entities import ( + DEFAULT_GTS_CONFIG, + GtsConfig, + GtsEntity, + GtsFile, ValidationError, ValidationResult, - GtsFile, - GtsEntity, - GtsConfig, - DEFAULT_GTS_CONFIG, +) +from .files_reader import ( + GtsFileReader, +) +from .gts import ( + GtsID, + GtsIdSegment, + GtsWildcard, ) from .path_resolver import GtsPathResolver from .store import ( GtsReader, GtsStore, ) -from .files_reader import ( - GtsFileReader, -) __all__ = [ - "GtsIdSegment", - "GtsID", - "GtsWildcard", - "ValidationError", - "ValidationResult", - "GtsFile", + "DEFAULT_GTS_CONFIG", + "GtsConfig", "GtsEntity", + "GtsFile", + "GtsFileReader", + "GtsID", + "GtsIdSegment", "GtsPathResolver", - "GtsConfig", - "DEFAULT_GTS_CONFIG", "GtsReader", "GtsStore", - "GtsFileReader", + "GtsWildcard", + "JsonEntity", # Backward compatibility aliases "JsonFile", - "JsonEntity", "JsonPathResolver", + "ValidationError", + "ValidationResult", ] # Backward compatibility aliases diff --git a/gts/src/gts/__main__.py b/gts/src/gts/__main__.py index f23a3de..bf054a7 100644 --- a/gts/src/gts/__main__.py +++ b/gts/src/gts/__main__.py @@ -2,7 +2,7 @@ Entry point for running gts as a module: python -m gts """ -from gts.cli import main +from gts._cli import main if __name__ == "__main__": main() diff --git a/gts/src/gts/cli.py b/gts/src/gts/_cli.py similarity index 96% rename from gts/src/gts/cli.py rename to gts/src/gts/_cli.py index 3106f9c..31c16f7 100644 --- a/gts/src/gts/cli.py +++ b/gts/src/gts/_cli.py @@ -1,13 +1,12 @@ from __future__ import annotations import argparse -import logging import json +import logging import sys -from typing import List +from ._server import GtsHttpServer from .ops import GtsOps -from .server import GtsHttpServer def build_parser() -> argparse.ArgumentParser: @@ -93,7 +92,7 @@ def build_parser() -> argparse.ArgumentParser: return p -def main(argv: List[str] | None = None) -> None: +def main(argv: list[str] | None = None) -> None: parser = build_parser() args = parser.parse_args(argv) @@ -129,9 +128,9 @@ def main(argv: List[str] | None = None) -> None: elif args.op == "openapi-spec": server = GtsHttpServer(ops=ops) spec = server.app.openapi() - with open(getattr(args, "out"), "w", encoding="utf-8") as f: + with open(args.out, "w", encoding="utf-8") as f: json.dump(spec, f, ensure_ascii=False, indent=2) - out = {"ok": True, "out": getattr(args, "out")} + out = {"ok": True, "out": args.out} json.dump(out, sys.stdout, ensure_ascii=False, indent=2) sys.stdout.write("\n") return diff --git a/gts/src/gts/server.py b/gts/src/gts/_server.py similarity index 72% rename from gts/src/gts/server.py rename to gts/src/gts/_server.py index 5bb667f..653e65b 100644 --- a/gts/src/gts/server.py +++ b/gts/src/gts/_server.py @@ -1,17 +1,19 @@ from __future__ import annotations -from typing import Any, Dict, List +import logging import sys +import time +from typing import Any -from fastapi import FastAPI, Body, Query +from fastapi import Body, FastAPI, Query from fastapi.responses import JSONResponse -from pydantic import BaseModel, Field +from pydantic import BaseModel, model_validator from starlette.middleware.base import BaseHTTPMiddleware -import time -import logging from .ops import GtsOps +logger = logging.getLogger(__name__) + # ANSI color codes class Colors: @@ -68,10 +70,14 @@ async def receive(): else: status_color = Colors.RED - # Log response at INFO level (verbose >= 1) - logging.info( + # Log response at INFO level (verbose >= 1). + # Neutralize CR/LF in the request-derived path to prevent log forging + # (CWE-117); ASGI percent-decodes scope["path"], so it may contain + # newlines that would otherwise inject forged log records. + safe_path = request.url.path.replace("\r", "\\r").replace("\n", "\\n") + logger.info( f"{Colors.CYAN}{request.method}{Colors.RESET} " - f"{Colors.BLUE}{request.url.path}{Colors.RESET} -> " + f"{Colors.BLUE}{safe_path}{Colors.RESET} -> " f"{status_color}{response.status_code}{Colors.RESET} " f"in {Colors.MAGENTA}{dur:.1f}ms{Colors.RESET}" ) @@ -83,13 +89,13 @@ async def receive(): body_json = json.loads(cached_body.decode("utf-8")) body_str = json.dumps(body_json, indent=2) - logging.debug( + logger.debug( f"{Colors.DIM}Request body:{Colors.RESET}\n" f"{Colors.GRAY}{body_str}{Colors.RESET}" ) - except Exception: + except Exception: # noqa: BLE001 - best-effort debug logging body_str = cached_body.decode("utf-8", errors="replace") - logging.debug( + logger.debug( f"{Colors.DIM}Request body (raw):{Colors.RESET}\n" f"{Colors.GRAY}{body_str}{Colors.RESET}" ) @@ -97,7 +103,7 @@ async def receive(): # Log response body at DEBUG level (verbose >= 2) if self.verbose >= 2: # Read response body - from starlette.responses import StreamingResponse, Response + from starlette.responses import Response, StreamingResponse if isinstance(response, (Response, StreamingResponse)): response_body = b"" @@ -110,13 +116,13 @@ async def receive(): body_json = json.loads(response_body.decode("utf-8")) body_str = json.dumps(body_json, indent=2) - logging.debug( + logger.debug( f"{Colors.DIM}Response body:{Colors.RESET}\n" f"{Colors.GRAY}{body_str}{Colors.RESET}" ) - except Exception: + except Exception: # noqa: BLE001 - best-effort debug logging body_str = response_body.decode("utf-8", errors="replace") - logging.debug( + logger.debug( f"{Colors.DIM}Response body (raw):{Colors.RESET}\n" f"{Colors.GRAY}{body_str}{Colors.RESET}" ) @@ -134,18 +140,39 @@ async def receive(): class SchemaRegister(BaseModel): type_id: str - schema_content: Dict[str, Any] = Field(..., alias="schema") + type_schema: dict[str, Any] class CastRequest(BaseModel): instance_id: str - to_schema_id: str + to_type_id: str class ValidateInstanceRequest(BaseModel): instance_id: str +class ValidateTypeSchemaRequest(BaseModel): + type_id: str + + +class ValidateEntityRequest(BaseModel): + entity_id: str | None = None + gts_id: str | None = None + + @model_validator(mode="after") + def validate_id(self) -> ValidateEntityRequest: + if not self.entity_id and not self.gts_id: + raise ValueError("entity_id (or gts_id) is required") + if self.entity_id and self.gts_id and self.entity_id != self.gts_id: + raise ValueError("entity_id and gts_id must match when both are provided") + return self + + @property + def resolved_id(self) -> str: + return self.entity_id or self.gts_id or "" + + class GtsHttpServer: def __init__( self, @@ -158,7 +185,7 @@ def __init__( self.host = host self.port = port self.base_url = f"http://{self.host}:{self.port}" - self.app = FastAPI(title="GTS Server", version="0.1.0") + self.app = FastAPI(title="GTS Server", version="0.13.0") self.app.add_middleware( _RequestLoggingMiddleware, verbose=self.ops.verbose, @@ -199,10 +226,10 @@ def _register_routes(self) -> None: response_class=JSONResponse, ) app.add_api_route( - "/schemas", + "/type-schemas", self.add_schema, methods=["POST"], - summary="Register schema by explicit type_id", + summary="Register a GTS Type Schema under an explicit type_id", response_class=JSONResponse, ) @@ -248,6 +275,20 @@ def _register_routes(self) -> None: methods=["POST"], summary="Validate instance by GTS ID", ) + # Op #12 - validate type schema + app.add_api_route( + "/validate-type-schema", + self.validate_type_schema, + methods=["POST"], + summary="Validate that a derived GTS Type Schema correctly extends its base chain", + ) + # validate entity (instance or schema) + app.add_api_route( + "/validate-entity", + self.validate_entity, + methods=["POST"], + summary="Validate entity (instance or type schema) by GTS Identifier", + ) # Op #7 - schema graph / relationships app.add_api_route( "/resolve-relationships", @@ -260,7 +301,7 @@ def _register_routes(self) -> None: "/compatibility", self.compatibility, methods=["GET"], - summary="Check minor version compatibility", + summary="Check Type Schema evolution compatibility", ) # Op #9 - cast app.add_api_route( @@ -286,71 +327,85 @@ def _register_routes(self) -> None: # Handlers as methods (no free functions) async def add_entity( - self, body: Dict[str, Any] = Body(...), validate: bool = Query(False) + self, + body: dict[str, Any] = Body(...), # noqa: B008 - FastAPI dependency pattern + validate: bool = Query(False), ) -> JSONResponse: result = self.ops.add_entity(body, validate=validate) status_code = 200 if result.ok else 422 return JSONResponse(result.to_dict(), status_code=status_code) async def add_entities( - self, body: List[Dict[str, Any]] = Body(...) + self, + body: list[dict[str, Any]] = Body(...), # noqa: B008 - FastAPI dependency pattern ) -> JSONResponse: return JSONResponse(self.ops.add_entities(body).to_dict()) async def add_schema(self, body: SchemaRegister) -> JSONResponse: return JSONResponse( - self.ops.add_schema(body.type_id, body.schema_content).to_dict() + self.ops.add_schema(body.type_id, body.type_schema).to_dict() ) - async def validate_id(self, id: str = Query(..., alias="gts_id")) -> Dict[str, Any]: + async def validate_id(self, id: str = Query(..., alias="gts_id")) -> dict[str, Any]: return self.ops.validate_id(id).to_dict() - async def extract_id(self, body: Dict[str, Any] = Body(...)) -> Dict[str, Any]: + async def extract_id( + self, + body: dict[str, Any] = Body(...), # noqa: B008 - FastAPI dependency pattern + ) -> dict[str, Any]: return self.ops.extract_id(body).to_dict() - async def parse(self, id: str = Query(..., alias="gts_id")) -> Dict[str, Any]: + async def parse(self, id: str = Query(..., alias="gts_id")) -> dict[str, Any]: return self.ops.parse_id(id).to_dict() async def match_id_pattern( self, candidate: str = Query(...), pattern: str = Query(...), - ) -> Dict[str, Any]: + ) -> dict[str, Any]: return self.ops.match_id_pattern(candidate, pattern).to_dict() - async def id_to_uuid(self, id: str = Query(..., alias="gts_id")) -> Dict[str, Any]: + async def id_to_uuid(self, id: str = Query(..., alias="gts_id")) -> dict[str, Any]: return self.ops.uuid(id).to_dict() - async def validate_instance(self, body: ValidateInstanceRequest) -> Dict[str, Any]: + async def validate_instance(self, body: ValidateInstanceRequest) -> dict[str, Any]: return self.ops.validate_instance(body.instance_id).to_dict() + async def validate_type_schema( + self, body: ValidateTypeSchemaRequest + ) -> dict[str, Any]: + return self.ops.validate_schema(body.type_id).to_dict() + + async def validate_entity(self, body: ValidateEntityRequest) -> dict[str, Any]: + return self.ops.validate_entity(body.resolved_id).to_dict() + async def schema_graph( self, id: str = Query(..., alias="gts_id") - ) -> Dict[str, Any]: + ) -> dict[str, Any]: return self.ops.schema_graph(id).to_dict() async def compatibility( self, - old: str = Query(..., alias="old_schema_id"), - new: str = Query(..., alias="new_schema_id"), - ) -> Dict[str, Any]: + old: str = Query(..., alias="old_type_id"), + new: str = Query(..., alias="new_type_id"), + ) -> dict[str, Any]: return self.ops.compatibility(old, new).to_dict() - async def cast(self, body: CastRequest) -> Dict[str, Any]: - return self.ops.cast(body.instance_id, body.to_schema_id).to_dict() + async def cast(self, body: CastRequest) -> dict[str, Any]: + return self.ops.cast(body.instance_id, body.to_type_id).to_dict() async def query( self, expr: str = Query(...), limit: int = Query(100, ge=1, le=1000) - ) -> Dict[str, Any]: + ) -> dict[str, Any]: return self.ops.query(expr, limit=limit).to_dict() - async def attr(self, gts_with_path: str = Query(...)) -> Dict[str, Any]: + async def attr(self, gts_with_path: str = Query(...)) -> dict[str, Any]: return self.ops.attr(gts_with_path).to_dict() - async def get_entity(self, gts_id: str) -> Dict[str, Any]: + async def get_entity(self, gts_id: str) -> dict[str, Any]: return self.ops.get_entity(gts_id).to_dict() async def get_entities( self, limit: int = Query(100, ge=1, le=1000) - ) -> Dict[str, Any]: + ) -> dict[str, Any]: return self.ops.get_entities(limit=limit).to_dict() diff --git a/gts/src/gts/compatibility.py b/gts/src/gts/compatibility.py new file mode 100644 index 0000000..3fbf801 --- /dev/null +++ b/gts/src/gts/compatibility.py @@ -0,0 +1,190 @@ +"""Type Schema evolution / derivation compatibility (spec sec 4, OP#8 & OP#12). + +The compatibility relations are defined by accepted-instance-set inclusion, +NOT by structural diffing: + +- backward compatibility: ``Valid(old) subset-of Valid(new)`` (new reads old data) +- forward compatibility: ``Valid(new) subset-of Valid(old)`` (old reads new data) + +Rather than re-implement the inclusion engine, this module delegates the +inclusion primitive to the ``jsonsubschema`` library and only adds the GTS +verdict vocabulary on top. Schema derivation (OP#12) reuses the same primitive +via :func:`check_accepted_set_inclusion`. +""" + +from __future__ import annotations + +from typing import Any + +from jsonschema.validators import validator_for +from jsonsubschema import isSubschema + +COMPATIBLE = "compatible" +INCOMPATIBLE = "incompatible" +UNKNOWN = "unknown" + +# JSON Schema meta keywords and GTS extensions that carry no assertion the +# inclusion engine understands. Stripping them keeps the comparison stable and +# avoids the library reasoning about identifiers or GTS-only annotations. +_META_KEYWORDS = {"$id", "$schema", "$comment", "$anchor", "$dynamicAnchor"} + +_NON_ASSERTION_KEYWORDS = { + "$anchor", + "$comment", + "$defs", + "$dynamicAnchor", + "$id", + "$schema", + "default", + "definitions", + "deprecated", + "description", + "examples", + "readOnly", + "title", + "writeOnly", +} + + +def boolean_schema_value(schema: Any) -> bool | None: + """Return True/False when a schema is boolean-equivalent, else None. + + ``{}`` == True and ``{"not": {}}`` == False; annotation keywords are ignored. + """ + if isinstance(schema, bool): + return schema + if isinstance(schema, dict): + assertions = [ + (k, v) for k, v in schema.items() if k not in _NON_ASSERTION_KEYWORDS + ] + if len(assertions) > 1: + return None + if not assertions: + return True + key, inner = assertions[0] + if key == "not": + iv = boolean_schema_value(inner) + return None if iv is None else (not iv) + return None + return None + + +def _finite_values(schema: Any) -> list[Any] | None: + if not isinstance(schema, dict): + return None + if "const" in schema: + return [schema["const"]] + enum = schema.get("enum") + return list(enum) if isinstance(enum, list) else None + + +def _value_constraint_makes_type_redundant(schema: dict[Any, Any]) -> bool: + if "type" not in schema: + return False + values = _finite_values(schema) + if values is None: + return False + try: + validator = validator_for({"type": schema["type"]})({"type": schema["type"]}) + return all(validator.is_valid(value) for value in values) + except Exception: # noqa: BLE001 - intentional broad fallback + return False + + +def _finite_subset(subset: Any, superset: Any) -> bool | None: + values = _finite_values(subset) + if values is None: + return None + try: + subset_validator = validator_for(subset)(subset) + superset_validator = validator_for(superset)(superset) + return all( + not subset_validator.is_valid(value) or superset_validator.is_valid(value) + for value in values + ) + except Exception: # noqa: BLE001 - intentional broad fallback + return None + + +def sanitize(schema: Any) -> Any: + """Return a copy of ``schema`` with meta/GTS-only keywords removed. + + ``$ref`` is deliberately preserved: callers resolve references before + comparing, and a surviving ``$ref`` means the target was unresolvable. + """ + if isinstance(schema, dict): + # ``jsonsubschema`` rejects some otherwise-valid const/enum schemas with + # a sibling type. The type can only be removed when it accepts every + # enumerated value, which leaves the accepted-instance set unchanged. + drop_type = _value_constraint_makes_type_redundant(schema) + result = {} + for key, value in schema.items(): + if key in _META_KEYWORDS: + continue + if isinstance(key, str) and key.startswith("x-gts-"): + continue + if key == "type" and drop_type: + continue + result[key] = sanitize(value) + return result + if isinstance(schema, list): + return [sanitize(item) for item in schema] + return schema + + +def _coerce_bool_schema(schema: Any) -> Any: + """Turn a top-level boolean schema into its object-equivalent. + + ``jsonsubschema`` only accepts object schemas as operands, so ``true`` and + ``false`` are expressed as ``{}`` and ``{"not": {}}`` respectively. + """ + if schema is True: + return {} + if schema is False: + return {"not": {}} + return schema + + +def _is_subschema(subset: Any, superset: Any) -> bool | None: + """``Valid(subset) subset-of Valid(superset)`` or ``None`` when unprovable.""" + finite_result = _finite_subset(subset, superset) + if finite_result is not None: + return finite_result + try: + return bool( + isSubschema( + _coerce_bool_schema(sanitize(subset)), + _coerce_bool_schema(sanitize(superset)), + ) + ) + except Exception: # noqa: BLE001 - intentional broad fallback + return None + + +def _verdict(result: bool | None) -> str: + if result is None: + return UNKNOWN + return COMPATIBLE if result else INCOMPATIBLE + + +def check_backward_compatibility(old_schema: Any, new_schema: Any) -> str: + """new consumers read old data: ``Valid(old) subset-of Valid(new)``.""" + return _verdict(_is_subschema(old_schema, new_schema)) + + +def check_forward_compatibility(old_schema: Any, new_schema: Any) -> str: + """old consumers read new data: ``Valid(new) subset-of Valid(old)``.""" + return _verdict(_is_subschema(new_schema, old_schema)) + + +def full_verdict(backward: str, forward: str) -> str: + if backward == INCOMPATIBLE or forward == INCOMPATIBLE: + return INCOMPATIBLE + if backward == COMPATIBLE and forward == COMPATIBLE: + return COMPATIBLE + return UNKNOWN + + +def check_accepted_set_inclusion(subset: Any, superset: Any) -> bool | None: + """Shared inclusion primitive used by OP#12 derivation admission.""" + return _is_subschema(subset, superset) diff --git a/gts/src/gts/derivation.py b/gts/src/gts/derivation.py new file mode 100644 index 0000000..a3ce5a5 --- /dev/null +++ b/gts/src/gts/derivation.py @@ -0,0 +1,358 @@ +"""OP#12 - Schema-vs-schema derivation admission (spec sec 4.1). + +Ported from the Rust reference (`schema_derivation.rs`). Admission requires +``Valid(derived) subset-of Valid(base)`` on the most-derived *declaration* of +each property, plus two admission rules that inclusion alone does not express: + +- a derivation may not switch off a base property with ``false`` +- a derivation may not close a nested object level in a way that orphans an + ancestor property under ``allOf`` composition + +The inclusion primitive is provided by :mod:`gts.compatibility` (backed by +``jsonsubschema``); this module only supplies the declaration reduction and the +admission rules. +""" + +from __future__ import annotations + +import copy +from typing import Any + +from .compatibility import boolean_schema_value, check_accepted_set_inclusion + +MAX_RECURSION_DEPTH = 64 +_ADDITIONAL = "additionalProperties" +_STRUCTURAL = {"properties", "required", _ADDITIONAL} + + +def validate_derivation_compatibility( + base_schema: Any, + derived_schema: Any, + base_id: str, + derived_id: str, +) -> list[str]: + """Full OP#12 admission check on resolved base/derived schemas.""" + errors = _validate_derivation(base_schema, derived_schema, base_id, derived_id) + errors.extend( + _validate_closed_descendant_branches( + base_schema, derived_schema, base_id, derived_id + ) + ) + return errors + + +def validate_derivation( + base_schema: Any, + derived_schema: Any, + base_id: str, + derived_id: str, +) -> list[str]: + """Declaration inclusion check without the closed-descendant branch rule.""" + return _validate_derivation(base_schema, derived_schema, base_id, derived_id) + + +def validate_closed_descendant_branches( + ancestor_schema: Any, + descendant_schema: Any, + ancestor_label: str, + descendant_label: str, +) -> list[str]: + return _validate_closed_descendant_branches( + ancestor_schema, descendant_schema, ancestor_label, descendant_label + ) + + +# --- declaration inclusion ------------------------------------------------- +def _validate_derivation( + base_schema: Any, + derived_schema: Any, + base_id: str, + derived_id: str, +) -> list[str]: + base = _declared_schema(base_schema, 0) + derived = _declared_schema(derived_schema, 0) + errors: list[str] = [] + + # An omitted additionalProperties inherits the base's constraint through + # allOf composition rather than reopening the level. + if ( + isinstance(derived, dict) + and _ADDITIONAL not in derived + and isinstance(base, dict) + and _ADDITIONAL in base + ): + derived[_ADDITIONAL] = copy.deepcopy(base[_ADDITIONAL]) + + if ( + isinstance(base, dict) + and boolean_schema_value(base.get(_ADDITIONAL)) is False + and isinstance(derived_schema, dict) + and _ADDITIONAL in derived_schema + and boolean_schema_value(derived_schema.get(_ADDITIONAL)) is not False + ): + errors.append( + f"derived schema '{derived_id}' loosens additionalProperties from a " + f"closed constraint in base '{base_id}'" + ) + + # Admission fails closed: an unprovable inclusion is rejected. + if check_accepted_set_inclusion(derived, base) is not True: + errors.append( + f"derived schema '{derived_id}' is not included in base '{base_id}': " + "the declared schema accepts instances the base rejects" + ) + + _collect_disabled_base_properties(base, derived, base_id, derived_id, errors) + return errors + + +def _declared_schema(schema: Any, depth: int) -> Any: + if not isinstance(schema, dict): + return copy.deepcopy(schema) + if depth >= MAX_RECURSION_DEPTH: + return copy.deepcopy(schema) + declared: dict[str, Any] = {} + additional: list[Any] = [None] + + all_of = schema.get("allOf") + if isinstance(all_of, list): + for branch in all_of: + branch_declared = _declared_schema(branch, depth + 1) + if isinstance(branch_declared, dict): + _absorb_declaration(declared, additional, branch_declared, depth) + _absorb_declaration(declared, additional, schema, depth) + + if additional[0] is not None: + declared[_ADDITIONAL] = additional[0] + return declared + + +def _absorb_declaration( + declared: dict[str, Any], + additional: list[Any], + source: dict[str, Any], + depth: int, +) -> None: + for keyword, value in source.items(): + if keyword == "allOf": + continue + if keyword == _ADDITIONAL: + _merge_additional_properties_constraint(additional, value) + elif keyword == "properties": + target = declared.setdefault("properties", {}) + if isinstance(target, dict) and isinstance(value, dict): + for name, prop in value.items(): + prop_declared = _declared_schema(prop, depth + 1) + if name in target: + target[name] = _absorb_property( + target[name], prop_declared, depth + 1 + ) + else: + target[name] = prop_declared + elif keyword == "required": + target = declared.setdefault("required", []) + if isinstance(target, list) and isinstance(value, list): + for name in value: + if name not in target: + target.append(name) + else: + declared[keyword] = copy.deepcopy(value) + + +def _absorb_property(inherited: Any, overlay: Any, depth: int) -> Any: + if ( + not isinstance(inherited, dict) + or not isinstance(overlay, dict) + or depth >= MAX_RECURSION_DEPTH + ): + return copy.deepcopy(overlay) + + composed: dict[str, Any] = { + k: copy.deepcopy(v) for k, v in overlay.items() if k not in _STRUCTURAL + } + additional: list[Any] = [inherited.get(_ADDITIONAL)] + for keyword in ("properties", "required"): + if keyword in inherited: + composed[keyword] = copy.deepcopy(inherited[keyword]) + _absorb_declaration(composed, additional, overlay, depth) + if additional[0] is not None: + composed[_ADDITIONAL] = additional[0] + return composed + + +def _merge_additional_properties_constraint( + additional: list[Any], candidate: Any +) -> None: + current = additional[0] + if boolean_schema_value(current) is False: + return + if boolean_schema_value(candidate) is True and current is not None: + return + additional[0] = copy.deepcopy(candidate) + + +def _collect_disabled_base_properties( + base: Any, + derived: Any, + base_id: str, + derived_id: str, + errors: list[str], +) -> None: + base_flat = flatten_schema(base) + derived_flat = flatten_schema(derived) + base_props = base_flat.get("properties") if isinstance(base_flat, dict) else None + derived_props = ( + derived_flat.get("properties") if isinstance(derived_flat, dict) else None + ) + if not isinstance(derived_props, dict): + return + for name, derived_property in derived_props.items(): + if ( + derived_property is False + and isinstance(base_props, dict) + and name in base_props + ): + errors.append( + f"property '{name}': derived schema '{derived_id}' disables property " + f"defined in base '{base_id}'" + ) + + +# --- closed-descendant branch admission ------------------------------------ +def _validate_closed_descendant_branches( + ancestor_schema: Any, + descendant_schema: Any, + ancestor_label: str, + descendant_label: str, +) -> list[str]: + errors: list[str] = [] + _collect_closed_descendant_branch_errors( + flatten_schema(ancestor_schema), + descendant_schema, + "", + 0, + ancestor_label, + descendant_label, + errors, + ) + return errors + + +def _collect_closed_descendant_branch_errors( + ancestor: Any, + descendant_schema: Any, + path: str, + depth: int, + ancestor_label: str, + descendant_label: str, + errors: list[str], +) -> None: + if depth >= MAX_RECURSION_DEPTH: + errors.append( + f"schema compatibility check exceeded maximum nesting depth at '{path}' " + f"between ancestor '{ancestor_label}' and descendant '{descendant_label}'" + ) + return + if not isinstance(descendant_schema, dict): + return + + ancestor_props = ancestor.get("properties") if isinstance(ancestor, dict) else None + descendant_props = descendant_schema.get("properties") + descendant_props = descendant_props if isinstance(descendant_props, dict) else None + + if boolean_schema_value(descendant_schema.get(_ADDITIONAL)) is False: + orphaned = sorted( + name + for name in (ancestor_props or {}) + if not (descendant_props and name in descendant_props) + ) + for name in orphaned: + property_path = _join_path(path, name) + errors.append( + f"property '{property_path}': descendant schema '{descendant_label}' " + "sets a closed additionalProperties constraint but does not restate " + f"property defined in ancestor '{ancestor_label}', making it unusable " + "under allOf composition" + ) + + if descendant_props: + common = sorted( + name + for name in descendant_props + if isinstance(ancestor_props, dict) and name in ancestor_props + ) + for name in common: + ancestor_prop = ancestor_props.get(name) # type: ignore[union-attr] + descendant_prop = descendant_props.get(name) + _collect_closed_descendant_branch_errors( + flatten_schema(ancestor_prop), + descendant_prop, + _join_path(path, name), + depth + 1, + ancestor_label, + descendant_label, + errors, + ) + + all_of = descendant_schema.get("allOf") + if isinstance(all_of, list): + for item in all_of: + _collect_closed_descendant_branch_errors( + ancestor, + item, + path, + depth + 1, + ancestor_label, + descendant_label, + errors, + ) + + +def _join_path(prefix: str, name: str) -> str: + return name if not prefix else f"{prefix}.{name}" + + +# --- allOf flattening ------------------------------------------------------ +def flatten_schema(schema: Any) -> Any: + """Merge ``allOf`` into one effective object schema (recursive on props).""" + if not isinstance(schema, dict): + return schema + result: dict[str, Any] = {} + all_of = schema.get("allOf") + if isinstance(all_of, list): + for branch in all_of: + _merge_flat(result, flatten_schema(branch)) + for key, value in schema.items(): + if key == "allOf": + continue + _merge_flat(result, {key: value}) + return result + + +def _merge_flat(target: dict[str, Any], source: dict[str, Any]) -> None: + for key, value in source.items(): + if key == "properties" and isinstance(value, dict): + props = target.setdefault("properties", {}) + for name, prop_schema in value.items(): + if ( + name in props + and isinstance(props[name], dict) + and isinstance(prop_schema, dict) + ): + props[name] = flatten_schema({"allOf": [props[name], prop_schema]}) + else: + props[name] = copy.deepcopy(prop_schema) + elif key == "required" and isinstance(value, list): + required = target.setdefault("required", []) + for name in value: + if name not in required: + required.append(name) + elif key == _ADDITIONAL: + current = target.get(_ADDITIONAL) + if boolean_schema_value(current) is False: + continue + if boolean_schema_value(value) is True and current is not None: + continue + target[_ADDITIONAL] = copy.deepcopy(value) + else: + target[key] = copy.deepcopy(value) diff --git a/gts/src/gts/entities.py b/gts/src/gts/entities.py index 6450b1d..4c92028 100644 --- a/gts/src/gts/entities.py +++ b/gts/src/gts/entities.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any, Dict, List, Optional, Set, Tuple +from typing import TYPE_CHECKING, Any from .gts import GtsID from .schema_cast import GtsEntityCastResult, SchemaCastError @@ -16,13 +16,13 @@ class ValidationError: schemaPath: str keyword: str message: str - params: Dict[str, Any] + params: dict[str, Any] data: Any | None = None @dataclass class ValidationResult: - errors: List[ValidationError] = field(default_factory=list) + errors: list[ValidationError] = field(default_factory=list) @dataclass @@ -31,7 +31,7 @@ class GtsFile: name: str content: Any sequencesCount: int = 0 - sequenceContent: Dict[int, Any] = field(default_factory=dict) + sequenceContent: dict[int, Any] = field(default_factory=dict) validation: ValidationResult = field(default_factory=ValidationResult) def __post_init__(self) -> None: @@ -43,8 +43,8 @@ def __post_init__(self) -> None: @dataclass class GtsConfig: - entity_id_fields: List[str] - schema_id_fields: List[str] + entity_id_fields: list[str] + schema_id_fields: list[str] DEFAULT_GTS_CONFIG = GtsConfig( @@ -74,33 +74,33 @@ class GtsConfig: @dataclass class GtsEntity: - gts_id: Optional[GtsID] = None + gts_id: GtsID | None = None is_schema: bool = False - file: Optional[GtsFile] = None - list_sequence: Optional[int] = None + file: GtsFile | None = None + list_sequence: int | None = None label: str = "" content: Any = None - gts_refs: List[Dict[str, str]] = field(default_factory=list) + gts_refs: list[dict[str, str]] = field(default_factory=list) validation: ValidationResult = field(default_factory=ValidationResult) - schemaId: Optional[str] = None - selected_entity_field: Optional[str] = None - selected_schema_id_field: Optional[str] = None + type_id: str | None = None + selected_entity_field: str | None = None + selected_type_id_field: str | None = None description: str = "" - raw_id: Optional[str] = None # Stores raw ID value (may be non-GTS) - schemaRefs: List[Dict[str, str]] = field(default_factory=list) + raw_id: str | None = None # Stores raw ID value (may be non-GTS) + schemaRefs: list[dict[str, str]] = field(default_factory=list) def __init__( self, *, - file: Optional[GtsFile] = None, - list_sequence: Optional[int] = None, + file: GtsFile | None = None, + list_sequence: int | None = None, content: Any = None, - cfg: Optional[GtsConfig] = None, - gts_id: Optional[GtsID] = None, + cfg: GtsConfig | None = None, + gts_id: GtsID | None = None, is_schema: bool = False, label: str = "", - validation: Optional[ValidationResult] = None, - schemaId: Optional[str] = None, + validation: ValidationResult | None = None, + type_id: str | None = None, ) -> None: self.file = file self.list_sequence = list_sequence @@ -109,9 +109,9 @@ def __init__( self.is_schema = is_schema self.label = label self.validation = validation or ValidationResult() - self.schemaId = schemaId + self.type_id = type_id self.selected_entity_field = None - self.selected_schema_id_field = None + self.selected_type_id_field = None self.gts_refs = [] self.schemaRefs = [] self.description = "" @@ -124,11 +124,12 @@ def __init__( if cfg is not None: idv = self._calc_json_entity_id(cfg) self.raw_id = idv # Store raw ID even if non-GTS - self.schemaId = self._calc_json_schema_id(cfg) + self.type_id = self._calc_json_schema_id(cfg) # If no valid GTS ID found in entity fields, use schema ID as fallback - if not (idv and GtsID.is_valid(idv)): - if self.schemaId and GtsID.is_valid(self.schemaId): - idv = self.schemaId + if not (idv and GtsID.is_valid(idv)) and ( + self.type_id and GtsID.is_valid(self.type_id) + ): + idv = self.type_id self.gts_id = GtsID(idv) if idv and GtsID.is_valid(idv) else None # Set label @@ -159,14 +160,10 @@ def _is_json_schema_entity(self) -> bool: url = self.content.get("$schema") if not isinstance(url, str): return False - if url.startswith("http://json-schema.org/"): - return True - if url.startswith("https://json-schema.org/"): - return True # Issue #25: strict check, no GTS IDs in $schema - return False + return url.startswith(("http://json-schema.org/", "https://json-schema.org/")) - def resolve_path(self, path: str) -> "GtsPathResolver": + def resolve_path(self, path: str) -> GtsPathResolver: from .path_resolver import GtsPathResolver resolver = GtsPathResolver(self.gts_id.id if self.gts_id else "", self.content) @@ -174,17 +171,20 @@ def resolve_path(self, path: str) -> "GtsPathResolver": def cast( self, - to_schema: "GtsEntity", - from_schema: "GtsEntity", - resolver: Optional[Any] = None, + to_schema: GtsEntity, + from_schema: GtsEntity, + resolver: Any | None = None, ) -> GtsEntityCastResult: - if self.is_schema: + if ( + self.is_schema + and from_schema.gts_id + and self.gts_id.id != from_schema.gts_id.id + ): # When casting a schema, from_schema might be a standard JSON Schema (no gts_id) # In that case, skip the sanity check - if from_schema.gts_id and self.gts_id.id != from_schema.gts_id.id: - raise SchemaCastError( - f"Internal error: {self.gts_id.id} != {from_schema.gts_id.id}" - ) + raise SchemaCastError( + f"Internal error: {self.gts_id.id} != {from_schema.gts_id.id}" + ) if not to_schema.is_schema: raise SchemaCastError("Target must be a schema") if not from_schema.is_schema: @@ -201,7 +201,7 @@ def cast( def _walk_and_collect( self, content: Any, - collector: List[Dict[str, str]], + collector: list[dict[str, str]], matcher: Any, # Callable but avoiding import ) -> None: """Generic tree walker that collects matching nodes. @@ -234,25 +234,24 @@ def walk(node: Any, current_path: str = "") -> None: walk(content) def _deduplicate_by_id_and_path( - self, items: List[Dict[str, str]] - ) -> List[Dict[str, str]]: + self, items: list[dict[str, str]] + ) -> list[dict[str, str]]: """Deduplicate items by their id and sourcePath.""" - uniq: Dict[str, Dict[str, str]] = {} + uniq: dict[str, dict[str, str]] = {} for item in items: key = f"{item['id']}|{item['sourcePath']}" uniq[key] = item return list(uniq.values()) - def _extract_gts_ids_with_paths(self) -> List[Dict[str, str]]: + def _extract_gts_ids_with_paths(self) -> list[dict[str, str]]: """Extract all GTS IDs from content with their paths.""" - found: List[Dict[str, str]] = [] + found: list[dict[str, str]] = [] - def gts_id_matcher(node: Any, path: str) -> Optional[Dict[str, str]]: + def gts_id_matcher(node: Any, path: str) -> dict[str, str] | None: """Match GTS ID strings.""" if isinstance(node, str): val = node - if val.startswith("gts://"): - val = val[6:] + val = val.removeprefix("gts://") if GtsID.is_valid(val): return {"id": val, "sourcePath": path or "root"} return None @@ -260,17 +259,16 @@ def gts_id_matcher(node: Any, path: str) -> Optional[Dict[str, str]]: self._walk_and_collect(self.content, found, gts_id_matcher) return self._deduplicate_by_id_and_path(found) - def _extract_ref_strings_with_paths(self) -> List[Dict[str, str]]: + def _extract_ref_strings_with_paths(self) -> list[dict[str, str]]: """Extract $ref strings with their paths (for schemas).""" - refs: List[Dict[str, str]] = [] + refs: list[dict[str, str]] = [] - def ref_matcher(node: Any, path: str) -> Optional[Dict[str, str]]: + def ref_matcher(node: Any, path: str) -> dict[str, str] | None: """Match $ref properties in dict nodes.""" if isinstance(node, dict) and isinstance(node.get("$ref"), str): val = node["$ref"] # Issue #32: handle gts:// prefix - if val.startswith("gts://"): - val = val[6:] + val = val.removeprefix("gts://") ref_path = f"{path}.$ref" if path else "$ref" return {"id": val, "sourcePath": ref_path} return None @@ -278,19 +276,18 @@ def ref_matcher(node: Any, path: str) -> Optional[Dict[str, str]]: self._walk_and_collect(self.content, refs, ref_matcher) return self._deduplicate_by_id_and_path(refs) - def _get_field_value(self, field: str) -> Optional[str]: + def _get_field_value(self, field: str) -> str | None: """Get string value from content field.""" if not isinstance(self.content, dict): return None v = self.content.get(field) if isinstance(v, str) and v.strip(): # Issue #31, #32: Handle gts:// prefix in fields (e.g. $id) - if v.startswith("gts://"): - v = v[6:] + v = v.removeprefix("gts://") return v return None - def _first_non_empty_field(self, fields: List[str]) -> Optional[Tuple[str, str]]: + def _first_non_empty_field(self, fields: list[str]) -> tuple[str, str] | None: """Find first non-empty field value in order. Returns the first non-empty string value without preferring GTS IDs. @@ -311,7 +308,7 @@ def _calc_json_entity_id(self, cfg: GtsConfig) -> str: return f"{self.file.path}#{self.list_sequence}" return self.file.path if self.file else "" - def _calc_json_schema_id(self, cfg: GtsConfig) -> Optional[str]: + def _calc_json_schema_id(self, cfg: GtsConfig) -> str | None: """Calculate schema_id based on entity type and content. Rules: @@ -324,29 +321,18 @@ def _calc_json_schema_id(self, cfg: GtsConfig) -> Optional[str]: # Get entity ID (the $id field for schemas) idv = self._get_field_value("$id") if idv and GtsID.is_valid(idv): - # Check if it's a chained ID (derived schema) - last_tilde = idv.rfind("~") + # For schemas, a chained $id means derivation. + # type_id is the parent (everything up to the second-to-last '~'). + # idv ends with '~' for schemas. + # Strip trailing '~' to find internal chain boundaries. + inner = idv.removesuffix("~") + last_tilde = inner.rfind("~") if last_tilde > 0: - # Find the previous segment (parent) - parent_end = last_tilde - # Check if there's another segment before this one - prefix = idv[:parent_end] - prev_tilde = prefix.rfind("~") - if prev_tilde > 0: - # Has a parent chain - return first segment (base type) - self.selected_schema_id_field = "$id" - return prefix[: prev_tilde + 1] - else: - # Single segment schema - base type, return $schema - schema_val = self._get_field_value("$schema") - if schema_val: - self.selected_schema_id_field = "$schema" - return schema_val - # Fallback to $schema for schemas - schema_val = self._get_field_value("$schema") - if schema_val: - self.selected_schema_id_field = "$schema" - return schema_val + # Has at least 2 segments - return parent chain + self.selected_type_id_field = "$id" + return inner[: last_tilde + 1] + # Base schema (single segment) - no GTS parent type. + # The $schema URL is NOT a GTS Type Identifier. return None # PRIORITY 1: Check entity_id_fields for a GTS ID (gtsId, id, etc.) @@ -362,53 +348,35 @@ def _calc_json_schema_id(self, cfg: GtsConfig) -> Optional[str]: idv = entity_id_cand[1] # If already a type id (ends with '~'), use it as-is if idv.endswith("~"): - self.selected_schema_id_field = entity_id_cand[0] + self.selected_type_id_field = entity_id_cand[0] return idv # For chained IDs (well-known instances), extract schema: # everything up to and including last '~' last_tilde = idv.rfind("~") if last_tilde > 0: - self.selected_schema_id_field = entity_id_cand[0] + self.selected_type_id_field = entity_id_cand[0] return idv[: last_tilde + 1] # PRIORITY 2: Fall back to explicit schema_id_fields (type, gtsTid, etc.) # Only check these if no chained GTS ID was found in entity_id_fields + # NOTE: Only use these for instances (non-schemas) - schemas use $id chain cand = self._first_non_empty_field(cfg.schema_id_fields) if cand: - self.selected_schema_id_field = cand[0] - schema_id = cand[1] - # If schema_id is a chained GTS ID, extract parent (base type) - if GtsID.is_valid(schema_id): - last_tilde = schema_id.rfind("~") - if last_tilde > 0 and not schema_id.endswith("~"): + self.selected_type_id_field = cand[0] + type_id_val = cand[1] + # If type_id is a chained GTS ID, extract parent (base type) + if GtsID.is_valid(type_id_val): + last_tilde = type_id_val.rfind("~") + if last_tilde > 0 and not type_id_val.endswith("~"): # It's an instance ID in type field - extract schema part - return schema_id[: last_tilde + 1] - return schema_id + return type_id_val[: last_tilde + 1] + return type_id_val # No schema reference found for instance return None - def _extract_uuid_from_content(self) -> Optional[str]: - """Extract a UUID value from content to use as instance identifier.""" - if not isinstance(self.content, dict): - return None - # Look for common UUID fields - for field_name in ["id", "uuid", "instanceId", "instance_id"]: - val = self.content.get(field_name) - if isinstance(val, str) and val.strip(): - # Check if it looks like a UUID (basic check) - import re - - if re.match( - r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", - val.lower(), - ): - # Convert UUID to a valid GTS segment format - return val.replace("-", "_") - return None - - def get_graph(self) -> Dict[str, Set[str]]: + def get_graph(self) -> dict[str, set[str]]: refs = {} for r in self.gts_refs: refs[r["sourcePath"]] = r["id"] - return {"id": self.gts_id.id, "schema_id": self.schemaId, "refs": refs} + return {"id": self.gts_id.id, "type_id": self.type_id, "refs": refs} diff --git a/gts/src/gts/files_reader.py b/gts/src/gts/files_reader.py index 10e370b..66cd24d 100644 --- a/gts/src/gts/files_reader.py +++ b/gts/src/gts/files_reader.py @@ -1,24 +1,26 @@ from __future__ import annotations import json -import yaml -from pathlib import Path +import logging import os -from typing import Iterator, List, Optional, Any - -from .store import GtsReader -from .entities import GtsEntity, GtsFile, DEFAULT_GTS_CONFIG, GtsConfig +from collections.abc import Iterator +from pathlib import Path +from typing import Any -import logging +import yaml +from .entities import DEFAULT_GTS_CONFIG, GtsConfig, GtsEntity, GtsFile +from .store import GtsReader EXCLUDE_LIST = ["node_modules", "dist", "build"] +logger = logging.getLogger(__name__) + class GtsFileReader(GtsReader): """Reads GTS entities from JSON and YAML files in directories specified by path.""" - def __init__(self, path: str | List[str], cfg: Optional[GtsConfig] = None) -> None: + def __init__(self, path: str | list[str], cfg: GtsConfig | None = None) -> None: """ Initialize FileReader with one or more paths. @@ -26,16 +28,16 @@ def __init__(self, path: str | List[str], cfg: Optional[GtsConfig] = None) -> No path: Single path string or list of paths (files or directories) cfg: GtsConfig for entity ID extraction (defaults to DEFAULT_GTS_CONFIG) """ - self.paths: List[Path] = [] + self.paths: list[Path] = [] if isinstance(path, str): self.paths = [Path(os.path.expanduser(path))] else: self.paths = [Path(os.path.expanduser(p)) for p in path] self.cfg = cfg or DEFAULT_GTS_CONFIG - self._files: List[Path] = [] + self._files: list[Path] = [] self._current_index = 0 - self._current_file_entities: List[GtsEntity] = [] + self._current_file_entities: list[GtsEntity] = [] self._current_entity_index = 0 self._initialized = False @@ -43,7 +45,7 @@ def _collect_files(self) -> None: """Collect all JSON and YAML files from the specified paths, following symlinks.""" valid_extensions = {".json", ".jsonc", ".gts", ".yaml", ".yml"} seen: set[str] = set() - collected: List[Path] = [] + collected: list[Path] = [] for path in self.paths: # Resolve symlinks and make absolute (non-strict to allow non-existing paths to be handled gracefully) @@ -54,7 +56,7 @@ def _collect_files(self) -> None: rp = str(resolved_path) if rp not in seen: seen.add(rp) - logging.debug(f"- discovered file: {resolved_path}") + logger.debug(f"- discovered file: {resolved_path}") collected.append(resolved_path) elif resolved_path.is_dir(): # Recursively scan for all valid file types, following symlinks @@ -69,7 +71,7 @@ def _collect_files(self) -> None: rp = str(fpath.resolve(strict=False)) if rp not in seen: seen.add(rp) - logging.debug(f"- discovered file: {fpath}") + logger.debug(f"- discovered file: {fpath}") collected.append(Path(rp)) self._files = collected @@ -82,9 +84,9 @@ def _load_file(self, file_path: Path) -> Any: else: return json.load(f) - def _process_file(self, file_path: Path) -> List[GtsEntity]: + def _process_file(self, file_path: Path) -> list[GtsEntity]: """Process a single JSON or YAML file and return list of GtsEntity objects.""" - entities: List[GtsEntity] = [] + entities: list[GtsEntity] = [] try: content = self._load_file(file_path) @@ -99,18 +101,17 @@ def _process_file(self, file_path: Path) -> List[GtsEntity]: file=json_file, list_sequence=idx, content=item, cfg=self.cfg ) if entity.gts_id: - logging.debug(f"- discovered entity: {entity.gts_id.id}") + logger.debug(f"- discovered entity: {entity.gts_id.id}") entities.append(entity) else: entity = GtsEntity( file=json_file, list_sequence=None, content=content, cfg=self.cfg ) if entity.gts_id: - logging.debug(f"- discovered entity: {entity.gts_id.id}") + logger.debug(f"- discovered entity: {entity.gts_id.id}") entities.append(entity) except Exception: - # Skip files that can't be parsed - pass + logger.debug("skipping unparsable file: %s", file_path, exc_info=True) return entities @@ -120,13 +121,11 @@ def __iter__(self) -> Iterator[GtsEntity]: self._collect_files() self._initialized = True - logging.debug(f"Processing {len(self._files)} files from {self.paths}") + logger.debug(f"Processing {len(self._files)} files from {self.paths}") for file_path in self._files: - entities = self._process_file(file_path) - for entity in entities: - yield entity + yield from self._process_file(file_path) - def read_by_id(self, entity_id: str) -> Optional[GtsEntity]: + def read_by_id(self, entity_id: str) -> GtsEntity | None: """ Read a GtsEntity by its ID. For FileReader, this returns None as we don't support random access by ID. diff --git a/gts/src/gts/gts.py b/gts/src/gts/gts.py index b100bb8..eeef6cf 100644 --- a/gts/src/gts/gts.py +++ b/gts/src/gts/gts.py @@ -3,18 +3,19 @@ import re import shlex import uuid -from typing import List, Optional, Tuple, Dict, Any +from typing import Any GTS_PREFIX = "gts." GTS_URI_PREFIX = "gts://" GTS_NS = uuid.uuid5(uuid.NAMESPACE_URL, "gts") GTS_SEGMENT_TOKEN_REGEX = re.compile(r"^[a-z_][a-z0-9_]*$") +UUID_REGEX = re.compile( + r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$" +) class GtsInvalidSegment(ValueError): - def __init__( - self, num: int, offset: int, segment: str, cause: Optional[str] = None - ): + def __init__(self, num: int, offset: int, segment: str, cause: str | None = None): if cause: super().__init__( f"Invalid GTS segment #{num} @ offset {offset}: '{segment}': {cause}" @@ -30,7 +31,7 @@ def __init__( class GtsInvalidId(ValueError): - def __init__(self, gts_id: str, cause: Optional[str] = None): + def __init__(self, gts_id: str, cause: str | None = None): if cause: super().__init__(f"Invalid GTS identifier: {gts_id}: {cause}") else: @@ -40,7 +41,7 @@ def __init__(self, gts_id: str, cause: Optional[str] = None): class GtsInvalidWildcard(ValueError): - def __init__(self, pattern: str, cause: Optional[str] = None): + def __init__(self, pattern: str, cause: str | None = None): if cause: super().__init__(f"Invalid GTS wildcard pattern: {pattern}: {cause}") else: @@ -66,7 +67,7 @@ def __init__(self, num: int, offset: int, segment: str): self.namespace: str = "" self.type: str = "" self.ver_major: int = 0 - self.ver_minor: Optional[int] = None + self.ver_minor: int | None = None self.is_type: bool = False self.is_wildcard: bool = False @@ -91,7 +92,7 @@ def _parse_segment_id(self, num: int, offset: int, segment: str): if len(tokens) < 5: raise GtsInvalidSegment(num, offset, segment, "Too few tokens") - for t in range(0, 4): + for t in range(4): if not GTS_SEGMENT_TOKEN_REGEX.match(tokens[t]): raise GtsInvalidSegment( num, offset, segment, "Invalid segment token: " + tokens[t] @@ -167,43 +168,83 @@ def _parse_segment_id(self, num: int, offset: int, segment: str): num, offset, segment, "Minor version must be an integer" ) + @classmethod + def _uuid_tail_segment(cls, num: int, offset: int, uuid_str: str) -> GtsIdSegment: + """Create a special UUID tail segment for combined anonymous instances.""" + seg = object.__new__(cls) + seg.num = num + seg.offset = offset + seg.segment = uuid_str + seg.vendor = "" + seg.package = "" + seg.namespace = "" + seg.type = "" + seg.ver_major = None + seg.ver_minor = None + seg.is_type = False + seg.is_wildcard = False + seg._is_uuid_tail = True + return seg + class GtsID: def __init__(self, id: str): raw = id.strip() # Strip gts:// URI prefix if present - if raw.startswith(GTS_URI_PREFIX): - raw = raw[len(GTS_URI_PREFIX) :] + raw = raw.removeprefix(GTS_URI_PREFIX) # Validate it's lower case if raw != raw.lower(): raise GtsInvalidId(id, "Must be lower case") - if "-" in raw: - raise GtsInvalidId(id, "Must not contain '-'") - if not raw.startswith(GTS_PREFIX): raise GtsInvalidId(id, f"Does not start with '{GTS_PREFIX}'") if len(raw) > 1024: raise GtsInvalidId(id, "Too long") self.id: str = raw - self.gts_id_segments: List[GtsIdSegment] = [] + self.gts_id_segments: list[GtsIdSegment] = [] + self.uuid_tail: str | None = None + + # Detect combined anonymous instance: last tilde-part is a UUID + remainder = raw[len(GTS_PREFIX) :] + tilde_parts = remainder.split("~") + last_part = tilde_parts[-1] if tilde_parts else "" + if UUID_REGEX.match(last_part) and len(tilde_parts) >= 2: + self.uuid_tail = last_part + # Hyphens are only allowed in the UUID tail + segments_portion = raw[: len(raw) - len(last_part) - 1] # strip ~ + if "-" in segments_portion: + raise GtsInvalidId(id, "Must not contain '-'") + else: + if "-" in raw: + raise GtsInvalidId(id, "Must not contain '-'") # split preserving empties to detect trailing '~' _parts = raw[len(GTS_PREFIX) :].split("~") - parts = [] - for i in range(0, len(_parts)): - if i < len(_parts) - 1: + + # If UUID tail, exclude it from segment parsing + if self.uuid_tail: + # All parts before UUID are type segments (end with ~) + seg_count = len(_parts) - 1 # exclude UUID tail + parts = [] + for i in range(seg_count): + if _parts[i] == "": + raise GtsInvalidId(id, f"GTS segment #{i + 1} is empty") parts.append(_parts[i] + "~") - if i == len(_parts) - 2 and _parts[i + 1] == "": - break - else: - parts.append(_parts[i]) + else: + parts = [] + for i in range(len(_parts)): + if i < len(_parts) - 1: + parts.append(_parts[i] + "~") + if i == len(_parts) - 2 and _parts[i + 1] == "": + break + else: + parts.append(_parts[i]) offset = len(GTS_PREFIX) - for i in range(0, len(parts)): + for i in range(len(parts)): if parts[i] == "": raise GtsInvalidId( id, f"GTS segment #{i + 1} @ offset {offset} is empty" @@ -212,41 +253,59 @@ def __init__(self, id: str): self.gts_id_segments.append(GtsIdSegment(i + 1, offset, parts[i])) offset += len(parts[i]) + # Add UUID tail as a special segment if present + if self.uuid_tail: + self.gts_id_segments.append( + GtsIdSegment._uuid_tail_segment( + len(self.gts_id_segments) + 1, offset, self.uuid_tail + ) + ) + # Issue #37: Single-segment instance IDs are not allowed # An instance ID (not ending with ~) must be chained (have at least 2 segments) - if not self.id.endswith("~") and len(self.gts_id_segments) == 1: + # UUID tail is exempt + non_uuid_segments = [ + s for s in self.gts_id_segments if not getattr(s, "_is_uuid_tail", False) + ] + if ( + not self.id.endswith("~") + and self.uuid_tail is None + and len(non_uuid_segments) == 1 + and not any(seg.is_wildcard for seg in self.gts_id_segments) + ): # Check if it's a wildcard (wildcards are allowed as single segment) - if not any(seg.is_wildcard for seg in self.gts_id_segments): - raise GtsInvalidId( - id, - "Single-segment instance IDs are not allowed. " - "Instance IDs must be chained (e.g., type~instance).", - ) + raise GtsInvalidId( + id, + "Single-segment instance IDs are not allowed. " + "Instance IDs must be chained (e.g., type~instance).", + ) @property def is_type(self) -> bool: return self.id.endswith("~") - def get_type_id(self) -> Optional[str]: + def get_type_id(self) -> str | None: if len(self.gts_id_segments) < 2: return None return GTS_PREFIX + "".join([s.segment for s in self.gts_id_segments[:-1]]) def to_uuid(self) -> uuid.UUID: + # For combined anonymous instances, return the embedded UUID directly + if self.uuid_tail: + return uuid.UUID(self.uuid_tail) return uuid.uuid5(GTS_NS, self.id) @classmethod def is_valid(cls, s: str) -> bool: # Strip gts:// URI prefix if present normalized = s - if normalized.startswith(GTS_URI_PREFIX): - normalized = normalized[len(GTS_URI_PREFIX) :] + normalized = normalized.removeprefix(GTS_URI_PREFIX) if not normalized.startswith(GTS_PREFIX): return False try: _ = cls(s) return True - except Exception: + except Exception: # noqa: BLE001 - any parsing failure means invalid ID return False def wildcard_match(self, pattern: GtsWildcard) -> bool: @@ -254,8 +313,19 @@ def wildcard_match(self, pattern: GtsWildcard) -> bool: # Helper function to match segments with version flexibility def match_segments( - pattern_segs: List[GtsIdSegment], candidate_segs: List[GtsIdSegment] + pattern_segs: list[GtsIdSegment], candidate_segs: list[GtsIdSegment] ) -> bool: + # Pattern ending with '~*' means "this type and any descendants". + # It should match both: + # - the base type itself (same prefix, no extra segment), and + # - instances/derived ids under that prefix. + if ( + pattern_segs + and pattern_segs[-1].is_wildcard + and len(pattern_segs) == len(candidate_segs) + 1 + ): + return match_segments(pattern_segs[:-1], candidate_segs) + # If pattern is longer than candidate, no match if len(pattern_segs) > len(candidate_segs): return False @@ -274,19 +344,18 @@ def match_segments( return False if p_seg.type and p_seg.type != c_seg.type: return False - # Check version fields if they are set in the pattern - if p_seg.ver_major != 0 and p_seg.ver_major != c_seg.ver_major: + # Check version fields when version is explicitly present in + # the wildcard segment (including v0.*). + if ".v" in p_seg.segment and p_seg.ver_major != c_seg.ver_major: return False if ( p_seg.ver_minor is not None and p_seg.ver_minor != c_seg.ver_minor ): return False - # Check is_type flag if set - if p_seg.is_type and p_seg.is_type != c_seg.is_type: - return False - # Wildcard matches - accept anything after this point - return True + # Check is_type flag if set; if it doesn't match, wildcard fails, + # otherwise wildcard matches - accept anything after this point + return not (p_seg.is_type and p_seg.is_type != c_seg.is_type) # Non-wildcard segment - all fields must match exactly # Check vendor, package, namespace, type match @@ -306,9 +375,8 @@ def match_segments( # Minor version: if pattern has no minor version, accept any minor in candidate # If pattern has minor version, it must match exactly - if p_seg.ver_minor is not None: - if p_seg.ver_minor != c_seg.ver_minor: - return False + if p_seg.ver_minor is not None and p_seg.ver_minor != c_seg.ver_minor: + return False # else: pattern has no minor version, so any minor version in candidate is OK # Check is_type flag matches @@ -330,10 +398,10 @@ def match_segments( # Use segment matching for wildcard patterns too return match_segments(pattern.gts_id_segments, self.gts_id_segments) - def parse_query(self, expr: str) -> Tuple[str, Dict[str, str]]: + def parse_query(self, expr: str) -> tuple[str, dict[str, str]]: base, _, filt = expr.partition("[") gts_base = base.strip() - conditions: Dict[str, str] = {} + conditions: dict[str, str] = {} if filt: filt = filt.rsplit("]", 1)[0] tokens = shlex.split(filt) @@ -343,7 +411,7 @@ def parse_query(self, expr: str) -> Tuple[str, Dict[str, str]]: conditions[k.strip()] = v.strip().strip('"') return gts_base, conditions - def match_query(self, obj: Dict[str, Any], gts_field: str, expr: str) -> bool: + def match_query(self, obj: dict[str, Any], gts_field: str, expr: str) -> bool: gts_base, cond = self.parse_query(expr) if not self.id.startswith(gts_base): return False @@ -356,7 +424,7 @@ def match_query(self, obj: Dict[str, Any], gts_field: str, expr: str) -> bool: return True @classmethod - def split_at_path(cls, gts_with_path: str) -> Tuple[str, Optional[str]]: + def split_at_path(cls, gts_with_path: str) -> tuple[str, str | None]: if "@" not in gts_with_path: return gts_with_path, None gts, path = gts_with_path.split("@", 1) diff --git a/gts/src/gts/ops.py b/gts/src/gts/ops.py index 1adc38e..4954c83 100644 --- a/gts/src/gts/ops.py +++ b/gts/src/gts/ops.py @@ -1,17 +1,17 @@ from __future__ import annotations -from typing import Dict, List, Optional, Any -from dataclasses import dataclass, field - +import builtins import json +from dataclasses import dataclass, field from pathlib import Path as SysPath +from typing import Any -from .gts import GtsID, GtsWildcard from .entities import DEFAULT_GTS_CONFIG, GtsConfig, GtsEntity from .files_reader import GtsFileReader +from .gts import GtsID, GtsWildcard from .path_resolver import GtsPathResolver -from .store import GtsStore, GtsStoreQueryResult from .schema_cast import GtsEntityCastResult +from .store import GtsStore, GtsStoreQueryResult # Interface helpers @@ -23,15 +23,20 @@ class GtsIdValidationResult: id: str valid: bool error: str = "" + is_type: bool | None = None is_wildcard: bool = False - def to_dict(self) -> Dict[str, Any]: - return { + def to_dict(self) -> dict[str, Any]: + d: dict[str, Any] = { "id": self.id, "valid": self.valid, - "error": self.error, "is_wildcard": self.is_wildcard, } + if self.error: + d["error"] = self.error + if self.is_type is not None: + d["is_type"] = self.is_type + return d @dataclass @@ -42,11 +47,11 @@ class GtsIdSegment: package: str namespace: str type: str - ver_major: Optional[int] - ver_minor: Optional[int] + ver_major: int | None + ver_minor: int | None is_type: bool - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: return { "vendor": self.vendor, "package": self.package, @@ -64,20 +69,23 @@ class GtsIdParseResult: id: str ok: bool - segments: List[GtsIdSegment] = field(default_factory=list) + segments: list[GtsIdSegment] = field(default_factory=list) error: str = "" + is_type: bool | None = None is_wildcard: bool = False - is_schema: bool = False - def to_dict(self) -> Dict[str, Any]: - return { + def to_dict(self) -> dict[str, Any]: + d: dict[str, Any] = { "id": self.id, "ok": self.ok, "segments": [s.to_dict() for s in self.segments], - "error": self.error, "is_wildcard": self.is_wildcard, - "is_schema": self.is_schema, } + if self.error: + d["error"] = self.error + if self.is_type is not None: + d["is_type"] = self.is_type + return d @dataclass @@ -89,7 +97,7 @@ class GtsIdMatchResult: match: bool error: str = "" - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: result = { "candidate": self.candidate, "pattern": self.pattern, @@ -107,7 +115,7 @@ class GtsUuidResult: id: str uuid: str - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: return {"id": self.id, "uuid": self.uuid} @@ -119,20 +127,38 @@ class GtsValidationResult: ok: bool error: str = "" - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: result = {"id": self.id, "ok": self.ok} if self.error: result["error"] = self.error return result +@dataclass +class GtsEntityValidationResult: + """Result of validating an entity (instance or type schema).""" + + id: str + ok: bool + entity_type: str = "" + error: str = "" + + def to_dict(self) -> dict[str, Any]: + result: dict[str, Any] = {"id": self.id, "ok": self.ok} + if self.entity_type: + result["entity_type"] = self.entity_type + if self.error: + result["error"] = self.error + return result + + @dataclass class GtsSchemaGraphResult: """Result of building a schema graph for an entity.""" - graph: Dict[str, Any] + graph: dict[str, Any] - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: return self.graph @@ -141,14 +167,14 @@ class GtsEntityInfo: """Information about a single entity.""" id: str - schema_id: Optional[str] - is_schema: bool + type_id: str | None + is_type_schema: bool - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: return { "id": self.id, - "schema_id": self.schema_id, - "is_schema": self.is_schema, + "type_id": self.type_id, + "is_type_schema": self.is_type_schema, } @@ -158,17 +184,17 @@ class GtsGetEntityResult: ok: bool id: str = "" - schema_id: Optional[str] = None - is_schema: bool = False + type_id: str | None = None + is_type_schema: bool = False content: Any = None error: str = "" - def to_dict(self) -> Dict[str, Any]: - result: Dict[str, Any] = {"ok": self.ok} + def to_dict(self) -> dict[str, Any]: + result: dict[str, Any] = {"ok": self.ok} if self.ok: result["id"] = self.id - result["schema_id"] = self.schema_id - result["is_schema"] = self.is_schema + result["type_id"] = self.type_id + result["is_type_schema"] = self.is_type_schema result["content"] = self.content else: result["error"] = self.error @@ -179,11 +205,11 @@ def to_dict(self) -> Dict[str, Any]: class GtsEntitiesListResult: """Result of listing entities.""" - entities: List[GtsEntityInfo] + entities: list[GtsEntityInfo] count: int total: int - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: return { "entities": [e.to_dict() for e in self.entities], "count": self.count, @@ -197,19 +223,19 @@ class GtsAddEntityResult: ok: bool id: str = "" - schema_id: Optional[str] = None - is_schema: bool = False + type_id: str | None = None + is_type_schema: bool = False error: str = "" - def to_dict(self) -> Dict[str, Any]: - result: Dict[str, Any] = {"ok": self.ok} + def to_dict(self) -> dict[str, Any]: + result: dict[str, Any] = {"ok": self.ok} if self.ok: result["id"] = self.id - result["schema_id"] = self.schema_id - result["is_schema"] = self.is_schema + result["type_id"] = self.type_id + result["is_type_schema"] = self.is_type_schema else: result["error"] = self.error - result["is_schema"] = self.is_schema + result["is_type_schema"] = self.is_type_schema return result @@ -218,9 +244,9 @@ class GtsAddEntitiesResult: """Result of adding multiple entities to the store.""" ok: bool - results: List[GtsAddEntityResult] + results: list[GtsAddEntityResult] - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: return { "ok": self.ok, "results": [r.to_dict() for r in self.results], @@ -235,8 +261,8 @@ class GtsAddSchemaResult: id: str = "" error: str = "" - def to_dict(self) -> Dict[str, Any]: - result: Dict[str, Any] = {"ok": self.ok} + def to_dict(self) -> dict[str, Any]: + result: dict[str, Any] = {"ok": self.ok} if self.ok: result["id"] = self.id else: @@ -249,18 +275,18 @@ class GtsExtractIdResult: """Result of extracting ID information from content.""" id: str - schema_id: Optional[str] - selected_entity_field: Optional[str] - selected_schema_id_field: Optional[str] - is_schema: bool + type_id: str | None + selected_entity_field: str | None + selected_type_id_field: str | None + is_type_schema: bool - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: return { "id": self.id, - "schema_id": self.schema_id, + "type_id": self.type_id, "selected_entity_field": self.selected_entity_field, - "selected_schema_id_field": self.selected_schema_id_field, - "is_schema": self.is_schema, + "selected_type_id_field": self.selected_type_id_field, + "is_type_schema": self.is_type_schema, } @@ -268,18 +294,18 @@ class GtsOps: def __init__( self, *, - path: Optional[str | List[str]] = None, - config: Optional[str] = None, + path: str | builtins.list[str] | None = None, + config: str | None = None, verbose: int = 0, ) -> None: self.verbose = verbose self.cfg = self._load_config(config) - self.path: Optional[str | List[str]] = path + self.path: str | list[str] | None = path self._reader = GtsFileReader(self.path, cfg=self.cfg) if self.path else None self.store = GtsStore(self._reader) if self._reader else GtsStore(reader=None) # type: ignore[arg-type] @staticmethod - def _create_config_from_data(data: Dict[str, Any]) -> GtsConfig: + def _create_config_from_data(data: dict[str, Any]) -> GtsConfig: """Create GtsConfig from JSON data with defaults.""" return GtsConfig( entity_id_fields=list( @@ -291,16 +317,16 @@ def _create_config_from_data(data: Dict[str, Any]) -> GtsConfig: ) @staticmethod - def _load_config_from_path(path: SysPath) -> Optional[GtsConfig]: + def _load_config_from_path(path: SysPath) -> GtsConfig | None: """Try to load config from a path, return None on failure.""" try: with path.open("r", encoding="utf-8") as f: data = json.load(f) return GtsOps._create_config_from_data(data) - except Exception: + except Exception: # noqa: BLE001 - fall back to defaults on any load failure return None - def _load_config(self, config_path: Optional[str]) -> GtsConfig: + def _load_config(self, config_path: str | None) -> GtsConfig: """Load config from user path, default path, or use defaults.""" # Try user-provided path if config_path: @@ -317,23 +343,26 @@ def _load_config(self, config_path: Optional[str]) -> GtsConfig: # Fall back to defaults return DEFAULT_GTS_CONFIG - def reload_from_path(self, path: str | List[str]) -> None: + def reload_from_path(self, path: str | builtins.list[str]) -> None: self.path = path self._reader = GtsFileReader(self.path, cfg=self.cfg) self.store = GtsStore(self._reader) def add_entity( - self, content: Dict[str, Any], validate: bool = False + self, content: dict[str, Any], validate: bool = False ) -> GtsAddEntityResult: entity = GtsEntity(content=content, cfg=self.cfg) - # For instances (non-schemas), require an id field - if not entity.is_schema: - # Instance must have an id from entity_id_fields (not just derived from schema) - if not entity.raw_id or not entity.selected_entity_field: - return GtsAddEntityResult( - ok=False, error="Instance must have an id field", is_schema=False - ) + # For instances (non-schemas), require an id field from entity_id_fields + # (not just derived from schema) + if not entity.is_schema and ( + not entity.raw_id or not entity.selected_entity_field + ): + return GtsAddEntityResult( + ok=False, + error="Instance must have an id field", + is_type_schema=False, + ) # Schemas MUST have a valid GTS ID if entity.is_schema and not entity.gts_id: @@ -344,57 +373,62 @@ def add_entity( # Validate $id prefix for schemas: must use gts:// URI, not plain gts. if entity.is_schema and validate: raw_id = content.get("$id", "") - if isinstance(raw_id, str): - # Reject plain gts. prefix (without gts://) - if raw_id.startswith("gts.") and not raw_id.startswith("gts://"): - return GtsAddEntityResult( - ok=False, - error="Schema $id must use gts:// URI format, not plain gts. prefix", - is_schema=True, - ) - - # Register the entity (use raw_id for non-GTS instances) - self.store.register(entity) - - # Always validate schemas - if entity.is_schema: - try: - self.store.validate_schema(entity.gts_id.id) - except Exception as e: + # Reject plain gts. prefix (without gts://) + if ( + isinstance(raw_id, str) + and raw_id.startswith("gts.") + and not raw_id.startswith("gts://") + ): return GtsAddEntityResult( - ok=False, error=f"Validation failed: {str(e)}" + ok=False, + error="Schema $id must use gts:// URI format, not plain gts. prefix", + is_type_schema=True, ) - # If validation is requested, validate the instance as well - if validate and not entity.is_schema and entity.gts_id: - try: - self.store.validate_instance(entity.gts_id.id) - except Exception as e: - return GtsAddEntityResult( - ok=False, error=f"Validation failed: {str(e)}" - ) + store_key = entity.gts_id.id if entity.is_schema else entity.raw_id + previous = self.store.get(store_key) + self.store.register(entity) + + try: + if entity.is_schema: + self.store.validate_schema_basic(entity.gts_id.id) + if validate: + self.store.validate_schema(entity.gts_id.id) + elif validate: + self.store.validate_instance(entity.raw_id or entity.gts_id.id) + except Exception as e: # noqa: BLE001 - converted to a result object at API boundary + self.store.unregister(store_key) + if previous: + self.store.register(previous) + return GtsAddEntityResult( + ok=False, + error=f"Validation failed: {e!s}", + is_type_schema=entity.is_schema, + ) # Return gts_id if available, otherwise raw_id entity_id = entity.gts_id.id if entity.gts_id else (entity.raw_id or "") return GtsAddEntityResult( ok=True, id=entity_id, - schema_id=entity.schemaId, - is_schema=entity.is_schema, + type_id=entity.type_id, + is_type_schema=entity.is_schema, ) - def add_entities(self, items: List[Dict[str, Any]]) -> GtsAddEntitiesResult: - results: List[GtsAddEntityResult] = [] + def add_entities( + self, items: builtins.list[dict[str, Any]] + ) -> GtsAddEntitiesResult: + results: list[GtsAddEntityResult] = [] for it in items: results.append(self.add_entity(it)) ok = all(r.ok for r in results) return GtsAddEntitiesResult(ok=ok, results=results) - def add_schema(self, type_id: str, schema: Dict[str, Any]) -> GtsAddSchemaResult: + def add_schema(self, type_id: str, schema: dict[str, Any]) -> GtsAddSchemaResult: try: self.store.register_schema(type_id, schema) return GtsAddSchemaResult(ok=True, id=type_id) - except Exception as e: + except Exception as e: # noqa: BLE001 - converted to a result object at API boundary return GtsAddSchemaResult(ok=False, error=str(e)) def validate_id(self, gts_id: str) -> GtsIdValidationResult: @@ -404,12 +438,21 @@ def validate_id(self, gts_id: str) -> GtsIdValidationResult: if is_wildcard: # For wildcards, try parsing as GtsWildcard _ = GtsWildcard(gts_id) + return GtsIdValidationResult( + id=gts_id, valid=True, is_type=False, is_wildcard=True + ) else: - _ = GtsID(gts_id) - return GtsIdValidationResult(id=gts_id, valid=True, is_wildcard=is_wildcard) - except Exception as e: + parsed = GtsID(gts_id) + return GtsIdValidationResult( + id=gts_id, valid=True, is_type=parsed.is_type, is_wildcard=False + ) + except Exception as e: # noqa: BLE001 - converted to a result object at API boundary return GtsIdValidationResult( - id=gts_id, valid=False, error=str(e), is_wildcard=is_wildcard + id=gts_id, + valid=False, + error=str(e), + is_type=None, + is_wildcard=is_wildcard, ) def parse_id(self, gts_id: str) -> GtsIdParseResult: @@ -418,33 +461,51 @@ def parse_id(self, gts_id: str) -> GtsIdParseResult: try: if is_wildcard: parsed = GtsWildcard(gts_id) + segs = parsed.gts_id_segments + segments = [ + GtsIdSegment( + vendor=s.vendor, + package=s.package, + namespace=s.namespace, + type=s.type, + ver_major=s.ver_major, + ver_minor=s.ver_minor, + is_type=s.is_type, + ) + for s in segs + ] + return GtsIdParseResult( + id=gts_id, + ok=True, + segments=segments, + is_type=False, + is_wildcard=True, + ) else: parsed = GtsID(gts_id) - segs = parsed.gts_id_segments - segments = [ - GtsIdSegment( - vendor=s.vendor, - package=s.package, - namespace=s.namespace, - type=s.type, - ver_major=s.ver_major, - ver_minor=s.ver_minor, - is_type=s.is_type, + segs = parsed.gts_id_segments + segments = [ + GtsIdSegment( + vendor=s.vendor, + package=s.package, + namespace=s.namespace, + type=s.type, + ver_major=s.ver_major, + ver_minor=s.ver_minor, + is_type=s.is_type, + ) + for s in segs + ] + return GtsIdParseResult( + id=gts_id, + ok=True, + segments=segments, + is_type=parsed.is_type, + is_wildcard=False, ) - for s in segs - ] - # is_schema: true if ends with ~ and not a wildcard ending with ~* - is_schema = gts_id.endswith("~") and not is_wildcard + except Exception as e: # noqa: BLE001 - converted to a result object at API boundary return GtsIdParseResult( - id=gts_id, - ok=True, - segments=segments, - is_wildcard=is_wildcard, - is_schema=is_schema, - ) - except Exception as e: - return GtsIdParseResult( - id=gts_id, ok=False, error=str(e), is_wildcard=is_wildcard + id=gts_id, ok=False, error=str(e), is_type=None, is_wildcard=is_wildcard ) def match_id_pattern(self, candidate: str, pattern: str) -> GtsIdMatchResult: @@ -458,7 +519,7 @@ def match_id_pattern(self, candidate: str, pattern: str) -> GtsIdMatchResult: p = GtsWildcard(pattern) match = c.wildcard_match(p) return GtsIdMatchResult(candidate=candidate, pattern=pattern, match=match) - except Exception as e: + except Exception as e: # noqa: BLE001 - converted to a result object at API boundary return GtsIdMatchResult( candidate=candidate, pattern=pattern, match=False, error=str(e) ) @@ -471,25 +532,34 @@ def validate_instance(self, gts_id: str) -> GtsValidationResult: try: self.store.validate_instance(gts_id) return GtsValidationResult(id=gts_id, ok=True) - except Exception as e: + except Exception as e: # noqa: BLE001 - converted to a result object at API boundary return GtsValidationResult(id=gts_id, ok=False, error=str(e)) def validate_schema(self, gts_id: str) -> GtsValidationResult: try: self.store.validate_schema(gts_id) return GtsValidationResult(id=gts_id, ok=True) - except Exception as e: + except Exception as e: # noqa: BLE001 - converted to a result object at API boundary return GtsValidationResult(id=gts_id, ok=False, error=str(e)) - def validate_entity(self, gts_id: str) -> GtsValidationResult: + def validate_entity(self, gts_id: str) -> GtsEntityValidationResult: try: - if gts_id.endswith("~"): - self.store.validate_schema(gts_id) - else: - self.store.validate_instance(gts_id) - return GtsValidationResult(id=gts_id, ok=True) - except Exception as e: - return GtsValidationResult(id=gts_id, ok=False, error=str(e)) + parsed = GtsID(gts_id) + except Exception as e: # noqa: BLE001 - converted to a result object at API boundary + return GtsEntityValidationResult( + id=gts_id, ok=False, entity_type="", error=str(e) + ) + + if parsed.is_type: + entity_type = "schema" + result = self.validate_schema(gts_id) + else: + entity_type = "instance" + result = self.validate_instance(gts_id) + + return GtsEntityValidationResult( + id=result.id, ok=result.ok, entity_type=entity_type, error=result.error + ) def schema_graph(self, gts_id: str) -> GtsSchemaGraphResult: graph = self.store.build_schema_graph(gts_id) @@ -503,7 +573,7 @@ def compatibility( def cast(self, from_id: str, to_schema_id: str) -> GtsEntityCastResult: try: return self.store.cast(from_id, to_schema_id) - except Exception as e: + except Exception as e: # noqa: BLE001 - converted to a result object at API boundary return GtsEntityCastResult(error=str(e)) def query(self, expr: str, limit: int = 100) -> GtsStoreQueryResult: @@ -522,18 +592,19 @@ def attr(self, gts_with_path: str) -> GtsPathResolver: ) return entity.resolve_path(path) - def extract_id(self, content: Dict[str, Any]) -> GtsExtractIdResult: + def extract_id(self, content: dict[str, Any]) -> GtsExtractIdResult: entity = GtsEntity(content=content, cfg=self.cfg) - # Always use raw_id - that's the actual value found in the entity_id_fields - # Note: gts_id may be derived from schemaId as fallback, but extract-id - # should return what was actually in the selected field - id_value = entity.raw_id or "" + # Use effective_id: raw_id for non-schemas, gts_id for schemas + if entity.is_schema: + id_value = entity.gts_id.id if entity.gts_id else (entity.raw_id or "") + else: + id_value = entity.raw_id or "" return GtsExtractIdResult( id=id_value, - schema_id=entity.schemaId, + type_id=entity.type_id, selected_entity_field=entity.selected_entity_field, - selected_schema_id_field=entity.selected_schema_id_field, - is_schema=entity.is_schema, + selected_type_id_field=entity.selected_type_id_field, + is_type_schema=entity.is_schema, ) def get_entity(self, gts_id: str) -> GtsGetEntityResult: @@ -554,11 +625,11 @@ def get_entity(self, gts_id: str) -> GtsGetEntityResult: return GtsGetEntityResult( ok=True, id=entity.gts_id.id if entity.gts_id else gts_id, - schema_id=entity.schemaId, - is_schema=entity.is_schema, + type_id=entity.type_id, + is_type_schema=entity.is_schema, content=entity.content, ) - except Exception as e: + except Exception as e: # noqa: BLE001 - converted to a result object at API boundary return GtsGetEntityResult(ok=False, error=str(e)) def get_entities(self, limit: int = 100) -> GtsEntitiesListResult: @@ -575,8 +646,8 @@ def get_entities(self, limit: int = 100) -> GtsEntitiesListResult: entities = [ GtsEntityInfo( id=entity_id, - schema_id=entity.schemaId, - is_schema=entity.is_schema, + type_id=entity.type_id, + is_type_schema=entity.is_schema, ) for entity_id, entity in all_entities[:limit] ] diff --git a/gts/src/gts/path_resolver.py b/gts/src/gts/path_resolver.py index 7dff2fc..e58a7df 100644 --- a/gts/src/gts/path_resolver.py +++ b/gts/src/gts/path_resolver.py @@ -1,7 +1,7 @@ from __future__ import annotations from dataclasses import dataclass -from typing import Any, List +from typing import Any @dataclass @@ -12,16 +12,16 @@ class GtsPathResolver: value: Any = None resolved: bool = False error: str | None = None - available_fields: List[str] = None # type: ignore + available_fields: list[str] = None # type: ignore def _normalize(self, path: str) -> str: return path.replace("/", ".") - def _split_raw_parts(self, norm: str) -> List[str]: + def _split_raw_parts(self, norm: str) -> list[str]: return [seg for seg in norm.split(".") if seg != ""] - def _parse_part(self, seg: str) -> List[str]: - out: List[str] = [] + def _parse_part(self, seg: str) -> list[str]: + out: list[str] = [] buf = "" i = 0 while i < len(seg): @@ -43,15 +43,15 @@ def _parse_part(self, seg: str) -> List[str]: out.append(buf) return out - def _parts(self, path: str) -> List[str]: + def _parts(self, path: str) -> list[str]: norm = self._normalize(path) raw = self._split_raw_parts(norm) - parts: List[str] = [] + parts: list[str] = [] for seg in raw: parts.extend(self._parse_part(seg)) return parts - def _list_available(self, node: Any, prefix: str, out: List[str]) -> None: + def _list_available(self, node: Any, prefix: str, out: list[str]) -> None: if isinstance(node, dict): for k, v in node.items(): p = f"{prefix}.{k}" if prefix else str(k) @@ -65,8 +65,8 @@ def _list_available(self, node: Any, prefix: str, out: List[str]) -> None: if isinstance(v, (dict, list)): self._list_available(v, p, out) - def _collect_from(self, node: Any) -> List[str]: - acc: List[str] = [] + def _collect_from(self, node: Any) -> list[str]: + acc: list[str] = [] self._list_available(node, "", acc) return acc diff --git a/gts/src/gts/schema_cast.py b/gts/src/gts/schema_cast.py index 09dbb89..ae42924 100644 --- a/gts/src/gts/schema_cast.py +++ b/gts/src/gts/schema_cast.py @@ -1,14 +1,17 @@ from __future__ import annotations +import copy +import logging from dataclasses import dataclass -from typing import Any, Dict, List, Optional, Tuple +from typing import Any -import copy -from jsonschema import validate as js_validate from jsonschema import exceptions as js_exceptions +from jsonschema import validate as js_validate from .gts import GtsID +logger = logging.getLogger(__name__) + class SchemaCastError(Exception): pass @@ -19,17 +22,22 @@ class GtsEntityCastResult: from_id: str = "" to_id: str = "" direction: str = "unknown" - added_properties: List[str] = None # type: ignore - removed_properties: List[str] = None # type: ignore - changed_properties: List[Dict[str, str]] = None # type: ignore + added_properties: list[str] = None # type: ignore + removed_properties: list[str] = None # type: ignore + changed_properties: list[dict[str, str]] = None # type: ignore is_fully_compatible: bool = False is_backward_compatible: bool = False is_forward_compatible: bool = False - incompatibility_reasons: List[str] = None # type: ignore - backward_errors: List[str] = None # type: ignore - forward_errors: List[str] = None # type: ignore - casted_entity: Optional[Dict[str, Any]] = None + incompatibility_reasons: list[str] = None # type: ignore + backward_errors: list[str] = None # type: ignore + forward_errors: list[str] = None # type: ignore + casted_entity: dict[str, Any] | None = None error: str = "" + # Optional explicit verdict strings ("compatible"/"incompatible"/"unknown"). + # When set, these take precedence over the boolean flags in to_dict(). + backward_verdict: str | None = None + forward_verdict: str | None = None + full_verdict: str | None = None def __post_init__(self): # Initialize list fields if None @@ -46,7 +54,14 @@ def __post_init__(self): if self.forward_errors is None: self.forward_errors = [] - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: + def _compat_str(val: bool) -> str: + return "compatible" if val else "incompatible" + + backward = self.backward_verdict or _compat_str(self.is_backward_compatible) + forward = self.forward_verdict or _compat_str(self.is_forward_compatible) + full = self.full_verdict or _compat_str(self.is_fully_compatible) + result = { "from": self.from_id, "to": self.to_id, @@ -56,6 +71,9 @@ def to_dict(self) -> Dict[str, Any]: "added_properties": self.added_properties, "removed_properties": self.removed_properties, "changed_properties": self.changed_properties, + "backward_compatibility": backward, + "forward_compatibility": forward, + "full_compatibility": full, "is_fully_compatible": self.is_fully_compatible, "is_backward_compatible": self.is_backward_compatible, "is_forward_compatible": self.is_forward_compatible, @@ -76,7 +94,7 @@ def cast( from_instance_content: dict, from_schema_content: dict, to_schema_content: dict, - resolver: Optional[Any] = None, + resolver: Any | None = None, ) -> GtsEntityCastResult: # Flatten target schema to merge allOf and get all properties including const values target_schema = cls._flatten_schema(to_schema_content) @@ -104,12 +122,12 @@ def cast( ) # Apply casting rules to the instance - added: List[str] = [] - removed: List[str] = [] - reasons: List[str] = [] + added: list[str] = [] + removed: list[str] = [] + reasons: list[str] = [] try: - casted, added, removed, incompatibility_reasons = ( + casted, added, removed, _incompatibility_reasons = ( cls._cast_instance_to_schema( copy.deepcopy(from_instance_content) if isinstance(from_instance_content, dict) @@ -123,8 +141,8 @@ def cast( from_id=from_instance_id, to_id=to_schema_id, direction=direction, - added_properties=sorted(list(dict.fromkeys(added))), - removed_properties=sorted(list(dict.fromkeys(removed))), + added_properties=sorted(dict.fromkeys(added)), + removed_properties=sorted(dict.fromkeys(removed)), changed_properties=[], is_fully_compatible=False, is_backward_compatible=is_backward, @@ -151,8 +169,8 @@ def cast( from_id=from_instance_id, to_id=to_schema_id, direction=direction, - added_properties=sorted(list(dict.fromkeys(added))), - removed_properties=sorted(list(dict.fromkeys(removed))), + added_properties=sorted(dict.fromkeys(added)), + removed_properties=sorted(dict.fromkeys(removed)), changed_properties=[], is_fully_compatible=is_fully_compatible, is_backward_compatible=is_backward, @@ -165,11 +183,19 @@ def cast( @staticmethod def _infer_direction(from_id: str, to_id: str) -> str: + def _last_versioned_segment(gid: GtsID): + # Skip the appended UUID-tail segment (ver_minor is None) so + # combined anonymous IDs still resolve to their versioned segment. + for seg in reversed(gid.gts_id_segments): + if not getattr(seg, "_is_uuid_tail", False): + return seg + return gid.gts_id_segments[-1] + try: gid_from = GtsID(from_id) gid_to = GtsID(to_id) - from_minor = gid_from.gts_id_segments[-1].ver_minor - to_minor = gid_to.gts_id_segments[-1].ver_minor + from_minor = _last_versioned_segment(gid_from).ver_minor + to_minor = _last_versioned_segment(gid_to).ver_minor if from_minor is not None and to_minor is not None: if to_minor > from_minor: return "up" @@ -177,11 +203,16 @@ def _infer_direction(from_id: str, to_id: str) -> str: return "down" return "none" except Exception: - pass + logger.debug( + "could not infer version direction for %s -> %s", + from_id, + to_id, + exc_info=True, + ) return "unknown" @staticmethod - def _effective_object_schema(s: Dict[str, Any]) -> Dict[str, Any]: + def _effective_object_schema(s: dict[str, Any]) -> dict[str, Any]: if not isinstance(s, dict): return {} if isinstance(s.get("properties"), dict) or isinstance(s.get("required"), list): @@ -197,11 +228,11 @@ def _effective_object_schema(s: Dict[str, Any]) -> Dict[str, Any]: @staticmethod def _cast_instance_to_schema( - instance: Dict[str, Any], - schema: Dict[str, Any], + instance: dict[str, Any], + schema: dict[str, Any], base_path: str = "", - incompatibility_reasons: List[str] = [], - ) -> Tuple[Dict[str, Any], List[str], List[str], List[str]]: + incompatibility_reasons: list[str] | None = None, + ) -> tuple[dict[str, Any], list[str], list[str], list[str]]: """Transform instance to conform to schema. Rules: @@ -210,9 +241,11 @@ def _cast_instance_to_schema( - Validate constraints via a final jsonschema validation step - Recursively handle nested objects (and arrays of objects) """ - added: List[str] = [] - removed: List[str] = [] - incompatibility_reasons: List[str] = [] + if incompatibility_reasons is None: + incompatibility_reasons = [] + added: list[str] = [] + removed: list[str] = [] + incompatibility_reasons: list[str] = [] if not isinstance(instance, dict): raise SchemaCastError("Instance must be an object for casting") @@ -230,7 +263,7 @@ def _cast_instance_to_schema( additional = schema.get("additionalProperties", True) # Start from current values - result: Dict[str, Any] = dict(instance) + result: dict[str, Any] = dict(instance) # 1) Ensure required properties exist (fill defaults if provided) for prop in required: @@ -270,12 +303,16 @@ def _cast_instance_to_schema( if prop in result: old_value = result[prop] # Only update if the const value is different and both are GTS IDs - if isinstance(const_value, str) and isinstance(old_value, str): - if GtsID.is_valid(const_value) and GtsID.is_valid(old_value): - if old_value != const_value: - result[prop] = const_value - path = f"{base_path}.{prop}" if base_path else prop - # Don't add to changed list, this is expected for version casting + if ( + isinstance(const_value, str) + and isinstance(old_value, str) + and GtsID.is_valid(const_value) + and GtsID.is_valid(old_value) + and old_value != const_value + ): + result[prop] = const_value + path = f"{base_path}.{prop}" if base_path else prop + # Don't add to changed list, this is expected for version casting # 3) Remove properties not present in target schema when additionalProperties is false if additional is False: @@ -316,7 +353,7 @@ def _cast_instance_to_schema( nested_schema = GtsEntityCastResult._effective_object_schema( items_schema ) - new_list: List[Any] = [] + new_list: list[Any] = [] for idx, item in enumerate(val): if isinstance(item, dict): new_item, add_sub, rem_sub, new_incompatibility_reasons = ( @@ -343,9 +380,9 @@ def _cast_instance_to_schema( @staticmethod def _validate_with_gts_id_tolerance( - instance: Dict[str, Any], - schema: Dict[str, Any], - resolver: Optional[Any] = None, + instance: dict[str, Any], + schema: dict[str, Any], + resolver: Any | None = None, ) -> None: """Validate instance against schema, but allow const values to differ if both are GTS IDs.""" # Create a modified schema that removes const constraints for GTS IDs @@ -385,7 +422,7 @@ def _remove_gts_const_constraints(schema: Any) -> Any: return result @staticmethod - def _flatten_schema(schema: Dict[str, Any]) -> Dict[str, Any]: + def _flatten_schema(schema: dict[str, Any]) -> dict[str, Any]: """Flatten a schema by merging allOf schemas.""" result = {"properties": {}, "required": []} @@ -413,12 +450,12 @@ def _flatten_schema(schema: Dict[str, Any]) -> Dict[str, Any]: @staticmethod def _check_min_max_constraint( prop: str, - old_schema: Dict[str, Any], - new_schema: Dict[str, Any], + old_schema: dict[str, Any], + new_schema: dict[str, Any], min_key: str, max_key: str, check_tightening: bool, - ) -> List[str]: + ) -> list[str]: """Check min/max constraint compatibility between schemas. Args: @@ -433,7 +470,7 @@ def _check_min_max_constraint( Returns: List of error messages """ - errors: List[str] = [] + errors: list[str] = [] # Check minimum constraint old_min = old_schema.get(min_key) @@ -474,10 +511,10 @@ def _check_min_max_constraint( @staticmethod def _check_constraint_compatibility( prop: str, - old_prop_schema: Dict[str, Any], - new_prop_schema: Dict[str, Any], + old_prop_schema: dict[str, Any], + new_prop_schema: dict[str, Any], check_tightening: bool = True, - ) -> List[str]: + ) -> list[str]: """Check if constraints are compatible between old and new property schemas. Args: @@ -490,7 +527,7 @@ def _check_constraint_compatibility( Returns: List of error messages """ - errors: List[str] = [] + errors: list[str] = [] prop_type = old_prop_schema.get("type") # Numeric constraints (for number/integer types) @@ -536,10 +573,10 @@ def _check_constraint_compatibility( @staticmethod def _check_schema_compatibility( - old_schema: Dict[str, Any], - new_schema: Dict[str, Any], + old_schema: dict[str, Any], + new_schema: dict[str, Any], check_backward: bool, - ) -> tuple[bool, List[str]]: + ) -> tuple[bool, list[str]]: """Unified compatibility checker for backward and forward compatibility. Args: @@ -551,7 +588,7 @@ def _check_schema_compatibility( Returns: Tuple of (is_compatible, list_of_errors) """ - errors: List[str] = [] + errors: list[str] = [] # Flatten schemas to handle allOf old_flat = GtsEntityCastResult._flatten_schema(old_schema) @@ -632,9 +669,9 @@ def _check_schema_compatibility( @staticmethod def _check_backward_compatibility( - old_schema: Dict[str, Any], - new_schema: Dict[str, Any], - ) -> tuple[bool, List[str]]: + old_schema: dict[str, Any], + new_schema: dict[str, Any], + ) -> tuple[bool, list[str]]: """Check if new schema is backward compatible with old schema. Backward compatibility: new consumers can read old data. @@ -654,9 +691,9 @@ def _check_backward_compatibility( @staticmethod def _check_forward_compatibility( - old_schema: Dict[str, Any], - new_schema: Dict[str, Any], - ) -> tuple[bool, List[str]]: + old_schema: dict[str, Any], + new_schema: dict[str, Any], + ) -> tuple[bool, list[str]]: """Check if new schema is forward compatible with old schema. Forward compatibility: old consumers can read new data. @@ -675,12 +712,12 @@ def _check_forward_compatibility( @staticmethod def _diff_objects( - obj_a: Dict[str, Any], - obj_b: Dict[str, Any], + obj_a: dict[str, Any], + obj_b: dict[str, Any], base: str, - added: List[str], - removed: List[str], - changed: List[Dict[str, str]], + added: list[str], + removed: list[str], + changed: list[dict[str, str]], ) -> None: a_props = obj_a.get("properties", {}) if isinstance(obj_a, dict) else {} b_props = obj_b.get("properties", {}) if isinstance(obj_b, dict) else {} @@ -721,16 +758,16 @@ def _path_label(path: str) -> str: return path if path else "root" @staticmethod - def _filtered(d: Dict[str, Any]) -> Dict[str, Any]: + def _filtered(d: dict[str, Any]) -> dict[str, Any]: exclude = ("properties", "required") return {k: v for k, v in d.items() if k not in exclude} @staticmethod def _only_optional_add_remove( - a: Dict[str, Any], - b: Dict[str, Any], + a: dict[str, Any], + b: dict[str, Any], path: str, - reasons: List[str], + reasons: list[str], ) -> bool: if not isinstance(a, dict) or not isinstance(b, dict): if a != b: @@ -760,8 +797,8 @@ def _only_optional_add_remove( set(b.get("required", [])) if isinstance(b.get("required"), list) else set() ) if a_req != b_req: - added_req = sorted(list(b_req - a_req)) - removed_req = sorted(list(a_req - b_req)) + added_req = sorted(b_req - a_req) + removed_req = sorted(a_req - b_req) if added_req: reasons.append( f"{GtsEntityCastResult._path_label(path)}: required added -> " diff --git a/gts/src/gts/store.py b/gts/src/gts/store.py index 4d3210a..7ff55b7 100644 --- a/gts/src/gts/store.py +++ b/gts/src/gts/store.py @@ -1,17 +1,23 @@ from __future__ import annotations +import logging +import uuid from abc import ABC, abstractmethod -from typing import Dict, Set, Tuple, List, Any, Optional, Iterator +from collections.abc import Iterator +from typing import Any -from jsonschema import validate as js_validate from jsonschema import RefResolver +from jsonschema.validators import validator_for +from referencing import Registry, Resource +from referencing.jsonschema import DRAFT202012 -from .gts import GtsID, GtsWildcard +from . import compatibility, derivation, traits from .entities import GtsEntity +from .gts import GtsID, GtsWildcard from .schema_cast import GtsEntityCastResult -from .x_gts_ref import XGtsRefValidator +from .x_gts_ref import XGtsRefValidator, _without_x_gts_ref -import logging +logger = logging.getLogger(__name__) class StoreGtsObjectNotFound(Exception): @@ -65,28 +71,18 @@ class GtsReader(ABC): @abstractmethod def __iter__(self) -> Iterator[GtsEntity]: """Return an iterator that yields JsonEntity objects.""" - pass @abstractmethod - def read_by_id(self, entity_id: str) -> Optional[GtsEntity]: + def read_by_id(self, entity_id: str) -> GtsEntity | None: """ Read a JsonEntity by its ID. Returns None if the entity is not found. Used for cache miss scenarios. """ - pass @abstractmethod def reset(self) -> None: """Reset the iterator to start from the beginning.""" - pass - - -class GtsStoreQueryResultEntry: - def __init__(self): - self.id = "" - self.schema_id = "" - self.is_schema = bool class GtsStoreQueryResult: @@ -94,9 +90,9 @@ def __init__(self): self.error = "" self.count = 0 self.limit = 0 - self.results: List[Dict[str, Any]] = [] + self.results: list[dict[str, Any]] = [] - def to_dict(self) -> Dict[str, Any]: + def to_dict(self) -> dict[str, Any]: if self.error: return {"error": self.error, "count": self.count, "limit": self.limit} return { @@ -115,14 +111,14 @@ def __init__(self, reader: GtsReader) -> None: Args: reader: GtsReader instance to populate entities from """ - self._by_id: Dict[str, GtsEntity] = {} + self._by_id: dict[str, GtsEntity] = {} self._reader = reader # Populate entities from reader if provided if self._reader: self._populate_from_reader() - logging.info(f"Populated GtsStore with {len(self._by_id)} entities") + logger.info(f"Populated GtsStore with {len(self._by_id)} entities") def _populate_from_reader(self) -> None: """Populate the store by iterating through the reader.""" @@ -139,6 +135,13 @@ def register(self, entity: GtsEntity) -> None: If entity has a valid gts_id, use that as the key. Otherwise, use raw_id for non-GTS entities. """ + # Instances should remain addressable by the id value they carry. + # For plain UUID anonymous instances, `gts_id` may be inferred from + # the `type` field while `raw_id` is the UUID we must look up by. + if not entity.is_schema and entity.raw_id: + self._by_id[entity.raw_id] = entity + return + if entity.gts_id and entity.gts_id.id: self._by_id[entity.gts_id.id] = entity elif entity.raw_id: @@ -147,7 +150,11 @@ def register(self, entity: GtsEntity) -> None: else: raise ValueError("Entity must have a valid gts_id or raw_id") - def register_schema(self, type_id: str, schema: Dict[str, Any]) -> None: + def unregister(self, entity_id: str) -> None: + """Remove an entity from the in-memory registry if it is present.""" + self._by_id.pop(entity_id, None) + + def register_schema(self, type_id: str, schema: dict[str, Any]) -> None: """ Register a schema (legacy method for backward compatibility). Creates a JsonEntity from the schema dict. @@ -159,7 +166,7 @@ def register_schema(self, type_id: str, schema: Dict[str, Any]) -> None: entity = GtsEntity(content=schema, gts_id=gts_id, is_schema=True) self._by_id[type_id] = entity - def get(self, entity_id: str) -> Optional[GtsEntity]: + def get(self, entity_id: str) -> GtsEntity | None: """ Get a JsonEntity by its ID. If not found in cache, try to fetch from reader. @@ -178,25 +185,24 @@ def get(self, entity_id: str) -> Optional[GtsEntity]: return None - def get_schema_content(self, type_id: str) -> Dict[str, Any]: + def get_schema_content(self, type_id: str) -> dict[str, Any]: """Get schema content as dict (legacy method for backward compatibility).""" entity = self.get(type_id) if entity and isinstance(entity.content, dict): return entity.content raise KeyError(f"Schema not found: {type_id}") - def _create_ref_resolver(self, schema: Dict[str, Any]) -> RefResolver: + def _create_ref_resolver(self, schema: dict[str, Any]) -> RefResolver: """Create a custom RefResolver that can resolve GTS ID references from the store.""" - def resolve_gts_ref(uri: str) -> Dict[str, Any]: + def resolve_gts_ref(uri: str) -> dict[str, Any]: """Resolve a GTS ID reference to its schema content.""" # Issue #32: handle gts:// prefix - if uri.startswith("gts://"): - uri = uri[6:] + uri = uri.removeprefix("gts://") try: return self.get_schema_content(uri) - except KeyError: - raise Exception(f"Unresolvable: {uri}") + except KeyError as e: + raise ValueError(f"Unresolvable: {uri}") from e # Create a store dict that maps GTS IDs to their schema content store = {} @@ -210,12 +216,23 @@ def resolve_gts_ref(uri: str) -> Dict[str, Any]: resolver = RefResolver.from_schema(schema, store=store, handlers=handlers) return resolver + def _create_reference_registry(self) -> Registry: + registry = Registry() + for entity_id, entity in self._by_id.items(): + if entity.is_schema and isinstance(entity.content, dict): + resource = Resource.from_contents( + _without_x_gts_ref(entity.content), + default_specification=DRAFT202012, + ) + registry = registry.with_resource(f"gts://{entity_id}", resource) + return registry + def items(self): """Return all entity ID and entity pairs.""" return self._by_id.items() @staticmethod - def _validate_schema_refs(schema: Dict[str, Any], path: str = "") -> None: + def _validate_schema_refs(schema: dict[str, Any], path: str = "") -> None: """ Validate all $ref values in a schema. @@ -285,7 +302,7 @@ def _validate_schema_x_gts_refs(self, gts_id: str) -> None: if not schema_entity.is_schema: raise ValueError(f"Entity '{gts_id}' is not a schema") - logging.info(f"Validating schema x-gts-ref fields for {gts_id}") + logger.info(f"Validating schema x-gts-ref fields for {gts_id}") # Validate x-gts-ref constraints in the schema x_gts_ref_validator = XGtsRefValidator(store=self) @@ -294,15 +311,300 @@ def _validate_schema_x_gts_refs(self, gts_id: str) -> None: error_messages = [ f"{err.field_path}: {err.reason}" for err in x_gts_ref_errors ] - raise Exception( + raise ValueError( f"Schema x-gts-ref validation failed: {'; '.join(error_messages)}" ) + @staticmethod + def _validate_gts_keywords(content: dict[str, Any]) -> None: + """Validate x-gts-final, x-gts-abstract, x-gts-traits, x-gts-traits-schema placement.""" + + def _contains_key_recursive(value: Any, key: str) -> bool: + if isinstance(value, dict): + if key in value: + return True + return any(_contains_key_recursive(v, key) for v in value.values()) + elif isinstance(value, list): + return any(_contains_key_recursive(v, key) for v in value) + return False + + # Validate x-gts-final + final_val = content.get("x-gts-final") + if final_val is not None and not isinstance(final_val, bool): + raise ValueError( + f"x-gts-final must be a boolean, got {type(final_val).__name__}" + ) + + # Validate x-gts-abstract + abstract_val = content.get("x-gts-abstract") + if abstract_val is not None and not isinstance(abstract_val, bool): + raise ValueError( + f"x-gts-abstract must be a boolean, got {type(abstract_val).__name__}" + ) + + # Mutual exclusion + if final_val is True and abstract_val is True: + raise ValueError( + "schema cannot declare both x-gts-final and x-gts-abstract as true" + ) + + # Check that x-gts-final/x-gts-abstract/x-gts-traits/x-gts-traits-schema + # appear only at the top level + top_level_keywords = { + "x-gts-final", + "x-gts-abstract", + "x-gts-traits", + "x-gts-traits-schema", + } + for key, value in content.items(): + if key in top_level_keywords: + continue + for kw in top_level_keywords: + if _contains_key_recursive(value, kw): + raise ValueError(f"{kw} must be at the schema top level") + + @staticmethod + def _content_is_abstract(content: dict[str, Any]) -> bool: + return content.get("x-gts-abstract") is True + + @staticmethod + def _content_is_final(content: dict[str, Any]) -> bool: + return content.get("x-gts-final") is True + + def _validate_schema_chain(self, gts_id: str) -> None: + """Validate OP#12: schema derivation chain compatibility.""" + gid = GtsID(gts_id) + segments = gid.gts_id_segments + + # Single-segment schemas have no parent to validate against + if len(segments) < 2: + return + + # Build chain IDs + chain_ids = [] + prefix = "gts." + for seg in segments: + chain_ids.append(prefix + seg.segment) + prefix = prefix + seg.segment + + # Validate each adjacent pair + for i in range(len(chain_ids) - 1): + base_id = chain_ids[i] + derived_id = chain_ids[i + 1] + + base_entity = self.get(base_id) + derived_entity = self.get(derived_id) + + # Check x-gts-final: if the base type is final, derivation is not allowed. + if ( + base_entity + and isinstance(base_entity.content, dict) + and self._content_is_final(base_entity.content) + ): + raise ValueError( + f"base type '{base_id}' is final and cannot be extended" + ) + + logger.info( + f"OP#12: Validating schema chain pair: base={base_id} derived={derived_id}" + ) + + if not base_entity or not isinstance(base_entity.content, dict): + raise ValueError( + f"Base schema '{base_id}' not found for chain validation" + ) + if not derived_entity or not isinstance(derived_entity.content, dict): + raise ValueError( + f"Derived schema '{derived_id}' not found for chain validation" + ) + + # Resolve both schemas (inline $refs) + base_resolved = self._resolve_schema_refs(base_entity.content) + derived_resolved = self._resolve_schema_refs(derived_entity.content) + + # Validate derivation compatibility (OP#12): accepted-instance-set + # inclusion on declared schemas plus GTS admission rules. + errors = derivation.validate_derivation_compatibility( + base_resolved, derived_resolved, base_id, derived_id + ) + if errors: + raise ValueError( + f"Schema '{derived_id}' is not compatible with base '{base_id}': " + + "; ".join(errors) + ) + + def _resolve_schema_refs(self, schema: Any) -> Any: + """Resolve $ref references in a schema by inlining referenced schemas. + + References are inlined recursively so that a schema reached through an + intermediate (A referenced via A~B) is fully expanded. Cyclic + references are left unresolved: the surviving $ref makes the effective + schema unprovable, which is the intended admission failure. + """ + import copy + + return self._inline_refs( + copy.deepcopy(schema), set(), self._supports_ref_siblings(schema) + ) + + @staticmethod + def _supports_ref_siblings(schema: Any) -> bool: + dialect = schema.get("$schema") if isinstance(schema, dict) else None + return isinstance(dialect, str) and ( + "/draft/2019-09/" in dialect or "/draft/2020-12/" in dialect + ) + + def _inline_refs( + self, node: Any, seen: set[str], supports_ref_siblings: bool + ) -> Any: + """Recursively inline $ref references, guarding against cycles.""" + if isinstance(node, dict): + ref_uri = node.get("$ref") + if isinstance(ref_uri, str): + ref_id: str | None = None + if ref_uri.startswith("gts://"): + ref_id = ref_uri[6:] + elif not ref_uri.startswith("#"): + ref_id = ref_uri + if ref_id is not None: + if ref_id in seen: + # Cycle detected: leave the $ref unresolved. + return node + try: + ref_schema = self.get_schema_content(ref_id) + except KeyError: + return node # Leave unresolved + import copy + + resolved = self._inline_refs( + copy.deepcopy(ref_schema), + seen | {ref_id}, + self._supports_ref_siblings(ref_schema), + ) + if supports_ref_siblings and len(node) > 1: + siblings = { + key: value for key, value in node.items() if key != "$ref" + } + return { + "allOf": [ + resolved, + self._inline_refs( + siblings, seen, supports_ref_siblings + ), + ] + } + return resolved + return { + key: self._inline_refs(value, seen, supports_ref_siblings) + for key, value in node.items() + } + if isinstance(node, list): + return [ + self._inline_refs(item, seen, supports_ref_siblings) for item in node + ] + return node + + def _build_effective_traits(self, gts_id: str) -> traits.EffectiveTraits: + """Build OP#13 EffectiveTraits by walking the type's chain (root -> leaf).""" + gid = GtsID(gts_id) + segments = gid.gts_id_segments + + chain_ids: list[str] = [] + prefix = "gts." + for seg in segments: + chain_ids.append(prefix + seg.segment) + prefix = prefix + seg.segment + + trait_schemas: list[Any] = [] + merged_traits: dict[str, Any] = {} + + for schema_id in chain_ids: + entity = self.get(schema_id) + if not entity or not isinstance(entity.content, dict): + continue + content = entity.content + + level_schemas: list[Any] = [] + traits.collect_trait_schema_from_value(content, level_schemas) + for ts in level_schemas: + # Inline local JSON Pointer refs against the host document, then + # resolve any gts:// refs so the composed schema is self-contained. + inlined = traits.inline_local_pointers(ts, content) + trait_schemas.append(self._resolve_schema_refs(inlined)) + + level_traits: dict[str, Any] = {} + traits.collect_traits_from_value(content, level_traits) + traits.merge_rfc7396_into(merged_traits, level_traits) + + leaf = self.get(chain_ids[-1]) if chain_ids else None + dialect = None + if leaf and isinstance(leaf.content, dict): + ds = leaf.content.get("$schema") + if isinstance(ds, str): + dialect = ds + + return traits.build_effective_traits(trait_schemas, merged_traits, dialect) + + def _validate_traits(self, gts_id: str, is_abstract: bool) -> None: + """Validate OP#13: schema traits for a type.""" + effective = self._build_effective_traits(gts_id) + errors = effective.validate(check_unresolved=not is_abstract) + if errors: + raise ValueError( + f"Schema '{gts_id}' trait validation failed: " + "; ".join(errors) + ) + + def validate_schema_basic(self, gts_id: str) -> None: + """Basic schema validation during registration (no chain validation). + + Checks: + 1. $ref URI format + 2. x-gts-ref field validation + 3. GTS keyword validation (x-gts-final, x-gts-abstract, placement) + 4. JSON Schema meta-schema validation + """ + if not gts_id.endswith("~"): + raise ValueError(f"ID '{gts_id}' is not a schema (must end with '~')") + + schema_entity = self.get(gts_id) + if not schema_entity: + raise StoreGtsSchemaNotFound(gts_id) + + if not schema_entity.is_schema: + raise ValueError(f"Entity '{gts_id}' is not a schema") + + schema_content = schema_entity.content + if not isinstance(schema_content, dict): + raise ValueError( # noqa: TRY004 - keep ValueError for API compatibility + f"Schema '{gts_id}' content must be a dictionary" + ) + + meta_schema_url = schema_content.get("$schema") + if ( + meta_schema_url + and isinstance(meta_schema_url, str) + and meta_schema_url.startswith(("gts.", "gts://")) + ): + raise ValueError( + f"Invalid $schema URL '{meta_schema_url}': must be a standard JSON Schema URL, not a GTS ID" + ) + + # 1. Validate $ref fields + self._validate_schema_refs(schema_content, "") + + # 2. Validate x-gts-ref fields + self._validate_schema_x_gts_refs(gts_id) + + # 3. Validate GTS keywords (x-gts-final, x-gts-abstract, placement) + self._validate_gts_keywords(schema_content) + def validate_schema(self, gts_id: str) -> None: """ Full schema validation including: 1. JSON Schema meta-schema validation 2. x-gts-ref field validation + 3. GTS keyword validation (x-gts-final, x-gts-abstract, placement) + 4. Schema chain derivation validation (OP#12) Args: gts_id: The GTS ID of the schema to validate @@ -319,43 +621,55 @@ def validate_schema(self, gts_id: str) -> None: schema_content = schema_entity.content if not isinstance(schema_content, dict): - raise ValueError(f"Schema '{gts_id}' content must be a dictionary") + raise ValueError( # noqa: TRY004 - keep ValueError for API compatibility + f"Schema '{gts_id}' content must be a dictionary" + ) # Issue #25: strict check, no GTS IDs in $schema meta_schema_url = schema_content.get("$schema") - if meta_schema_url and isinstance(meta_schema_url, str): - if meta_schema_url.startswith("gts.") or meta_schema_url.startswith( - "gts://" - ): - raise ValueError( - f"Invalid $schema URL '{meta_schema_url}': must be a standard JSON Schema URL, not a GTS ID" - ) + if ( + meta_schema_url + and isinstance(meta_schema_url, str) + and meta_schema_url.startswith(("gts.", "gts://")) + ): + raise ValueError( + f"Invalid $schema URL '{meta_schema_url}': must be a standard JSON Schema URL, not a GTS ID" + ) - logging.info(f"Validating schema {gts_id}") + logger.info(f"Validating schema {gts_id}") # 1. Validate $ref fields - must be local (#...) or gts:// URIs - # Issue #32: This validation must happen first to enforce strict $ref format self._validate_schema_refs(schema_content, "") - # 2. Validate x-gts-ref fields (before JSON Schema validation) + # 2. Validate x-gts-ref fields self._validate_schema_x_gts_refs(gts_id) - # 3. Validate against JSON Schema meta-schema + # 3. Validate GTS keywords (x-gts-final, x-gts-abstract, placement) + self._validate_gts_keywords(schema_content) + + # 4. Validate schema derivation chain (OP#12) + self._validate_schema_chain(gts_id) + + # 5. Validate against JSON Schema meta-schema try: from jsonschema import Draft7Validator from jsonschema.validators import validator_for if meta_schema_url: - # Use the appropriate validator for the schema version validator_class = validator_for({"$schema": meta_schema_url}) validator_class.check_schema(schema_content) else: - # Default to Draft7 if no $schema specified Draft7Validator.check_schema(schema_content) - logging.info(f"Schema {gts_id} passed JSON Schema meta-schema validation") + logger.info(f"Schema {gts_id} passed JSON Schema meta-schema validation") except Exception as e: - raise Exception(f"JSON Schema validation failed for '{gts_id}': {str(e)}") + raise ValueError( + f"JSON Schema validation failed for '{gts_id}': {e!s}" + ) from e + + # 6. Validate traits (OP#13) + is_abstract = self._content_is_abstract(schema_content) + self._validate_traits(gts_id, is_abstract) def validate_instance( self, @@ -368,31 +682,57 @@ def validate_instance( obj: The object to validate gts_id: The GTS ID of the object (used to find the schema) """ - gid = GtsID(gts_id) - obj = self.get(gid.id) + obj = None + # Well-known and combined-anonymous IDs are valid GTS IDs. + if GtsID.is_valid(gts_id): + gid = GtsID(gts_id) + obj = self.get(gid.id) + lookup_id = gid.id + else: + # Anonymous instance ID path: allow plain UUID and resolve by raw id. + try: + _ = uuid.UUID(gts_id) + except Exception as e: + raise StoreGtsObjectNotFound(gts_id) from e + obj = self.get(gts_id) + lookup_id = gts_id + if not obj: raise StoreGtsObjectNotFound(gts_id) - if not obj.schemaId: - raise StoreGtsSchemaForInstanceNotFound(gid.id) + if not obj.type_id: + raise StoreGtsSchemaForInstanceNotFound(lookup_id) try: - schema = self.get_schema_content(obj.schemaId) - except KeyError: - raise StoreGtsSchemaNotFound(obj.schemaId) + schema = self.get_schema_content(obj.type_id) + except KeyError as e: + raise StoreGtsSchemaNotFound(obj.type_id) from e - logging.info(f"Validating instance {gts_id} against schema {obj.schemaId}") + logger.info(f"Validating instance {gts_id} against schema {obj.type_id}") - # Create custom RefResolver to resolve GTS ID references - resolver = self._create_ref_resolver(schema) - js_validate(instance=obj.content, schema=schema, resolver=resolver) + # Check if the schema is abstract - abstract types cannot have direct instances + if isinstance(schema, dict) and self._content_is_abstract(schema): + raise ValueError( + f"type '{obj.type_id}' is abstract and cannot have direct instances" + ) - # Validate x-gts-ref constraints + schema_for_validation = _without_x_gts_ref(schema) + validator_class = validator_for(schema_for_validation) + validator = validator_class( + schema_for_validation, registry=self._create_reference_registry() + ) + validator.validate(obj.content) + + # Validate x-gts-ref constraints against the ref-resolved schema. x_gts_ref_validator = XGtsRefValidator(store=self) - x_gts_ref_errors = x_gts_ref_validator.validate_instance(obj.content, schema) + x_gts_ref_errors = x_gts_ref_validator.validate_instance( + obj.content, self._resolve_schema_refs(schema) + ) if x_gts_ref_errors: error_messages = [ f"{err.field_path}: {err.reason}" for err in x_gts_ref_errors ] - raise Exception(f"x-gts-ref validation failed: {'; '.join(error_messages)}") + raise ValueError( + f"x-gts-ref validation failed: {'; '.join(error_messages)}" + ) def cast( self, @@ -415,7 +755,7 @@ def cast( from_schema = from_entity from_schema_id = from_entity.gts_id.id else: - from_schema_id = from_entity.schemaId + from_schema_id = from_entity.type_id if not from_schema_id: raise StoreGtsSchemaForInstanceNotFound(from_id) from_schema = self.get(from_schema_id) @@ -465,13 +805,17 @@ def is_minor_compatible( old_schema = old_entity.content if isinstance(old_entity.content, dict) else {} new_schema = new_entity.content if isinstance(new_entity.content, dict) else {} - # Use the cast method's compatibility checking logic - is_backward, backward_errors = ( - GtsEntityCastResult._check_backward_compatibility(old_schema, new_schema) - ) - is_forward, forward_errors = GtsEntityCastResult._check_forward_compatibility( - old_schema, new_schema + # Compatibility follows accepted-instance-set inclusion on the effective + # (ref-resolved) schemas. Resolve $ref first so the verdict reflects the + # referenced targets (spec sec 4.3). + old_resolved = self._resolve_schema_refs(old_schema) + new_resolved = self._resolve_schema_refs(new_schema) + + backward = compatibility.check_backward_compatibility( + old_resolved, new_resolved ) + forward = compatibility.check_forward_compatibility(old_resolved, new_resolved) + full = compatibility.full_verdict(backward, forward) # Determine direction direction = GtsEntityCastResult._infer_direction(old_schema_id, new_schema_id) @@ -483,19 +827,22 @@ def is_minor_compatible( added_properties=[], removed_properties=[], changed_properties=[], - is_fully_compatible=is_backward and is_forward, - is_backward_compatible=is_backward, - is_forward_compatible=is_forward, + is_fully_compatible=full == compatibility.COMPATIBLE, + is_backward_compatible=backward == compatibility.COMPATIBLE, + is_forward_compatible=forward == compatibility.COMPATIBLE, incompatibility_reasons=[], - backward_errors=backward_errors, - forward_errors=forward_errors, + backward_errors=[], + forward_errors=[], casted_entity=None, + backward_verdict=backward, + forward_verdict=forward, + full_verdict=full, ) - def build_schema_graph(self, gts_id: str) -> Tuple[Dict[str, Set[str]], List[str]]: + def build_schema_graph(self, gts_id: str) -> tuple[dict[str, set[str]], list[str]]: seen_gts_ids = set() - def gts2node(gts_id: str, seen_gts_ids: Set[str]) -> str: + def gts2node(gts_id: str, seen_gts_ids: set[str]) -> str: ret = {"id": gts_id} if gts_id in seen_gts_ids: @@ -516,11 +863,11 @@ def gts2node(gts_id: str, seen_gts_ids: Set[str]) -> str: refs[r["sourcePath"]] = gts2node(r["id"], seen_gts_ids) if refs: ret["refs"] = refs - if entity.schemaId: - if not entity.schemaId.startswith( + if entity.type_id: + if not entity.type_id.startswith( "http://json-schema.org" - ) and not entity.schemaId.startswith("https://json-schema.org"): - ret["schema_id"] = gts2node(entity.schemaId, seen_gts_ids) + ) and not entity.type_id.startswith("https://json-schema.org"): + ret["type_id"] = gts2node(entity.type_id, seen_gts_ids) else: ret["errors"] = ret.get("errors", []) + ["Schema not recognized"] else: @@ -530,7 +877,7 @@ def gts2node(gts_id: str, seen_gts_ids: Set[str]) -> str: return gts2node(gts_id, seen_gts_ids) - def _parse_query_filters(self, filter_str: str) -> Dict[str, str]: + def _parse_query_filters(self, filter_str: str) -> dict[str, str]: """Parse filter expressions from query string. Args: @@ -539,7 +886,7 @@ def _parse_query_filters(self, filter_str: str) -> Dict[str, str]: Returns: Dictionary of filter key-value pairs """ - filters: Dict[str, str] = {} + filters: dict[str, str] = {} if not filter_str: return filters @@ -555,7 +902,7 @@ def _parse_query_filters(self, filter_str: str) -> Dict[str, str]: def _validate_query_pattern( self, base_pattern: str, is_wildcard: bool - ) -> Tuple[Optional[GtsWildcard], Optional[GtsID], str]: + ) -> tuple[GtsWildcard | None, GtsID | None, str]: """Validate and parse the query pattern. Args: @@ -567,7 +914,7 @@ def _validate_query_pattern( """ if is_wildcard: # Wildcard pattern must end with .* or ~* - if not (base_pattern.endswith(".*") or base_pattern.endswith("~*")): + if not base_pattern.endswith((".*", "~*")): return ( None, None, @@ -576,8 +923,8 @@ def _validate_query_pattern( try: wildcard_pattern = GtsWildcard(base_pattern) return wildcard_pattern, None, "" - except Exception as e: - return None, None, f"Invalid query: {str(e)}" + except Exception as e: # noqa: BLE001 - error surfaced in return value + return None, None, f"Invalid query: {e!s}" else: # Non-wildcard pattern must be a complete valid GTS ID try: @@ -585,16 +932,16 @@ def _validate_query_pattern( if not exact_gts_id.gts_id_segments: return None, None, "Invalid query: GTS ID has no valid segments" return None, exact_gts_id, "" - except Exception as e: - return None, None, f"Invalid query: {str(e)}" + except Exception as e: # noqa: BLE001 - error surfaced in return value + return None, None, f"Invalid query: {e!s}" def _matches_id_pattern( self, entity_id: GtsID, base_pattern: str, is_wildcard: bool, - wildcard_pattern: Optional[GtsWildcard], - exact_gts_id: Optional[GtsID], + wildcard_pattern: GtsWildcard | None, + exact_gts_id: GtsID | None, ) -> bool: """Check if entity ID matches the query pattern. @@ -609,7 +956,14 @@ def _matches_id_pattern( True if entity ID matches the pattern """ if is_wildcard and wildcard_pattern: - return entity_id.wildcard_match(wildcard_pattern) + matched = entity_id.wildcard_match(wildcard_pattern) + if not matched: + return False + if base_pattern.endswith("~*"): + base_depth = max(0, len(wildcard_pattern.gts_id_segments) - 1) + if len(entity_id.gts_id_segments) <= base_depth: + return False + return True # For non-wildcard patterns, use wildcard_match to support version flexibility # This allows patterns like "gts.x.test.v1~" to match "gts.x.test.v1.0~" @@ -617,14 +971,13 @@ def _matches_id_pattern( try: pattern_as_wildcard = GtsWildcard(base_pattern) return entity_id.wildcard_match(pattern_as_wildcard) - except Exception: - # If it can't be converted to wildcard, fall back to exact match + except Exception: # noqa: BLE001 - fall back to exact match return entity_id.id == base_pattern return entity_id.id == base_pattern def _matches_filters( - self, entity_content: Dict[str, Any], filters: Dict[str, str] + self, entity_content: dict[str, Any], filters: dict[str, str] ) -> bool: """Check if entity content matches all filter criteria. diff --git a/gts/src/gts/traits.py b/gts/src/gts/traits.py new file mode 100644 index 0000000..bf4367d --- /dev/null +++ b/gts/src/gts/traits.py @@ -0,0 +1,402 @@ +"""OP#13 - Schema Traits Validation (``x-gts-traits-schema`` / ``x-gts-traits``). + +Ported from the Rust reference (`schema_traits.rs`). Validates that trait values +supplied by derived schemas conform to the effective trait schema built from the +whole inheritance chain. + +Algorithm: +1. Walk the chain root -> leaf. For each schema collect ``x-gts-traits-schema`` + subschemas (compose via ``allOf``) and ``x-gts-traits`` values (RFC 7396 + merge). +2. Materialize absent trait properties from their ``default`` (never ``const``). +3. Validate the effective values against the effective schema (JSON Schema + + ``x-gts-ref`` + required-trait completeness, the last only for non-abstract + types). +""" + +from __future__ import annotations + +import copy +from typing import Any + +from jsonschema.validators import validator_for + +from . import derivation +from .x_gts_ref import XGtsRefValidator + +X_GTS_TRAITS_SCHEMA = "x-gts-traits-schema" +X_GTS_TRAITS = "x-gts-traits" +MAX_RECURSION_DEPTH = 64 +_MISSING = object() + + +class EffectiveTraits: + """Built trait artifacts plus the raw inputs they were composed from.""" + + def __init__( + self, + schema: Any, + values: Any, + resolved_trait_schemas: list[Any], + merged_traits: dict[str, Any], + ) -> None: + self.schema = schema + self.values = values + self.resolved_trait_schemas = resolved_trait_schemas + self.merged_traits = merged_traits + + def _has_schema(self) -> bool: + return len(self.resolved_trait_schemas) > 0 + + def _has_explicit_values(self) -> bool: + return isinstance(self.merged_traits, dict) and len(self.merged_traits) > 0 + + def validate(self, check_unresolved: bool) -> list[str]: + """Return a list of error strings (empty means valid).""" + errors = _validate_trait_schema_integrity(self.resolved_trait_schemas) + if errors: + return errors + errors = _validate_trait_schema_compatibility(self.resolved_trait_schemas) + if errors: + return errors + + if not self._has_schema(): + if self._has_explicit_values(): + return [ + f"{X_GTS_TRAITS} values provided but no {X_GTS_TRAITS_SCHEMA} " # noqa: ISC004 + "is defined in the inheritance chain" + ] + return [] + + if _effective_schema_is_false(self.schema): + if self._has_explicit_values(): + return [ + f"{X_GTS_TRAITS_SCHEMA} resolves to `false` in the chain - " # noqa: ISC004 + f"{X_GTS_TRAITS} values are prohibited" + ] + return [] + + return _validate_trait_values(self.schema, self.values, check_unresolved) + + +# --- collection ------------------------------------------------------------ +def collect_trait_schema_from_value(value: Any, out: list[Any], depth: int = 0) -> None: + if depth >= MAX_RECURSION_DEPTH or not isinstance(value, dict): + return + if X_GTS_TRAITS_SCHEMA in value: + out.append(copy.deepcopy(value[X_GTS_TRAITS_SCHEMA])) + all_of = value.get("allOf") + if isinstance(all_of, list): + for item in all_of: + collect_trait_schema_from_value(item, out, depth + 1) + + +def collect_traits_from_value( + value: Any, merged: dict[str, Any], depth: int = 0 +) -> None: + if depth >= MAX_RECURSION_DEPTH or not isinstance(value, dict): + return + traits = value.get(X_GTS_TRAITS) + if isinstance(traits, dict): + for k, v in traits.items(): + merged[k] = copy.deepcopy(v) + all_of = value.get("allOf") + if isinstance(all_of, list): + for item in all_of: + collect_traits_from_value(item, merged, depth + 1) + + +def inline_local_pointers(fragment: Any, root: Any, depth: int = 0) -> Any: + """Inline JSON Pointer ``#/...`` refs against the host document ``root``.""" + if depth >= MAX_RECURSION_DEPTH: + return copy.deepcopy(fragment) + if isinstance(fragment, dict): + ref = fragment.get("$ref") + if isinstance(ref, str) and ref.startswith("#/"): + target = _resolve_json_pointer(root, ref[1:]) + if target is not None: + resolved = inline_local_pointers(target, root, depth + 1) + if len(fragment) > 1 and isinstance(resolved, dict): + for k, v in fragment.items(): + if k != "$ref": + resolved[k] = inline_local_pointers(v, root, depth + 1) + return resolved + return { + k: inline_local_pointers(v, root, depth + 1) for k, v in fragment.items() + } + if isinstance(fragment, list): + return [inline_local_pointers(item, root, depth + 1) for item in fragment] + return copy.deepcopy(fragment) + + +def _resolve_json_pointer(root: Any, pointer: str) -> Any: + # pointer begins with '/' + parts = [p for p in pointer.split("/") if p != ""] + current = root + for part in parts: + part = part.replace("~1", "/").replace("~0", "~") + if isinstance(current, dict) and part in current: + current = current[part] + elif isinstance(current, list): + try: + current = current[int(part)] + except (ValueError, IndexError): + return None + else: + return None + return current + + +# --- RFC 7396 merge -------------------------------------------------------- +def merge_rfc7396_into( + target: dict[str, Any], patch: dict[str, Any], depth: int = 0 +) -> None: + if depth >= MAX_RECURSION_DEPTH: + return + for k, v in patch.items(): + if v is None: + target.pop(k, None) + elif isinstance(v, dict): + existing = target.get(k) + if isinstance(existing, dict): + merge_rfc7396_into(existing, v, depth + 1) + else: + fresh: dict[str, Any] = {} + merge_rfc7396_into(fresh, v, depth + 1) + target[k] = fresh + else: + target[k] = copy.deepcopy(v) + + +# --- composition ----------------------------------------------------------- +def build_effective_traits_schema(schemas: list[Any]) -> Any: + if len(schemas) == 0: + return {} + if len(schemas) == 1: + return copy.deepcopy(schemas[0]) + return {"type": "object", "allOf": [copy.deepcopy(s) for s in schemas]} + + +def build_effective_traits( + resolved_trait_schemas: list[Any], + merged_traits: dict[str, Any], + dialect: str | None, +) -> EffectiveTraits: + effective_schema = build_effective_traits_schema(resolved_trait_schemas) + if dialect and isinstance(effective_schema, dict): + effective_schema["$schema"] = dialect + values = _materialize_traits(effective_schema, merged_traits) + return EffectiveTraits( + schema=effective_schema, + values=values, + resolved_trait_schemas=list(resolved_trait_schemas), + merged_traits=copy.deepcopy(merged_traits), + ) + + +def _effective_schema_is_false(schema: Any, depth: int = 0) -> bool: + if depth >= MAX_RECURSION_DEPTH: + return False + if schema is False: + return True + if isinstance(schema, dict): + all_of = schema.get("allOf") + if isinstance(all_of, list): + return any(_effective_schema_is_false(i, depth + 1) for i in all_of) + return False + + +# --- materialization ------------------------------------------------------- +def _collect_props(schema: Any, props: list[tuple[str, Any]], depth: int = 0) -> None: + if depth >= MAX_RECURSION_DEPTH or not isinstance(schema, dict): + return + p = schema.get("properties") + if isinstance(p, dict): + for k, v in p.items(): + props.append((k, v)) + all_of = schema.get("allOf") + if isinstance(all_of, list): + for item in all_of: + _collect_props(item, props, depth + 1) + + +def _collect_all_properties(schema: Any) -> list[tuple[str, Any]]: + props: list[tuple[str, Any]] = [] + _collect_props(schema, props, 0) + # keep last occurrence of each name (rightmost wins) + seen = set() + deduped: list[tuple[str, Any]] = [] + for name, sch in reversed(props): + if name not in seen: + seen.add(name) + deduped.append((name, sch)) + deduped.reverse() + return deduped + + +def _collect_all_required(schema: Any, req=None, depth: int = 0): + if req is None: + req = set() + if depth >= MAX_RECURSION_DEPTH or not isinstance(schema, dict): + return req + required = schema.get("required") + if isinstance(required, list): + for item in required: + if isinstance(item, str): + req.add(item) + all_of = schema.get("allOf") + if isinstance(all_of, list): + for item in all_of: + _collect_all_required(item, req, depth + 1) + return req + + +def _materialize_traits(trait_schema: Any, traits: Any, depth: int = 0) -> Any: + if depth >= MAX_RECURSION_DEPTH: + return copy.deepcopy(traits) + result: dict[str, Any] = dict(traits) if isinstance(traits, dict) else {} + + all_props: list[tuple[str, Any]] = [] + _collect_props(trait_schema, all_props, 0) + + # Resolve each property once; nearest (most-derived) default wins (leaf->root). + order: list[str] = [] + resolved: dict[str, tuple[Any, Any]] = {} + for name, sch in reversed(all_props): + if name not in resolved: + order.append(name) + resolved[name] = (sch, _MISSING) + prop_schema, nearest_default = resolved[name] + if nearest_default is _MISSING and isinstance(sch, dict) and "default" in sch: + resolved[name] = (prop_schema, sch["default"]) + + for name in order: + prop_schema, nearest_default = resolved[name] + if name not in result: + if nearest_default is not _MISSING: + result[name] = copy.deepcopy(nearest_default) + elif ( + isinstance(result.get(name), dict) + and isinstance(prop_schema, dict) + and prop_schema.get("type") == "object" + and "properties" in prop_schema + ): + result[name] = _materialize_traits(prop_schema, result[name], depth + 1) + + return result + + +# --- validation ------------------------------------------------------------ +def _validate_trait_schema_integrity(resolved_trait_schemas: list[Any]) -> list[str]: + for i, ts in enumerate(resolved_trait_schemas): + if isinstance(ts, bool): + continue + if isinstance(ts, dict): + try: + cls = validator_for(ts) + cls.check_schema(ts) + except Exception as e: # noqa: BLE001 - surfaced as validation error message + return [f"{X_GTS_TRAITS_SCHEMA}[{i}] is not a valid JSON Schema: {e}"] + else: + return [ + f"{X_GTS_TRAITS_SCHEMA}[{i}] must be an object subschema or a " # noqa: ISC004 + f"boolean; got {ts}" + ] + return [] + + +def _validate_trait_schema_compatibility( + resolved_trait_schemas: list[Any], +) -> list[str]: + errors: list[str] = [] + for i in range(1, len(resolved_trait_schemas)): + ancestor_schema = build_effective_traits_schema(resolved_trait_schemas[:i]) + descendant_schema = build_effective_traits_schema( + resolved_trait_schemas[: i + 1] + ) + for err in derivation.validate_derivation( + ancestor_schema, + descendant_schema, + "ancestor trait schema", + "descendant trait schema", + ): + errors.append( + f"{X_GTS_TRAITS_SCHEMA}[{i}] is incompatible with ancestor trait " + f"schema: {err}" + ) + for err in derivation.validate_closed_descendant_branches( + ancestor_schema, + resolved_trait_schemas[i], + "ancestor trait schema", + "descendant trait schema", + ): + errors.append( + f"{X_GTS_TRAITS_SCHEMA}[{i}] is incompatible with ancestor trait " + f"schema: {err}" + ) + return errors + + +def _strip_required(schema: Any, depth: int = 0) -> Any: + if depth >= MAX_RECURSION_DEPTH or not isinstance(schema, dict): + return schema + out = dict(schema) + out.pop("required", None) + all_of = out.get("allOf") + if isinstance(all_of, list): + out["allOf"] = [_strip_required(i, depth + 1) for i in all_of] + return out + + +def _validate_traits_against_schema( + trait_schema: Any, effective_traits: Any, check_unresolved: bool +) -> list[str]: + errors: list[str] = [] + validation_schema = ( + trait_schema if check_unresolved else _strip_required(trait_schema) + ) + + try: + cls = validator_for(validation_schema) + validator = cls(validation_schema) + for error in validator.iter_errors(effective_traits): + errors.append(f"trait validation: {error.message}") + except Exception as e: # noqa: BLE001 - surfaced as validation error message + errors.append(f"failed to compile trait schema: {e}") + + if not check_unresolved: + return errors + + all_props = _collect_all_properties(trait_schema) + required = _collect_all_required(trait_schema) + traits_obj = effective_traits if isinstance(effective_traits, dict) else {} + + for prop_name, prop_schema in all_props: + if prop_name not in required: + continue + has_value = prop_name in traits_obj + has_default = isinstance(prop_schema, dict) and "default" in prop_schema + if not has_value and not has_default: + expected_type = "any" + if isinstance(prop_schema, dict) and isinstance( + prop_schema.get("type"), str + ): + expected_type = prop_schema["type"] + errors.append( + f"trait property '{prop_name}' (type: {expected_type}) is not " + "resolved: no value provided and no default defined in the trait " + "schema" + ) + return errors + + +def _validate_trait_values( + effective_traits_schema: Any, effective_traits: Any, check_unresolved: bool +) -> list[str]: + errors = _validate_traits_against_schema( + effective_traits_schema, effective_traits, check_unresolved + ) + xref = XGtsRefValidator() + for err in xref.validate_instance(effective_traits, effective_traits_schema, ""): + errors.append(f"trait x-gts-ref: {err.reason}") + return errors diff --git a/gts/src/gts/x_gts_ref.py b/gts/src/gts/x_gts_ref.py index 445a61c..ddfc0ac 100644 --- a/gts/src/gts/x_gts_ref.py +++ b/gts/src/gts/x_gts_ref.py @@ -11,9 +11,51 @@ """ from __future__ import annotations -from typing import Any, Dict, List, Optional -from .gts import GtsID, GTS_URI_PREFIX +from typing import Any + +from jsonschema.validators import validator_for + +from .gts import GTS_URI_PREFIX, GtsID + + +def _without_x_gts_ref(schema: Any) -> Any: + if isinstance(schema, dict): + stripped = { + key: _without_x_gts_ref(value) + for key, value in schema.items() + if key != "x-gts-ref" + } + for keyword in ("oneOf", "anyOf", "allOf"): + branches = stripped.get(keyword) + if ( + isinstance(branches, list) + and branches + and all(isinstance(branch, dict) and not branch for branch in branches) + ): + stripped.pop(keyword, None) + return stripped + if isinstance(schema, list): + return [_without_x_gts_ref(value) for value in schema] + return schema + + +def _is_x_gts_ref_only_combinator(branches: list[Any]) -> bool: + if not branches: + return False + for branch in branches: + stripped = _without_x_gts_ref(branch) + if not isinstance(stripped, dict) or stripped: + return False + return True + + +def _is_structurally_valid(instance: Any, schema: Any) -> bool: + try: + validator = validator_for(schema)(_without_x_gts_ref(schema)) + return validator.is_valid(instance) + except Exception: # noqa: BLE001 - treat any validation error as "not valid" + return False class XGtsRefValidationError(Exception): @@ -32,7 +74,7 @@ def __init__(self, field_path: str, value: Any, ref_pattern: str, reason: str): class XGtsRefValidator: """Validator for x-gts-ref constraints in GTS schemas.""" - def __init__(self, store: Optional[Any] = None): + def __init__(self, store: Any | None = None): """ Initialize validator. @@ -42,8 +84,8 @@ def __init__(self, store: Optional[Any] = None): self.store = store def validate_instance( - self, instance: Dict[str, Any], schema: Dict[str, Any], instance_path: str = "" - ) -> List[XGtsRefValidationError]: + self, instance: dict[str, Any], schema: dict[str, Any], instance_path: str = "" + ) -> list[XGtsRefValidationError]: """ Validate an instance against x-gts-ref constraints in schema. @@ -55,43 +97,115 @@ def validate_instance( Returns: List of validation errors (empty if valid) """ - errors = [] + errors: list[XGtsRefValidationError] = [] - def visit_instance(inst, sch, path): + def visit_instance(inst, sch, path, errs): """Visit instance nodes and validate x-gts-ref constraints.""" if not isinstance(sch, dict): return - # Check for x-gts-ref constraint if "x-gts-ref" in sch and isinstance(inst, str): error = self._validate_ref_value(inst, sch["x-gts-ref"], path, schema) if error: - errors.append(error) - - # Recurse into object properties - if sch.get("type") == "object" and "properties" in sch: - if isinstance(inst, dict): - for prop_name, prop_schema in sch["properties"].items(): - if prop_name in inst: - prop_path = f"{path}.{prop_name}" if path else prop_name - visit_instance(inst[prop_name], prop_schema, prop_path) - - # Recurse into array items - if sch.get("type") == "array" and "items" in sch: - if isinstance(inst, list): - for idx, item in enumerate(inst): - item_path = f"{path}[{idx}]" - visit_instance(item, sch["items"], item_path) - - visit_instance(instance, schema, instance_path) + errs.append(error) + + one_of = sch.get("oneOf") + if isinstance(one_of, list): + if _is_x_gts_ref_only_combinator(one_of): + branch_errors = [ + _validate_branch(inst, branch, path) for branch in one_of + ] + matching = sum(not branch for branch in branch_errors) + if matching == 0: + errs.append( + XGtsRefValidationError( + path, inst, "", "oneOf: no branch matched" + ) + ) + elif matching > 1: + errs.append( + XGtsRefValidationError( + path, + inst, + "", + f"oneOf: {matching} branches matched, expected exactly 1", + ) + ) + else: + matching_branches = [ + branch + for branch in one_of + if _is_structurally_valid(inst, branch) + ] + if len(matching_branches) == 1: + errs.extend(_validate_branch(inst, matching_branches[0], path)) + + any_of = sch.get("anyOf") + if isinstance(any_of, list): + if _is_x_gts_ref_only_combinator(any_of): + branch_errors = [ + _validate_branch(inst, branch, path) for branch in any_of + ] + if not any(not branch for branch in branch_errors): + errs.append( + XGtsRefValidationError( + path, inst, "", "anyOf: no branch matched" + ) + ) + else: + matching_branches = [ + branch + for branch in any_of + if _is_structurally_valid(inst, branch) + ] + branch_errors = [ + _validate_branch(inst, branch, path) + for branch in matching_branches + ] + if matching_branches and not any( + not branch for branch in branch_errors + ): + errs.append( + XGtsRefValidationError( + path, inst, "", "anyOf: no branch matched" + ) + ) + + all_of = sch.get("allOf") + if isinstance(all_of, list): + for branch in all_of: + if _is_structurally_valid(inst, branch): + errs.extend(_validate_branch(inst, branch, path)) + + if ( + sch.get("type") == "object" + and "properties" in sch + and isinstance(inst, dict) + ): + for prop_name, prop_schema in sch["properties"].items(): + if prop_name in inst: + prop_path = f"{path}.{prop_name}" if path else prop_name + visit_instance(inst[prop_name], prop_schema, prop_path, errs) + + if sch.get("type") == "array" and "items" in sch and isinstance(inst, list): + for idx, item in enumerate(inst): + item_path = f"{path}[{idx}]" + visit_instance(item, sch["items"], item_path, errs) + + def _validate_branch(inst, branch, path): + branch_errors: list[XGtsRefValidationError] = [] + visit_instance(inst, branch, path, branch_errors) + return branch_errors + + visit_instance(instance, schema, instance_path, errors) return errors def validate_schema( self, - schema: Dict[str, Any], + schema: dict[str, Any], schema_path: str = "", - root_schema: Optional[Dict[str, Any]] = None, - ) -> List[XGtsRefValidationError]: + root_schema: dict[str, Any] | None = None, + ) -> list[XGtsRefValidationError]: """ Validate x-gts-ref fields in a schema definition. @@ -137,8 +251,8 @@ def visit_schema(sch, path): return errors def _validate_ref_value( - self, value: str, ref_pattern: str, field_path: str, schema: Dict[str, Any] - ) -> Optional[XGtsRefValidationError]: + self, value: str, ref_pattern: str, field_path: str, schema: dict[str, Any] + ) -> XGtsRefValidationError | None: """ Validate an instance value against its x-gts-ref constraint. @@ -184,8 +298,8 @@ def _validate_ref_value( return self._validate_gts_pattern(value, ref_pattern, field_path) def _validate_ref_pattern( - self, ref_pattern: str, field_path: str, root_schema: Dict[str, Any] - ) -> Optional[XGtsRefValidationError]: + self, ref_pattern: str, field_path: str, root_schema: dict[str, Any] + ) -> XGtsRefValidationError | None: """ Validate an x-gts-ref pattern in a schema definition. @@ -237,7 +351,7 @@ def _validate_ref_pattern( def _validate_gts_id_or_pattern( self, pattern: str, field_path: str - ) -> Optional[XGtsRefValidationError]: + ) -> XGtsRefValidationError | None: """Validate a GTS ID or pattern in schema definition.""" if pattern == "gts.*": return None # Valid wildcard @@ -263,7 +377,7 @@ def _validate_gts_id_or_pattern( def _validate_gts_pattern( self, value: str, pattern: str, field_path: str - ) -> Optional[XGtsRefValidationError]: + ) -> XGtsRefValidationError | None: """ Validate value matches a GTS pattern. @@ -323,7 +437,7 @@ def _normalize_gts_value(self, value: str) -> str: return value[len(GTS_URI_PREFIX) :] return value - def _resolve_pointer(self, schema: Dict[str, Any], pointer: str) -> Optional[str]: + def _resolve_pointer(self, schema: dict[str, Any], pointer: str) -> str | None: """ Resolve a JSON Pointer in the schema. diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..ae7177d --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,60 @@ +"""Tests for the command-line interface dispatch.""" + +import json + +import pytest + +from gts._cli import main + + +@pytest.mark.parametrize( + "arguments", + [ + ["validate-id", "--gts-id", "gts.vendor.package.namespace.type.v1~"], + ["parse-id", "--gts-id", "gts.vendor.package.namespace.type.v1~"], + [ + "match-id-pattern", + "--candidate", + "gts.vendor.package.namespace.type.v1~", + "--pattern", + "gts.vendor.package.*", + ], + ["uuid", "--gts-id", "gts.vendor.package.namespace.type.v1~"], + [ + "validate-instance", + "--gts-id", + "gts.vendor.package.namespace.type.v1~vendor.package.namespace.item.v1", + ], + ["resolve-relationships", "--gts-id", "gts.vendor.package.namespace.type.v1~"], + [ + "compatibility", + "--old-schema-id", + "gts.vendor.package.namespace.type.v1~", + "--new-schema-id", + "gts.vendor.package.namespace.type.v1.1~", + ], + [ + "cast", + "--from-id", + "gts.vendor.package.namespace.type.v1~vendor.package.namespace.item.v1", + "--to-schema-id", + "gts.vendor.package.namespace.type.v1.1~", + ], + ["query", "--expr", "gts.vendor.package.*"], + ["attr", "--gts-with-path", "gts.vendor.package.namespace.type.v1~@name"], + ["list"], + ], +) +def test_cli_operations_emit_json(arguments, capsys): + main(arguments) + + assert json.loads(capsys.readouterr().out) + + +def test_cli_writes_openapi_spec(tmp_path, capsys): + output_path = tmp_path / "openapi.json" + + main(["openapi-spec", "--out", str(output_path)]) + + assert json.loads(capsys.readouterr().out) == {"ok": True, "out": str(output_path)} + assert json.loads(output_path.read_text())["openapi"] diff --git a/tests/test_compatibility.py b/tests/test_compatibility.py new file mode 100644 index 0000000..7ba6083 --- /dev/null +++ b/tests/test_compatibility.py @@ -0,0 +1,123 @@ +"""Tests for gts.compatibility (spec sec 4, OP#8 & OP#12 inclusion primitive).""" + +from gts.compatibility import ( + COMPATIBLE, + INCOMPATIBLE, + UNKNOWN, + boolean_schema_value, + sanitize, + check_backward_compatibility, + check_forward_compatibility, + full_verdict, + check_accepted_set_inclusion, +) + + +class TestBooleanSchemaValue: + def test_bool_passthrough(self): + assert boolean_schema_value(True) is True + assert boolean_schema_value(False) is False + + def test_empty_dict_is_true(self): + assert boolean_schema_value({}) is True + + def test_annotations_only_is_true(self): + assert boolean_schema_value({"title": "x", "description": "y"}) is True + + def test_not_empty_is_false(self): + assert boolean_schema_value({"not": {}}) is False + + def test_double_negation_is_true(self): + assert boolean_schema_value({"not": {"not": {}}}) is True + + def test_multiple_assertions_is_none(self): + assert boolean_schema_value({"type": "string", "minLength": 1}) is None + + def test_non_bool_non_dict_is_none(self): + assert boolean_schema_value("nope") is None + assert boolean_schema_value(42) is None + + def test_not_of_unprovable_is_none(self): + assert boolean_schema_value({"not": {"type": "string", "minLength": 1}}) is None + + +class TestSanitize: + def test_strips_meta_keywords(self): + schema = {"$id": "x", "$schema": "y", "type": "string"} + assert sanitize(schema) == {"type": "string"} + + def test_strips_x_gts_keys(self): + schema = {"x-gts-ref": "gts.*", "type": "string"} + assert sanitize(schema) == {"type": "string"} + + def test_recurses_into_lists_and_nested_dicts(self): + schema = { + "allOf": [{"$id": "a", "type": "string"}], + "properties": {"p": {"$comment": "x", "type": "integer"}}, + } + result = sanitize(schema) + assert result["allOf"] == [{"type": "string"}] + assert result["properties"]["p"] == {"type": "integer"} + + def test_non_dict_non_list_passthrough(self): + assert sanitize("abc") == "abc" + assert sanitize(5) == 5 + + def test_drops_redundant_type_with_enum(self): + schema = {"enum": ["a", "b"], "type": "string"} + result = sanitize(schema) + assert "type" not in result + + def test_keeps_type_when_enum_has_other_types(self): + schema = {"enum": ["a", 1], "type": "string"} + result = sanitize(schema) + assert result.get("type") == "string" + + +class TestCompatibilityVerdicts: + def test_backward_compatible_widened_enum(self): + # old accepted set must be subset of new + result = check_backward_compatibility( + {"type": "string", "enum": ["a"]}, + {"type": "string", "enum": ["a", "b"]}, + ) + assert result == COMPATIBLE + + def test_backward_incompatible_narrowed_type(self): + result = check_backward_compatibility( + {"type": "string"}, + {"type": "integer"}, + ) + assert result == INCOMPATIBLE + + def test_forward_compatible_case(self): + result = check_forward_compatibility( + {"type": "string", "enum": ["a", "b"]}, + {"type": "string", "enum": ["a"]}, + ) + assert result == COMPATIBLE + + def test_full_verdict_incompatible_dominates(self): + assert full_verdict(INCOMPATIBLE, COMPATIBLE) == INCOMPATIBLE + assert full_verdict(COMPATIBLE, INCOMPATIBLE) == INCOMPATIBLE + + def test_full_verdict_compatible_both(self): + assert full_verdict(COMPATIBLE, COMPATIBLE) == COMPATIBLE + + def test_full_verdict_unknown_otherwise(self): + assert full_verdict(UNKNOWN, COMPATIBLE) == UNKNOWN + assert full_verdict(COMPATIBLE, UNKNOWN) == UNKNOWN + + def test_accepted_set_inclusion_true(self): + assert check_accepted_set_inclusion({"type": "string"}, {}) is True + + def test_accepted_set_inclusion_false(self): + assert ( + check_accepted_set_inclusion({"type": "string"}, {"type": "integer"}) + is False + ) + + def test_boolean_schema_operands_are_coerced(self): + # True == accept everything, False == accept nothing + assert check_accepted_set_inclusion(False, True) is True + assert check_accepted_set_inclusion(True, False) is False diff --git a/tests/test_derivation.py b/tests/test_derivation.py new file mode 100644 index 0000000..20c40b1 --- /dev/null +++ b/tests/test_derivation.py @@ -0,0 +1,185 @@ +"""Tests for gts.derivation (OP#12 schema-vs-schema derivation admission).""" + +from gts.derivation import ( + flatten_schema, + validate_derivation, + validate_derivation_compatibility, + validate_closed_descendant_branches, +) + + +class TestFlattenSchema: + def test_non_dict_returned_as_is(self): + assert flatten_schema("not-a-dict") == "not-a-dict" + assert flatten_schema(True) is True + + def test_merges_allof_properties(self): + schema = { + "allOf": [ + {"properties": {"a": {"type": "string"}}, "required": ["a"]}, + {"properties": {"b": {"type": "integer"}}, "required": ["b"]}, + ] + } + flat = flatten_schema(schema) + assert set(flat["properties"].keys()) == {"a", "b"} + assert set(flat["required"]) == {"a", "b"} + + def test_merges_same_property_via_nested_allof(self): + schema = { + "allOf": [ + {"properties": {"a": {"type": "string", "minLength": 1}}}, + {"properties": {"a": {"maxLength": 10}}}, + ] + } + flat = flatten_schema(schema) + assert flat["properties"]["a"]["minLength"] == 1 + assert flat["properties"]["a"]["maxLength"] == 10 + + def test_additional_properties_false_sticky(self): + schema = { + "allOf": [ + {"additionalProperties": False}, + {"additionalProperties": True}, + ] + } + flat = flatten_schema(schema) + assert flat["additionalProperties"] is False + + def test_additional_properties_true_does_not_override_existing(self): + schema = {"allOf": [{"additionalProperties": {"type": "string"}}]} + flat = flatten_schema(schema) + # top-level has no additionalProperties key, so allOf value applies + assert flat["additionalProperties"] == {"type": "string"} + + def test_scalar_keys_overwritten(self): + schema = {"allOf": [{"title": "a"}], "title": "b"} + flat = flatten_schema(schema) + assert flat["title"] == "b" + + +class TestValidateDerivation: + def test_compatible_extension_no_errors(self): + base = {"type": "object", "properties": {"a": {"type": "string"}}} + derived = { + "allOf": [base], + "type": "object", + "properties": {"b": {"type": "integer"}}, + } + errors = validate_derivation(base, derived, "base", "derived") + assert errors == [] + + def test_loosening_additional_properties_flagged(self): + base = {"type": "object", "additionalProperties": False} + derived = {"type": "object", "additionalProperties": True} + errors = validate_derivation(base, derived, "base", "derived") + assert any("loosens additionalProperties" in e for e in errors) + + def test_incompatible_type_change_flagged(self): + base = {"type": "string"} + derived = {"type": "integer"} + errors = validate_derivation(base, derived, "base", "derived") + assert any("not included in base" in e for e in errors) + + def test_disabling_base_property_flagged(self): + base = { + "type": "object", + "properties": {"a": {"type": "string"}}, + } + derived = { + "type": "object", + "properties": {"a": False}, + } + errors = validate_derivation(base, derived, "base", "derived") + assert any("disables property" in e for e in errors) + + +class TestValidateClosedDescendantBranches: + def test_no_errors_when_descendant_restates_property(self): + ancestor = { + "type": "object", + "properties": {"a": {"type": "string"}}, + } + descendant = { + "type": "object", + "properties": {"a": {"type": "string"}}, + "additionalProperties": False, + } + errors = validate_closed_descendant_branches( + ancestor, descendant, "ancestor", "descendant" + ) + assert errors == [] + + def test_orphaned_property_flagged(self): + ancestor = { + "type": "object", + "properties": {"a": {"type": "string"}}, + } + descendant = { + "type": "object", + "properties": {}, + "additionalProperties": False, + } + errors = validate_closed_descendant_branches( + ancestor, descendant, "ancestor", "descendant" + ) + assert any("unusable under allOf composition" in e for e in errors) + + def test_recurses_into_allof_branches(self): + ancestor = { + "type": "object", + "properties": {"a": {"type": "string"}}, + } + descendant = { + "allOf": [ + {"type": "object", "additionalProperties": False}, + ] + } + errors = validate_closed_descendant_branches( + ancestor, descendant, "ancestor", "descendant" + ) + assert any("a" in e for e in errors) + + def test_recurses_into_nested_common_properties(self): + ancestor = { + "type": "object", + "properties": { + "nested": { + "type": "object", + "properties": {"x": {"type": "string"}}, + } + }, + } + descendant = { + "type": "object", + "properties": { + "nested": { + "type": "object", + "properties": {}, + "additionalProperties": False, + } + }, + } + errors = validate_closed_descendant_branches( + ancestor, descendant, "ancestor", "descendant" + ) + assert any("nested.x" in e for e in errors) + + +class TestValidateDerivationCompatibility: + def test_combines_declaration_and_closed_branch_checks(self): + base = { + "type": "object", + "properties": {"a": {"type": "string"}}, + } + derived = { + "allOf": [base], + "type": "object", + "properties": {}, + "additionalProperties": False, + } + errors = validate_derivation_compatibility(base, derived, "base", "derived") + assert any("unusable under allOf composition" in e for e in errors) + + def test_non_dict_schema_handled(self): + errors = validate_derivation_compatibility(True, True, "base", "derived") + assert errors == [] diff --git a/tests/test_entities.py b/tests/test_entities.py index a55e1b5..0148569 100644 --- a/tests/test_entities.py +++ b/tests/test_entities.py @@ -175,9 +175,9 @@ def test_entity_schema_id_calculation(self): cfg=DEFAULT_GTS_CONFIG, ) - # Issue #25: $schema is no longer used for schema_id, use 'type' field instead - assert entity.schemaId == "gts.vendor.package.namespace.type.v1~" - assert entity.selected_schema_id_field == "type" + # Issue #25: $schema is no longer used for type_id, use 'type' field instead + assert entity.type_id == "gts.vendor.package.namespace.type.v1~" + assert entity.selected_type_id_field == "type" def test_entity_label_from_file(self): """Test entity label derived from file.""" @@ -330,10 +330,10 @@ def test_get_graph_basic(self): "ref": "gts.vendor.package.namespace.other.v1~", }, gts_id=gts_id, - schemaId="gts.vendor.package.namespace.type.v1~", + type_id="gts.vendor.package.namespace.type.v1~", ) graph = entity.get_graph() assert graph["id"] == "gts.vendor.package.namespace.type.v1~" - assert graph["schema_id"] == "gts.vendor.package.namespace.type.v1~" + assert graph["type_id"] == "gts.vendor.package.namespace.type.v1~" assert "refs" in graph diff --git a/tests/test_files_reader_coverage.py b/tests/test_files_reader_coverage.py new file mode 100644 index 0000000..2583029 --- /dev/null +++ b/tests/test_files_reader_coverage.py @@ -0,0 +1,67 @@ +"""Additional public behavior coverage for file-backed GTS discovery.""" + +import json + +from gts.files_reader import GtsFileReader + + +def test_reader_discovers_json_yaml_and_list_entities_and_skips_invalid_files(tmp_path): + source = tmp_path / "source" + source.mkdir() + (source / "entities.json").write_text( + json.dumps( + [ + {"id": "gts.acme.catalog._.item.v1~acme.catalog._.one.v1"}, + {"id": "not-a-gts-id"}, + ] + ), + encoding="utf-8", + ) + (source / "schema.yaml").write_text( + """$id: gts://gts.acme.catalog._.item.v1~ +$schema: https://json-schema.org/draft/2020-12/schema +type: object +""", + encoding="utf-8", + ) + (source / "broken.json").write_text("{not json", encoding="utf-8") + (source / "notes.txt").write_text("ignored", encoding="utf-8") + excluded = source / "node_modules" + excluded.mkdir() + (excluded / "ignored.json").write_text( + json.dumps({"id": "gts.acme.catalog._.item.v1~acme.catalog._.ignored.v1"}), + encoding="utf-8", + ) + + entities = list(GtsFileReader(str(source))) + + assert [entity.gts_id.id for entity in entities] == [ + "gts.acme.catalog._.item.v1~acme.catalog._.one.v1", + "gts.acme.catalog._.item.v1~", + ] + assert entities[0].label == "entities.json#0" + assert entities[0].file.sequencesCount == 2 + assert entities[1].file.name == "schema.yaml" + + +def test_reader_accepts_multiple_paths_and_reset_recollects_files(tmp_path): + first = tmp_path / "first.gts" + second = tmp_path / "second.jsonc" + first.write_text( + json.dumps({"id": "gts.acme.catalog._.item.v1~acme.catalog._.first.v1"}), + encoding="utf-8", + ) + second.write_text( + json.dumps({"id": "gts.acme.catalog._.item.v1~acme.catalog._.second.v1"}), + encoding="utf-8", + ) + reader = GtsFileReader([str(first), str(second)]) + + assert [entity.raw_id for entity in reader] == [ + "gts.acme.catalog._.item.v1~acme.catalog._.first.v1", + "gts.acme.catalog._.item.v1~acme.catalog._.second.v1", + ] + assert reader.read_by_id("anything") is None + + reader.reset() + assert [entity.file.name for entity in reader] == ["first.gts", "second.jsonc"] diff --git a/tests/test_gts_id.py b/tests/test_gts_id.py index 4732725..f5ba85c 100644 --- a/tests/test_gts_id.py +++ b/tests/test_gts_id.py @@ -241,3 +241,45 @@ def test_underscore_in_tokens_allowed(self): """Test that underscores are allowed in tokens.""" gts_id = GtsID("gts.my_vendor.my_package.my_namespace.my_type.v1~") assert gts_id.gts_id_segments[0].vendor == "my_vendor" + + def test_combined_anonymous_id_uses_embedded_uuid(self): + embedded_uuid = "7a1d2f34-5678-49ab-9012-abcdef123456" + gts_id = GtsID( + "gts.vendor.package.namespace.type.v1~" + embedded_uuid + ) + + assert gts_id.uuid_tail == embedded_uuid + assert gts_id.to_uuid() == uuid.UUID(embedded_uuid) + assert len(gts_id.gts_id_segments) == 2 + + @pytest.mark.parametrize( + "segment", + [ + "vendor.package.namespace.type.v01", + "vendor.package.namespace.type.v1.01", + "vendor.package.namespace.type.v-1", + ], + ) + def test_rejects_noncanonical_versions(self, segment): + with pytest.raises(GtsInvalidSegment): + GtsIdSegment(1, 0, segment) + + def test_query_helpers_parse_and_match_filters(self): + gts_id = GtsID( + "gts.vendor.package.namespace.type.v1~vendor.package.namespace.item.v1" + ) + base, filters = gts_id.parse_query( + 'gts.vendor.package.namespace.type.v1~[status="active"]' + ) + + assert base == "gts.vendor.package.namespace.type.v1~" + assert filters == {"status": "active"} + assert gts_id.match_query( + {"gtsId": gts_id.id, "status": "active"}, + "gtsId", + 'gts.vendor.package.namespace.type.v1~[status="active"]', + ) + + def test_split_at_path_rejects_empty_selector(self): + with pytest.raises(ValueError, match="cannot be empty"): + GtsID.split_at_path("gts.vendor.package.namespace.type.v1~@") diff --git a/tests/test_ops.py b/tests/test_ops.py new file mode 100644 index 0000000..95ca333 --- /dev/null +++ b/tests/test_ops.py @@ -0,0 +1,319 @@ +"""Tests for gts.ops.GtsOps (the high-level CLI/HTTP operations facade).""" + +import pytest + +from gts.ops import GtsOps + + +SCHEMA = { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "gts.x.test._.foo.v1~", + "type": "object", + "properties": {"name": {"type": "string"}}, + "required": ["name"], +} + +INSTANCE = { + "$id": "gts.x.test._.foo.v1~x.test._.inst.v1", + "type": "gts.x.test._.foo.v1~", + "name": "hi", +} + + +@pytest.fixture +def ops(): + return GtsOps(path=None) + + +class TestConstructionAndConfig: + def test_default_config_used_when_no_path(self, ops): + assert "$id" in ops.cfg.entity_id_fields + + def test_config_from_invalid_path_falls_back(self): + o = GtsOps(path=None, config="/nonexistent/path/config.json") + assert o.cfg is not None + + def test_reload_from_path_missing_dir_raises(self, ops, tmp_path): + empty_dir = tmp_path / "empty" + empty_dir.mkdir() + ops.reload_from_path(str(empty_dir)) + assert ops.store is not None + + +class TestAddEntity: + def test_add_schema_success(self, ops): + result = ops.add_entity(SCHEMA) + assert result.ok is True + assert result.is_type_schema is True + assert result.id == "gts.x.test._.foo.v1~" + + def test_add_schema_missing_gts_id(self, ops): + bad_schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + } + result = ops.add_entity(bad_schema) + assert result.ok is False + assert "Unable to detect GTS ID" in result.error + + def test_add_schema_plain_gts_prefix_rejected_when_validate(self, ops): + schema = dict(SCHEMA) + schema["$id"] = "gts.x.test._.foo.v1~" + result = ops.add_entity(schema, validate=True) + # $id doesn't start with gts:// -> rejected only if raw $id startswith "gts." + assert result.ok is False + assert "gts:// URI format" in result.error + + def test_add_instance_without_id_field_rejected(self, ops): + result = ops.add_entity({"name": "hi"}) + assert result.ok is False + assert "must have an id field" in result.error + + def test_add_instance_success(self, ops): + ops.add_entity(SCHEMA) + result = ops.add_entity(INSTANCE) + assert result.ok is True + assert result.is_type_schema is False + + def test_add_instance_validate_failure_restores_previous(self, ops): + ops.add_entity(SCHEMA) + bad_instance = { + "$id": "gts.x.test._.foo.v1~x.test._.inst.v1", + "gtsType": "gts.x.test._.foo.v1~", + } # missing required "name" + result = ops.add_entity(bad_instance, validate=True) + assert result.ok is False + assert "Validation failed" in result.error + + def test_add_schema_validate_basic_failure(self, ops): + bad_schema = { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "gts.x.test._.foo.v1~", + "type": "object", + "x-gts-ref": "notgts.*", + } + result = ops.add_entity(bad_schema) + assert result.ok is False + assert "Validation failed" in result.error + + def test_add_entities_batch(self, ops): + result = ops.add_entities([SCHEMA, INSTANCE]) + assert result.ok is True + assert len(result.results) == 2 + + +class TestAddSchemaLegacy: + def test_add_schema_legacy_success(self, ops): + result = ops.add_schema("gts.x.test._.legacy.v1~", {"type": "object"}) + assert result.ok is True + assert result.id == "gts.x.test._.legacy.v1~" + + def test_add_schema_legacy_failure(self, ops): + result = ops.add_schema("gts.x.test._.legacy.v1", {"type": "object"}) + assert result.ok is False + assert result.error + + +class TestValidateId: + def test_valid_wildcard(self, ops): + result = ops.validate_id("gts.x.test.*") + assert result.valid is True + assert result.is_wildcard is True + + def test_valid_exact(self, ops): + result = ops.validate_id("gts.x.test._.foo.v1~") + assert result.valid is True + assert result.is_type is True + + def test_invalid_id(self, ops): + result = ops.validate_id("not a valid id") + assert result.valid is False + assert result.error + + def test_to_dict_variants(self, ops): + result = ops.validate_id("gts.x.test._.foo.v1~") + d = result.to_dict() + assert d["valid"] is True + assert d["is_type"] is True + + def test_invalid_wildcard(self, ops): + result = ops.validate_id("notgts*") + assert result.valid is False + + +class TestParseId: + def test_parse_exact(self, ops): + result = ops.parse_id("gts.x.test._.foo.v1~") + assert result.ok is True + assert len(result.segments) == 1 + assert result.segments[0].vendor == "x" + d = result.to_dict() + assert d["segments"][0]["vendor"] == "x" + + def test_parse_wildcard(self, ops): + result = ops.parse_id("gts.x.test.*") + assert result.ok is True + assert result.is_wildcard is True + + def test_parse_invalid(self, ops): + result = ops.parse_id("not valid") + assert result.ok is False + assert result.error + + +class TestMatchIdPattern: + def test_match_true(self, ops): + result = ops.match_id_pattern("gts.x.test._.foo.v1~", "gts.x.test.*") + assert result.match is True + + def test_match_false(self, ops): + result = ops.match_id_pattern( + "gts.x.other._.foo.v1~", "gts.x.test.*" + ) + assert result.match is False + + def test_match_malformed_wildcard_candidate(self, ops): + result = ops.match_id_pattern("a*b", "gts.x.test.*") + assert result.match is False + assert result.error + + def test_to_dict_with_error(self, ops): + result = ops.match_id_pattern("bad id", "gts.x.test.*") + d = result.to_dict() + assert "error" in d + + +class TestUuid: + def test_uuid_deterministic(self, ops): + r1 = ops.uuid("gts.x.test._.foo.v1~") + r2 = ops.uuid("gts.x.test._.foo.v1~") + assert r1.uuid == r2.uuid + d = r1.to_dict() + assert d["id"] == "gts.x.test._.foo.v1~" + + +class TestValidateInstanceSchemaEntity: + def test_validate_schema_ok(self, ops): + ops.add_entity(SCHEMA) + result = ops.validate_schema("gts.x.test._.foo.v1~") + assert result.ok is True + d = result.to_dict() + assert d["ok"] is True + + def test_validate_schema_error(self, ops): + result = ops.validate_schema("gts.x.test._.missing.v1~") + assert result.ok is False + assert result.error + + def test_validate_instance_ok(self, ops): + ops.add_entity(SCHEMA) + ops.add_entity(INSTANCE) + result = ops.validate_instance(INSTANCE["$id"]) + assert result.ok is True + + def test_validate_instance_error(self, ops): + result = ops.validate_instance("gts.x.test._.foo.v1~x.test._.missing.v1") + assert result.ok is False + + def test_validate_entity_schema(self, ops): + ops.add_entity(SCHEMA) + result = ops.validate_entity("gts.x.test._.foo.v1~") + assert result.entity_type == "schema" + assert result.ok is True + d = result.to_dict() + assert d["entity_type"] == "schema" + + def test_validate_entity_instance(self, ops): + ops.add_entity(SCHEMA) + ops.add_entity(INSTANCE) + result = ops.validate_entity(INSTANCE["$id"]) + assert result.entity_type == "instance" + assert result.ok is True + + def test_validate_entity_invalid_id(self, ops): + result = ops.validate_entity("not a valid id") + assert result.ok is False + assert result.entity_type == "" + + +class TestSchemaGraphCompatibilityCast: + def test_schema_graph(self, ops): + ops.add_entity(SCHEMA) + result = ops.schema_graph("gts.x.test._.foo.v1~") + assert result.graph["id"] == "gts.x.test._.foo.v1~" + assert result.to_dict() == result.graph + + def test_compatibility(self, ops): + ops.add_entity(SCHEMA) + result = ops.compatibility("gts.x.test._.foo.v1~", "gts.x.test._.foo.v1~") + assert result.is_fully_compatible is True + + def test_cast_success(self, ops): + ops.add_entity(SCHEMA) + ops.add_entity(INSTANCE) + result = ops.cast(INSTANCE["$id"], "gts.x.test._.foo.v1~") + assert result.error == "" + + def test_cast_error_wrapped(self, ops): + result = ops.cast("gts.x.test._.foo.v1~x.test._.missing.v1", "gts.x.test._.foo.v1~") + assert result.error != "" + + +class TestQueryAttrExtractGetEntities: + def test_query(self, ops): + ops.add_entity(SCHEMA) + ops.add_entity(INSTANCE) + result = ops.query("gts.x.test._.foo.v1~*") + assert result.count >= 1 + + def test_attr_no_path(self, ops): + result = ops.attr("gts.x.test._.foo.v1~") + assert result.error + + def test_attr_entity_not_found(self, ops): + result = ops.attr("gts.x.test._.missing.v1~@name") + assert result.error + + def test_attr_success(self, ops): + ops.add_entity(SCHEMA) + ops.add_entity(INSTANCE) + result = ops.attr(f"{INSTANCE['$id']}@name") + assert result.resolved is True + assert result.value == "hi" + + def test_extract_id_schema(self, ops): + result = ops.extract_id(SCHEMA) + assert result.is_type_schema is True + assert result.id == "gts.x.test._.foo.v1~" + d = result.to_dict() + assert d["is_type_schema"] is True + + def test_extract_id_instance(self, ops): + result = ops.extract_id(INSTANCE) + assert result.is_type_schema is False + assert result.id == INSTANCE["$id"] + + def test_get_entity_found(self, ops): + ops.add_entity(SCHEMA) + result = ops.get_entity("gts.x.test._.foo.v1~") + assert result.ok is True + d = result.to_dict() + assert d["ok"] is True + + def test_get_entity_not_found(self, ops): + result = ops.get_entity("gts.x.test._.missing.v1~") + assert result.ok is False + d = result.to_dict() + assert "error" in d + + def test_get_entities_and_list(self, ops): + ops.add_entity(SCHEMA) + ops.add_entity(INSTANCE) + result = ops.get_entities(limit=1) + assert result.count == 1 + assert result.total == 2 + d = result.to_dict() + assert len(d["entities"]) == 1 + + result2 = ops.list(limit=100) + assert result2.count == 2 diff --git a/tests/test_regressions.py b/tests/test_regressions.py new file mode 100644 index 0000000..e905f41 --- /dev/null +++ b/tests/test_regressions.py @@ -0,0 +1,138 @@ +"""Regression tests for schema validation and compatibility behavior.""" + +import pytest +from jsonschema import ValidationError + +from gts.compatibility import INCOMPATIBLE, check_backward_compatibility +from gts.entities import DEFAULT_GTS_CONFIG, GtsEntity +from gts.ops import GtsOps +from gts._server import ValidateEntityRequest +from gts.store import GtsStore +from gts.traits import build_effective_traits +from gts.x_gts_ref import XGtsRefValidator + + +def _schema_entity(gts_id, content): + return GtsEntity( + content={ + "$id": f"gts://{gts_id}", + "$schema": "https://json-schema.org/draft/2020-12/schema", + **content, + }, + cfg=DEFAULT_GTS_CONFIG, + ) + + +class TestXGtsRefCombinators: + def test_plain_one_of_is_left_to_jsonschema(self): + errors = XGtsRefValidator().validate_instance( + "value", {"oneOf": [{"type": "string"}, {"type": "integer"}]} + ) + + assert errors == [] + + def test_x_gts_ref_only_one_of_still_requires_one_match(self): + schema = { + "oneOf": [ + {"x-gts-ref": "gts.x.test._.first.v1~"}, + {"x-gts-ref": "gts.x.test._.second.v1~"}, + ] + } + + valid = "gts.x.test._.first.v1~x.test._.item.v1" + invalid = "gts.x.test._.other.v1~x.test._.item.v1" + + assert XGtsRefValidator().validate_instance(valid, schema) == [] + assert [ + error.reason + for error in XGtsRefValidator().validate_instance(invalid, schema) + ] == ["oneOf: no branch matched"] + + +class TestReferenceResolution: + def test_boolean_schema_does_not_require_reference_resolution(self): + assert GtsStore(reader=None)._resolve_schema_refs(True) is True + + def test_modern_ref_siblings_are_preserved_during_instance_validation(self): + store = GtsStore(reader=None) + target_id = "gts.x.test._.target.v1~" + source_id = "gts.x.test._.source.v1~" + instance_id = "gts.x.test._.source.v1~x.test._.item.v1" + + store.register(_schema_entity(target_id, {"type": "string"})) + store.register( + _schema_entity( + source_id, + { + "type": "object", + "properties": { + "value": { + "$ref": f"gts://{target_id}", + "minLength": 3, + } + }, + }, + ) + ) + store.register( + GtsEntity( + content={"id": instance_id, "value": "x"}, + cfg=DEFAULT_GTS_CONFIG, + ) + ) + + with pytest.raises(ValidationError): + store.validate_instance(instance_id) + + +class TestCompatibility: + def test_type_constraint_is_not_dropped_when_enum_contains_other_types(self): + result = check_backward_compatibility( + {"enum": ["x", 1]}, + {"enum": ["x", 1], "type": "string"}, + ) + + assert result == INCOMPATIBLE + + +class TestTraits: + def test_null_default_is_materialized_and_overrides_ancestor_default(self): + effective = build_effective_traits( + [ + {"properties": {"value": {"default": "ancestor"}}}, + {"properties": {"value": {"default": None}}}, + ], + {}, + None, + ) + + assert effective.values == {"value": None} + + +class TestRegistrationAndRequestValidation: + def test_failed_schema_registration_rolls_back_the_candidate(self): + ops = GtsOps() + base_id = "gts.x.test._.base.v1~" + derived_id = "gts.x.test._.base.v1~x.test._.child.v1~" + + assert ops.add_entity( + { + "$id": f"gts://{base_id}", + "$schema": "http://json-schema.org/draft-07/schema#", + "x-gts-final": True, + }, + validate=True, + ).ok + assert not ops.add_entity( + { + "$id": f"gts://{derived_id}", + "$schema": "http://json-schema.org/draft-07/schema#", + }, + validate=True, + ).ok + + assert ops.store.get(derived_id) is None + + def test_validate_entity_request_requires_an_identifier(self): + with pytest.raises(ValueError, match="entity_id"): + ValidateEntityRequest() diff --git a/tests/test_schema_cast.py b/tests/test_schema_cast.py new file mode 100644 index 0000000..09aa0c4 --- /dev/null +++ b/tests/test_schema_cast.py @@ -0,0 +1,489 @@ +"""Tests for gts.schema_cast (OP#9 version casting).""" + +from gts.schema_cast import GtsEntityCastResult, SchemaCastError + + +class TestToDict: + def test_to_dict_with_verdict_strings(self): + result = GtsEntityCastResult( + from_id="a", + to_id="b", + backward_verdict="compatible", + forward_verdict="unknown", + full_verdict="unknown", + ) + d = result.to_dict() + assert d["backward_compatibility"] == "compatible" + assert d["forward_compatibility"] == "unknown" + assert d["full_compatibility"] == "unknown" + + def test_to_dict_falls_back_to_bool_flags(self): + result = GtsEntityCastResult( + from_id="a", + to_id="b", + is_backward_compatible=True, + is_forward_compatible=False, + is_fully_compatible=False, + ) + d = result.to_dict() + assert d["backward_compatibility"] == "compatible" + assert d["forward_compatibility"] == "incompatible" + + def test_to_dict_includes_error_when_present(self): + result = GtsEntityCastResult(error="boom") + d = result.to_dict() + assert d["error"] == "boom" + + def test_to_dict_omits_error_when_absent(self): + result = GtsEntityCastResult() + d = result.to_dict() + assert "error" not in d + + +class TestInferDirection: + def test_up_direction(self): + assert ( + GtsEntityCastResult._infer_direction( + "gts.x.test._.foo.v1.0~x.test._.bar.v1.0", + "gts.x.test._.foo.v1.0~x.test._.bar.v1.5", + ) + == "up" + ) + + def test_down_direction(self): + assert ( + GtsEntityCastResult._infer_direction( + "gts.x.test._.foo.v1.0~x.test._.bar.v1.5", + "gts.x.test._.foo.v1.0~x.test._.bar.v1.0", + ) + == "down" + ) + + def test_none_direction_same_minor(self): + assert ( + GtsEntityCastResult._infer_direction( + "gts.x.test._.foo.v1.0~x.test._.bar.v1.0", + "gts.x.test._.foo.v1.0~x.test._.bar.v1.0", + ) + == "none" + ) + + def test_unknown_on_invalid_id(self): + assert GtsEntityCastResult._infer_direction("not-an-id", "also-not") == "unknown" + + def test_combined_anonymous_id_uses_versioned_segment(self): + # Regression: the appended UUID-tail segment has ver_minor=None; the + # direction must be inferred from the last versioned segment instead. + assert ( + GtsEntityCastResult._infer_direction( + "gts.x.test._.foo.v1.0~123e4567-e89b-12d3-a456-426614174000", + "gts.x.test._.foo.v1.5~", + ) + == "up" + ) + + +class TestEffectiveObjectSchema: + def test_non_dict_returns_empty(self): + assert GtsEntityCastResult._effective_object_schema("x") == {} + + def test_direct_properties_returned(self): + s = {"properties": {"a": {}}} + assert GtsEntityCastResult._effective_object_schema(s) == s + + def test_allof_branch_with_properties_found(self): + s = {"allOf": [{"title": "x"}, {"properties": {"a": {}}}]} + result = GtsEntityCastResult._effective_object_schema(s) + assert result == {"properties": {"a": {}}} + + def test_no_match_returns_schema_itself(self): + s = {"type": "string"} + assert GtsEntityCastResult._effective_object_schema(s) == s + + +class TestFlattenSchema: + def test_merges_allof(self): + schema = { + "allOf": [ + {"properties": {"a": {}}, "required": ["a"]}, + ], + "properties": {"b": {}}, + "required": ["b"], + } + flat = GtsEntityCastResult._flatten_schema(schema) + assert set(flat["properties"].keys()) == {"a", "b"} + assert set(flat["required"]) == {"a", "b"} + + def test_additional_properties_top_level_overrides(self): + schema = { + "allOf": [{"additionalProperties": False}], + "additionalProperties": True, + } + flat = GtsEntityCastResult._flatten_schema(schema) + assert flat["additionalProperties"] is True + + +class TestCastInstanceToSchema: + def test_non_dict_instance_raises(self): + try: + GtsEntityCastResult._cast_instance_to_schema("nope", {}) + assert False, "expected SchemaCastError" + except SchemaCastError: + pass + + def test_missing_required_without_default_reports_reason(self): + schema = {"properties": {"a": {"type": "string"}}, "required": ["a"]} + result, added, removed, reasons = GtsEntityCastResult._cast_instance_to_schema( + {}, schema + ) + assert "a" not in result + assert any("Missing required property" in r for r in reasons) + + def test_missing_required_with_default_added(self): + schema = { + "properties": {"a": {"type": "string", "default": "x"}}, + "required": ["a"], + } + result, added, removed, reasons = GtsEntityCastResult._cast_instance_to_schema( + {}, schema + ) + assert result["a"] == "x" + assert "a" in added + + def test_optional_default_added_when_missing(self): + schema = {"properties": {"b": {"default": 5}}} + result, added, removed, reasons = GtsEntityCastResult._cast_instance_to_schema( + {}, schema + ) + assert result["b"] == 5 + assert "b" in added + + def test_const_gts_id_updated(self): + schema = { + "properties": { + "type": {"const": "gts.x.test._.foo.v2~"}, + } + } + instance = {"type": "gts.x.test._.foo.v1~"} + result, added, removed, reasons = GtsEntityCastResult._cast_instance_to_schema( + instance, schema + ) + assert result["type"] == "gts.x.test._.foo.v2~" + + def test_additional_properties_false_removes_extra(self): + schema = { + "properties": {"a": {"type": "string"}}, + "additionalProperties": False, + } + instance = {"a": "x", "extra": "y"} + result, added, removed, reasons = GtsEntityCastResult._cast_instance_to_schema( + instance, schema + ) + assert "extra" not in result + assert "extra" in removed + + def test_nested_object_recursion(self): + schema = { + "properties": { + "child": { + "type": "object", + "properties": {"x": {"type": "string", "default": "d"}}, + } + } + } + instance = {"child": {}} + result, added, removed, reasons = GtsEntityCastResult._cast_instance_to_schema( + instance, schema + ) + assert result["child"]["x"] == "d" + assert "child.x" in added + + def test_nested_array_of_objects_recursion(self): + schema = { + "properties": { + "items": { + "type": "array", + "items": { + "type": "object", + "properties": {"x": {"default": "d"}}, + }, + } + } + } + instance = {"items": [{}]} + result, added, removed, reasons = GtsEntityCastResult._cast_instance_to_schema( + instance, schema + ) + assert result["items"][0]["x"] == "d" + assert "items[0].x" in added + + +class TestRemoveGtsConstConstraints: + def test_replaces_const_gts_id_with_type_string(self): + schema = {"const": "gts.x.test._.foo.v1~"} + result = GtsEntityCastResult._remove_gts_const_constraints(schema) + assert result == {"type": "string"} + + def test_non_gts_const_left_intact(self): + schema = {"const": "plainvalue"} + result = GtsEntityCastResult._remove_gts_const_constraints(schema) + assert result == {"const": "plainvalue"} + + def test_recurses_into_nested_dict_and_list(self): + schema = { + "properties": {"a": {"const": "gts.x.test._.foo.v1~"}}, + "allOf": [{"const": "gts.x.test._.bar.v1~"}], + } + result = GtsEntityCastResult._remove_gts_const_constraints(schema) + assert result["properties"]["a"] == {"type": "string"} + assert result["allOf"][0] == {"type": "string"} + + def test_non_dict_passthrough(self): + assert GtsEntityCastResult._remove_gts_const_constraints("abc") == "abc" + + +class TestCheckMinMaxConstraint: + def test_backward_tightened_minimum_flagged(self): + errors = GtsEntityCastResult._check_min_max_constraint( + "p", {"minimum": 1}, {"minimum": 5}, "minimum", "maximum", True + ) + assert any("increased" in e for e in errors) + + def test_backward_added_minimum_flagged(self): + errors = GtsEntityCastResult._check_min_max_constraint( + "p", {}, {"minimum": 5}, "minimum", "maximum", True + ) + assert any("added" in e for e in errors) + + def test_forward_relaxed_minimum_flagged(self): + errors = GtsEntityCastResult._check_min_max_constraint( + "p", {"minimum": 5}, {"minimum": 1}, "minimum", "maximum", False + ) + assert any("decreased" in e for e in errors) + + def test_forward_removed_minimum_flagged(self): + errors = GtsEntityCastResult._check_min_max_constraint( + "p", {"minimum": 5}, {}, "minimum", "maximum", False + ) + assert any("removed" in e for e in errors) + + def test_backward_tightened_maximum_flagged(self): + errors = GtsEntityCastResult._check_min_max_constraint( + "p", {"maximum": 10}, {"maximum": 5}, "minimum", "maximum", True + ) + assert any("decreased" in e for e in errors) + + def test_forward_relaxed_maximum_flagged(self): + errors = GtsEntityCastResult._check_min_max_constraint( + "p", {"maximum": 5}, {"maximum": 10}, "minimum", "maximum", False + ) + assert any("increased" in e for e in errors) + + +class TestCheckConstraintCompatibility: + def test_numeric_constraints_checked(self): + errors = GtsEntityCastResult._check_constraint_compatibility( + "p", {"type": "number", "minimum": 1}, {"type": "number", "minimum": 5} + ) + assert errors + + def test_string_constraints_checked(self): + errors = GtsEntityCastResult._check_constraint_compatibility( + "p", {"type": "string", "minLength": 1}, {"type": "string", "minLength": 5} + ) + assert errors + + def test_array_constraints_checked(self): + errors = GtsEntityCastResult._check_constraint_compatibility( + "p", {"type": "array", "minItems": 1}, {"type": "array", "minItems": 5} + ) + assert errors + + def test_other_types_no_errors(self): + errors = GtsEntityCastResult._check_constraint_compatibility( + "p", {"type": "boolean"}, {"type": "boolean"} + ) + assert errors == [] + + +class TestCheckSchemaCompatibility: + def test_backward_added_required_flagged(self): + old = {"properties": {"a": {}}} + new = {"properties": {"a": {}}, "required": ["a"]} + ok, errors = GtsEntityCastResult._check_backward_compatibility(old, new) + assert not ok + assert any("Added required" in e for e in errors) + + def test_forward_removed_required_flagged(self): + old = {"properties": {"a": {}}, "required": ["a"]} + new = {"properties": {"a": {}}} + ok, errors = GtsEntityCastResult._check_forward_compatibility(old, new) + assert not ok + assert any("Removed required" in e for e in errors) + + def test_type_change_flagged(self): + old = {"properties": {"a": {"type": "string"}}} + new = {"properties": {"a": {"type": "integer"}}} + ok, errors = GtsEntityCastResult._check_backward_compatibility(old, new) + assert not ok + assert any("type changed" in e for e in errors) + + def test_backward_added_enum_values_flagged(self): + old = {"properties": {"a": {"enum": ["x"]}}} + new = {"properties": {"a": {"enum": ["x", "y"]}}} + ok, errors = GtsEntityCastResult._check_backward_compatibility(old, new) + assert not ok + assert any("added enum values" in e for e in errors) + + def test_forward_removed_enum_values_flagged(self): + old = {"properties": {"a": {"enum": ["x", "y"]}}} + new = {"properties": {"a": {"enum": ["x"]}}} + ok, errors = GtsEntityCastResult._check_forward_compatibility(old, new) + assert not ok + assert any("removed enum values" in e for e in errors) + + def test_nested_object_errors_prefixed(self): + old = {"properties": {"a": {"type": "object", "properties": {"b": {"type": "string"}}}}} + new = {"properties": {"a": {"type": "object", "properties": {"b": {"type": "integer"}}}}} + ok, errors = GtsEntityCastResult._check_backward_compatibility(old, new) + assert not ok + assert any("Property 'a':" in e for e in errors) + + def test_fully_compatible_returns_true(self): + old = {"properties": {"a": {"type": "string"}}} + new = {"properties": {"a": {"type": "string"}}, "properties2": {}} + ok, errors = GtsEntityCastResult._check_backward_compatibility(old, {"properties": {"a": {"type": "string"}}}) + assert ok + assert errors == [] + + +class TestDiffObjects: + def test_added_and_removed_properties(self): + added, removed, changed = [], [], [] + GtsEntityCastResult._diff_objects( + {"properties": {"a": {}}}, + {"properties": {"b": {}}}, + "", + added, + removed, + changed, + ) + assert removed == ["a"] + assert added == ["b"] + + def test_type_and_format_changes(self): + added, removed, changed = [], [], [] + GtsEntityCastResult._diff_objects( + {"properties": {"a": {"type": "string", "format": "date"}}}, + {"properties": {"a": {"type": "integer", "format": "int32"}}}, + "", + added, + removed, + changed, + ) + change_strs = [c["change"] for c in changed] + assert any("type:" in c for c in change_strs) + assert any("format:" in c for c in change_strs) + + def test_required_added_and_removed(self): + added, removed, changed = [], [], [] + GtsEntityCastResult._diff_objects( + {"required": ["a"]}, + {"required": ["b"]}, + "", + added, + removed, + changed, + ) + change_strs = {(c["path"], c["change"]) for c in changed} + assert ("a", "required: removed") in change_strs + assert ("b", "required: added") in change_strs + + +class TestOnlyOptionalAddRemove: + def test_identical_schemas_true(self): + assert GtsEntityCastResult._only_optional_add_remove( + {"type": "string"}, {"type": "string"}, "", [] + ) + + def test_value_mismatch_for_non_dicts(self): + reasons = [] + result = GtsEntityCastResult._only_optional_add_remove(1, 2, "path", reasons) + assert not result + assert any("value changed" in r for r in reasons) + + def test_keyword_change_detected(self): + reasons = [] + result = GtsEntityCastResult._only_optional_add_remove( + {"type": "string"}, {"type": "integer"}, "path", reasons + ) + assert not result + assert any("keyword 'type' changed" in r for r in reasons) + + def test_required_added_and_removed_detected(self): + reasons = [] + result = GtsEntityCastResult._only_optional_add_remove( + {"required": ["a"]}, {"required": ["b"]}, "path", reasons + ) + assert not result + assert any("required added" in r for r in reasons) + assert any("required removed" in r for r in reasons) + + def test_nested_property_recursion(self): + reasons = [] + a = {"properties": {"x": {"type": "string"}}} + b = {"properties": {"x": {"type": "integer"}}} + result = GtsEntityCastResult._only_optional_add_remove(a, b, "", reasons) + assert not result + assert any("properties.x" in r for r in reasons) + + +class TestCastClassmethod: + def test_cast_backward_incompatible_and_validation_error(self): + from_schema = { + "type": "object", + "properties": {"a": {"type": "string"}}, + } + to_schema = { + "type": "object", + "properties": {"a": {"type": "string"}, "b": {"type": "string"}}, + "required": ["b"], + } + result = GtsEntityCastResult.cast( + "gts.x.test._.foo.v1~a.b._.c.v1", + "gts.x.test._.foo.v2~", + {"a": "x"}, + from_schema, + to_schema, + ) + assert result.is_fully_compatible is False + assert result.incompatibility_reasons + + def test_cast_fully_compatible(self): + from_schema = { + "type": "object", + "properties": {"a": {"type": "string"}}, + } + to_schema = { + "type": "object", + "properties": {"a": {"type": "string"}}, + } + result = GtsEntityCastResult.cast( + "gts.x.test._.foo.v1~a.b._.c.v1", + "gts.x.test._.foo.v1~", + {"a": "x"}, + from_schema, + to_schema, + ) + assert result.is_fully_compatible is True + assert result.casted_entity == {"a": "x"} + + def test_cast_with_non_dict_instance_content_defaults_to_empty(self): + result = GtsEntityCastResult.cast( + "gts.x.test._.foo.v1.0~x.test._.bar.v1.0", + "gts.x.test._.foo.v1.0~", + "not-a-dict", + {}, + {}, + ) + assert result.casted_entity == {} diff --git a/tests/test_server.py b/tests/test_server.py new file mode 100644 index 0000000..da5e1c0 --- /dev/null +++ b/tests/test_server.py @@ -0,0 +1,196 @@ +"""Tests for gts._server (FastAPI route handlers), called directly as async +coroutines via asyncio.run to avoid pulling in an HTTP test client dependency. +""" + +import asyncio + +import pytest + +from gts.ops import GtsOps +from gts._server import GtsHttpServer, ValidateEntityRequest, _RequestLoggingMiddleware + + +SCHEMA = { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": "gts.x.test._.foo.v1~", + "type": "object", + "properties": {"name": {"type": "string"}}, +} + +INSTANCE = { + "$id": "gts.x.test._.foo.v1~x.test._.inst.v1", + "type": "gts.x.test._.foo.v1~", + "name": "hi", +} + + +def run(coro): + return asyncio.run(coro) + + +@pytest.fixture +def server(): + ops = GtsOps(path=None) + return GtsHttpServer(ops=ops) + + +class TestServerConstruction: + def test_app_and_routes_registered(self, server): + assert server.app is not None + assert server.base_url == "http://127.0.0.1:8000" + paths = {route.path for route in server.app.routes} + assert "/entities" in paths + assert "/query" in paths + + +class TestValidateEntityRequestModel: + def test_requires_id(self): + with pytest.raises(Exception): + ValidateEntityRequest() + + def test_mismatched_ids_raise(self): + with pytest.raises(Exception): + ValidateEntityRequest(entity_id="a", gts_id="b") + + def test_entity_id_used(self): + req = ValidateEntityRequest(entity_id="gts.x.test._.foo.v1~") + assert req.resolved_id == "gts.x.test._.foo.v1~" + + def test_gts_id_used_when_entity_id_absent(self): + req = ValidateEntityRequest(gts_id="gts.x.test._.foo.v1~") + assert req.resolved_id == "gts.x.test._.foo.v1~" + + def test_matching_ids_ok(self): + req = ValidateEntityRequest(entity_id="a", gts_id="a") + assert req.resolved_id == "a" + + +class TestHandlers: + def test_add_entity_success(self, server): + resp = run(server.add_entity(body=SCHEMA, validate=False)) + assert resp.status_code == 200 + + def test_add_entity_failure(self, server): + resp = run(server.add_entity(body={"no": "id"}, validate=False)) + assert resp.status_code == 422 + + def test_add_entities(self, server): + resp = run(server.add_entities(body=[SCHEMA, INSTANCE])) + assert resp.status_code == 200 + + def test_add_schema(self, server): + from gts._server import SchemaRegister + + body = SchemaRegister(type_id="gts.x.test._.bar.v1~", type_schema={"type": "object"}) + resp = run(server.add_schema(body)) + assert resp.status_code == 200 + + def test_validate_id(self, server): + result = run(server.validate_id(id="gts.x.test._.foo.v1~")) + assert result["valid"] is True + + def test_extract_id(self, server): + result = run(server.extract_id(body=SCHEMA)) + assert result["is_type_schema"] is True + + def test_parse(self, server): + result = run(server.parse(id="gts.x.test._.foo.v1~")) + assert result["ok"] is True + + def test_match_id_pattern(self, server): + result = run( + server.match_id_pattern(candidate="gts.x.test._.foo.v1~", pattern="gts.x.test.*") + ) + assert result["match"] is True + + def test_id_to_uuid(self, server): + result = run(server.id_to_uuid(id="gts.x.test._.foo.v1~")) + assert "uuid" in result + + def test_validate_instance(self, server): + from gts._server import ValidateInstanceRequest + + run(server.add_entity(body=SCHEMA, validate=False)) + run(server.add_entity(body=INSTANCE, validate=False)) + result = run( + server.validate_instance(ValidateInstanceRequest(instance_id=INSTANCE["$id"])) + ) + assert result["ok"] is True + + def test_validate_type_schema(self, server): + from gts._server import ValidateTypeSchemaRequest + + run(server.add_entity(body=SCHEMA, validate=False)) + result = run( + server.validate_type_schema( + ValidateTypeSchemaRequest(type_id="gts.x.test._.foo.v1~") + ) + ) + assert result["ok"] is True + + def test_validate_entity(self, server): + run(server.add_entity(body=SCHEMA, validate=False)) + result = run( + server.validate_entity(ValidateEntityRequest(entity_id="gts.x.test._.foo.v1~")) + ) + assert result["ok"] is True + + def test_schema_graph(self, server): + run(server.add_entity(body=SCHEMA, validate=False)) + result = run(server.schema_graph(id="gts.x.test._.foo.v1~")) + assert result["id"] == "gts.x.test._.foo.v1~" + + def test_compatibility(self, server): + run(server.add_entity(body=SCHEMA, validate=False)) + result = run( + server.compatibility( + old="gts.x.test._.foo.v1~", new="gts.x.test._.foo.v1~" + ) + ) + assert result["is_fully_compatible"] is True + + def test_cast(self, server): + from gts._server import CastRequest + + run(server.add_entity(body=SCHEMA, validate=False)) + run(server.add_entity(body=INSTANCE, validate=False)) + result = run( + server.cast( + CastRequest(instance_id=INSTANCE["$id"], to_type_id="gts.x.test._.foo.v1~") + ) + ) + assert "error" not in result + + def test_query(self, server): + run(server.add_entity(body=SCHEMA, validate=False)) + run(server.add_entity(body=INSTANCE, validate=False)) + result = run(server.query(expr="gts.x.test._.foo.v1~*", limit=10)) + assert result["count"] >= 1 + + def test_attr(self, server): + run(server.add_entity(body=SCHEMA, validate=False)) + run(server.add_entity(body=INSTANCE, validate=False)) + result = run(server.attr(gts_with_path=f"{INSTANCE['$id']}@name")) + assert result["value"] == "hi" + + def test_get_entity(self, server): + run(server.add_entity(body=SCHEMA, validate=False)) + result = run(server.get_entity(gts_id="gts.x.test._.foo.v1~")) + assert result["ok"] is True + + def test_get_entities(self, server): + run(server.add_entity(body=SCHEMA, validate=False)) + run(server.add_entity(body=INSTANCE, validate=False)) + result = run(server.get_entities(limit=10)) + assert result["total"] == 2 + + +class TestRequestLoggingMiddlewareVerboseOff: + def test_dispatch_skips_when_not_verbose(self, server): + middleware = _RequestLoggingMiddleware(server.app, verbose=0) + + async def call_next(request): + return "response-sentinel" + + result = run(middleware.dispatch(request=None, call_next=call_next)) + assert result == "response-sentinel" diff --git a/tests/test_store_extra.py b/tests/test_store_extra.py new file mode 100644 index 0000000..4b42ec2 --- /dev/null +++ b/tests/test_store_extra.py @@ -0,0 +1,451 @@ +"""Additional coverage-focused tests for gts.store.GtsStore.""" + +import pytest +from typing import Iterator, Optional + +from gts.store import GtsStore, GtsReader, StoreGtsEntityNotFound, StoreGtsObjectNotFound +from gts.entities import GtsEntity, DEFAULT_GTS_CONFIG +from gts.gts import GtsID + + +class MockGtsReader(GtsReader): + def __init__(self, entities, extra_by_id=None): + self._entities = entities + self._extra_by_id = extra_by_id or {} + self._index = 0 + + def __iter__(self) -> Iterator[GtsEntity]: + self._index = 0 + return self + + def __next__(self) -> GtsEntity: + if self._index >= len(self._entities): + raise StopIteration + entity = self._entities[self._index] + self._index += 1 + return entity + + def read_by_id(self, entity_id: str) -> Optional[GtsEntity]: + for entity in self._entities: + if entity.gts_id and entity.gts_id.id == entity_id: + return entity + return self._extra_by_id.get(entity_id) + + def reset(self) -> None: + self._index = 0 + + +def _schema_entity(gts_id: str, content_extra=None): + content = { + "$schema": "http://json-schema.org/draft-07/schema#", + "$id": gts_id, + "type": "object", + "properties": {"name": {"type": "string"}}, + } + if content_extra: + content.update(content_extra) + return GtsEntity(content=content, gts_id=GtsID(gts_id), is_schema=True) + + +class TestRegisterEdgeCases: + def test_register_raises_without_id(self): + store = GtsStore(reader=None) + entity = GtsEntity(content={"a": 1}) + with pytest.raises(ValueError): + store.register(entity) + + def test_get_falls_back_to_reader_not_in_initial_iter(self): + schema = _schema_entity("gts.x.test._.foo.v1~") + extra = _schema_entity("gts.x.test._.bar.v1~") + reader = MockGtsReader([schema], extra_by_id={"gts.x.test._.bar.v1~": extra}) + store = GtsStore(reader) + result = store.get("gts.x.test._.bar.v1~") + assert result is not None + assert result.content["$id"] == "gts.x.test._.bar.v1~" + + def test_get_returns_none_when_reader_absent_and_missing(self): + store = GtsStore(reader=None) + assert store.get("gts.x.test._.missing.v1~") is None + + def test_unregister_missing_id_noop(self): + store = GtsStore(reader=None) + store.unregister("gts.x.test._.missing.v1~") # should not raise + + +class TestValidateSchemaRefs: + def test_local_ref_valid(self): + GtsStore._validate_schema_refs({"$ref": "#/defs/foo"}) + + def test_gts_ref_valid(self): + GtsStore._validate_schema_refs({"$ref": "gts://gts.x.test._.foo.v1~"}) + + def test_gts_ref_invalid_id_raises(self): + with pytest.raises(ValueError, match="invalid GTS identifier"): + GtsStore._validate_schema_refs({"$ref": "gts://not a valid id"}) + + def test_other_ref_raises(self): + with pytest.raises(ValueError, match="must be a local ref"): + GtsStore._validate_schema_refs({"$ref": "http://example.com/schema"}) + + def test_recurses_into_list(self): + with pytest.raises(ValueError): + GtsStore._validate_schema_refs( + {"allOf": [{"$ref": "http://example.com/schema"}]} + ) + + +class TestValidateGtsKeywords: + def test_final_must_be_bool(self): + with pytest.raises(ValueError, match="x-gts-final must be a boolean"): + GtsStore._validate_gts_keywords({"x-gts-final": "yes"}) + + def test_abstract_must_be_bool(self): + with pytest.raises(ValueError, match="x-gts-abstract must be a boolean"): + GtsStore._validate_gts_keywords({"x-gts-abstract": "yes"}) + + def test_mutual_exclusion(self): + with pytest.raises(ValueError, match="cannot declare both"): + GtsStore._validate_gts_keywords( + {"x-gts-final": True, "x-gts-abstract": True} + ) + + def test_nested_keyword_placement_raises(self): + with pytest.raises(ValueError, match="must be at the schema top level"): + GtsStore._validate_gts_keywords( + {"properties": {"a": {"x-gts-final": True}}} + ) + + def test_valid_top_level_keywords_pass(self): + GtsStore._validate_gts_keywords({"x-gts-final": True}) + GtsStore._validate_gts_keywords({"x-gts-abstract": True}) + + def test_content_is_abstract_and_final(self): + assert GtsStore._content_is_abstract({"x-gts-abstract": True}) is True + assert GtsStore._content_is_abstract({}) is False + assert GtsStore._content_is_final({"x-gts-final": True}) is True + assert GtsStore._content_is_final({}) is False + + +class TestValidateSchemaXGtsRefs: + def test_non_schema_id_raises(self): + store = GtsStore(reader=None) + with pytest.raises(ValueError, match="not a schema"): + store._validate_schema_x_gts_refs("gts.x.test._.foo.v1") + + def test_missing_schema_raises(self): + store = GtsStore(reader=None) + from gts.store import StoreGtsSchemaNotFound + + with pytest.raises(StoreGtsSchemaNotFound): + store._validate_schema_x_gts_refs("gts.x.test._.missing.v1~") + + def test_entity_not_schema_raises(self): + entity = GtsEntity( + content={"a": 1}, gts_id=GtsID("gts.x.test._.foo.v1~"), is_schema=False + ) + store = GtsStore(reader=None) + store.register(entity) + with pytest.raises(ValueError, match="is not a schema"): + store._validate_schema_x_gts_refs("gts.x.test._.foo.v1~") + + def test_invalid_x_gts_ref_raises(self): + schema = _schema_entity( + "gts.x.test._.foo.v1~", {"x-gts-ref": "notgts.*"} + ) + store = GtsStore(reader=None) + store.register(schema) + with pytest.raises(Exception, match="x-gts-ref validation failed"): + store._validate_schema_x_gts_refs("gts.x.test._.foo.v1~") + + +class TestValidateSchemaChain: + def test_single_segment_no_parent_ok(self): + store = GtsStore(reader=None) + store._validate_schema_chain("gts.x.test._.foo.v1~") + + def test_final_base_blocks_derivation(self): + base = _schema_entity("gts.x.test._.base.v1~", {"x-gts-final": True}) + derived = _schema_entity("gts.x.test._.base.v1~x.test._.derived.v1~") + store = GtsStore(reader=None) + store.register(base) + store.register(derived) + with pytest.raises(ValueError, match="is final"): + store._validate_schema_chain("gts.x.test._.base.v1~x.test._.derived.v1~") + + def test_missing_base_schema_raises(self): + derived = _schema_entity("gts.x.test._.base.v1~x.test._.derived.v1~") + store = GtsStore(reader=None) + store.register(derived) + with pytest.raises(ValueError, match="not found for chain validation"): + store._validate_schema_chain("gts.x.test._.base.v1~x.test._.derived.v1~") + + def test_incompatible_derivation_raises(self): + base = _schema_entity( + "gts.x.test._.base.v1~", + {"properties": {"name": {"type": "string"}, "a": {"type": "string"}}}, + ) + derived = _schema_entity( + "gts.x.test._.base.v1~x.test._.derived.v1~", + {"properties": {"a": {"type": "integer"}}}, + ) + store = GtsStore(reader=None) + store.register(base) + store.register(derived) + with pytest.raises(ValueError, match="is not compatible with base"): + store._validate_schema_chain("gts.x.test._.base.v1~x.test._.derived.v1~") + + +class TestResolveSchemaRefsAndInline: + def test_resolves_gts_ref(self): + target = _schema_entity("gts.x.test._.target.v1~") + store = GtsStore(reader=None) + store.register(target) + schema = {"$ref": "gts://gts.x.test._.target.v1~"} + resolved = store._resolve_schema_refs(schema) + assert resolved["type"] == "object" + + def test_unresolvable_ref_left_unresolved(self): + store = GtsStore(reader=None) + schema = {"$ref": "gts://gts.x.test._.missing.v1~"} + resolved = store._resolve_schema_refs(schema) + assert resolved == schema + + def test_cyclic_ref_left_unresolved(self): + a = _schema_entity( + "gts.x.test._.a.v1~", {"$ref": "gts://gts.x.test._.b.v1~"} + ) + b = _schema_entity( + "gts.x.test._.b.v1~", {"$ref": "gts://gts.x.test._.a.v1~"} + ) + store = GtsStore(reader=None) + store.register(a) + store.register(b) + resolved = store._resolve_schema_refs(a.content) + assert "$ref" in str(resolved) + + def test_ref_with_siblings_creates_allof(self): + target = _schema_entity("gts.x.test._.target.v1~") + store = GtsStore(reader=None) + store.register(target) + schema = { + "$schema": "https://json-schema.org/draft/2020-12/schema", + "$ref": "gts://gts.x.test._.target.v1~", + "title": "sibling", + } + resolved = store._resolve_schema_refs(schema) + assert "allOf" in resolved + + def test_supports_ref_siblings_false_for_non_dict(self): + assert GtsStore._supports_ref_siblings("nope") is False + + def test_inline_refs_list_recursion(self): + store = GtsStore(reader=None) + node = [{"a": 1}, {"b": 2}] + result = store._inline_refs(node, set(), False) + assert result == node + + def test_inline_refs_scalar_passthrough(self): + store = GtsStore(reader=None) + assert store._inline_refs(5, set(), False) == 5 + + +class TestCastAndCompatibility: + def _build_store(self): + old_schema = _schema_entity("gts.x.test._.foo.v1.0~") + new_schema = _schema_entity( + "gts.x.test._.foo.v1.5~", + {"properties": {"name": {"type": "string"}, "extra": {"type": "string", "default": "d"}}}, + ) + instance = GtsEntity( + content={ + "$id": "gts.x.test._.foo.v1.0~x.test._.inst.v1.0", + "gtsType": "gts.x.test._.foo.v1.0~", + "name": "hi", + }, + cfg=DEFAULT_GTS_CONFIG, + ) + store = GtsStore(reader=None) + store.register(old_schema) + store.register(new_schema) + store.register(instance) + return store, old_schema, new_schema, instance + + def test_cast_success(self): + store, old_schema, new_schema, instance = self._build_store() + result = store.cast(instance.raw_id, "gts.x.test._.foo.v1.5~") + assert result.casted_entity is not None + assert result.casted_entity["extra"] == "d" + + def test_cast_from_missing_entity_raises(self): + store, *_ = self._build_store() + with pytest.raises(StoreGtsEntityNotFound): + store.cast("gts.x.test._.foo.v1.0~x.test._.missing.v1.0", "gts.x.test._.foo.v1.5~") + + def test_cast_from_schema_raises(self): + store, old_schema, new_schema, instance = self._build_store() + from gts.store import StoreGtsCastFromSchemaNotAllowed + + with pytest.raises(StoreGtsCastFromSchemaNotAllowed): + store.cast(old_schema.gts_id.id, new_schema.gts_id.id) + + def test_cast_to_missing_schema_raises(self): + store, old_schema, new_schema, instance = self._build_store() + with pytest.raises(StoreGtsObjectNotFound): + store.cast(instance.raw_id, "gts.x.test._.missing.v1~") + + def test_is_minor_compatible_missing_entity(self): + store = GtsStore(reader=None) + result = store.is_minor_compatible("gts.x.test._.a.v1~", "gts.x.test._.b.v1~") + assert result.is_fully_compatible is False + assert "Schema not found" in result.incompatibility_reasons + + def test_is_minor_compatible_valid(self): + store, old_schema, new_schema, instance = self._build_store() + result = store.is_minor_compatible(old_schema.gts_id.id, new_schema.gts_id.id) + assert result.backward_verdict is not None + + +class TestBuildSchemaGraphWithRefs: + def test_graph_includes_refs_and_type_id(self): + schema = _schema_entity("gts.x.test._.foo.v1~") + instance = GtsEntity( + content={ + "$id": "gts.x.test._.foo.v1~x.test._.inst.v1", + "gtsType": "gts.x.test._.foo.v1~", + "name": "hi", + }, + cfg=DEFAULT_GTS_CONFIG, + ) + store = GtsStore(reader=None) + store.register(schema) + store.register(instance) + graph = store.build_schema_graph(instance.gts_id.id) + assert graph["id"] == instance.gts_id.id + assert "type_id" in graph + + def test_graph_skips_json_schema_org_refs(self): + schema = _schema_entity("gts.x.test._.foo.v1~") + store = GtsStore(reader=None) + store.register(schema) + graph = store.build_schema_graph("gts.x.test._.foo.v1~") + assert "refs" not in graph or "http://json-schema.org" not in str(graph) + + +class TestQueryEdgeCases: + def test_query_filter_wildcard_value(self): + entity = GtsEntity( + content={ + "$id": "gts.x.test._.foo.v1~x.test._.a.v1", + "status": "active", + }, + cfg=DEFAULT_GTS_CONFIG, + ) + store = GtsStore(reader=None) + store.register(entity) + result = store.query("gts.x.test._.foo.*[status=*]") + assert result.count == 1 + + def test_query_filter_wildcard_value_excludes_empty(self): + entity = GtsEntity( + content={ + "$id": "gts.x.test._.foo.v1~x.test._.a.v1", + }, + cfg=DEFAULT_GTS_CONFIG, + ) + store = GtsStore(reader=None) + store.register(entity) + result = store.query("gts.x.test._.foo.*[status=*]") + assert result.count == 0 + + def test_parse_query_filters_empty_string(self): + store = GtsStore(reader=None) + assert store._parse_query_filters("") == {} + + def test_query_result_to_dict_error(self): + from gts.store import GtsStoreQueryResult + + r = GtsStoreQueryResult() + r.error = "bad" + r.count = 0 + r.limit = 10 + assert r.to_dict() == {"error": "bad", "count": 0, "limit": 10} + + def test_query_result_to_dict_ok(self): + from gts.store import GtsStoreQueryResult + + r = GtsStoreQueryResult() + r.results = [{"a": 1}] + r.count = 1 + d = r.to_dict() + assert d["results"] == [{"a": 1}] + + +class TestValidateSchemaFullFlow: + def test_meta_schema_url_rejects_gts_id(self): + schema = _schema_entity( + "gts.x.test._.foo.v1~", {"$schema": "gts.x.other.v1~"} + ) + store = GtsStore(reader=None) + store.register(schema) + with pytest.raises(ValueError, match="must be a standard JSON Schema URL"): + store.validate_schema("gts.x.test._.foo.v1~") + + def test_validate_schema_basic_success(self): + schema = _schema_entity("gts.x.test._.foo.v1~") + store = GtsStore(reader=None) + store.register(schema) + store.validate_schema_basic("gts.x.test._.foo.v1~") + + def test_validate_schema_with_traits_error(self): + schema = _schema_entity( + "gts.x.test._.foo.v1~", + { + "x-gts-traits-schema": { + "type": "object", + "properties": {"a": {"type": "string"}}, + "required": ["a"], + } + }, + ) + store = GtsStore(reader=None) + store.register(schema) + with pytest.raises(ValueError, match="trait validation failed"): + store.validate_schema("gts.x.test._.foo.v1~") + + def test_validate_instance_abstract_type_rejected(self): + schema = _schema_entity( + "gts.x.test._.foo.v1~", {"x-gts-abstract": True} + ) + instance = GtsEntity( + content={ + "$id": "gts.x.test._.foo.v1~x.test._.inst.v1", + "gtsType": "gts.x.test._.foo.v1~", + "name": "hi", + }, + cfg=DEFAULT_GTS_CONFIG, + ) + store = GtsStore(reader=None) + store.register(schema) + store.register(instance) + with pytest.raises(ValueError, match="is abstract"): + store.validate_instance(instance.gts_id.id) + + def test_validate_instance_by_uuid(self): + schema = _schema_entity("gts.x.test._.foo.v1~") + store = GtsStore(reader=None) + store.register(schema) + entity = GtsEntity( + content={ + "type": "gts.x.test._.foo.v1~", + "id": "12345678-1234-5678-1234-567812345678", + "name": "hi", + }, + cfg=DEFAULT_GTS_CONFIG, + ) + store.register(entity) + store.validate_instance(entity.raw_id) + + def test_validate_instance_invalid_non_uuid_non_gts_raises(self): + store = GtsStore(reader=None) + with pytest.raises(StoreGtsObjectNotFound): + store.validate_instance("totally-not-valid") diff --git a/tests/test_traits.py b/tests/test_traits.py new file mode 100644 index 0000000..576389f --- /dev/null +++ b/tests/test_traits.py @@ -0,0 +1,215 @@ +"""Tests for gts.traits (OP#13 schema traits validation).""" + +from gts.traits import ( + build_effective_traits, + build_effective_traits_schema, + collect_trait_schema_from_value, + collect_traits_from_value, + inline_local_pointers, + merge_rfc7396_into, +) + + +class TestCollection: + def test_collect_trait_schema_from_value_direct(self): + out = [] + collect_trait_schema_from_value( + {"x-gts-traits-schema": {"type": "object"}}, out + ) + assert out == [{"type": "object"}] + + def test_collect_trait_schema_from_value_via_allof(self): + out = [] + collect_trait_schema_from_value( + {"allOf": [{"x-gts-traits-schema": {"type": "object"}}]}, out + ) + assert out == [{"type": "object"}] + + def test_collect_trait_schema_from_value_ignores_non_dict(self): + out = [] + collect_trait_schema_from_value("not-a-dict", out) + assert out == [] + + def test_collect_traits_from_value_merges_allof(self): + merged = {} + collect_traits_from_value( + { + "x-gts-traits": {"a": 1}, + "allOf": [{"x-gts-traits": {"b": 2}}], + }, + merged, + ) + assert merged == {"b": 2, "a": 1} + + +class TestInlineLocalPointers: + def test_resolves_local_pointer(self): + root = {"defs": {"foo": {"type": "string"}}} + fragment = {"$ref": "#/defs/foo"} + result = inline_local_pointers(fragment, root) + assert result == {"type": "string"} + + def test_unresolved_pointer_returns_fragment(self): + root = {} + fragment = {"$ref": "#/missing"} + result = inline_local_pointers(fragment, root) + assert result == fragment + + def test_sibling_keys_merged_with_resolved_ref(self): + root = {"defs": {"foo": {"type": "string"}}} + fragment = {"$ref": "#/defs/foo", "minLength": 2} + result = inline_local_pointers(fragment, root) + assert result == {"type": "string", "minLength": 2} + + def test_non_ref_dict_recurses(self): + root = {} + fragment = {"a": {"b": 1}} + result = inline_local_pointers(fragment, root) + assert result == {"a": {"b": 1}} + + def test_list_recursion(self): + root = {} + fragment = [{"a": 1}, {"$ref": "#/missing"}] + result = inline_local_pointers(fragment, root) + assert result == [{"a": 1}, {"$ref": "#/missing"}] + + def test_array_index_pointer(self): + root = {"items": [{"type": "string"}, {"type": "integer"}]} + fragment = {"$ref": "#/items/1"} + result = inline_local_pointers(fragment, root) + assert result == {"type": "integer"} + + def test_invalid_array_index_returns_none(self): + root = {"items": [{"type": "string"}]} + fragment = {"$ref": "#/items/notanumber"} + result = inline_local_pointers(fragment, root) + # unresolved, fragment unchanged + assert result == fragment + + +class TestMergeRfc7396: + def test_merge_overwrites_scalar(self): + target = {"a": 1} + merge_rfc7396_into(target, {"a": 2}) + assert target == {"a": 2} + + def test_merge_null_removes_key(self): + target = {"a": 1, "b": 2} + merge_rfc7396_into(target, {"a": None}) + assert target == {"b": 2} + + def test_merge_nested_dict(self): + target = {"a": {"x": 1}} + merge_rfc7396_into(target, {"a": {"y": 2}}) + assert target == {"a": {"x": 1, "y": 2}} + + def test_merge_replaces_non_dict_with_dict(self): + target = {"a": 1} + merge_rfc7396_into(target, {"a": {"y": 2}}) + assert target == {"a": {"y": 2}} + + +class TestBuildEffectiveTraitsSchema: + def test_empty_list_returns_empty_dict(self): + assert build_effective_traits_schema([]) == {} + + def test_single_schema_returned_as_is(self): + schema = {"type": "object"} + assert build_effective_traits_schema([schema]) == schema + + def test_multiple_schemas_composed_via_allof(self): + result = build_effective_traits_schema([{"a": 1}, {"b": 2}]) + assert result["type"] == "object" + assert result["allOf"] == [{"a": 1}, {"b": 2}] + + +class TestBuildEffectiveTraits: + def test_no_schema_no_values(self): + effective = build_effective_traits([], {}, None) + assert effective.validate(check_unresolved=True) == [] + + def test_values_without_schema_is_error(self): + effective = build_effective_traits([], {"a": 1}, None) + errors = effective.validate(check_unresolved=True) + assert any("no x-gts-traits-schema is defined" in e for e in errors) + + def test_schema_false_prohibits_values(self): + effective = build_effective_traits([False], {"a": 1}, None) + errors = effective.validate(check_unresolved=True) + assert any("values are prohibited" in e for e in errors) + + def test_schema_false_with_no_values_is_ok(self): + effective = build_effective_traits([False], {}, None) + assert effective.validate(check_unresolved=True) == [] + + def test_default_materialized(self): + effective = build_effective_traits( + [{"type": "object", "properties": {"a": {"default": "x"}}}], {}, None + ) + assert effective.values == {"a": "x"} + + def test_valid_trait_values_pass(self): + schema = { + "type": "object", + "properties": {"a": {"type": "string"}}, + "required": ["a"], + } + effective = build_effective_traits([schema], {"a": "hi"}, None) + assert effective.validate(check_unresolved=True) == [] + + def test_invalid_trait_type_fails(self): + schema = { + "type": "object", + "properties": {"a": {"type": "string"}}, + } + effective = build_effective_traits([schema], {"a": 5}, None) + errors = effective.validate(check_unresolved=True) + assert any("trait validation" in e for e in errors) + + def test_required_without_default_unresolved(self): + schema = { + "type": "object", + "properties": {"a": {"type": "string"}}, + "required": ["a"], + } + effective = build_effective_traits([schema], {}, None) + errors = effective.validate(check_unresolved=True) + assert any("is not resolved" in e for e in errors) + + def test_abstract_skips_unresolved_check(self): + schema = { + "type": "object", + "properties": {"a": {"type": "string"}}, + "required": ["a"], + } + effective = build_effective_traits([schema], {}, None) + errors = effective.validate(check_unresolved=False) + assert errors == [] + + def test_incompatible_trait_schema_chain_flagged(self): + # Second schema narrows type incompatibly with the ancestor. + effective = build_effective_traits( + [{"type": "string"}, {"type": "integer"}], {}, None + ) + errors = effective.validate(check_unresolved=True) + assert any("incompatible with ancestor trait schema" in e for e in errors) + + def test_invalid_trait_schema_integrity_flagged(self): + effective = build_effective_traits([{"type": "not-a-real-type"}], {}, None) + errors = effective.validate(check_unresolved=True) + assert any("not a valid JSON Schema" in e for e in errors) + + def test_dialect_applied_to_effective_schema(self): + effective = build_effective_traits( + [{"type": "object"}], {}, "https://json-schema.org/draft/2020-12/schema" + ) + assert effective.schema["$schema"] == "https://json-schema.org/draft/2020-12/schema" + + def test_x_gts_ref_errors_prefixed(self): + schema = { + "type": "object", + "properties": {"ref": {"type": "string", "x-gts-ref": "not-valid"}}, + } + effective = build_effective_traits([schema], {"ref": "also-not-valid"}, None) + errors = effective.validate(check_unresolved=True) + assert any(e.startswith("trait x-gts-ref:") for e in errors) diff --git a/tests/test_x_gts_ref.py b/tests/test_x_gts_ref.py new file mode 100644 index 0000000..397fad9 --- /dev/null +++ b/tests/test_x_gts_ref.py @@ -0,0 +1,204 @@ +"""Tests for gts.x_gts_ref (x-gts-ref schema & instance validation, spec sec 9.5).""" + +from gts.x_gts_ref import XGtsRefValidator + + +class TestValidateSchema: + def test_valid_absolute_pattern(self): + errors = XGtsRefValidator().validate_schema( + {"x-gts-ref": "gts.x.test._.foo.v1~"} + ) + assert errors == [] + + def test_valid_wildcard_pattern(self): + errors = XGtsRefValidator().validate_schema({"x-gts-ref": "gts.*"}) + assert errors == [] + + def test_valid_prefix_wildcard(self): + errors = XGtsRefValidator().validate_schema({"x-gts-ref": "gts.x.test.*"}) + assert errors == [] + + def test_invalid_wildcard_prefix_direct(self): + error = XGtsRefValidator()._validate_gts_id_or_pattern("notgts*", "path") + assert error is not None + assert "Invalid GTS wildcard pattern" in error.reason + + def test_invalid_specific_gts_id(self): + errors = XGtsRefValidator().validate_schema({"x-gts-ref": "gts.bad id"}) + assert len(errors) == 1 + assert "Invalid GTS identifier" in errors[0].reason + + def test_non_string_ref_value(self): + errors = XGtsRefValidator().validate_schema({"x-gts-ref": 123}) + assert len(errors) == 1 + assert "must be a string" in errors[0].reason + + def test_invalid_prefix_value(self): + errors = XGtsRefValidator().validate_schema({"x-gts-ref": "nope"}) + assert len(errors) == 1 + assert "must start with" in errors[0].reason + + def test_relative_pointer_resolves_to_valid_id(self): + schema = { + "$id": "gts.x.test._.foo.v1~", + "properties": { + "ref_field": {"x-gts-ref": "/$id"}, + }, + } + errors = XGtsRefValidator().validate_schema(schema) + assert errors == [] + + def test_relative_pointer_unresolvable(self): + schema = {"properties": {"ref_field": {"x-gts-ref": "/missing/path"}}} + errors = XGtsRefValidator().validate_schema(schema) + assert len(errors) == 1 + assert "Cannot resolve reference path" in errors[0].reason + + def test_relative_pointer_resolves_to_invalid_id(self): + schema = { + "not_gts": "definitely not a gts id !!", + "properties": {"ref_field": {"x-gts-ref": "/not_gts"}}, + } + errors = XGtsRefValidator().validate_schema(schema) + assert len(errors) == 1 + assert "is not a valid GTS identifier" in errors[0].reason + + def test_recurses_into_nested_structures(self): + schema = { + "properties": { + "child": {"x-gts-ref": "notgts.*"}, + } + } + errors = XGtsRefValidator().validate_schema(schema) + assert len(errors) == 1 + assert "properties/child/x-gts-ref" in errors[0].field_path + + def test_recurses_into_list_of_dicts(self): + schema = {"allOf": [{"x-gts-ref": "notgts.*"}]} + errors = XGtsRefValidator().validate_schema(schema) + assert len(errors) == 1 + assert "allOf[0]/x-gts-ref" in errors[0].field_path + + +class TestValidateInstanceValue: + def test_non_string_instance_value_error(self): + error = XGtsRefValidator()._validate_ref_value(123, "gts.*", "ref", {}) + assert error is not None + assert "Value must be a string" in error.reason + + def test_relative_ref_pattern_resolution_on_instance(self): + schema = { + "$id": "gts.x.test._.foo.v1~", + "type": "object", + "properties": {"ref": {"x-gts-ref": "/$id"}}, + } + errors = XGtsRefValidator().validate_instance( + {"ref": "gts.x.test._.foo.v1~x.test._.bar.v1"}, schema + ) + assert errors == [] + + def test_relative_ref_pattern_resolution_fails_when_not_gts_prefix(self): + schema = { + "other": "not-gts-value", + "type": "object", + "properties": {"ref": {"x-gts-ref": "/other"}}, + } + errors = XGtsRefValidator().validate_instance( + {"ref": "gts.x.test._.foo.v1~"}, schema + ) + assert len(errors) == 1 + assert "is not a GTS pattern" in errors[0].reason + + def test_wildcard_pattern_matches_prefix(self): + errors = XGtsRefValidator().validate_instance( + "gts.x.test._.foo.v1~", {"x-gts-ref": "gts.x.test.*"} + ) + assert errors == [] + + def test_wildcard_pattern_mismatch(self): + errors = XGtsRefValidator().validate_instance( + "gts.x.other._.foo.v1~", {"x-gts-ref": "gts.x.test.*"} + ) + assert len(errors) == 1 + assert "does not match pattern" in errors[0].reason + + def test_exact_pattern_mismatch(self): + errors = XGtsRefValidator().validate_instance( + "gts.x.other._.foo.v1~", {"x-gts-ref": "gts.x.test._.foo.v1~"} + ) + assert len(errors) == 1 + assert "does not match pattern" in errors[0].reason + + def test_store_lookup_missing_entity(self): + class FakeStore: + def get(self, value): + return None + + errors = XGtsRefValidator(store=FakeStore()).validate_instance( + "gts.x.test._.foo.v1~", {"x-gts-ref": "gts.*"} + ) + assert len(errors) == 1 + assert "not found in registry" in errors[0].reason + + def test_store_lookup_found_entity(self): + class FakeStore: + def get(self, value): + return object() + + errors = XGtsRefValidator(store=FakeStore()).validate_instance( + "gts.x.test._.foo.v1~", {"x-gts-ref": "gts.*"} + ) + assert errors == [] + + def test_array_items_recursion(self): + schema = { + "type": "array", + "items": {"x-gts-ref": "gts.x.test.*"}, + } + errors = XGtsRefValidator().validate_instance( + ["gts.x.other.v1~"], schema + ) + assert len(errors) == 1 + + def test_object_properties_recursion(self): + schema = { + "type": "object", + "properties": {"ref": {"x-gts-ref": "gts.x.test.*"}}, + } + errors = XGtsRefValidator().validate_instance( + {"ref": "gts.x.other.v1~"}, schema + ) + assert len(errors) == 1 + + def test_any_of_no_branch_matched(self): + schema = { + "anyOf": [ + {"x-gts-ref": "gts.x.test._.a.v1~"}, + {"x-gts-ref": "gts.x.test._.b.v1~"}, + ] + } + errors = XGtsRefValidator().validate_instance( + "gts.x.test._.c.v1~x.test._.item.v1", schema + ) + assert any("anyOf: no branch matched" in e.reason for e in errors) + + def test_any_of_matches_one_branch(self): + schema = { + "anyOf": [ + {"x-gts-ref": "gts.x.test._.a.v1~"}, + {"x-gts-ref": "gts.x.test._.b.v1~"}, + ] + } + errors = XGtsRefValidator().validate_instance( + "gts.x.test._.a.v1~x.test._.item.v1", schema + ) + assert errors == [] + + def test_all_of_validates_each_matching_branch(self): + schema = { + "allOf": [ + {"type": "string", "x-gts-ref": "gts.x.test.*"}, + ] + } + errors = XGtsRefValidator().validate_instance("gts.x.other.v1~", schema) + assert len(errors) == 1