diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index faed42e1df..868385a506 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -23,6 +23,7 @@ jobs: djqs: ${{ steps.filter.outputs.djqs }} djrs: ${{ steps.filter.outputs.djrs }} ui: ${{ steps.filter.outputs.ui }} + workflow: ${{ steps.filter.outputs.workflow }} steps: - uses: actions/checkout@v4 @@ -46,12 +47,33 @@ jobs: ui: - datajunction-ui/** - '!datajunction-ui/package.json' + workflow: + - .github/workflows/test.yml predicate-quantifier: every build: needs: changes runs-on: ubuntu-latest + # One Postgres for the whole job. Without it each xdist worker starts its + # own container and rebuilds the same template, N times over, all at + # startup. Services cannot be conditional, so the non-server legs get one + # too; it costs a couple of seconds to start and nothing to ignore. + services: + postgres: + image: postgres:latest + env: + POSTGRES_USER: dj + POSTGRES_PASSWORD: dj + POSTGRES_DB: dj + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U dj" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + strategy: fail-fast: false matrix: @@ -71,11 +93,15 @@ jobs: run: uv python install ${{ matrix.python-version }} - name: Run Tests + # Editing this workflow has to exercise this workflow. Without the + # `workflow` clause a change to test.yml matches no filter, every test + # step is skipped, and the PR goes green having run nothing -- so a + # broken workflow reaches main and surfaces on somebody else's PR. if: | - (matrix.library == 'client' && (needs.changes.outputs.client == 'true' || needs.changes.outputs.server == 'true')) || - (matrix.library == 'server' && needs.changes.outputs.server == 'true') || - (matrix.library == 'djqs' && needs.changes.outputs.djqs == 'true') || - (matrix.library == 'djrs' && needs.changes.outputs.djrs == 'true') + (matrix.library == 'client' && (needs.changes.outputs.client == 'true' || needs.changes.outputs.server == 'true' || needs.changes.outputs.workflow == 'true')) || + (matrix.library == 'server' && (needs.changes.outputs.server == 'true' || needs.changes.outputs.workflow == 'true')) || + (matrix.library == 'djqs' && (needs.changes.outputs.djqs == 'true' || needs.changes.outputs.workflow == 'true')) || + (matrix.library == 'djrs' && (needs.changes.outputs.djrs == 'true' || needs.changes.outputs.workflow == 'true')) run: | echo "Testing ${{ matrix.library }} ..." export TEST_DIR=${{ matrix.library == 'server' && './datajunction-server' || matrix.library == 'client' && './datajunction-clients/python' || matrix.library == 'djqs' && './datajunction-query' || matrix.library == 'djrs' && './datajunction-reflection'}} @@ -94,9 +120,190 @@ jobs: # Run linters uv run pre-commit run --all-files + # Build the template databases once for the whole job. Each xdist + # worker then clones them (~90ms) instead of building its own. + if [ "${{ matrix.library }}" = "server" ]; then + PG="postgresql://dj:dj@localhost:5432" + PGX="postgresql+psycopg://dj:dj@localhost:5432" + export DJ_TEST_POSTGRES_URL="$PGX/dj" + + # The suite creates this role for containers it starts itself; on a + # server it does not own, it cannot. + psql "$PG/dj" -c "CREATE ROLE readonly_user WITH LOGIN PASSWORD 'readonly'" + + psql "$PG/dj" -c "CREATE DATABASE template_all_examples" + uv run python tests/helpers/populate_template.py "$PGX/template_all_examples" + + psql "$PG/dj" -c "CREATE DATABASE template_preaggs TEMPLATE template_all_examples" + uv run python tests/helpers/populate_preaggs_template.py "$PGX/template_preaggs" + + psql "$PG/dj" -c "CREATE DATABASE template_dimension_links" + uv run python tests/helpers/populate_template.py \ + "$PGX/template_dimension_links" COMPLEX_DIMENSION_LINK + fi + # Run tests export MODULE=${{ matrix.library == 'server' && 'datajunction_server' || matrix.library == 'client' && 'datajunction' || matrix.library == 'djqs' && 'djqs' || matrix.library == 'djrs' && 'datajunction_reflection'}} - uv run pytest ${{ (matrix.library == 'server' || matrix.library == 'client') && '-n auto --dist=loadscope' || '' }} --cov-fail-under=100 --cov=$MODULE --cov-report term-missing -vv tests/ --doctest-modules $MODULE --without-integration --without-slow-integration --ignore=datajunction_server/alembic/env.py + + # Only the server skips coverage here. It is the expensive one -- + # 28.8min instrumented against ~15min not -- so the `coverage-shard` + # matrix below measures it in parallel pieces and the `coverage` job + # enforces its threshold. + # + # The other three are instrumented in place. Their suites are small + # enough that it costs the gate nothing, and they have nowhere else + # to be measured, because the shards are server-only. An earlier + # version of this workflow moved coverage off every leg at once, + # which silently took client, djqs and djrs from a 100% gate to no + # gate at all. + COV_ARGS="" + if [ "${{ matrix.library }}" != "server" ]; then + COV_ARGS="--cov-fail-under=100 --cov-report=term-missing --cov=$MODULE" + fi + + uv run pytest ${{ (matrix.library == 'server' || matrix.library == 'client') && '-n auto --dist=loadscope' || '' }} $COV_ARGS -vv tests/ --doctest-modules $MODULE --without-integration --without-slow-integration --ignore=datajunction_server/alembic/env.py + + # Coverage roughly doubles the suite, so measure it across shards in parallel + # rather than serially inside a build leg. Each shard only produces data; the + # `coverage` job below combines them and enforces the threshold. + coverage-shard: + needs: changes + runs-on: ubuntu-latest + + # Same shared Postgres as the build job. This matters more here: without it + # every one of the five shards starts its own containers and rebuilds every + # template, so the duplicated startup cost is paid five times over. + services: + postgres: + image: postgres:latest + env: + POSTGRES_USER: dj + POSTGRES_PASSWORD: dj + POSTGRES_DB: dj + ports: + - 5432:5432 + options: >- + --health-cmd "pg_isready -U dj" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + if: needs.changes.outputs.server == 'true' || needs.changes.outputs.workflow == 'true' + + strategy: + fail-fast: false + matrix: + # The last shard is a catch-all: `tests/` minus the others. Listing + # every directory explicitly means a newly added one belongs to no + # shard, and the only symptom is combined coverage quietly dropping + # below the threshold. + include: + - shard: 1 + paths: "tests/api" + - shard: 2 + paths: "tests/construction" + - shard: 3 + paths: "tests/sql" + - shard: 4 + paths: "tests/internal" + # Also carries --doctest-modules. The build legs pass it, and it is + # what imports every module in the package -- without it the + # import-time lines of modules no test imports directly (models/ + # table.py, __about__.py, a couple of __init__.py) are never + # executed, and the combined total lands just under the threshold. + - shard: 5 + paths: >- + tests + --ignore=tests/api + --ignore=tests/construction + --ignore=tests/sql + --ignore=tests/internal + --doctest-modules datajunction_server + --ignore=datajunction_server/alembic/env.py + + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + version: "latest" + enable-cache: true + + - name: Set up Python + run: uv python install 3.13 + + - name: Run shard ${{ matrix.shard }} + working-directory: ./datajunction-server + # Write to a name coverage's own erase() will not match. On startup + # pytest-cov erases every `.coverage.*` file in the directory, not just + # its own, so a fragment named that way can be destroyed by another + # shard sharing the workspace -- or, worse on a reused runner, a stale + # one can survive and skew the combined total. + env: + COVERAGE_FILE: .cov-shard${{ matrix.shard }} + run: | + uv sync --group test --python 3.13 + + PG="postgresql://dj:dj@localhost:5432" + PGX="postgresql+psycopg://dj:dj@localhost:5432" + export DJ_TEST_POSTGRES_URL="$PGX/dj" + + # The suite creates this role for containers it starts itself; on a + # server it does not own, it cannot. + psql "$PG/dj" -c "CREATE ROLE readonly_user WITH LOGIN PASSWORD 'readonly'" + + psql "$PG/dj" -c "CREATE DATABASE template_all_examples" + uv run python tests/helpers/populate_template.py "$PGX/template_all_examples" + + psql "$PG/dj" -c "CREATE DATABASE template_preaggs TEMPLATE template_all_examples" + uv run python tests/helpers/populate_preaggs_template.py "$PGX/template_preaggs" + + psql "$PG/dj" -c "CREATE DATABASE template_dimension_links" + uv run python tests/helpers/populate_template.py \ + "$PGX/template_dimension_links" COMPLEX_DIMENSION_LINK + + uv run pytest -n auto --dist=loadscope \ + --cov=datajunction_server --cov-report= \ + -q ${{ matrix.paths }} \ + --without-integration --without-slow-integration + + - uses: actions/upload-artifact@v4 + with: + name: coverage-shard-${{ matrix.shard }} + path: datajunction-server/.cov-shard${{ matrix.shard }} + include-hidden-files: true + retention-days: 1 + + # This is the job branch protection should require; the shards only produce + # data and enforce nothing on their own. + coverage: + needs: coverage-shard + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - name: Install uv + uses: astral-sh/setup-uv@v4 + with: + version: "latest" + enable-cache: true + + - name: Set up Python + run: uv python install 3.13 + + - uses: actions/download-artifact@v4 + with: + pattern: coverage-shard-* + merge-multiple: true + path: datajunction-server + + - name: Combine and enforce + working-directory: ./datajunction-server + run: | + uv sync --group test --python 3.13 + ls -la .cov-shard* || { echo "no coverage fragments downloaded"; exit 1; } + uv run coverage combine .cov-shard* + uv run coverage report --show-missing --fail-under=100 build-javascript: runs-on: ubuntu-latest diff --git a/datajunction-clients/javascript/package-lock.json b/datajunction-clients/javascript/package-lock.json index 46c8a7a59a..6a26df6c1c 100644 --- a/datajunction-clients/javascript/package-lock.json +++ b/datajunction-clients/javascript/package-lock.json @@ -1,12 +1,12 @@ { "name": "datajunction", - "version": "0.0.223", + "version": "0.0.244", "lockfileVersion": 2, "requires": true, "packages": { "": { "name": "datajunction", - "version": "0.0.223", + "version": "0.0.244", "license": "MIT", "dependencies": { "@babel/core": "^7.22.5", diff --git a/datajunction-clients/javascript/package.json b/datajunction-clients/javascript/package.json index aee20a14f4..daa4e37e9b 100644 --- a/datajunction-clients/javascript/package.json +++ b/datajunction-clients/javascript/package.json @@ -1,6 +1,6 @@ { "name": "datajunction", - "version": "0.0.223", + "version": "0.0.244", "description": "A Javascript client for interacting with a DataJunction server", "module": "src/index.js", "scripts": { diff --git a/datajunction-clients/python/datajunction/__about__.py b/datajunction-clients/python/datajunction/__about__.py index 0ff8e74b1a..727425f40a 100644 --- a/datajunction-clients/python/datajunction/__about__.py +++ b/datajunction-clients/python/datajunction/__about__.py @@ -2,4 +2,4 @@ Version for Hatch """ -__version__ = "0.0.223" +__version__ = "0.0.244" diff --git a/datajunction-clients/python/datajunction/cli.py b/datajunction-clients/python/datajunction/cli.py index b2de2ed001..e8b8f29d9e 100644 --- a/datajunction-clients/python/datajunction/cli.py +++ b/datajunction-clients/python/datajunction/cli.py @@ -47,6 +47,7 @@ def push( verbose: bool = False, force: bool = False, allow_empty: bool = False, + format: str = "text", ): """ Alias for deploy without dryrun. @@ -57,6 +58,7 @@ def push( verbose=verbose, force=force, allow_empty=allow_empty, + format=format, ) def dryrun( @@ -916,7 +918,7 @@ def create_parser(self): type=str, default="text", choices=["text", "json"], - help="Output format for dry run (default: text)", + help="Output format for the deployment result (default: text)", ) # Deployment source tracking flags push_parser.add_argument( @@ -1523,12 +1525,17 @@ def dispatch_command(self, args, parser): verbose=args.verbose, force=args.force, allow_empty=args.allow_empty, + format=args.format, ) except DJDeploymentFailure: - # Errors already displayed in the deployment panel + # Errors already displayed in the deployment panel (or, in + # --format json mode, in the JSON already printed to stdout) raise SystemExit(1) except DJClientException as exc: - Console().print(f"[red bold]ERROR:[/red bold] {exc}") + if args.format == "json": + print(json.dumps({"error": str(exc)}, indent=2)) + else: + Console().print(f"[red bold]ERROR:[/red bold] {exc}") raise SystemExit(1) elif args.command == "generate-codeowners": count = DeploymentService.build_codeowners( diff --git a/datajunction-clients/python/datajunction/deployment.py b/datajunction-clients/python/datajunction/deployment.py index d4c04d3018..250d7c038d 100644 --- a/datajunction-clients/python/datajunction/deployment.py +++ b/datajunction-clients/python/datajunction/deployment.py @@ -140,11 +140,13 @@ def push( verbose: bool = False, force: bool = False, allow_empty: bool = False, + format: str = "text", ): """ Push a local project to a namespace. """ console = console or self.console + as_json = format == "json" deployment_spec, file_errors = self._reconstruct_deployment_spec(source_path) @@ -161,13 +163,14 @@ def push( branch, ) - print_deployment_header( - mode="push", - namespace=deployment_spec["namespace"], - console=console, - repo=source.get("repository"), - branch=source.get("branch") or branch, - ) + if not as_json: + print_deployment_header( + mode="push", + namespace=deployment_spec["namespace"], + console=console, + repo=source.get("repository"), + branch=source.get("branch") or branch, + ) # Apply git config in the normal push flow. Skip only when the # caller passed an explicit ``namespace`` equal to the project's @@ -186,10 +189,11 @@ def push( parent_namespace=parent_namespace or None, ) except Exception as e: # pylint: disable=broad-except - console.print( - f"[yellow]Warning: could not set git config on namespace " - f"'{deployment_spec['namespace']}': {e}[/yellow]", - ) + if not as_json: + console.print( + f"[yellow]Warning: could not set git config on namespace " + f"'{deployment_spec['namespace']}': {e}[/yellow]", + ) if force: deployment_spec["force"] = True if allow_empty: @@ -211,25 +215,30 @@ def push( raise DJClientException("Deployment timed out after 5 minutes") deployment = DeploymentInfo.from_dict(deployment_data) - print_results( - deployment_uuid, - deployment, - console, - verbose=verbose, - ) + if as_json: + print(json.dumps(deployment_data, indent=2)) + else: + print_results( + deployment_uuid, + deployment, + console, + verbose=verbose, + ) if deployment.status == DeploymentStatus.SUCCESS: invalid_results = [ r for r in deployment.results if r.status == ResultStatus.INVALID ] if invalid_results: - console.print( - "\nDeployment finished: [bold yellow]SUCCESS with invalid nodes[/bold yellow]", - ) + if not as_json: + console.print( + "\nDeployment finished: [bold yellow]SUCCESS with invalid nodes[/bold yellow]", + ) raise DJDeploymentFailure( project_name=deployment_spec.get("namespace", source_path), errors=[r.__dict__ for r in invalid_results], ) - console.print("\nDeployment finished: [bold green]SUCCESS[/bold green]") + if not as_json: + console.print("\nDeployment finished: [bold green]SUCCESS[/bold green]") if deployment.status == DeploymentStatus.FAILED: errors = [ r @@ -241,10 +250,11 @@ def push( errors=[r.__dict__ for r in (errors if errors else deployment.results)], ) if file_errors: - console.print() - console.rule("[red bold]Errors[/red bold]", style="red", align="left") - for err in file_errors: - console.print(f"[red] {err}[/red]") + if not as_json: + console.print() + console.rule("[red bold]Errors[/red bold]", style="red", align="left") + for err in file_errors: + console.print(f"[red] {err}[/red]") raise DJClientException( "Fix file name mismatches before deploying.", ) @@ -562,9 +572,24 @@ def _reconstruct_deployment_spec( or project_metadata.get("prefix", ""), "nodes": nodes, "tags": project_metadata.get("tags", []), + # Upsert-only on the server: an empty list is a no-op, never a + # retirement, so defaulting to [] is safe here in a way it is not + # for custom_metadata_schemas below. + "hierarchies": project_metadata.get("hierarchies", []), "preaggregations": preaggregations, } + # Forwarded only when dj.yaml declares it, because absent and empty mean + # different things to the server: absent leaves registered schemas alone, + # while an empty list says this manifest manages them and declares none, + # which retires them. Defaulting to [] here would silently retire every + # schema in the namespace on the next push from a manifest that never + # mentioned them. + if "custom_metadata_schemas" in project_metadata: + deployment_spec["custom_metadata_schemas"] = project_metadata[ + "custom_metadata_schemas" + ] + # Add deployment source if available from env vars source = self._build_deployment_source(cwd=base_dir) if source: # pragma: no branch diff --git a/datajunction-clients/python/datajunction/models.py b/datajunction-clients/python/datajunction/models.py index bfa0cf9c6a..8eacc38194 100644 --- a/datajunction-clients/python/datajunction/models.py +++ b/datajunction-clients/python/datajunction/models.py @@ -3,8 +3,9 @@ from __future__ import annotations import enum +import re from dataclasses import dataclass, field -from typing import TYPE_CHECKING, Any +from typing import TYPE_CHECKING, Any, Literal, TypeAlias from datajunction._base import SerializableMixin @@ -361,6 +362,38 @@ def from_dict( # --------------------------------------------------------------------------- +@dataclass +class SemanticFingerprint: + """A semantic node digest returned by the server.""" + + digest: str + version: int = 1 + + def __post_init__(self) -> None: + if re.fullmatch(r"[0-9a-f]{64}", self.digest) is None: + raise ValueError( + "Semantic fingerprint digest must be 64 lowercase hexadecimal characters", + ) + + @classmethod + def from_dict(cls, d: dict) -> SemanticFingerprint: + return cls( + version=d.get("version", 1), + digest=d.get("digest", ""), + ) + + +SemanticFingerprintValue: TypeAlias = SemanticFingerprint | Literal["unknown"] + + +def _parse_semantic_fingerprint(value: Any) -> SemanticFingerprintValue | None: + if value is None or value == "unknown": + return value + if not isinstance(value, dict): + raise ValueError("Semantic fingerprint must be an object or 'unknown'") + return SemanticFingerprint.from_dict(value) + + @dataclass class DeploymentResult: """A single node-level result within a deployment.""" @@ -370,15 +403,26 @@ class DeploymentResult: status: str message: str = "" changed_fields: list[str] = field(default_factory=list) + deploy_type: str = "" + change_tier: Literal["none", "minor", "major"] | None = None + semantic_fingerprint: SemanticFingerprintValue | None = None + # True when the node was re-deployed only to retry a pre-existing failure. + # None on responses from servers that predate the field. + revalidation_only: bool | None = None @classmethod def from_dict(cls, d: dict) -> DeploymentResult: + fingerprint = d.get("semantic_fingerprint") return cls( name=d.get("name", ""), operation=d.get("operation", ""), status=d.get("status", ""), message=d.get("message", ""), changed_fields=d.get("changed_fields") or [], + deploy_type=d.get("deploy_type", ""), + change_tier=d.get("change_tier"), + semantic_fingerprint=_parse_semantic_fingerprint(fingerprint), + revalidation_only=d.get("revalidation_only"), ) @@ -392,9 +436,11 @@ class DownstreamImpact: caused_by: list[str] = field(default_factory=list) depth: int = 0 impact_type: str = "" + semantic_fingerprint: SemanticFingerprintValue | None = None @classmethod def from_dict(cls, d: dict) -> DownstreamImpact: + fingerprint = d.get("semantic_fingerprint") return cls( name=d.get("name", ""), node_type=d.get("node_type", ""), @@ -402,6 +448,7 @@ def from_dict(cls, d: dict) -> DownstreamImpact: caused_by=d.get("caused_by") or [], depth=d.get("depth", 0), impact_type=d.get("impact_type", ""), + semantic_fingerprint=_parse_semantic_fingerprint(fingerprint), ) diff --git a/datajunction-clients/python/tests/test_cli.py b/datajunction-clients/python/tests/test_cli.py index 2595db7f9d..49dcf90db0 100644 --- a/datajunction-clients/python/tests/test_cli.py +++ b/datajunction-clients/python/tests/test_cli.py @@ -2447,6 +2447,28 @@ def test_push_exits_1_on_generic_client_error(self, tmp_path): cli.run() assert exc_info.value.code == 1 + def test_push_generic_client_error_format_json(self, tmp_path, capsys): + """A non-deployment error in --format json mode prints a JSON error object.""" + from datajunction.cli import DJCLI + from datajunction.exceptions import DJClientException + + cli = DJCLI(builder_client=mock.MagicMock()) + with patch.object( + cli, + "push", + side_effect=DJClientException("Connection refused"), + ): + with patch.object( + sys, + "argv", + ["dj", "push", str(tmp_path), "--format", "json"], + ): + with pytest.raises(SystemExit) as exc_info: + cli.run() + assert exc_info.value.code == 1 + printed = json.loads(capsys.readouterr().out) + assert printed == {"error": "Connection refused"} + class TestGenerateCodeowners: """Tests for `dj generate-codeowners` and DeploymentService.build_codeowners.""" diff --git a/datajunction-clients/python/tests/test_deploy.py b/datajunction-clients/python/tests/test_deploy.py index c89487ea96..f3b6bec1e3 100644 --- a/datajunction-clients/python/tests/test_deploy.py +++ b/datajunction-clients/python/tests/test_deploy.py @@ -1,21 +1,23 @@ import importlib.metadata import io -from pathlib import Path +import json import time import zipfile +from pathlib import Path from unittest import mock -import pytest from unittest.mock import MagicMock, patch + +import pytest +import yaml from datajunction.deployment import DeploymentService from datajunction.exceptions import DJClientException, DJDeploymentFailure -from datajunction.models import DeploymentInfo +from datajunction.models import DeploymentInfo, SemanticFingerprint from datajunction.rendering import ( _render_error_bullets, _strip_summary_lines, print_deployment_header, print_results, ) -import yaml from rich.console import Console @@ -75,6 +77,131 @@ def test_reconstruct_deployment_spec_separates_preaggregations(tmp_path): assert "kind" not in preagg +def test_reconstruct_deployment_spec_forwards_hierarchies(tmp_path): + """A `hierarchies:` block in dj.yaml reaches the deployment payload. + + Same defect as custom_metadata_schemas: the payload names its keys one by one, + so an unnamed manifest section is read and dropped. Hierarchies are upsert-only + on the server, so an absent block is a no-op and [] is a safe default. + """ + (tmp_path / "dj.yaml").write_text( + "namespace: ns\n" + "hierarchies:\n" + " - name: geography\n" + " display_name: Geography\n" + " levels:\n" + " - name: country\n" + " dimension_node: ns.country\n" + " - name: city\n" + " dimension_node: ns.city\n", + ) + (tmp_path / "revenue.yaml").write_text( + "name: ns.revenue\nnode_type: metric\nquery: SELECT SUM(amount) FROM ns.fct\n", + ) + + svc = DeploymentService(MagicMock()) + spec, _ = svc._reconstruct_deployment_spec(tmp_path) + + assert spec["hierarchies"] == [ + { + "name": "geography", + "display_name": "Geography", + "levels": [ + {"name": "country", "dimension_node": "ns.country"}, + {"name": "city", "dimension_node": "ns.city"}, + ], + }, + ] + + +def test_reconstruct_deployment_spec_defaults_hierarchies_to_empty(tmp_path): + """A manifest with no hierarchies sends [], which the server treats as a no-op.""" + (tmp_path / "dj.yaml").write_text("namespace: ns\n") + (tmp_path / "revenue.yaml").write_text( + "name: ns.revenue\nnode_type: metric\nquery: SELECT SUM(amount) FROM ns.fct\n", + ) + + svc = DeploymentService(MagicMock()) + spec, _ = svc._reconstruct_deployment_spec(tmp_path) + + assert spec["hierarchies"] == [] + + +def test_reconstruct_deployment_spec_forwards_custom_metadata_schemas(tmp_path): + """A `custom_metadata_schemas:` block in dj.yaml reaches the deployment payload. + + The payload is assembled key by key, so a manifest section the client does not + name is read and dropped -- the deploy then succeeds having registered nothing, + which is indistinguishable from success. + """ + (tmp_path / "dj.yaml").write_text( + "namespace: ns\n" + "custom_metadata_schemas:\n" + " - key: system\n" + " description: Governance metadata.\n" + " json_schema:\n" + " type: object\n" + " properties:\n" + " lifecycle:\n" + " type: string\n" + " enum: [experimental, active, retired]\n", + ) + (tmp_path / "revenue.yaml").write_text( + "name: ns.revenue\nnode_type: metric\nquery: SELECT SUM(amount) FROM ns.fct\n", + ) + + svc = DeploymentService(MagicMock()) + spec, _ = svc._reconstruct_deployment_spec(tmp_path) + + assert spec["custom_metadata_schemas"] == [ + { + "key": "system", + "description": "Governance metadata.", + "json_schema": { + "type": "object", + "properties": { + "lifecycle": { + "type": "string", + "enum": ["experimental", "active", "retired"], + }, + }, + }, + }, + ] + + +def test_reconstruct_deployment_spec_omits_absent_custom_metadata_schemas(tmp_path): + """A manifest that never mentions schemas must not send the key at all. + + The server reads absent as "this manifest does not manage schemas" and an empty + list as "it manages them and declares none", which retires every schema in the + namespace. Sending [] for a manifest that simply has no opinion would delete + registrations that something else owns. + """ + (tmp_path / "dj.yaml").write_text("namespace: ns\n") + (tmp_path / "revenue.yaml").write_text( + "name: ns.revenue\nnode_type: metric\nquery: SELECT SUM(amount) FROM ns.fct\n", + ) + + svc = DeploymentService(MagicMock()) + spec, _ = svc._reconstruct_deployment_spec(tmp_path) + + assert "custom_metadata_schemas" not in spec + + +def test_reconstruct_deployment_spec_forwards_empty_custom_metadata_schemas(tmp_path): + """An explicitly empty list is forwarded, because it means "retire them".""" + (tmp_path / "dj.yaml").write_text("namespace: ns\ncustom_metadata_schemas: []\n") + (tmp_path / "revenue.yaml").write_text( + "name: ns.revenue\nnode_type: metric\nquery: SELECT SUM(amount) FROM ns.fct\n", + ) + + svc = DeploymentService(MagicMock()) + spec, _ = svc._reconstruct_deployment_spec(tmp_path) + + assert spec["custom_metadata_schemas"] == [] + + def test_reconstruct_deployment_spec_rejects_unknown_kind(tmp_path): """An unrecognized ``kind`` fails loudly instead of silently becoming a node.""" (tmp_path / "dj.yaml").write_text("namespace: ns\n") @@ -766,6 +893,37 @@ def test_push_waits_until_success(monkeypatch, tmp_path): client.check_deployment.assert_called() +def test_push_format_json_prints_deployment_data(monkeypatch, tmp_path, capsys): + """push(format="json") must print the raw deployment dict to stdout + instead of the rich text panel, so a caller (e.g. a CI script posting + a PR comment) can parse the wet-run result.""" + (tmp_path / "dj.yaml").write_text(yaml.safe_dump({"namespace": "foo"})) + (tmp_path / "bar.yaml").write_text(yaml.safe_dump({"name": "foo.bar"})) + + client = MagicMock() + client.deploy.return_value = { + "uuid": "abc", + "status": "success", + "results": [ + { + "name": "foo.bar", + "operation": "update", + "status": "success", + "changed_fields": ["query"], + }, + ], + "namespace": "foo", + } + + svc = DeploymentService(client, console=Console(file=io.StringIO())) + monkeypatch.setattr(time, "sleep", lambda _: None) + + svc.push(tmp_path, format="json") + + printed = json.loads(capsys.readouterr().out) + assert printed == client.deploy.return_value + + def test_push_force_sets_flag_in_spec(monkeypatch, tmp_path): """push(force=True) must include {"force": True} in the spec sent to client.deploy.""" (tmp_path / "dj.yaml").write_text(yaml.safe_dump({"namespace": "foo"})) @@ -913,6 +1071,38 @@ def test_push_raises_on_success_with_invalid_nodes(monkeypatch, tmp_path): assert exc_info.value.errors[0]["name"] == "foo.bar" +def test_push_format_json_raises_on_success_with_invalid_nodes(monkeypatch, tmp_path): + """Same as above in --format json mode: still raises, and skips the rich + text warning (the JSON dump already carries the invalid-node status).""" + (tmp_path / "dj.yaml").write_text(yaml.safe_dump({"namespace": "foo"})) + (tmp_path / "bar.yaml").write_text(yaml.safe_dump({"name": "foo.bar"})) + + invalid_results = [ + { + "deploy_type": "node", + "name": "foo.bar", + "operation": "create", + "status": "invalid", + "message": "One or more metrics are INVALID", + }, + ] + client = MagicMock() + client.deploy.return_value = { + "uuid": "456", + "status": "success", + "results": invalid_results, + "namespace": "foo", + } + + svc = DeploymentService(client, console=Console(file=io.StringIO())) + monkeypatch.setattr(time, "sleep", lambda _: None) + + with pytest.raises(DJDeploymentFailure) as exc_info: + svc.push(tmp_path, format="json") + + assert exc_info.value.errors[0]["name"] == "foo.bar" + + @pytest.mark.timeout(2) def test_push_raises_after_polling_to_failure(monkeypatch, tmp_path): (tmp_path / "dj.yaml").write_text(yaml.safe_dump({"namespace": "ns"})) @@ -1212,24 +1402,42 @@ def test_get_impact_calls_api(self, tmp_path, monkeypatch): monkeypatch.delenv("DJ_DEPLOY_REPO", raising=False) # Create mock client - mock_client = MagicMock() - mock_client.get_deployment_impact.return_value = { + response = { "uuid": "dry_run", "namespace": "test.ns", "status": "success", "results": [ { "name": "test.ns.my_node", + "deploy_type": "node", "operation": "noop", "status": "success", "message": "", + "change_tier": "none", + "semantic_fingerprint": { + "version": 1, + "digest": "a" * 64, + }, + "revalidation_only": True, + }, + ], + "downstream_impacts": [ + { + "name": "external.metric", + "node_type": "metric", + "predicted_status": "valid", + "semantic_fingerprint": { + "version": 1, + "digest": "b" * 64, + }, }, ], - "downstream_impacts": [], } + mock_client = MagicMock() + mock_client.get_deployment_impact.return_value = response svc = DeploymentService(mock_client) - result = svc.get_impact(tmp_path) + result = svc.get_impact(tmp_path, display=False) # Verify the API was called mock_client.get_deployment_impact.assert_called_once() @@ -1237,9 +1445,85 @@ def test_get_impact_calls_api(self, tmp_path, monkeypatch): assert call_args["namespace"] == "test.ns" assert "nodes" in call_args - # Verify the result is returned - assert result["namespace"] == "test.ns" - assert result["uuid"] == "dry_run" + assert result is response + parsed = DeploymentInfo.from_dict(result) + assert parsed.results[0].deploy_type == "node" + assert parsed.results[0].change_tier == "none" + assert parsed.results[0].revalidation_only is True + assert parsed.results[0].semantic_fingerprint == SemanticFingerprint( + digest="a" * 64, + ) + assert parsed.downstream_impacts[0].semantic_fingerprint == SemanticFingerprint( + digest="b" * 64, + ) + + def test_deployment_info_parses_unknown_fingerprints(self): + parsed = DeploymentInfo.from_dict( + { + "uuid": "dry_run", + "namespace": "test.ns", + "status": "success", + "results": [{"semantic_fingerprint": "unknown"}], + "downstream_impacts": [{"semantic_fingerprint": "unknown"}], + }, + ) + + assert parsed.results[0].semantic_fingerprint == "unknown" + assert parsed.downstream_impacts[0].semantic_fingerprint == "unknown" + + with pytest.raises( + ValueError, + match="Semantic fingerprint must be an object or 'unknown'", + ): + DeploymentInfo.from_dict( + { + "results": [{"semantic_fingerprint": "invalid"}], + }, + ) + + def test_deployment_info_parses_older_impact_response(self): + parsed = DeploymentInfo.from_dict( + { + "uuid": "dry_run", + "namespace": "test.ns", + "status": "success", + "results": [ + { + "name": "test.ns.my_node", + "operation": "noop", + "status": "skipped", + }, + ], + }, + ) + assert parsed.results[0].deploy_type == "" + assert parsed.results[0].change_tier is None + assert parsed.results[0].semantic_fingerprint is None + assert parsed.downstream_impacts == [] + + @pytest.mark.parametrize("digest", ["a" * 63, "A" * 64, "g" * 64]) + def test_semantic_fingerprint_rejects_invalid_digest(self, digest): + with pytest.raises(ValueError, match="64 lowercase hexadecimal"): + SemanticFingerprint(digest=digest) + + def test_semantic_fingerprint_preserves_unknown_version(self): + parsed = DeploymentInfo.from_dict( + { + "results": [ + { + "semantic_fingerprint": { + "version": 2, + "digest": "a" * 64, + }, + }, + ], + }, + ) + + assert parsed.results[0].semantic_fingerprint == SemanticFingerprint( + version=2, + digest="a" * 64, + ) def test_get_impact_with_namespace_override(self, tmp_path, monkeypatch): """get_impact should respect namespace override.""" @@ -1579,6 +1863,47 @@ def test_push_git_config_failure_is_warned_not_raised(self, monkeypatch, tmp_pat assert "Warning" in out.getvalue() client.deploy.assert_called_once() + def test_push_git_config_failure_is_silent_in_format_json( + self, + monkeypatch, + tmp_path, + ): + """Same as above in --format json mode: the warning is suppressed so + it doesn't pollute the JSON on stdout.""" + (tmp_path / "dj.yaml").write_text(yaml.safe_dump({"namespace": "project.main"})) + (tmp_path / "my_node.yaml").write_text( + yaml.safe_dump({"name": "project.my_node"}), + ) + + monkeypatch.delenv("DJ_DEPLOY_REPO", raising=False) + monkeypatch.setattr( + DeploymentService, + "_detect_git_branch", + staticmethod(lambda cwd=None: "main"), + ) + monkeypatch.setattr( + DeploymentService, + "_detect_git_repo", + staticmethod(lambda cwd=None: None), + ) + monkeypatch.setattr(time, "sleep", lambda _: None) + + client = MagicMock() + client._set_namespace_git_config.side_effect = Exception("network error") + client.deploy.return_value = {"uuid": "abc", "status": "success", "results": []} + client.check_deployment.return_value = { + "uuid": "abc", + "status": "success", + "results": [], + } + + out = io.StringIO() + svc = DeploymentService(client, console=Console(file=out)) + svc.push(tmp_path, format="json") + + assert "Warning" not in out.getvalue() + client.deploy.assert_called_once() + def test_djdeploymentfailure_str_with_errors(): exc = DJDeploymentFailure( @@ -1623,6 +1948,28 @@ def test_push_raises_on_file_name_mismatch(monkeypatch, tmp_path): svc.push(tmp_path) +def test_push_format_json_raises_on_file_name_mismatch(monkeypatch, tmp_path): + """Same as above in --format json mode: still raises, and skips the rich + text error rule/listing (nothing to print — the JSON on stdout is the + deployment result, not the file-name-mismatch warnings).""" + (tmp_path / "dj.yaml").write_text(yaml.safe_dump({"namespace": "foo"})) + (tmp_path / "wrong.yaml").write_text(yaml.safe_dump({"name": "foo.bar"})) + + client = MagicMock() + client.deploy.return_value = { + "uuid": "abc", + "status": "success", + "results": [], + "namespace": "foo", + } + + svc = DeploymentService(client, console=Console(file=io.StringIO())) + monkeypatch.setattr(time, "sleep", lambda _: None) + + with pytest.raises(DJClientException, match="Fix file name mismatches"): + svc.push(tmp_path, format="json") + + def test_collect_nodes_skips_validation_for_unnamed_node(tmp_path): """A YAML file with no 'name' field should still be collected without warnings.""" (tmp_path / "dj.yaml").write_text(yaml.safe_dump({"namespace": "foo"})) diff --git a/datajunction-query/djqs/__about__.py b/datajunction-query/djqs/__about__.py index 0ff8e74b1a..727425f40a 100644 --- a/datajunction-query/djqs/__about__.py +++ b/datajunction-query/djqs/__about__.py @@ -2,4 +2,4 @@ Version for Hatch """ -__version__ = "0.0.223" +__version__ = "0.0.244" diff --git a/datajunction-query/djqs/api/queries.py b/datajunction-query/djqs/api/queries.py index de44c03e63..de25cb099e 100644 --- a/datajunction-query/djqs/api/queries.py +++ b/datajunction-query/djqs/api/queries.py @@ -5,7 +5,7 @@ import json import logging import uuid -from dataclasses import asdict +from dataclasses import asdict, replace from http import HTTPStatus from typing import Any @@ -35,10 +35,85 @@ decode_results, encode_results, ) +from djqs.result_cache import ( + build_result_cache_key, + get_cached_result, + has_stale_while_revalidate, + parse_cache_control, +) from djqs.utils import get_settings _logger = logging.getLogger(__name__) router = APIRouter(tags=["SQL Queries"]) +_pending_refresh_keys: set[str] = set() + + +async def _process_refresh( + settings: Settings, + postgres_pool: AsyncConnectionPool, + query: Query, + headers: dict[str, str] | None, + result_cache_key: str | None, + result_cache_timeout: int | None, + refresh_key: str, +) -> QueryResults: + """Run a result refresh and always release its in-process lease.""" + try: + return await process_query( + settings=settings, + postgres_pool=postgres_pool, + query=query, + headers=headers, + result_cache_key=result_cache_key, + result_cache_timeout=result_cache_timeout, + ) + finally: + _pending_refresh_keys.discard(refresh_key) + + +async def get_cached_or_schedule_refresh( + create_query: QueryCreate, + settings: Settings, + response: Response, + background_tasks: BackgroundTasks, + postgres_pool: AsyncConnectionPool, + request: Request, +) -> QueryResults | None: + """Return an SWR result or schedule its refresh when the caller opted in.""" + cache_control = request.headers.get("cache-control") + if not has_stale_while_revalidate(cache_control): + return None + + no_cache, no_store, retention = parse_cache_control(cache_control) + if no_cache: + return None + + cache_key = build_result_cache_key(create_query, dict(request.headers)) + cached_result, is_fresh = get_cached_result(settings.results_backend, cache_key) + if ( + cached_result is not None + and not is_fresh + and not no_store + and cache_key not in _pending_refresh_keys + ): + _pending_refresh_keys.add(cache_key) + try: + await save_query_and_run( + create_query=replace(create_query, async_=True), + settings=settings, + response=response, + background_tasks=background_tasks, + postgres_pool=postgres_pool, + headers=request.headers, + result_cache_key=cache_key, + result_cache_timeout=retention, + refresh_key=cache_key, + ) + except Exception: + _pending_refresh_keys.discard(cache_key) + raise + response.status_code = HTTPStatus.OK + return cached_result @router.post( @@ -75,6 +150,15 @@ async def submit_query( # pylint: disable=too-many-arguments This endpoint is different from others in that it accepts both JSON and msgpack, and can also return JSON or msgpack, depending on HTTP headers. + + Request cache policy: + - ``Cache-Control: stale-while-revalidate`` opts into reusable completed-result + caching. Fresh results are returned immediately; retained stale results are + returned while an asynchronous refresh runs. + - ``max-age=N`` overrides result retention in seconds (up to one week) without + changing the 12-hour freshness window. + - ``no-cache`` bypasses reusable results and ``no-store`` prevents writing or + refreshing them. """ content_type = request.headers.get("content-type") if content_type == "application/json": @@ -102,15 +186,36 @@ async def submit_query( # pylint: disable=too-many-arguments data["catalog_name"] = data.get("catalog_name") or settings.default_catalog create_query = QueryCreate(**data) - - query_with_results = await save_query_and_run( - create_query=create_query, - settings=settings, - response=response, - background_tasks=background_tasks, - postgres_pool=postgres_pool, - headers=request.headers, + query_with_results = await get_cached_or_schedule_refresh( + create_query, + settings, + response, + background_tasks, + postgres_pool, + request, ) + if query_with_results is None: + use_result_cache = has_stale_while_revalidate( + request.headers.get("cache-control"), + ) + no_store = False + retention = None + cache_key = None + if use_result_cache: + _, no_store, retention = parse_cache_control( + request.headers.get("cache-control"), + ) + cache_key = build_result_cache_key(create_query, dict(request.headers)) + query_with_results = await save_query_and_run( + create_query=create_query, + settings=settings, + response=response, + background_tasks=background_tasks, + postgres_pool=postgres_pool, + headers=request.headers, + result_cache_key=None if no_store else cache_key, + result_cache_timeout=retention, + ) return_type = get_best_match(accept, ["application/json", "application/msgpack"]) if not return_type: @@ -141,6 +246,9 @@ async def save_query_and_run( # pylint: disable=R0913 background_tasks: BackgroundTasks, postgres_pool: AsyncConnectionPool, headers: dict[str, str] | None = None, + result_cache_key: str | None = None, + result_cache_timeout: int | None = None, + refresh_key: str | None = None, ) -> QueryResults: """ Store a new query to the DB and run it. @@ -176,13 +284,27 @@ async def save_query_and_run( # pylint: disable=R0913 ) if query.async_: - background_tasks.add_task( - process_query, - settings, - postgres_pool, - query, - headers, - ) + if refresh_key: + background_tasks.add_task( + _process_refresh, + settings, + postgres_pool, + query, + headers, + result_cache_key, + result_cache_timeout, + refresh_key, + ) + else: + background_tasks.add_task( + process_query, + settings, + postgres_pool, + query, + headers, + result_cache_key, + result_cache_timeout, + ) response.status_code = HTTPStatus.CREATED return QueryResults( @@ -202,6 +324,8 @@ async def save_query_and_run( # pylint: disable=R0913 postgres_pool=postgres_pool, query=query, headers=headers, + result_cache_key=result_cache_key, + result_cache_timeout=result_cache_timeout, ) return query_results diff --git a/datajunction-query/djqs/engine.py b/datajunction-query/djqs/engine.py index 138588eb5c..fd8b941040 100644 --- a/datajunction-query/djqs/engine.py +++ b/datajunction-query/djqs/engine.py @@ -25,6 +25,7 @@ QueryState, StatementResults, ) +from djqs.result_cache import CachedQueryResult from djqs.typing import ColumnType, Description, SQLADialect, Stream, TypeEnum from djqs.utils import get_settings @@ -224,6 +225,8 @@ async def process_query( postgres_pool: AsyncConnectionPool, query: Query, headers: dict[str, str] | None = None, + result_cache_key: str | None = None, + result_cache_timeout: int | None = None, ) -> QueryResults: """ Process a query. @@ -265,7 +268,7 @@ async def process_query( .save_query( query_id=query.id, submitted_query=query.submitted_query, - state=QueryState.FINISHED.value, + state=query.state.value, async_=query.async_, ) .execute(conn=conn) @@ -283,7 +286,7 @@ async def process_query( ), ) - return QueryResults( + query_results = QueryResults( id=query.id, catalog_name=query.catalog_name, engine_name=query.engine_name, @@ -298,3 +301,13 @@ async def process_query( results=results, errors=errors, ) + if query.state == QueryState.FINISHED and result_cache_key: + settings.results_backend.set( + result_cache_key, + CachedQueryResult( + result=query_results, + created_at=datetime.now(timezone.utc).timestamp(), + ), + timeout=result_cache_timeout, + ) + return query_results diff --git a/datajunction-query/djqs/result_cache.py b/datajunction-query/djqs/result_cache.py new file mode 100644 index 0000000000..18c801b7b7 --- /dev/null +++ b/datajunction-query/djqs/result_cache.py @@ -0,0 +1,100 @@ +""" +Stale-while-revalidate caching for completed query results. +""" + +import hashlib +import json +import time +from dataclasses import dataclass +from typing import Any + +from cachelib.base import BaseCache + +from djqs.constants import SQLALCHEMY_URI +from djqs.models.query import QueryCreate, QueryResults + +DEFAULT_FRESHNESS_SECONDS = 12 * 60 * 60 +DEFAULT_RETENTION_SECONDS = 24 * 60 * 60 +MAX_RETENTION_SECONDS = 7 * 24 * 60 * 60 + + +@dataclass +class CachedQueryResult: + """A successful query result together with its freshness timestamp.""" + + result: QueryResults + created_at: float + + +def parse_cache_control(cache_control: str | None) -> tuple[bool, bool, int]: + """ + Return ``(no_cache, no_store, retention_seconds)`` for Cache-Control. + + ``max-age`` extends or shortens result retention only. It never changes the + shared freshness window, which prevents one caller from making a shared + result appear fresh indefinitely. + """ + directives = [ + directive.strip().lower() for directive in (cache_control or "").split(",") + ] + no_cache = "no-cache" in directives + no_store = "no-store" in directives + retention = DEFAULT_RETENTION_SECONDS + for directive in directives: + name, separator, value = directive.partition("=") + if name != "max-age" or not separator: + continue + try: + candidate = int(value.strip().strip('"')) + except ValueError: + continue + if candidate > 0: + retention = min(candidate, MAX_RETENTION_SECONDS) + break + return no_cache, no_store, retention + + +def has_stale_while_revalidate(cache_control: str | None) -> bool: + """Return whether the caller explicitly opts into result SWR.""" + return any( + directive.strip().lower().partition("=")[0] == "stale-while-revalidate" + for directive in (cache_control or "").split(",") + ) + + +def build_result_cache_key( + query: QueryCreate, + headers: dict[str, str] | None = None, +) -> str: + """Build a stable key for a query's execution-affecting inputs.""" + sqlalchemy_uri = next( + ( + value + for name, value in (headers or {}).items() + if name.lower() == SQLALCHEMY_URI.lower() + ), + None, + ) + payload = json.dumps( + { + "catalog_name": query.catalog_name, + "engine_name": query.engine_name, + "engine_version": query.engine_version, + "submitted_query": query.submitted_query, + "sqlalchemy_uri": sqlalchemy_uri, + }, + sort_keys=True, + separators=(",", ":"), + ) + return f"query-results:{hashlib.sha256(payload.encode()).hexdigest()}" + + +def get_cached_result( + cache: BaseCache, + key: str, +) -> tuple[QueryResults | None, bool]: + """Return a cached result and whether it is still within freshness.""" + cached: Any = cache.get(key) + if not isinstance(cached, CachedQueryResult): + return None, False + return cached.result, (time.time() - cached.created_at) < DEFAULT_FRESHNESS_SECONDS diff --git a/datajunction-query/tests/api/queries_test.py b/datajunction-query/tests/api/queries_test.py index 3ed812cddb..969e44cf35 100644 --- a/datajunction-query/tests/api/queries_test.py +++ b/datajunction-query/tests/api/queries_test.py @@ -10,15 +10,19 @@ from unittest import mock import msgpack +import pytest +from fastapi import BackgroundTasks, Request, Response from fastapi.testclient import TestClient from freezegun import freeze_time from pytest_mock import MockerFixture +from djqs.api import queries as queries_api from djqs.config import Settings from djqs.engine import process_query from djqs.models.query import ( Query, QueryCreate, + QueryResults, QueryState, StatementResults, decode_results, @@ -459,6 +463,273 @@ def test_submit_query_async( assert isinstance(arguments[3], Query) +def test_stale_swr_forces_async_refresh( + mocker: MockerFixture, + client: TestClient, +) -> None: + """A stale SWR hit returns immediately and refreshes asynchronously.""" + stale_result = QueryResults( + submitted_query="SELECT 1 AS col", + state=QueryState.FINISHED, + ) + add_task = mocker.patch("fastapi.BackgroundTasks.add_task") + mocker.patch( + "djqs.api.queries.get_cached_result", + return_value=(stale_result, False), + ) + mocker.patch( + "djqs.api.queries.build_result_cache_key", + return_value="cache-key", + ) + query_create = QueryCreate( + catalog_name="warehouse_inmemory", + engine_name="duckdb_inmemory", + engine_version="0.7.1", + submitted_query="SELECT 1 AS col", + ) + + try: + response = client.post( + "/queries/", + data=json.dumps(asdict(query_create)), + headers={ + "Content-Type": "application/json", + "Accept": "application/json", + "Cache-Control": "stale-while-revalidate", + }, + ) + assert response.status_code == HTTPStatus.OK + assert response.json()["id"] == str(stale_result.id) + arguments = add_task.call_args.args + assert arguments[0] is queries_api._process_refresh + assert arguments[3].async_ is True + assert arguments[5] == "cache-key" + assert arguments[7] == "cache-key" + finally: + queries_api._pending_refresh_keys.discard("cache-key") + + +def test_swr_cache_miss_stores_result(client: TestClient) -> None: + """An SWR cache miss writes the completed result for a later request.""" + query_create = QueryCreate( + catalog_name="warehouse_inmemory", + engine_name="duckdb_inmemory", + engine_version="0.7.1", + submitted_query="SELECT 2 AS col", + ) + headers = { + "Content-Type": "application/json", + "Accept": "application/json", + "Cache-Control": "stale-while-revalidate", + } + cache_key = queries_api.build_result_cache_key(query_create, headers) + settings = get_settings() + settings.results_backend.delete(cache_key) + + response = client.post( + "/queries/", + data=json.dumps(asdict(query_create)), + headers=headers, + ) + + assert response.status_code == HTTPStatus.OK + cached_result, is_fresh = queries_api.get_cached_result( + settings.results_backend, + cache_key, + ) + assert cached_result is not None + assert str(cached_result.id) == response.json()["id"] + assert is_fresh is True + + +@pytest.mark.asyncio +async def test_failed_refresh_releases_its_lease(mocker: MockerFixture) -> None: + """A failed refresh does not prevent a later refresh attempt.""" + queries_api._pending_refresh_keys.add("cache-key") + mocker.patch( + "djqs.api.queries.process_query", + side_effect=RuntimeError("refresh failed"), + ) + try: + with pytest.raises(RuntimeError, match="refresh failed"): + await queries_api._process_refresh( + settings=mocker.MagicMock(), + postgres_pool=mocker.MagicMock(), + query=Query(), + headers=None, + result_cache_key="cache-key", + result_cache_timeout=60, + refresh_key="cache-key", + ) + assert "cache-key" not in queries_api._pending_refresh_keys + finally: + queries_api._pending_refresh_keys.discard("cache-key") + + +@pytest.mark.asyncio +async def test_no_cache_skips_swr_lookup(mocker: MockerFixture) -> None: + """A no-cache request bypasses reading and refreshing result-cache entries.""" + request = Request( + { + "type": "http", + "headers": [ + (b"cache-control", b"stale-while-revalidate, no-cache"), + ], + }, + ) + get_cached_result = mocker.patch("djqs.api.queries.get_cached_result") + + result = await queries_api.get_cached_or_schedule_refresh( + create_query=QueryCreate( + catalog_name="warehouse_inmemory", + engine_name="duckdb_inmemory", + engine_version="0.7.1", + submitted_query="SELECT 1", + ), + settings=mocker.MagicMock(), + response=Response(), + background_tasks=BackgroundTasks(), + postgres_pool=mocker.MagicMock(), + request=request, + ) + + assert result is None + get_cached_result.assert_not_called() + + +@pytest.mark.asyncio +async def test_cache_miss_does_not_schedule_refresh(mocker: MockerFixture) -> None: + """A cache miss leaves execution to the normal query submission path.""" + request = Request( + { + "type": "http", + "headers": [ + (b"cache-control", b"stale-while-revalidate"), + ], + }, + ) + mocker.patch( + "djqs.api.queries.build_result_cache_key", + return_value="cache-key", + ) + mocker.patch( + "djqs.api.queries.get_cached_result", + return_value=(None, False), + ) + save_query_and_run = mocker.patch("djqs.api.queries.save_query_and_run") + + result = await queries_api.get_cached_or_schedule_refresh( + create_query=QueryCreate( + catalog_name="warehouse_inmemory", + engine_name="duckdb_inmemory", + engine_version="0.7.1", + submitted_query="SELECT 1", + ), + settings=mocker.MagicMock(), + response=Response(), + background_tasks=BackgroundTasks(), + postgres_pool=mocker.MagicMock(), + request=request, + ) + + assert result is None + save_query_and_run.assert_not_awaited() + + +@pytest.mark.asyncio +async def test_refresh_scheduling_failure_releases_its_lease( + mocker: MockerFixture, +) -> None: + """A scheduling failure releases the lease before propagating the error.""" + request = Request( + { + "type": "http", + "headers": [ + (b"cache-control", b"stale-while-revalidate"), + ], + }, + ) + query = QueryCreate( + catalog_name="warehouse_inmemory", + engine_name="duckdb_inmemory", + engine_version="0.7.1", + submitted_query="SELECT 1", + ) + mocker.patch( + "djqs.api.queries.get_cached_result", + return_value=(QueryResults(submitted_query="SELECT 1"), False), + ) + mocker.patch( + "djqs.api.queries.build_result_cache_key", + return_value="cache-key", + ) + mocker.patch( + "djqs.api.queries.save_query_and_run", + side_effect=RuntimeError("could not schedule refresh"), + ) + + with pytest.raises(RuntimeError, match="could not schedule refresh"): + await queries_api.get_cached_or_schedule_refresh( + create_query=query, + settings=mocker.MagicMock(), + response=Response(), + background_tasks=BackgroundTasks(), + postgres_pool=mocker.MagicMock(), + request=request, + ) + + assert "cache-key" not in queries_api._pending_refresh_keys + + +@pytest.mark.asyncio +async def test_submit_query_cache_miss_uses_result_cache(mocker: MockerFixture) -> None: + """An SWR cache miss executes and caches the query using its retention policy.""" + request = Request( + { + "type": "http", + "headers": [ + (b"content-type", b"application/json"), + (b"cache-control", b"stale-while-revalidate, max-age=60"), + ], + }, + ) + result = QueryResults(submitted_query="SELECT 1", state=QueryState.FINISHED) + mocker.patch( + "djqs.api.queries.get_cached_or_schedule_refresh", + return_value=None, + ) + mocker.patch( + "djqs.api.queries.build_result_cache_key", + return_value="cache-key", + ) + save_query_and_run = mocker.patch( + "djqs.api.queries.save_query_and_run", + return_value=result, + ) + + response = await queries_api.submit_query( + accept="application/json", + settings=mocker.MagicMock(), + request=request, + response=Response(), + postgres_pool=mocker.MagicMock(), + background_tasks=BackgroundTasks(), + body={ + "catalog_name": "warehouse_inmemory", + "engine_name": "duckdb_inmemory", + "engine_version": "0.7.1", + "submitted_query": "SELECT 1", + }, + ) + + assert response.status_code == HTTPStatus.OK + assert json.loads(response.body) == json.loads( + json.dumps(asdict(result), default=str), + ) + assert save_query_and_run.await_args.kwargs["result_cache_key"] == "cache-key" + assert save_query_and_run.await_args.kwargs["result_cache_timeout"] == 60 + + def test_submit_query_error(client: TestClient) -> None: """ Test submitting invalid query to ``POST /queries/``. diff --git a/datajunction-query/tests/result_cache_test.py b/datajunction-query/tests/result_cache_test.py new file mode 100644 index 0000000000..452b529f4e --- /dev/null +++ b/datajunction-query/tests/result_cache_test.py @@ -0,0 +1,78 @@ +from unittest.mock import patch + +from cachelib.simple import SimpleCache + +from djqs.models.query import QueryCreate, QueryResults +from djqs.result_cache import ( + CachedQueryResult, + DEFAULT_FRESHNESS_SECONDS, + MAX_RETENTION_SECONDS, + build_result_cache_key, + get_cached_result, + has_stale_while_revalidate, + parse_cache_control, +) + + +def test_cache_control_uses_positive_max_age_for_retention(): + assert parse_cache_control("max-age=172800") == (False, False, 172800) + assert parse_cache_control("max-age=999999999") == ( + False, + False, + MAX_RETENTION_SECONDS, + ) + assert parse_cache_control("no-cache, no-store, max-age=0") == ( + True, + True, + 24 * 60 * 60, + ) + assert parse_cache_control("max-age=invalid") == (False, False, 24 * 60 * 60) + assert has_stale_while_revalidate("max-age=60, stale-while-revalidate") is True + assert has_stale_while_revalidate("stale-while-revalidate=60") is True + assert has_stale_while_revalidate("max-age=60") is False + + +def test_result_cache_key_ignores_async_execution_mode(): + first = QueryCreate( + catalog_name="warehouse", + engine_name="trino", + engine_version="1", + submitted_query="SELECT 1", + async_=True, + ) + second = QueryCreate( + catalog_name="warehouse", + engine_name="trino", + engine_version="1", + submitted_query="SELECT 1", + async_=False, + ) + assert build_result_cache_key(first) == build_result_cache_key(second) + assert build_result_cache_key(first, {"SQLALCHEMY_URI": "postgres://one"}) != ( + build_result_cache_key(first, {"SQLALCHEMY_URI": "postgres://two"}) + ) + + +def test_get_cached_result_distinguishes_fresh_and_stale_entries(): + cache = SimpleCache() + key = "result" + result = QueryResults(submitted_query="SELECT 1") + + with patch("djqs.result_cache.time.time", return_value=100): + cache.set(key, CachedQueryResult(result=result, created_at=100)) + cached, is_fresh = get_cached_result(cache, key) + assert cached == result + assert is_fresh is True + + with patch( + "djqs.result_cache.time.time", + return_value=100 + DEFAULT_FRESHNESS_SECONDS, + ): + cached, is_fresh = get_cached_result(cache, key) + assert cached == result + assert is_fresh is False + + +def test_get_cached_result_returns_miss_for_absent_entry(): + """Non-cache values cannot be treated as reusable query results.""" + assert get_cached_result(SimpleCache(), "missing") == (None, False) diff --git a/datajunction-reflection/datajunction_reflection/__about__.py b/datajunction-reflection/datajunction_reflection/__about__.py index 0ff8e74b1a..727425f40a 100644 --- a/datajunction-reflection/datajunction_reflection/__about__.py +++ b/datajunction-reflection/datajunction_reflection/__about__.py @@ -2,4 +2,4 @@ Version for Hatch """ -__version__ = "0.0.223" +__version__ = "0.0.244" diff --git a/datajunction-server/datajunction_server/__about__.py b/datajunction-server/datajunction_server/__about__.py index 0ff8e74b1a..727425f40a 100644 --- a/datajunction-server/datajunction_server/__about__.py +++ b/datajunction-server/datajunction_server/__about__.py @@ -2,4 +2,4 @@ Version for Hatch """ -__version__ = "0.0.223" +__version__ = "0.0.244" diff --git a/datajunction-server/datajunction_server/alembic/versions/2026_08_30_0000-cm0003dropowner_drop_owner_from_custommetadataschema.py b/datajunction-server/datajunction_server/alembic/versions/2026_08_30_0000-cm0003dropowner_drop_owner_from_custommetadataschema.py new file mode 100644 index 0000000000..c48fdb6445 --- /dev/null +++ b/datajunction-server/datajunction_server/alembic/versions/2026_08_30_0000-cm0003dropowner_drop_owner_from_custommetadataschema.py @@ -0,0 +1,26 @@ +"""drop owner from custommetadataschema + +Revision ID: cm0003dropowner +Revises: wf0001names +Create Date: 2026-08-30 00:00:00.000000+00:00 + +""" + +import sqlalchemy as sa +from alembic import op + +revision = "cm0003dropowner" +down_revision = "wf0001names" +branch_labels = None +depends_on = None + + +def upgrade(): + op.drop_column("custommetadataschema", "owner") + + +def downgrade(): + op.add_column( + "custommetadataschema", + sa.Column("owner", sa.String(), nullable=True), + ) diff --git a/datajunction-server/datajunction_server/api/custom_metadata.py b/datajunction-server/datajunction_server/api/custom_metadata.py index b6be6c5e0a..92e5932bd3 100644 --- a/datajunction-server/datajunction_server/api/custom_metadata.py +++ b/datajunction-server/datajunction_server/api/custom_metadata.py @@ -11,7 +11,12 @@ from datajunction_server.database.custom_metadata_schema import CustomMetadataSchema from datajunction_server.api.helpers import check_namespace_not_git_only from datajunction_server.internal.access.authentication.http import SecureAPIRouter -from datajunction_server.internal.custom_metadata import ensure_expression_index +from datajunction_server.internal.custom_metadata import ( + assert_not_reserved_globally, + check_json_schema, + ensure_expression_index, + upsert_schema_row, +) from datajunction_server.models.custom_metadata import ( CustomMetadataSchemaCreate, CustomMetadataSchemaOutput, @@ -31,12 +36,6 @@ router = SecureAPIRouter(tags=["metadata-schemas"]) -def _value_kind(json_schema: dict) -> str | None: - """Extract a single-string type from a JSON Schema, or None.""" - t = json_schema.get("type") - return t if isinstance(t, str) else None - - @router.post("/metadata-schemas/", response_model=CustomMetadataSchemaOutput) async def register_schema( data: CustomMetadataSchemaCreate, @@ -46,29 +45,18 @@ async def register_schema( access_checker: AccessChecker = Depends(get_access_checker), ) -> CustomMetadataSchema: """Create or upsert a custom_metadata schema registration.""" - # Validate that json_schema is itself a valid JSON Schema - try: - jsonschema.Draft202012Validator.check_schema(data.json_schema) - except jsonschema.exceptions.SchemaError as exc: - raise HTTPException( - status_code=422, - detail=f"Invalid JSON Schema: {exc.message}", - ) + check_json_schema(data.key, data.json_schema) - # --- Two-tier authorization --- - # 1. Global key (namespace is None) → require admin + # Two-tier authorization: a global key or a reserved one is an admin's to + # register; a namespace-scoped one needs write access to that namespace. if data.namespace is None and not current_user.is_admin: raise DJAuthorizationException( message="Only administrators may register global schema keys.", ) - - # 2. reserved=True → require admin if data.reserved and not current_user.is_admin: raise DJAuthorizationException( message="Only administrators may register reserved schema keys.", ) - - # 3. Namespace-scoped → require write access to the namespace if data.namespace is not None: access_checker.add_namespace(data.namespace, ResourceAction.WRITE) await access_checker.check(on_denied=AccessDenialMode.RAISE) @@ -84,72 +72,19 @@ async def register_schema( ), ) - # 4. Check if a reserved global row exists for this key - if data.namespace is not None: - reserved_global = ( - await session.execute( - select(CustomMetadataSchema).where( - CustomMetadataSchema.key == data.key, - CustomMetadataSchema.namespace.is_(None), - CustomMetadataSchema.reserved.is_(True), - CustomMetadataSchema.deactivated_at.is_(None), - ), - ) - ).scalar_one_or_none() - if reserved_global is not None: - raise HTTPException( - status_code=409, - detail=f"Key '{data.key}' is reserved globally and cannot be registered at namespace scope.", - ) - - node_type_val = data.node_type.value if data.node_type else None - - # Upsert on (key, node_type, namespace) — use select-then-update to handle NULLs - node_type_clause = ( - CustomMetadataSchema.node_type.is_(None) - if node_type_val is None - else CustomMetadataSchema.node_type == node_type_val - ) - namespace_clause = ( - CustomMetadataSchema.namespace.is_(None) - if data.namespace is None - else CustomMetadataSchema.namespace == data.namespace + await assert_not_reserved_globally(session, data.key, data.namespace) + + row = await upsert_schema_row( + session, + key=data.key, + namespace=data.namespace, + node_type=data.node_type.value if data.node_type else None, + json_schema=data.json_schema, + filterable=data.filterable, + description=data.description, + reserved=data.reserved, + current_user_id=current_user.id, ) - existing = ( - await session.execute( - select(CustomMetadataSchema).where( - CustomMetadataSchema.key == data.key, - node_type_clause, - namespace_clause, - CustomMetadataSchema.deactivated_at.is_(None), - ), - ) - ).scalar_one_or_none() - - if existing: - existing.json_schema = data.json_schema - existing.value_kind = _value_kind(data.json_schema) - existing.filterable = data.filterable - existing.description = data.description - existing.owner = data.owner - existing.reserved = data.reserved - existing.updated_by_id = current_user.id - row = existing - else: - row = CustomMetadataSchema( - key=data.key, - node_type=node_type_val, - namespace=data.namespace, - json_schema=data.json_schema, - value_kind=_value_kind(data.json_schema), - filterable=data.filterable, - description=data.description, - owner=data.owner, - reserved=data.reserved, - created_by_id=current_user.id, - updated_by_id=current_user.id, - ) - session.add(row) await session.commit() if row.filterable: await ensure_expression_index(session, row.key, row.value_kind) diff --git a/datajunction-server/datajunction_server/api/djsql.py b/datajunction-server/datajunction_server/api/djsql.py index 616fdde924..2206d30ee5 100644 --- a/datajunction-server/datajunction_server/api/djsql.py +++ b/datajunction-server/datajunction_server/api/djsql.py @@ -3,6 +3,8 @@ """ import logging +from dataclasses import dataclass +from typing import Literal from fastapi import Depends, Query, Request from pydantic import BaseModel @@ -49,19 +51,39 @@ class TranslatedDJSQL(BaseModel): dialect: str -def selects_from_metrics(select: ast.SelectExpression) -> bool: - """Check if a SELECT sources from the 'metrics' table.""" - return ( - select.from_ is not None - and len(select.from_.relations) == 1 - and len(select.from_.relations[0].extensions) == 0 - and str(select.from_.relations[0].primary).lower() == "metrics" - ) +@dataclass(frozen=True) +class ParsedDJSQL: + """Validated DJ SQL components and selected pseudo-table.""" + + source: Literal["metrics", "dimensions"] + metrics: list[str] + dimensions: list[str] + filters: list[str] + orderby: list[str] + limit: int | None + + +def get_pseudo_table( + select: ast.SelectExpression, +) -> Literal["metrics", "dimensions"] | None: + """Return the single unjoined pseudo-table selected by the query.""" + if ( + select.from_ is None + or len(select.from_.relations) != 1 + or select.from_.relations[0].extensions + ): + return None + table = str(select.from_.relations[0].primary).lower() + if table == "metrics": + return "metrics" + if table == "dimensions": + return "dimensions" + return None def parse_dj_sql( query: str, -) -> tuple[list[str], list[str], list[str], list[str], int | None]: +) -> ParsedDJSQL: """ Parse a DJ SQL query and extract metrics, dimensions, filters, orderby, limit. @@ -75,17 +97,17 @@ def parse_dj_sql( LIMIT 10 Returns: - Tuple of (metrics, dimensions, filters, orderby, limit) + Parsed pseudo-table and query components. Note: Validation of metric/dimension nodes is delegated to build_metrics_sql. """ tree = parse(query) select = tree.select - if not selects_from_metrics(select): + source = get_pseudo_table(select) + if source is None: raise DJInvalidInputException( - "DJ SQL queries must SELECT FROM metrics. " - "Example: SELECT metric1, dim1 FROM metrics GROUP BY dim1", + "DJ SQL queries must SELECT FROM metrics or dimensions.", ) # Validate no unsupported clauses @@ -94,21 +116,41 @@ def parse_dj_sql( "HAVING, LATERAL VIEWS, and SET OPERATIONS are not allowed in DJ SQL queries.", ) - # Extract dimensions from GROUP BY - dimensions = [str(exp) for exp in select.group_by] + group_by = [str(exp) for exp in select.group_by] - # Extract metrics: projection columns that are not in GROUP BY dimensions - # Validation that these are actual metric nodes is delegated to build_metrics_sql - metrics = [] + projected = [] for col in select.projection: - if not isinstance(col, ast.Column): + # Remove the alias first, if one was set + if isinstance(col, ast.Alias): + col = col.child + + if isinstance(col, ast.Column): + col_ident = col.identifier(False) + # A role-qualified attribute like ``dim.attr[role]`` parses as a Subscript + # wrapping a Column, and stringifies exactly like its GROUP BY counterpart. + elif isinstance(col, ast.Subscript) and isinstance(col.expr, ast.Column): + col_ident = str(col) + else: raise DJInvalidInputException( f"Only direct columns are allowed in DJ SQL queries, found: {col}", ) - col_ident = col.identifier(False) - if col_ident not in dimensions: - metrics.append(col_ident) + projected.append(col_ident) + + if source == "metrics": + dimensions = group_by + metrics = [column for column in projected if column not in dimensions] + if not metrics: + raise DJInvalidInputException( + "DJ SQL queries selecting FROM metrics require at least one metric", + ) + else: + dimensions = projected + metrics = [] + if group_by and set(group_by) != set(dimensions): + raise DJInvalidInputException( + "GROUP BY for dimension queries must match the projected dimensions", + ) # Extract filters from WHERE filters = [str(select.where)] if select.where else [] @@ -130,7 +172,7 @@ def parse_dj_sql( f"LIMIT must be an integer, got: {select.limit}", ) from exc - return metrics, dimensions, filters, orderby, limit + return ParsedDJSQL(source, metrics, dimensions, filters, orderby, limit) @router.get("/djsql/", response_model=TranslatedDJSQL) @@ -156,10 +198,13 @@ async def get_sql_for_djsql( LIMIT 10 ``` + Dimension domains use ``SELECT FROM dimensions``; ``DISTINCT`` is + implicit and ``GROUP BY`` may be omitted. + Returns the generated SQL that can be executed against your data warehouse. """ # Parse the DJ SQL query (validation delegated to build_metrics_sql) - metrics, dimensions, filters, orderby, limit = parse_dj_sql(query) + parsed = parse_dj_sql(query) # Map dialect string to enum (None means use builder default) dialect_enum: Dialect | None = None @@ -174,11 +219,11 @@ async def get_sql_for_djsql( # Build SQL using v3 builder result = await build_metrics_sql( session=session, - metrics=metrics, - dimensions=dimensions, - filters=filters, - orderby=orderby if orderby else None, - limit=limit, + metrics=parsed.metrics, + dimensions=parsed.dimensions, + filters=parsed.filters, + orderby=parsed.orderby or None, + limit=parsed.limit, dialect=dialect_enum, ) _logger.info( @@ -214,23 +259,23 @@ async def _build_djsql_query( ``/djsql/data``, ``/djsql/stream/``, ``/data/``) emit identical SQL for the same metrics + dimensions. """ - metrics, dimensions, filters, orderby, limit = parse_dj_sql(query) + parsed = parse_dj_sql(query) execution_ctx = await resolve_dialect_and_engine_for_metrics( session=session, - metrics=metrics, - dimensions=dimensions, + metrics=parsed.metrics, + dimensions=parsed.dimensions, use_materialized=use_materialized, engine_name=engine_name, engine_version=engine_version, - filters=filters if filters else None, + filters=parsed.filters or None, ) generated_sql = await build_metrics_sql( session=session, - metrics=metrics, - dimensions=dimensions, - filters=filters if filters else None, - orderby=orderby if orderby else None, - limit=limit, + metrics=parsed.metrics, + dimensions=parsed.dimensions, + filters=parsed.filters or None, + orderby=parsed.orderby or None, + limit=parsed.limit, dialect=execution_ctx.dialect, use_materialized=use_materialized, ) diff --git a/datajunction-server/datajunction_server/api/helpers.py b/datajunction-server/datajunction_server/api/helpers.py index 6659b54009..d19b5ad0f7 100644 --- a/datajunction-server/datajunction_server/api/helpers.py +++ b/datajunction-server/datajunction_server/api/helpers.py @@ -317,7 +317,9 @@ def _resolve_required_dimensions( invalid_required_dimensions: set[str] = set() matched_columns: list[Column] = [] - parent_col_map = {col.name: col for col in parent_columns} + parent_cols_by_name: dict[str, list[Column]] = {} + for col in parent_columns: + parent_cols_by_name.setdefault(col.name, []).append(col) # Separate full paths from short names # full_paths: {dim_node_name: [(full_path, col_name), ...]} @@ -337,10 +339,13 @@ def _resolve_required_dimensions( short_names.append(required_dim) for short_name in short_names: - if short_name in parent_col_map: - matched_columns.append(parent_col_map[short_name]) + matches = parent_cols_by_name.get(short_name, []) + if len(matches) == 1: + matched_columns.append(matches[0]) else: - invalid_required_dimensions.add(short_name) # pragma: no cover + # No match, or the same short name exists on more than one direct + # parent -- ambiguous, so it must be qualified as `node.column`. + invalid_required_dimensions.add(short_name) for dim_node_name, paths in full_paths.items(): dim_node = dim_nodes.get(dim_node_name) @@ -1030,6 +1035,7 @@ def get_node_revision_materialization( MaterializationConfigInfoUnified( **materialization_config_output.model_dump(), **info.model_dump(), + node_version=node_revision.version, ), ) return materializations diff --git a/datajunction-server/datajunction_server/api/namespaces.py b/datajunction-server/datajunction_server/api/namespaces.py index 1eeee1eb7f..27c76e6912 100644 --- a/datajunction-server/datajunction_server/api/namespaces.py +++ b/datajunction-server/datajunction_server/api/namespaces.py @@ -33,6 +33,9 @@ fetch_existing_yaml_map, generate_namespace_yaml_files, ) +from datajunction_server.internal.namespace_locks import ( + lock_namespace_boundary_lifecycle, +) from datajunction_server.internal.namespaces import ( create_or_reactivate_namespace, detect_parent_cycle, @@ -46,7 +49,6 @@ hard_delete_namespace, mark_namespace_deactivated, mark_namespace_restored, - namespace_boundary_scope_targets, namespaces_to_authorize, provision_namespace_boundary, resolve_git_config, @@ -55,7 +57,10 @@ ) from datajunction_server.internal.nodes import activate_node, deactivate_node from datajunction_server.models import access -from datajunction_server.models.access import ResourceAction +from datajunction_server.models.access import ( + ResourceAction, + namespace_boundary_scope_targets, +) from datajunction_server.models.deployment import ( BulkNamespaceSourcesRequest, BulkNamespaceSourcesResponse, @@ -490,19 +495,60 @@ async def hard_delete_node_namespace( is set to true. If cascade is set to false, we'll raise an error. This should be used with caution, as the impact may be large. """ - access_checker.add_namespace(namespace, ResourceAction.DELETE) - await access_checker.check(on_denied=AccessDenialMode.RAISE) - - # Only apply the default-branch guard when the namespace exists. Git config - # is inherited from ancestors, so a missing namespace under a git-backed root - # still resolves is_default_branch=True (no branch -> treated as default) and - # would wrongly 422 instead of falling through to the 404 path below. + # Hold the provisioning lock until deletion commits so a new governed + # boundary cannot appear between authorization and removing the subtree. + await lock_namespace_boundary_lifecycle(session) namespace_exists = await NodeNamespace.get( session, namespace, raise_if_not_exists=False, ) + # Hard-deleting a boundary removes its enforcement policy. + action = ( + ResourceAction.MANAGE + if namespace_exists and namespace_exists.is_governed_boundary + else ResourceAction.DELETE + ) + access_checker.add_namespace(namespace, action) + await access_checker.check(on_denied=AccessDenialMode.RAISE) + + # Every retained boundary in the deleted subtree requires MANAGE, including + # empty and deactivated descendants deleted without cascade. Explicit checks + # also cover boundaries provisioned after the request's context was loaded. + governed_boundaries = ( + ( + await session.execute( + select(NodeNamespace.namespace).where( + NodeNamespace.is_governed_boundary.is_(True), + or_( + NodeNamespace.namespace == namespace, + NodeNamespace.namespace.startswith( + f"{namespace}.", + autoescape=True, + ), + ), + ), + ) + ) + .scalars() + .all() + ) + if governed_boundaries: + boundary_checker = AccessChecker(access_checker.auth_context) + boundary_checker.add_namespaces( + list(governed_boundaries), + ResourceAction.MANAGE, + ) + await boundary_checker.check( + on_denied=AccessDenialMode.RAISE, + require_explicit_grant=True, + ) + + # Only apply the default-branch guard when the namespace exists. Git config + # is inherited from ancestors, so a missing namespace under a git-backed root + # still resolves is_default_branch=True (no branch -> treated as default) and + # would wrongly 422 instead of falling through to the 404 path below. git_info = await get_git_info_for_namespace(session, namespace) if ( namespace_exists diff --git a/datajunction-server/datajunction_server/api/semantic_layer.py b/datajunction-server/datajunction_server/api/semantic_layer.py index befe77862e..dc9737f4a7 100644 --- a/datajunction-server/datajunction_server/api/semantic_layer.py +++ b/datajunction-server/datajunction_server/api/semantic_layer.py @@ -21,7 +21,10 @@ from datajunction_server.database.user import User from datajunction_server.errors import DJException from datajunction_server.internal.access.authentication.http import SecureAPIRouter -from datajunction_server.internal.sql import generate_metrics_sql +from datajunction_server.internal.sql import ( + generate_dimensions_sql, + generate_metrics_sql, +) from datajunction_server.models.node_type import NodeType from datajunction_server.utils import get_current_user, get_session @@ -99,9 +102,30 @@ def _cube_column_type_map(cube: NodeRevision) -> dict[str, str | None]: } +def _cube_metadata_map(cube: NodeRevision) -> dict[str, dict[str, str | None]]: + """Return column metadata for the metric/dimension ids exposed by a cube.""" + return { + column.cube_element_name: {"display_name": column.display_name} + for column in cube.columns + } + + +def _generated_column_arrow_type_name(column: Any) -> str: + """Return the semantic-layer Arrow type name for a generated SQL column.""" + arrow_type = _arrow_type_name(getattr(column, "type", None)) + if arrow_type: + return arrow_type + + semantic_type = str(getattr(column, "semantic_type", "") or "").lower() + if semantic_type == "dimension": + return DIMENSION_FALLBACK_ARROW_TYPE_NAME + return METRIC_FALLBACK_ARROW_TYPE_NAME + + def _metrics_payload(cube: NodeRevision) -> list["MetricInfo"]: """Spec ``metrics`` list. ``definition`` is display-only.""" type_by_name = _cube_column_type_map(cube) + metadata_by_name = _cube_metadata_map(cube) return [ MetricInfo( id=metric_name, @@ -110,6 +134,12 @@ def _metrics_payload(cube: NodeRevision) -> list["MetricInfo"]: definition=metric_name, description=None, aggregation="OTHER", + metadata=MetricsMetadata( + display_name=metadata_by_name.get(metric_name, {}).get( + "display_name", + "", + ), + ), ) for metric_name in cube.cube_node_metrics ] @@ -118,6 +148,7 @@ def _metrics_payload(cube: NodeRevision) -> list["MetricInfo"]: def _dimensions_payload(cube: NodeRevision) -> list["DimensionInfo"]: """Spec ``dimensions`` list. Grain detection is deferred.""" type_by_name = _cube_column_type_map(cube) + metadata_by_name = _cube_metadata_map(cube) return [ DimensionInfo( id=dim_ref, @@ -126,6 +157,9 @@ def _dimensions_payload(cube: NodeRevision) -> list["DimensionInfo"]: definition=dim_ref, description=None, grain=None, + metadata=DimensionMetadata( + display_name=metadata_by_name.get(dim_ref, {}).get("display_name", ""), + ), ) for dim_ref in cube.cube_node_dimensions ] @@ -234,6 +268,12 @@ class QueryRequest(BaseModel): # --------------------------------------------------------------------------- +class MetricsMetadata(BaseModel): + """Metric-specific field metadata""" + + display_name: str + + class MetricInfo(BaseModel): """A metric exposed by a semantic view.""" @@ -243,6 +283,13 @@ class MetricInfo(BaseModel): definition: str description: str | None aggregation: str + metadata: MetricsMetadata + + +class DimensionMetadata(BaseModel): + """Dimension-specific field metadata""" + + display_name: str class DimensionInfo(BaseModel): @@ -254,6 +301,7 @@ class DimensionInfo(BaseModel): definition: str description: str | None grain: str | None + metadata: DimensionMetadata class ViewSummary(BaseModel): @@ -353,22 +401,27 @@ async def _generate_sql( request_filters = [_filter_to_sql(f) for f in payload.filters] - # Delegate to the shared metrics-SQL core (the same helper ``/sql/metrics/v3`` - # uses). Pinning the view's cube via ``matched_cube`` makes the dialect - # resolve from that cube's own availability and applies its stored - # ``cube_filters``; ``dialect=None`` lets the helper do that resolution. - generated_sql = await generate_metrics_sql( - session, - metrics=payload.metrics, - dimensions=payload.dimensions, - filters=request_filters, - matched_cube=cube_rev, - orderby=orderby, - limit=limit, - use_materialized=True, - dialect=None, - endpoint="/semantic-layer/views/sql", - ) + if payload.metrics: + generated_sql = await generate_metrics_sql( + session, + metrics=payload.metrics, + dimensions=payload.dimensions, + filters=request_filters, + matched_cube=cube_rev, + orderby=orderby, + limit=limit, + endpoint="/semantic-layer/views/sql", + ) + else: + generated_sql = await generate_dimensions_sql( + session, + dimensions=payload.dimensions, + filters=request_filters, + matched_cube=cube_rev, + orderby=orderby, + limit=limit, + endpoint="/semantic-layer/views/sql", + ) # ``generated_sql.sql`` renders via ``to_sql(query, dialect)``, which already # applies DJ's dialect rules and transpiles to the resolved dialect, so it is # execution-ready for the caller (which runs it directly). We return the @@ -377,7 +430,10 @@ async def _generate_sql( return GeneratedSQLResponse( sql=generated_sql.sql, dialect=generated_sql.dialect.value, - columns=[ColumnInfo(name=col.name, type=str(col.type)) for col in columns], + columns=[ + ColumnInfo(name=col.name, type=_generated_column_arrow_type_name(col)) + for col in columns + ], cube_name=generated_sql.cube_name, ) @@ -403,12 +459,8 @@ async def generate_query_sql( 400, f"`limit` {payload.limit} exceeds the maximum of {MAX_ROW_LIMIT}.", ) - if not payload.metrics: - return _problem( - 400, - "A query must request at least one metric. Dimension-only queries " - "(distinct values) are not supported on this endpoint.", - ) + if not payload.metrics and not payload.dimensions: + return _problem(400, "A query must request at least one metric or dimension.") try: cube_node = await Node.get_cube_by_name(session, view_name) if cube_node is None or cube_node.current is None: diff --git a/datajunction-server/datajunction_server/api/sql.py b/datajunction-server/datajunction_server/api/sql.py index 3e1fca9387..d39607b698 100644 --- a/datajunction-server/datajunction_server/api/sql.py +++ b/datajunction-server/datajunction_server/api/sql.py @@ -30,7 +30,10 @@ QueryCacheManager, QueryRequestParams, ) -from datajunction_server.internal.sql import generate_metrics_sql +from datajunction_server.internal.sql import ( + generate_dimensions_sql, + generate_metrics_sql, +) from datajunction_server.models.dialect import Dialect from datajunction_server.models.metric import TranslatedSQL, V3TranslatedSQL from datajunction_server.models.node_type import NodeType @@ -566,6 +569,68 @@ async def get_combined_measures_sql_v3( ) +@router.get( + "/sql/dimensions/v3/", + response_model=V3TranslatedSQL, + name="Get Dimensions SQL V3", + tags=["sql", "v3"], +) +async def get_dimensions_sql_v3( + dimensions: list[str] = Query([]), + filters: list[str] = Query( + [], + description="Filters layered on top of any stored cube filters", + ), + cube: str | None = Query( + None, + description="Optional cube whose metrics define the reachable value domain", + ), + orderby: list[str] = Query([]), + limit: int | None = Query(None), + use_materialized: bool = Query(True), + dialect: Dialect | None = Query(None), + query_params: str = Query("{}", description="Query parameters"), + *, + session: AsyncSession = Depends(get_session), + current_user: User = Depends(get_current_user), +) -> V3TranslatedSQL: + """Generate distinct dimension SQL. + + Without a cube, queries the dimension-bearing node directly. With a cube, + uses all cube metrics and combines stored and request filters with ``AND``. + """ + if not dimensions: + raise DJInvalidInputException("At least one dimension is required") + + result = await generate_dimensions_sql( + session, + dimensions=dimensions, + filters=filters, + cube=cube, + orderby=orderby or None, + limit=limit, + use_materialized=use_materialized, + dialect=dialect, + query_parameters=json.loads(query_params) or None, + ) + return V3TranslatedSQL( + sql=result.sql, + columns=[ + V3ColumnMetadata( + name=col.name, + type=str(col.type), + semantic_name=col.semantic_name, + semantic_type=col.semantic_type, + ) + for col in result.columns + ], + dialect=result.dialect, + cube_name=result.cube_name, + scan_estimate=result.scan_estimate, + warnings=result.warnings, + ) + + @router.get( "/sql/metrics/v3/", response_model=V3TranslatedSQL, @@ -644,6 +709,9 @@ async def get_metrics_sql_v3( Set to False when generating SQL for materialization refresh to avoid circular references. """ + if not metrics and not cube: + raise DJInvalidInputException("At least one metric is required") + # Shared metrics-SQL core (cube pinning, cube_filters prepend, dialect # auto-resolve, build_metrics_sql, and the build-latency metrics + [SQL] log). # Also used by the semantic-layer endpoint. diff --git a/datajunction-server/datajunction_server/api/tags.py b/datajunction-server/datajunction_server/api/tags.py index 4dc9685ad2..185a270474 100644 --- a/datajunction-server/datajunction_server/api/tags.py +++ b/datajunction-server/datajunction_server/api/tags.py @@ -3,9 +3,10 @@ """ from collections.abc import Callable +from http import HTTPStatus -from fastapi import Depends -from sqlalchemy import select +from fastapi import Depends, Response +from sqlalchemy import func, select from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import noload @@ -14,7 +15,11 @@ from datajunction_server.database.history import History from datajunction_server.database.tag import Tag, TagNodeRelationship from datajunction_server.database.user import User -from datajunction_server.errors import DJAlreadyExistsException, DJDoesNotExistException +from datajunction_server.errors import ( + DJActionNotAllowedException, + DJAlreadyExistsException, + DJDoesNotExistException, +) from datajunction_server.internal.access.authentication.http import SecureAPIRouter from datajunction_server.internal.history import ActivityType, EntityType from datajunction_server.models.node import NodeMinimumDetail @@ -63,7 +68,7 @@ async def get_tag_by_name( ) tag = (await session.execute(statement)).scalars().one_or_none() if not tag and raise_if_not_exists: - raise DJDoesNotExistException( # pragma: no cover + raise DJDoesNotExistException( message=(f"A tag with name `{name}` does not exist."), http_status_code=404, ) @@ -193,6 +198,54 @@ async def update_a_tag( return tag +@router.delete("/tags/{name}/", status_code=HTTPStatus.NO_CONTENT) +async def delete_a_tag( + name: str, + session: AsyncSession = Depends(get_session), + current_user: User = Depends(get_current_user), + save_history: Callable = Depends(get_save_history), +): + """ + Delete a tag. Only tags with no nodes attached can be deleted. + """ + tag = await get_tag_by_name( + session, + name, + raise_if_not_exists=True, + for_update=True, + ) + attached_nodes = ( + await session.execute( + select(func.count()) + .select_from(TagNodeRelationship) + .join(Node, Node.id == TagNodeRelationship.node_id) + .where(TagNodeRelationship.tag_id == tag.id) + .where(Node.deactivated_at.is_(None)), + ) + ).scalar_one() + if attached_nodes: + raise DJActionNotAllowedException( + message=( + f"Cannot delete tag `{name}` as it is still attached to " + f"{attached_nodes} node(s). Remove the tag from these nodes first." + ), + http_status_code=HTTPStatus.CONFLICT, + ) + + await save_history( + event=History( + entity_type=EntityType.TAG, + entity_name=tag.name, + activity_type=ActivityType.DELETE, + user=current_user.username, + ), + session=session, + ) + await session.delete(tag) + await session.commit() + return Response(status_code=HTTPStatus.NO_CONTENT) + + @router.get("/tags/{name}/nodes/", response_model=list[NodeMinimumDetail]) async def list_nodes_for_a_tag( name: str, diff --git a/datajunction-server/datajunction_server/construction/build_v3/builder.py b/datajunction-server/datajunction_server/construction/build_v3/builder.py index 9b1b5fa7a3..403641d3c0 100644 --- a/datajunction-server/datajunction_server/construction/build_v3/builder.py +++ b/datajunction-server/datajunction_server/construction/build_v3/builder.py @@ -270,7 +270,6 @@ async def setup_build_context( use_materialized: Whether to use materialized tables include_temporal_filters: Whether to include temporal partition filters from cube lookback_window: Lookback window for temporal filters - Returns: Fully initialized BuildContext """ @@ -311,9 +310,8 @@ async def setup_build_context( # Load all required nodes (metrics + explicit dimensions + filter dimensions) await load_nodes(ctx) - # Validate we have at least one metric if not ctx.metrics: - raise DJInvalidInputException("At least one metric is required") + return ctx # Decompose metrics and group by parent node ctx.metric_groups, ctx.decomposed_metrics = await decompose_and_group_metrics(ctx) @@ -384,6 +382,9 @@ async def build_measures_sql( GeneratedMeasuresSQL with one GrainGroupSQL per aggregation level, plus context and decomposed metrics for efficient reuse by build_metrics_sql """ + if not metrics: + raise DJInvalidInputException("At least one metric is required") + # Setup context (loads nodes, decomposes metrics, adds dimensions from expressions) ctx = await setup_build_context( session=session, @@ -508,6 +509,8 @@ async def build_metrics_sql( """ Build metrics SQL for a set of metrics and dimensions. + With no metrics, returns distinct attributes from one dimension-bearing node. + Metrics SQL applies final metric expressions on top of measures, including handling derived metrics. It produces a single executable query with the following layers: @@ -569,6 +572,15 @@ async def build_metrics_sql( lookback_window=lookback_window, ) + if not metrics: + # Imported lazily because node_query also uses the shared ordering helper + # from this module. + from datajunction_server.construction.build_v3.node_query import ( + build_dimension_sql_v3, + ) + + return build_dimension_sql_v3(ctx, orderby, limit, query_parameters) + # Frame-aware live window lookback: when a requested metric carries a row/ # range window frame and a filter narrows the order dimension, expand the # scan to feed the frame and return a plan carrying the output restriction diff --git a/datajunction-server/datajunction_server/construction/build_v3/cte.py b/datajunction-server/datajunction_server/construction/build_v3/cte.py index 2e26ca7ff6..8a5b93e3db 100644 --- a/datajunction-server/datajunction_server/construction/build_v3/cte.py +++ b/datajunction-server/datajunction_server/construction/build_v3/cte.py @@ -20,7 +20,10 @@ GrainGroupSQL, PushdownFilters, ) -from datajunction_server.construction.build_v3.utils import get_cte_name +from datajunction_server.construction.build_v3.utils import ( + column_table_name, + get_cte_name, +) from datajunction_server.database.node import Node from datajunction_server.models.node_type import NodeType from datajunction_server.sql.decompose import wrap_divisions_in_nullif @@ -800,65 +803,431 @@ def filter_cte_projection( if not query_ast.select.projection: # pragma: no cover return query_ast - projection = query_ast.select.projection + _apply_keep_positions( + query_ast.select, + _keep_positions(query_ast.select, columns_to_select), + ) + return query_ast + + +def _is_star(expr: object) -> bool: + """Whether a projection entry is ``*`` or ``.*``.""" + if isinstance(expr, ast.Wildcard): + return True + return isinstance(expr, ast.Column) and str(expr.name.name) == "*" + - def _col_name(expr: object) -> str | None: - if isinstance(expr, ast.Alias): - return str(expr.alias.name) if expr.alias else None - if isinstance(expr, ast.Column): - return str(expr.alias.name) if expr.alias else str(expr.name.name) +def _projection_name(expr: object) -> str | None: + """The output name of one projection entry, or ``None`` if it has none.""" + if isinstance(expr, ast.Alias): + return str(expr.alias.name) if expr.alias else None + if _is_star(expr): + # A star stands for whatever its source has; it names nothing itself. return None + if isinstance(expr, ast.Column): + return str(expr.alias.name) if expr.alias else str(expr.name.name) + return None + + +def _output_alias_clauses(select: ast.SelectExpression) -> list[ast.Node]: + """ + The clauses that may name the select's own output, one expression each. + + GROUP BY, HAVING and ORDER BY can all address a projection entry by the + alias it is given or by its position. WHERE deliberately does not appear: + a column it uses resolves against the FROM, not against the projection, so + a column only the WHERE reads is still free to go. + """ + parts: list[ast.Node] = list(select.group_by) + if select.having is not None: + parts.append(select.having) + if select.organization is not None: + parts.extend(item.expr for item in select.organization.order) + parts.extend(item.expr for item in select.organization.sort) + return parts + + +def _keep_positions( + select: ast.SelectExpression, + columns_to_select: set[str], +) -> set[int]: + """ + The 1-indexed projection positions a select must keep. + + A position survives when its output name is wanted downstream, when a + clause that can address the projection names or points at it, or when it + has no readable output name. Names are matched without regard to case, + the way every dialect DJ targets resolves them. + + Under DISTINCT every position survives: the projection is the dedup key, so + narrowing it folds together rows that were distinct and silently changes + the row count. + """ + projection = select.projection + if (select.quantifier or "").upper() == "DISTINCT": + return set(range(1, len(projection) + 1)) - # Protect columns that GROUP BY depends on so we never prune them. - # Positional references (integers) protect the column at that SELECT position; - # named references protect by column name. - effective_cols = set(columns_to_select) - positional_refs: set[int] = set() # 1-indexed positions from GROUP BY - for item in query_ast.select.group_by: + effective_cols = {name.casefold() for name in columns_to_select} + for item in _output_alias_clauses(select): if isinstance(item, ast.Number) and isinstance(item.value, int): pos = int(item.value) if 1 <= pos <= len(projection): - positional_refs.add(pos) - if name := _col_name(projection[pos - 1]): - effective_cols.add(name) - elif isinstance(item, ast.Column): - name = str(item.alias.name) if item.alias else str(item.name.name) - effective_cols.add(name) - - # Build the filtered projection, tracking old to new position for renumbering. + if name := _projection_name(projection[pos - 1]): + effective_cols.add(name.casefold()) + continue + for column in item.find_all(ast.Column): + named = column.alias.name if column.alias else column.name.name + effective_cols.add(str(named).casefold()) + + keep: set[int] = set() + for i, expr in enumerate(projection): + col_name = _projection_name(expr) + if col_name is None or col_name.casefold() in effective_cols: + keep.add(i + 1) + return keep + + +def _apply_keep_positions(select: ast.SelectExpression, keep: set[int]) -> None: + """ + Drop projection entries outside ``keep`` and renumber positional references. + + GROUP BY and ORDER BY positions are renumbered rather than rewritten to + alias names, which is invalid in dialects like Trino. Left alone they would + quietly point at whichever column slid into that slot. An empty result + leaves the projection alone: an empty SELECT is never an improvement. + """ new_projection = [] old_to_new: dict[int, int] = {} # 1-indexed old pos to 1-indexed new pos - for i, expr in enumerate(projection): - old_pos = i + 1 - col_name = _col_name(expr) - if col_name is None: # pragma: no cover - # Keep expressions we can't analyze (defensive) + for i, expr in enumerate(select.projection): + if i + 1 in keep: new_projection.append(expr) - old_to_new[old_pos] = len(new_projection) - elif col_name in effective_cols: - new_projection.append(expr) - old_to_new[old_pos] = len(new_projection) - # else: pruned — no mapping entry + old_to_new[i + 1] = len(new_projection) - # If we filtered everything, keep original (shouldn't happen) if new_projection: - query_ast.select.projection = new_projection - - # Renumber positional GROUP BY references if any columns were removed before them. - if positional_refs: - new_group_by: list[ast.Expression] = [] - for item in query_ast.select.group_by: - if isinstance(item, ast.Number) and isinstance(item.value, int): - old_pos = int(item.value) - new_pos = old_to_new.get(old_pos, old_pos) - new_group_by.append( - ast.Number(value=new_pos) if new_pos != old_pos else item, + select.projection = new_projection + + new_group_by: list[ast.Expression] = [] + for item in select.group_by: + if isinstance(item, ast.Number) and isinstance(item.value, int): + old_pos = int(item.value) + new_pos = old_to_new.get(old_pos, old_pos) + new_group_by.append( + ast.Number(value=new_pos) if new_pos != old_pos else item, + ) + else: + new_group_by.append(item) + select.group_by = new_group_by + + if select.organization is not None: + for sort_item in (*select.organization.order, *select.organization.sort): + position = sort_item.expr + if isinstance(position, ast.Number) and isinstance(position.value, int): + position.value = old_to_new.get( + int(position.value), + int(position.value), ) - else: - new_group_by.append(item) - query_ast.select.group_by = new_group_by - return query_ast + +def _set_op_arms(select: ast.SelectExpression) -> list[ast.SelectExpression]: + """Each arm of a set operation, or the select itself when there is none.""" + arms = [] + arm: ast.SelectExpression | None = select + while arm is not None: + arms.append(arm) + arm = arm.set_op.right if arm.set_op else None + return arms + + +def _compares_whole_rows(select: ast.SelectExpression) -> bool: + """ + Whether this select picks its rows by comparing them against each other. + + DISTINCT does — it keeps a row only if no earlier row equals it. So does + every set operation but UNION ALL: UNION, INTERSECT, EXCEPT, EXCEPT ALL + and MINUS all decide a row's fate by looking at the other arm's rows. + UNION ALL alone just concatenates, and is the one safe kind. Read per + arm, so an arm that is itself DISTINCT under a UNION ALL counts too. + + This matters because pruning rests on an unread column being free to + drop, and a row-to-row comparison uses every column projected, read or + not. Drop ``drop_me`` from + + SELECT grp, keep, drop_me FROM t WHERE id <= 2 + UNION + SELECT grp, keep, drop_me FROM t WHERE id >= 3 + + and two rows fold into one, so SUM(keep) answers 30 where it owes 40. + Nothing raises; the number just gets quieter. + + Callers put such a scope in ``unprunable``, which keeps its projection + whole and stops a narrowed demand reaching a source it stars. + """ + return any( + (arm.quantifier or "").upper() == "DISTINCT" + or ( + arm.set_op is not None + and " ".join((arm.set_op.kind or "").upper().split()) != "UNION ALL" + ) + for arm in _set_op_arms(select) + ) + + +def _arm_parts(arm: ast.SelectExpression) -> list[ast.Node]: + """The clauses of one arm, excluding the arms unioned after it.""" + return [ + *arm.projection, + *arm.group_by, + *arm.lateral_views, + *([arm.from_] if arm.from_ else []), + *([arm.where] if arm.where else []), + *([arm.having] if arm.having else []), + *([arm.organization] if arm.organization else []), + ] + + +def _table_alias(table: ast.Table) -> ast.Name | None: + """The alias a table carries, whether on the table itself or wrapping it.""" + if table.alias: + return table.alias + if isinstance(table.parent, ast.Alias): + return table.parent.alias + return None + + +def _joins_naturally(arm: ast.SelectExpression) -> bool: + """Whether the arm has a NATURAL JOIN.""" + return any(join.natural for join in arm.find_all(ast.Join)) + + +def _enclosing_selects( + node: ast.Node, + outermost: ast.SelectExpression, +) -> list[ast.SelectExpression]: + """The selects a node sits in, innermost first, ending at outermost.""" + chain: list[ast.SelectExpression] = [] + current = node.parent + while current is not None: + if isinstance(current, ast.SelectExpression): + chain.append(current) + if current is outermost: + break + current = current.parent + return chain + + +def _select_bindings( + arm: ast.SelectExpression, + cte_names: set[str], +) -> tuple[dict[ast.SelectExpression, dict[str, str]], set[str]]: + """ + Map each select to its CTE bindings, and the CTEs the statement reads. + + A select's map covers only its own FROM, keyed by alias and table name, so + a subquery reusing an alias shadows the outer binding instead of + overwriting it: + + FROM a AS x WHERE EXISTS (SELECT 1 FROM b AS x ...) + {outer: {"a": "a", "x": "a"}, EXISTS: {"b": "b", "x": "b"}} + """ + bindings: dict[ast.SelectExpression, dict[str, str]] = {arm: {}} + read: set[str] = set() + for part in _arm_parts(arm): + for table in part.find_all(ast.Table): + name = table.name.identifier(quotes=False) + if name not in cte_names: + continue + read.add(name) + owner = _enclosing_selects(table, arm) + # Detached tables go in the outermost map. + bound = bindings.setdefault(owner[0] if owner else arm, {}) + bound[name] = name + if alias := _table_alias(table): + bound[alias.identifier(quotes=False)] = name + return bindings, read + + +def _record_column_use( + node: ast.Node, + qualifier: str | None, + wanted: str, + arm: ast.SelectExpression, + bindings: dict[ast.SelectExpression, dict[str, str]], + read: set[str], + live: dict[str, set[str]], +) -> None: + """ + Resolve a column to its CTE and mark it live. + """ + if qualifier is not None: + for select in _enclosing_selects(node, arm): + bound = bindings.get(select, {}).get(qualifier) + if bound is not None: + live[bound].add(wanted) + return + for target in read: + live[target].add(wanted) + + +def _record_select_uses( + arm: ast.SelectExpression, + bindings: dict[ast.SelectExpression, dict[str, str]], + read: set[str], + live: dict[str, set[str]], +) -> None: + """ + Resolve every column a select reads to its CTE and mark it live. + + The column wanted is the segment right after the qualifier -- for a struct + path like p.line_item.target_sets the producer projects line_item, not the + leaf. + + A qualifier reaches us two ways. Dotted into the name is the parsed form, + and it can carry further struct segments. Hung off the column as a table is + what the builder produces for its own projections and GROUP BY, and there + the column name stands alone. + + When a column is referenced in USING, both sides of the join must project + it. + """ + for part in _arm_parts(arm): + for column in part.find_all(ast.Column): + segments = column.identifier(quotes=False).split(SEPARATOR) + qualifier = segments[0] if len(segments) > 1 else None + wanted = segments[1] if len(segments) > 1 else segments[0] + if qualifier is None: + qualifier = column_table_name(column) + _record_column_use( + column, + qualifier, + wanted, + arm, + bindings, + read, + live, + ) + for criteria in part.find_all(ast.JoinCriteria): + for joined in criteria.using or []: + _record_column_use( + criteria, + None, + joined.identifier(quotes=False), + arm, + bindings, + read, + live, + ) + + +def _scopes_readers_first( + scopes: dict[str, ast.Query], + cte_names: set[str], +) -> list[str]: + """ + Topologically sort the scopes based on read order. + + A CTE is only reached once every scope reading it has been, so the demand + on it is complete by then. Pruning a reader only narrows what it asks of + its own sources, so one pass in this order settles every projection. + + Returns the CTE names plus ``""`` for the outer select. No scope reads + the outer select, so it always comes first. + """ + reads: dict[str, set[str]] = {} + for key, scope in scopes.items(): + producers: set[str] = set() + for arm in _set_op_arms(scope.select): + producers.update(_select_bindings(arm, cte_names)[1]) + producers.discard(key) + reads[key] = producers + + remaining = dict.fromkeys(scopes, 0) + for producers in reads.values(): + for producer in producers: + remaining[producer] += 1 + + ready = [key for key, count in remaining.items() if not count] + order: list[str] = [] + while ready: + key = ready.pop() + order.append(key) + for producer in reads[key]: + remaining[producer] -= 1 + if not remaining[producer]: + ready.append(producer) + + if len(order) < len(scopes): # pragma: no cover + # A reference cycle among CTEs; fall back to declaration order reversed. + settled = set(order) + order.extend(key for key in reversed(list(scopes)) if key not in settled) + return order + + +def prune_cte_projections(query: ast.Query) -> None: + """ + Prune unused columns from the query, once it's fully assembled. + + 1. Topologically sort the scopes based on read order (the CTEs plus the + outer select, keyed ``""``, which is the answer and so never pruned), + readers before what they read — :func:`_scopes_readers_first`. + 2. Walk that order once, keeping two records: ``live[cte]``, columns used + by a downstream scope, and ``unprunable``, for CTEs that cannot be + pruned. + 3. At each scope: mark it unprunable if it compares whole rows + (:func:`_compares_whole_rows`), prune to what it actually needs, then + add the columns it reads to each of its sources' ``live``. + + Pruning a scope before recording its reads is what makes one pass enough: + trimming a reader narrows what it goes on to ask of the CTEs beneath it. + Using ``*`` makes its source unprunable, unless the star's own scope has + demand to pass down. A reference whose qualifier names no CTE in scope is + added to every CTE in scope — which one supplies it needs compilation to + know, and asking a CTE for a name it has not got is a no-op. + """ + ctes = {cte.alias_or_name.name: cte for cte in query.ctes} + if not ctes: + return + cte_names = set(ctes) + # The outer select is a scope too, and the only one nothing else reads. + scopes: dict[str, ast.Query] = {"": query, **ctes} + + live: dict[str, set[str]] = {name: set() for name in cte_names} + unprunable: set[str] = set() + + for key in _scopes_readers_first(scopes, cte_names): + scope = scopes[key] + if key and _compares_whole_rows(scope.select): + # Its row is its match key, so it also can't narrow what it stars. + unprunable.add(key) + if key and live[key] and key not in unprunable: + arms = _set_op_arms(scope.select) + if len(arms) == 1: + filter_cte_projection(scope, live[key]) + else: + # Arms are unioned by position, so they all keep the same ones. + keep: set[int] = set() + for arm in arms: + keep.update(_keep_positions(arm, live[key])) + for arm in arms: + _apply_keep_positions(arm, keep) + + for arm in _set_op_arms(scope.select): + bindings, read = _select_bindings(arm, cte_names) + _record_select_uses(arm, bindings, read, live) + if _joins_naturally(arm): + # Its join key is the columns both sides share. + unprunable.update(read) + if not any(_is_star(item) for item in arm.projection): + continue + # A star names no columns of its own: it re-exposes whatever its + # source has. So a CTE that stars a source passes its own demand + # straight through. Where that demand is unknown — the outer + # select, or a CTE nobody reads — the source keeps everything. + starred = set(read) + if key and key not in unprunable and live[key]: + for target in starred: + live[target].update(live[key]) + else: + unprunable.update(starred) def flatten_inner_ctes( @@ -2072,7 +2441,6 @@ def _build_select_projection_map( def collect_node_ctes( ctx: BuildContext, nodes_to_include: list[Node], - needed_columns_by_node: dict[str, set[str]] | None = None, injected_filters: dict[str, ast.Expression] | None = None, pushdown: PushdownFilters | None = None, ) -> tuple[list[tuple[str, ast.Query]], list[str], dict[str, set[str]]]: @@ -2091,8 +2459,6 @@ def collect_node_ctes( Args: ctx: Build context nodes_to_include: List of nodes to create CTEs for - needed_columns_by_node: Optional dict of node_name -> set of column names - If provided, CTEs will only select the needed columns. injected_filters: Optional dict of node_name -> filter expression to inject as a WHERE clause into that node's CTE. Used to push temporal partition filters down into upstream CTEs (e.g. a date-spine) rather than applying @@ -2208,14 +2574,6 @@ def collect_refs(node: Node, visited: set[str]) -> None: inner_cte_renames, ) - # Apply column filtering if specified - needed_cols = None - if needed_columns_by_node: # pragma: no branch - needed_cols = needed_columns_by_node.get(node.name) - - if needed_cols and not _cte_has_set_operation(query_ast): # pragma: no branch - query_ast = filter_cte_projection(query_ast, needed_cols) - # Inject filters into this CTE's WHERE clause from two sources: # (1) Temporal/explicit filters targeted at this node by name # (2) User dimension filters that reference columns this CTE outputs diff --git a/datajunction-server/datajunction_server/construction/build_v3/cube_matcher.py b/datajunction-server/datajunction_server/construction/build_v3/cube_matcher.py index 07fead205c..2314aa0b5e 100644 --- a/datajunction-server/datajunction_server/construction/build_v3/cube_matcher.py +++ b/datajunction-server/datajunction_server/construction/build_v3/cube_matcher.py @@ -303,8 +303,8 @@ async def resolve_dialect_and_engine_for_metrics( matching cube with availability exists: - Use the cube's availability catalog's engine matching dialect_override (or the first engine if none match) - 2. Otherwise, fall back to the first metric's catalog's default engine, - filtered by dialect_override if provided + 2. Otherwise, fall back to the first metric's catalog, or for a metricless + query the first dimension node's catalog, filtered by dialect_override. Args: session: Database session @@ -383,16 +383,19 @@ async def resolve_dialect_and_engine_for_metrics( cube=cube, ) - if not metrics: + if not metrics and not dimensions: raise DJInvalidInputException( - "At least one metric is required.", + "At least one metric or dimension is required.", http_status_code=422, ) - # Fallback: use first metric's catalog's default engine + # Fall back to the first requested metric or dimension's catalog. + anchor_name = ( + metrics[0] if metrics else parse_dimension_ref(dimensions[0]).node_name + ) node = await Node.get_by_name( session, - metrics[0], + anchor_name, raise_if_not_exists=True, options=[ joinedload(Node.current).options( @@ -403,11 +406,11 @@ async def resolve_dialect_and_engine_for_metrics( ], ) if not node: # pragma: no cover - raise ValueError(f"Metric not found: {metrics[0]}") + raise ValueError(f"Query node not found: {anchor_name}") catalog_name = node.current.catalog.name if node.current.catalog else None if not catalog_name: # pragma: no cover - raise ValueError(f"Metric {metrics[0]} has no catalog") + raise ValueError(f"Query node {anchor_name} has no catalog") # Resolve engine: when no dialect is explicitly requested, prefer Trino. # Fall back to the catalog's first engine if no Trino engine exists. @@ -430,10 +433,10 @@ async def resolve_dialect_and_engine_for_metrics( Dialect(engine.dialect) if engine.dialect else Dialect.TRINO ) logger.info( - "[BuildV3] Resolved dialect=%s engine=%s from metric %s catalog=%s", + "[BuildV3] Resolved dialect=%s engine=%s from query node %s catalog=%s", dialect, engine.name, - metrics[0], + anchor_name, catalog_name, ) @@ -524,6 +527,9 @@ async def build_sql_from_cube( Returns: GeneratedSQL with the query and column metadata. """ + if not metrics: + raise DJInvalidInputException("At least one metric is required") + # Import here to avoid circular dependency from datajunction_server.construction.build_v3.builder import setup_build_context diff --git a/datajunction-server/datajunction_server/construction/build_v3/measures.py b/datajunction-server/datajunction_server/construction/build_v3/measures.py index 5fceb1de64..f6912654ca 100644 --- a/datajunction-server/datajunction_server/construction/build_v3/measures.py +++ b/datajunction-server/datajunction_server/construction/build_v3/measures.py @@ -20,8 +20,8 @@ _fk_key_column_names, collect_node_ctes, extract_dimension_node, - get_table_references_from_ast, inject_filter_into_select, + prune_cte_projections, references_filter_only_dimension, strip_role_suffix, ) @@ -64,8 +64,7 @@ ResolvedDimension, ) from datajunction_server.construction.build_v3.utils import ( - extract_columns_from_expression, - extract_columns_referenced_from_node, + column_table_name, get_column_type, get_cte_name, get_short_name, @@ -101,31 +100,23 @@ def _rewrite_col_refs(expr: Any, table_alias: str) -> None: def _resolve_dim_namespace_refs( expressions: list[ast.Expression], dim_node_to_alias: dict[str, str], -) -> dict[str, set[str]]: +) -> None: """Resolve dim-namespaced column refs in metric/grain expressions. A metric or grain expression may contain a fully-qualified column like ``v3.customer.tier`` referring to a column on a dimension node that the - parent fact joins to. In one walk per expression, this function: - - 1. Rewrites the namespace from the dim node name to the dim's joined - table alias (so the renderer emits ``t2.tier`` instead of the literal - node name, which is not a valid table reference). - 2. Returns ``{dim_node_name: {col, ...}}`` so the caller can keep those - columns in the dim's CTE projection (otherwise filter_cte_projection - drops them as unused). + parent fact joins to. Rewrites the namespace from the dim node name to the + dim's joined table alias, so the renderer emits ``t2.tier`` instead of the + literal node name, which is not a valid table reference. """ - dim_cols: dict[str, set[str]] = {} for expr in expressions: for nc in iter_namespaced_columns(expr): if nc.node not in dim_node_to_alias: continue - dim_cols.setdefault(nc.node, set()).add(nc.name) nc.column.name = ast.Name( nc.name, namespace=ast.Name(dim_node_to_alias[nc.node]), ) - return dim_cols # Mapping from type string to ColumnType instance @@ -328,37 +319,6 @@ def add_prefixes(node: ast.Expression) -> None: add_prefixes(filter_ast) -def extract_join_columns_for_node(join_sql: str, node_name: str) -> set[str]: - """ - Extract column names from join SQL that belong to a specific node. - - Parses the join_sql (e.g., "v3.order_details.customer_id = v3.customer.customer_id") - and returns the short column names for columns belonging to the given node. - - Args: - join_sql: The join condition SQL string - node_name: The fully qualified node name to filter by - - Returns: - Set of short column names (e.g., {"customer_id"}) - - Examples: - extract_join_columns_for_node( - "v3.order_details.customer_id = v3.customer.customer_id", - "v3.order_details" - ) -> {"customer_id"} - """ - result: set[str] = set() - join_expr = parse(f"SELECT 1 WHERE {join_sql}").select.where - if join_expr: # pragma: no branch - prefix = node_name + SEPARATOR - for col in join_expr.find_all(ast.Column): - col_id = col.identifier() - if col_id.startswith(prefix): - result.add(get_short_name(col_id)) - return result - - def get_dimension_table_alias( resolved_dim: ResolvedDimension, main_alias: str, @@ -393,81 +353,26 @@ def get_dimension_table_alias( return main_alias # pragma: no cover -def collect_cte_nodes_and_needed_columns( +def collect_cte_nodes( ctx: BuildContext, parent_node: Node, resolved_dimensions: list[ResolvedDimension], - grain_col_specs: list[tuple[ast.Expression, str]], - metric_expressions: list[tuple[str, ast.Expression]], -) -> tuple[list[Node], dict[str, set[str]]]: +) -> list[Node]: """ - Determine which nodes need CTEs and the minimal set of columns each must project. - - Returns a tuple of: - - nodes_for_ctes: ordered list of non-source nodes that require CTEs - - needed_columns_by_node: mapping of node name -> set of column names that - must remain in that node's CTE projection after filter_cte_projection runs - - The needed columns for each node are gathered from: - - For parent_node: local dimension columns, grain columns, metric expression - columns, join key columns, and temporal partition columns - - For each dimension node: the requested dimension attribute, join key columns, - and any columns referenced from that node in parent_node's or other dimension - nodes' SQL (including aliased references like CROSS JOIN node AS alias) - """ - nodes_for_ctes: list[Node] = [] - needed_columns_by_node: dict[str, set[str]] = {} - - # Collect columns needed from parent node - parent_needed_cols: set[str] = set() - - # Add local dimension columns - for resolved_dim in resolved_dimensions: - if resolved_dim.is_local: - parent_needed_cols.add(resolved_dim.column_name) - - # Add grain columns for LIMITED aggregability. - # For complex expressions, extract the actual leaf columns they reference. - for gc_expr, _ in grain_col_specs: - if isinstance(gc_expr, ast.Column): - parent_needed_cols.add(gc_expr.name.name) - else: - parent_needed_cols.update(extract_columns_from_expression(gc_expr)) + Determine which nodes need CTEs. - # Add columns from metric expressions - for _, expr in metric_expressions: - parent_needed_cols.update(extract_columns_from_expression(expr)) - - # Add join key columns (from the left side of joins) - for resolved_dim in resolved_dimensions: - if resolved_dim.join_path: - for link in resolved_dim.join_path.links: - if link.join_sql: # pragma: no branch - parent_needed_cols.update( - extract_join_columns_for_node(link.join_sql, parent_node.name), - ) + Returns the ordered list of non-source nodes that require CTEs: the parent + node, plus every dimension node sitting on a resolved dimension's join path + (including intermediate hops, which a multi-hop chain routes through). - # Add temporal partition columns from cube if linked to this parent - # This ensures the columns are available in the CTE for the WHERE clause - if ctx.temporal_partition_columns and parent_node.current: - for partition_col_ref in ctx.temporal_partition_columns: - dimension_ref = parse_dimension_ref(partition_col_ref) - - # Match the exact dimension link. The same parent can reach one - # dimension node through multiple roles. - if parent_node.current.dimension_links: # pragma: no branch - for link in parent_node.current.dimension_links: # pragma: no branch - if ( - link.dimension.name == dimension_ref.node_name - and (link.role or None) == dimension_ref.role - ): - parent_needed_cols.add(dimension_ref.column_name) - break + What each CTE has to project is settled later, by + :func:`prune_cte_projections` on the assembled query. + """ + nodes_for_ctes: list[Node] = [] # Parent node needs CTE if it's not a source if parent_node.type != NodeType.SOURCE: # pragma: no branch nodes_for_ctes.append(parent_node) - needed_columns_by_node[parent_node.name] = parent_needed_cols # Dimension nodes from joins need CTEs for resolved_dim in resolved_dimensions: @@ -479,155 +384,7 @@ def collect_cte_nodes_and_needed_columns( if dim_node not in nodes_for_ctes: nodes_for_ctes.append(dim_node) - # Collect needed columns for this dimension - dim_cols: set[str] = set() - - # Add the dimension column being selected - if resolved_dim.join_path.target_node_name == dim_node.name: - dim_cols.add(resolved_dim.column_name) - - # Add join key columns from this dimension (right side of this link) - if link.join_sql: # pragma: no branch - dim_cols.update( - extract_join_columns_for_node(link.join_sql, dim_node.name), - ) - - # Case 1: parent_node's query directly selects from dim_node - nodes_to_scan: list[Node] = [] - if parent_node.current and parent_node.current.query: - nodes_to_scan.append(parent_node) - # Case 2: another dimension node's query references dim_node - for other_rdim in resolved_dimensions: - if other_rdim.join_path: - for other_link in other_rdim.join_path.links: - other_dim = ctx.nodes.get( - other_link.dimension.name, - other_link.dimension, - ) - if ( - other_dim - and other_dim.name != dim_node.name - and other_dim.type != NodeType.SOURCE - and other_dim.current - and other_dim.current.query - and other_dim not in nodes_to_scan - ): - nodes_to_scan.append(other_dim) - for referencing_node in nodes_to_scan: - rq = ctx.get_parsed_query(referencing_node) - found = extract_columns_referenced_from_node(rq, dim_node.name) - if found: - _logger.info( - "filter_cte_projection: %s references cols %s from %s", - referencing_node.name, - sorted(found), - dim_node.name, - ) - dim_cols.update(found) - - # Merge with existing if any - if dim_node.name in needed_columns_by_node: - needed_columns_by_node[dim_node.name].update(dim_cols) - else: - needed_columns_by_node[dim_node.name] = dim_cols - _logger.info( - "filter_cte_projection: keeping cols %s for %s", - sorted(dim_cols), - dim_node.name, - ) - - # For multi-hop joins: the left side of this link is an intermediate - # dimension node that also needs the left-side join key columns. - # (For the first link, the left side is parent_node, already handled - # above in parent_needed_cols.) - left_node_name = link.node_revision.name - if left_node_name != parent_node.name and link.join_sql: - left_node = ctx.nodes.get(left_node_name) - if ( - left_node and left_node.type != NodeType.SOURCE - ): # pragma: no branch - left_join_cols = extract_join_columns_for_node( - link.join_sql, - left_node_name, - ) - if left_node_name in needed_columns_by_node: - needed_columns_by_node[left_node_name].update( - left_join_cols, - ) - else: # pragma: no cover - needed_columns_by_node[left_node_name] = left_join_cols - - _add_transitive_consumer_columns(ctx, nodes_for_ctes, needed_columns_by_node) - - return nodes_for_ctes, needed_columns_by_node - - -def _add_transitive_consumer_columns( - ctx: BuildContext, - nodes_for_ctes: list[Node], - needed_columns_by_node: dict[str, set[str]], -) -> None: - """ - Widen each node's needed columns to cover every CTE that reads from it. - - The loops above only look at the parent node and the dimension nodes sitting - on a resolved join path. ``collect_node_ctes`` builds its CTE set from the - full transitive closure of the dependency graph, so a node can end up as a - CTE without ever being a resolved dimension — an intermediate transform, or a - dimension pulled in only as somebody else's parent. Those nodes are never - scanned, so the columns they read stay out of ``needed_columns_by_node`` and - ``filter_cte_projection`` prunes them away, leaving the consuming CTE - referencing a column its source no longer projects. - - Walking the same closure here and unioning in what each CTE actually reads - keeps the two sets consistent. Only nodes already in - ``needed_columns_by_node`` are widened: a node with no entry is left alone so - it stays unpruned, which is what callers expect. - """ - closure: dict[str, Node] = {} - - def collect(node: Node) -> None: - if node.name in closure or node.type == NodeType.SOURCE: - return - closure[node.name] = node - if not (node.current and node.current.query): # pragma: no cover - return - try: - refs = get_table_references_from_ast(ctx.get_parsed_query(node)) - except Exception: # pragma: no cover - return - for ref in refs: - if ref_node := ctx.nodes.get(ref): - collect(ref_node) - - for node in nodes_for_ctes: - collect(node) - - # Only nodes that are already pruned can be under-projected. - targets = [ - name for name in needed_columns_by_node if name in closure or name in ctx.nodes - ] - - for consumer in closure.values(): - if not (consumer.current and consumer.current.query): # pragma: no cover - continue - try: - consumer_ast = ctx.get_parsed_query(consumer) - except Exception: # pragma: no cover - continue - for target in targets: - if target == consumer.name: - continue - referenced = extract_columns_referenced_from_node(consumer_ast, target) - if referenced - needed_columns_by_node[target]: - _logger.info( - "filter_cte_projection: %s references cols %s from %s " - "(transitive consumer)", - consumer.name, - sorted(referenced), - target, - ) - needed_columns_by_node[target].update(referenced) + return nodes_for_ctes def _build_temporal_pushdown( @@ -830,26 +587,6 @@ def _apply_outer_where_atoms( select.where = atom -def _col_table_name(col: ast.Column) -> str | None: - """Return the table-qualifier short name for a column, or None. - - Handles both qualification styles: - - ``_table`` (set by :func:`make_column_ref`) — projection / GROUP BY. - - ``name.namespace`` (set by ``_add_table_prefixes_to_filter``) — - filter atoms. - """ - tbl = col._table - if tbl is not None: - tname = getattr(tbl, "name", None) - if tname is None: # pragma: no cover - return None - return tname.name if hasattr(tname, "name") else str(tname) - if col.name and col.name.namespace: - ns = col.name.namespace - return ns.name if hasattr(ns, "name") else str(ns) - return None # pragma: no cover - - def _set_col_table_alias(col: ast.Column, new_alias: str) -> None: """Rewrite the table-qualifier on a column to ``new_alias``, matching the qualification style already on the column. @@ -984,7 +721,7 @@ def _absorb_filtered_joins_for_outer_safety( def _process(node: ast.Node) -> None: for col in node.find_all(ast.Column): - tname_str = _col_table_name(col) + tname_str = column_table_name(col) if tname_str is not None and tname_str in absorbed_aliases: absorbed_cols[tname_str].add(col.name.name) _set_col_table_alias(col, main_alias) @@ -1435,9 +1172,8 @@ def build_select_ast( grain_col_refs.append(ast.Column(name=ast.Name(gc_alias))) # Resolve dim-namespaced refs in metric expressions (e.g. ``v3.customer.tier``) - # to the dim's joined table alias, and remember which columns each dim - # CTE must keep. Done up-front so the rewrite is visible to both the - # projection loop below and CTE pruning. + # to the dim's joined table alias. Done up-front so the rewrite is visible + # to both the projection loop below and CTE pruning. dim_node_to_alias: dict[str, str] = {} for (dim_node_name, role), alias in dim_aliases.items(): if dim_node_name not in dim_node_to_alias or not role: @@ -1446,7 +1182,7 @@ def build_select_ast( # (COUNT DISTINCT level) expressions. The grain specs hold the same AST # objects already placed in the projection above, so rewriting them here is # reflected in the emitted SQL. - metric_dim_cols = _resolve_dim_namespace_refs( + _resolve_dim_namespace_refs( [expr for _, expr in metric_expressions] + [expr for expr, _ in grain_col_specs], dim_node_to_alias, @@ -1467,16 +1203,8 @@ def build_select_ast( parent_node_name=parent_node.name, ) - # Collect all nodes that need CTEs and the minimal columns each must project. - nodes_for_ctes, needed_columns_by_node = collect_cte_nodes_and_needed_columns( - ctx, - parent_node, - resolved_dimensions, - grain_col_specs, - metric_expressions, - ) - for dim_name, cols in metric_dim_cols.items(): - needed_columns_by_node.setdefault(dim_name, set()).update(cols) + # Collect all nodes that need CTEs. + nodes_for_ctes = collect_cte_nodes(ctx, parent_node, resolved_dimensions) temporal_filter_ast, injected_cte_filters = _build_temporal_pushdown( ctx, @@ -1566,7 +1294,6 @@ def build_select_ast( ctes, scanned_sources, consumed_by_node = collect_node_ctes( ctx, nodes_for_ctes, - needed_columns_by_node, injected_filters=injected_cte_filters or None, pushdown=PushdownFilters( filters=all_filters, @@ -1666,6 +1393,9 @@ def build_select_ast( cte_list.append(cte_query) query.ctes = cte_list + # Now that every reader exists, each CTE can be trimmed to what they read. + prune_cte_projections(query) + return query, scanned_sources @@ -1894,18 +1624,11 @@ def _preagg_dimension_ctes( the join straight at its table instead. """ dim_nodes: list[Node] = [] - needed_columns: dict[str, set[str]] = {} for coverage in join_back: - rdim = coverage.dimension - link = coverage.link - dim_node = ctx.nodes.get(link.dimension.name, link.dimension) + dim_node = ctx.nodes.get(coverage.link.dimension.name, coverage.link.dimension) if dim_node not in dim_nodes: dim_nodes.append(dim_node) - cols = needed_columns.setdefault(dim_node.name, set()) - cols.add(rdim.column_name) - if link.join_sql: # pragma: no branch - cols.update(extract_join_columns_for_node(link.join_sql, dim_node.name)) - ctes, scanned_sources, _ = collect_node_ctes(ctx, dim_nodes, needed_columns) + ctes, scanned_sources, _ = collect_node_ctes(ctx, dim_nodes) return ctes, scanned_sources @@ -2210,6 +1933,9 @@ def build_grain_group_from_preagg( cte_list.append(cte_query) query.ctes = cte_list + # Now that every reader exists, each CTE can be trimmed to what they read. + prune_cte_projections(query) + # Pre-aggregation path: the only raw sources scanned are those behind the # dimensions joined back onto the pre-agg. # TODO: Consider tracking the pre-agg table itself as a "materialized source" diff --git a/datajunction-server/datajunction_server/construction/build_v3/node_query.py b/datajunction-server/datajunction_server/construction/build_v3/node_query.py index 15a72e2539..6f652ef282 100644 --- a/datajunction-server/datajunction_server/construction/build_v3/node_query.py +++ b/datajunction-server/datajunction_server/construction/build_v3/node_query.py @@ -13,6 +13,7 @@ """ import logging +from copy import deepcopy from typing import Any, cast from sqlalchemy import select @@ -32,6 +33,9 @@ parse_dimension_ref, resolve_dimensions, ) +from datajunction_server.construction.build_v3.filters import ( + parse_and_resolve_filters, +) from datajunction_server.construction.build_v3.loaders import ( batch_load_nodes_with_dependencies, find_upstream_node_names, @@ -43,9 +47,12 @@ build_dimension_joins, build_filter_column_aliases, build_outer_where, - collect_cte_nodes_and_needed_columns, + collect_cte_nodes, outer_only_filter_refs, ) +from datajunction_server.construction.build_v3.materialization import ( + get_table_reference_parts_with_materialization, +) from datajunction_server.construction.build_v3.types import ( BuildContext, ColumnMetadata, @@ -55,6 +62,7 @@ ) from datajunction_server.construction.build_v3.utils import ( add_dimensions_from_filters, + get_column_type, get_cte_name, ) from datajunction_server.database.column import Column as DBColumn @@ -72,6 +80,143 @@ logger = logging.getLogger(__name__) +def project_dimension_values_sql( + generated: GeneratedSQL, + dimensions: list[str], + orderby: list[str] | None = None, + limit: int | None = None, +) -> GeneratedSQL: + """Project distinct dimensions from a fact-scoped metrics query.""" + columns_by_name = {column.semantic_name: column for column in generated.columns} + columns = [columns_by_name[dimension] for dimension in dimensions] + + inner = deepcopy(generated.query) + inner.parenthesized = True + inner.alias = ast.Name("dimension_values") + inner.as_ = True + query = ast.Query( + select=ast.Select( + projection=[ + ast.Column( + name=ast.Name(column.name), + _table=ast.Table(ast.Name("dimension_values")), + ) + for column in columns + ], + from_=ast.From(relations=[ast.Relation(primary=inner)]), + quantifier="DISTINCT", + ), + ) + return apply_orderby_limit( + GeneratedSQL( + query=query, + columns=columns, + dialect=generated.dialect, + cube_name=generated.cube_name, + scan_estimate=generated.scan_estimate, + warnings=generated.warnings, + ), + orderby, + limit, + ) + + +def build_dimension_sql_v3( + ctx: BuildContext, + orderby: list[str] | None = None, + limit: int | None = None, + query_parameters: dict[str, Any] | None = None, +) -> GeneratedSQL: + """Build ``SELECT DISTINCT`` for attributes from one dimension node.""" + requested = [dim for dim in ctx.dimensions if dim not in ctx.filter_dimensions] + refs = [parse_dimension_ref(dim) for dim in ctx.dimensions] + node_names = {ref.node_name for ref in refs} + if not requested or len(node_names) != 1: + raise DJInvalidInputException( + "Metricless queries require dimension attributes from exactly one node", + ) + + node_name = refs[0].node_name + dimension_node = ctx.nodes.get(node_name) + if dimension_node is None: + raise DJInvalidInputException(f"Dimension node `{node_name}` does not exist") + # Direct-domain queries need a relational node. Metrics are aggregate + # expressions, while unmaterialized cubes have no query body of their own. + if dimension_node.type not in { + NodeType.SOURCE, + NodeType.TRANSFORM, + NodeType.DIMENSION, + }: + raise DJInvalidInputException( + f"Metricless queries cannot select attributes from `{node_name}`", + ) + + requested_refs = [parse_dimension_ref(dim) for dim in requested] + available = {column.name for column in dimension_node.current.columns} # type: ignore[union-attr] + missing = [ref.column_name for ref in refs if ref.column_name not in available] + if missing: + raise DJInvalidInputException( + f"Dimension `{node_name}` does not contain columns: {missing}", + ) + + cte_pairs, _, _ = collect_node_ctes(ctx, [dimension_node]) + table_parts, _ = get_table_reference_parts_with_materialization( + ctx, + dimension_node, + ) + dimension_table = ".".join(table_parts) + aliases = {dim: parse_dimension_ref(dim).column_name for dim in ctx.dimensions} + output_aliases = [ctx.alias_registry.register(dim) for dim in requested] + projection: list[Any] = [] + for ref, output_alias in zip(requested_refs, output_aliases): + column = ast.Column(name=ast.Name(ref.column_name)) + if output_alias != ref.column_name: + column.set_alias(ast.Name(output_alias)) + column.set_as(True) + projection.append(column) + query = ast.Query( + select=ast.Select( + projection=projection, + from_=ast.From.Table(dimension_table), + where=parse_and_resolve_filters( + ctx.filters, + aliases, + nodes=ctx.nodes, + ) + if ctx.filters + else None, + quantifier="DISTINCT", + ), + ) + for cte_name, cte_body in cte_pairs: + cte_body.to_cte(ast.Name(cte_name), query) + query.ctes = [cte_body for _, cte_body in cte_pairs] + + if query_parameters: + substitute_query_params(query, query_parameters) + return apply_orderby_limit( + GeneratedSQL( + query=query, + columns=[ + ColumnMetadata( + name=output_alias, + semantic_name=dim, + type=get_column_type(dimension_node, ref.column_name), + semantic_type="dimension", + ) + for dim, ref, output_alias in zip( + requested, + requested_refs, + output_aliases, + ) + ], + dialect=ctx.dialect, + ), + orderby, + limit, + ) + + async def build_node_sql_v3( session: AsyncSession, node_name: str, @@ -404,19 +549,12 @@ def _build_with_dimensions( those CTEs via ``PushdownFilters`` (handled inside ``collect_node_ctes``); everything else is applied at the outer ``WHERE``. """ - # ``collect_cte_nodes_and_needed_columns`` walks every link in every - # resolved dim's ``join_path`` and adds each intermediate hop's - # dimension to the CTE list — exactly what we need for multi-hop - # chains where a dim link routes through an intermediate transform/dim. - # Reused from measures.py so we don't drift. We pass empty grain / - # metric args because non-metric nodes don't decompose into components. - nodes_for_ctes, _needed_columns = collect_cte_nodes_and_needed_columns( - ctx, - starting, - resolved_dims, - grain_col_specs=[], - metric_expressions=[], - ) + # ``collect_cte_nodes`` walks every link in every resolved dim's + # ``join_path`` and adds each intermediate hop's dimension to the CTE + # list — exactly what we need for multi-hop chains where a dim link + # routes through an intermediate transform/dim. Reused from measures.py + # so we don't drift. + nodes_for_ctes = collect_cte_nodes(ctx, starting, resolved_dims) # Build the filter-column-alias map up front so ``PushdownFilters`` # can resolve user filter refs to the right CTE columns. @@ -430,10 +568,10 @@ def _build_with_dimensions( ) # ``collect_node_ctes`` skips sources (they get inlined as physical refs) - # and produces bodies in dep order. We deliberately don't pass - # ``needed_columns_by_node`` — the v3 metric path uses it for column - # trimming, but for ``/sql/{node}`` we want each node's full projection - # in the CTE so the user gets every column the node defines. + # and produces bodies in dep order. We deliberately don't prune the + # projections afterwards — the v3 metric path trims columns, but for + # ``/sql/{node}`` we want each node's full projection in the CTE so the + # user gets every column the node defines. cte_pairs, _, _ = collect_node_ctes( ctx, nodes_for_ctes, diff --git a/datajunction-server/datajunction_server/construction/build_v3/utils.py b/datajunction-server/datajunction_server/construction/build_v3/utils.py index 9d678372bb..9496d1a42b 100644 --- a/datajunction-server/datajunction_server/construction/build_v3/utils.py +++ b/datajunction-server/datajunction_server/construction/build_v3/utils.py @@ -104,6 +104,26 @@ def get_column_type(node: Node, column_name: str) -> str: return "string" # pragma: no cover +def column_table_name(col: ast.Column) -> str | None: + """Return the table-qualifier short name for a column, or None. + + Handles both qualification styles: + - ``_table`` (set by :func:`make_column_ref`) — projection / GROUP BY. + - ``name.namespace`` (set by ``_add_table_prefixes_to_filter``) — + filter atoms. + """ + tbl = col._table + if tbl is not None: + tname = getattr(tbl, "name", None) + if tname is None: # pragma: no cover + return None + return tname.name if hasattr(tname, "name") else str(tname) + if col.name and col.name.namespace: + ns = col.name.namespace + return ns.name if hasattr(ns, "name") else str(ns) + return None # pragma: no cover + + def extract_columns_from_expression(expr: ast.Expression) -> set[str]: """ Extract all column names referenced in an expression. @@ -144,54 +164,6 @@ def iter_namespaced_columns(expr: ast.Expression) -> Iterator[NamespacedColumn]: ) -def extract_columns_referenced_from_node( - query_ast: ast.Query, - node_name: str, -) -> set[str]: - """ - Extract column names that a query uses from a specific node. - - Handles both aliased references (e.g. ``CROSS JOIN node AS alias`` where - columns appear as ``alias.col``) and unaliased references (where columns - appear as ``node_short_name.col`` or ``full.node.name.col``). - - Args: - query_ast: The parsed query to scan. - node_name: Full node name (e.g. ``common.dimensions.xp.max_observation_end``). - - Returns: - Set of short column names used from that node. - """ - # Build the set of table-reference prefixes that map to this node. - # A table may be aliased (AS tbl_alias) or bare — we collect all identifiers - # by which columns may be qualified. - prefixes: set[str] = set() - for table in query_ast.find_all(ast.Table): - if str(table.name) == node_name: - if table.alias: - prefixes.add(str(table.alias)) - else: - # No alias: SQL can qualify columns by the full name or the - # last segment only (most common in practice). - prefixes.add(node_name) - prefixes.add(node_name.split(SEPARATOR)[-1]) - - if not prefixes: - return set() - - result: set[str] = set() - for col in query_ast.find_all(ast.Column): - # col.identifier() returns the full namespace-prefixed name - # (e.g. "alias.column_name") without requiring col.table to be set, - # since that is only populated after compilation. - col_id = col.identifier() - for prefix in prefixes: - if col_id.startswith(prefix + SEPARATOR): - result.add(get_short_name(col_id)) - break - return result - - def collect_required_dimensions( nodes: dict[str, Node], metrics: list[str], diff --git a/datajunction-server/datajunction_server/database/column.py b/datajunction-server/datajunction_server/database/column.py index da26d8bee6..74fe2de4ec 100644 --- a/datajunction-server/datajunction_server/database/column.py +++ b/datajunction-server/datajunction_server/database/column.py @@ -137,6 +137,31 @@ def to_spec(self, *, include_cube_role: bool = False): unit=self.unit, ) + def to_reference_link_spec(self): + """Build the DimensionReferenceLinkSpec for this column's reference + dimension link. Only valid when dimension_id and dimension_column + are set (i.e. this column has a reference-type dimension link). + """ + from datajunction_server.construction.build_v3.dimensions import ( + parse_dimension_ref, + ) + from datajunction_server.models.deployment import DimensionReferenceLinkSpec + from datajunction_server.utils import SEPARATOR + + # `dimension_column` carries an optional "[role]" suffix (set by + # _create_or_update_dimension_link); parse_dimension_ref splits it + # back out into `role` so this round-trips to the same spec the role + # was authored with, rather than a bare-role, bracket-suffixed one + # that never compares equal to it. + ref = parse_dimension_ref( + f"{self.dimension.name}{SEPARATOR}{self.dimension_column}", + ) + return DimensionReferenceLinkSpec( + node_column=self.name, + dimension=f"{ref.node_name}{SEPARATOR}{ref.column_name}", + role=ref.role, + ) + def identifier(self) -> tuple[str, ColumnType]: """ Unique identifier for this column. diff --git a/datajunction-server/datajunction_server/database/custom_metadata_schema.py b/datajunction-server/datajunction_server/database/custom_metadata_schema.py index 8a53ed9872..456b481109 100644 --- a/datajunction-server/datajunction_server/database/custom_metadata_schema.py +++ b/datajunction-server/datajunction_server/database/custom_metadata_schema.py @@ -41,7 +41,6 @@ class CustomMetadataSchema(Base): ForeignKey("users.id"), default=None, ) - owner: Mapped[str | None] = mapped_column(String, default=None) updated_by_id: Mapped[int | None] = mapped_column( BigInteger, ForeignKey("users.id"), diff --git a/datajunction-server/datajunction_server/database/dimensionlink.py b/datajunction-server/datajunction_server/database/dimensionlink.py index 9e59d58f8c..7ea37782a5 100644 --- a/datajunction-server/datajunction_server/database/dimensionlink.py +++ b/datajunction-server/datajunction_server/database/dimensionlink.py @@ -101,6 +101,7 @@ def to_spec(self): join_on=self.join_sql, join_type=self.join_type if self.join_type else JoinType.LEFT, join_cardinality=self.join_cardinality, + default_value=self.default_value, spark_hints=self.spark_hints, ) diff --git a/datajunction-server/datajunction_server/database/node.py b/datajunction-server/datajunction_server/database/node.py index d93acc7530..1527388399 100644 --- a/datajunction-server/datajunction_server/database/node.py +++ b/datajunction-server/datajunction_server/database/node.py @@ -64,7 +64,6 @@ from datajunction_server.models.base import labelize from datajunction_server.models.deployment import ( CubeSpec, - DimensionReferenceLinkSpec, DimensionSpec, MaterializationSpec, MetricSpec, @@ -613,10 +612,7 @@ async def to_spec(self, session: AsyncSession) -> NodeSpec: for link in self.current.dimension_links # type: ignore ] ref_link_specs = [ - DimensionReferenceLinkSpec( - node_column=col.name, - dimension=f"{col.dimension.name}{SEPARATOR}{col.dimension_column}", - ) + col.to_reference_link_spec() for col in sorted_columns if col.dimension_id and col.dimension_column ] @@ -659,23 +655,10 @@ async def to_spec(self, session: AsyncSession) -> NodeSpec: legacy_from_md, ) - # A required dimension that is a direct column on a parent node is - # exported as its bare column name (portable as-is). One that lives on - # a node elsewhere on the graph (e.g. a linked dimension) is exported - # as its fully-qualified `node.column` path, so a re-deploy into a - # different namespace can still find it — the bare name would fail to - # resolve against the parents. Prefix parameterization (`${prefix}`) is - # applied later, in get_node_specs_for_export, and only for in-deploy - # nodes. Only touch ``parents`` when there are required dims to classify. - required_dimensions_spec: list[str] = [] - if self.current.required_dimensions: - parent_names = {parent.name for parent in self.current.parents} - required_dimensions_spec = sorted( - col.name - if col.node_revision.name in parent_names - else col.full_name() - for col in self.current.required_dimensions - ) + # Every required dimension is exported as its fully-qualified `node.column` path. + required_dimensions_spec: list[str] = sorted( + col.full_name() for col in self.current.required_dimensions + ) extra_kwargs.update( required_dimensions=required_dimensions_spec, direction=self.current.metric_metadata.direction diff --git a/datajunction-server/datajunction_server/internal/access/authorization/context.py b/datajunction-server/datajunction_server/internal/access/authorization/context.py index 46531ea5f2..bb184fe1d7 100644 --- a/datajunction-server/datajunction_server/internal/access/authorization/context.py +++ b/datajunction-server/datajunction_server/internal/access/authorization/context.py @@ -9,6 +9,7 @@ from sqlalchemy.ext.asyncio import AsyncSession from sqlalchemy.orm import selectinload +from datajunction_server.database.namespace import NodeNamespace from datajunction_server.database.rbac import Role, RoleAssignment, RoleScope from datajunction_server.database.user import User from datajunction_server.internal.access.group_membership import ( @@ -47,6 +48,7 @@ class AuthContext: # Scopes from the configured default-access role, evaluated as a fallback # alongside the user's own grants. default_scopes: list[RoleScope] = field(default_factory=list) + governed_boundaries: tuple[str, ...] = () @classmethod async def from_user( @@ -72,6 +74,7 @@ async def from_user( user=user, ) default_scopes = await cls.get_default_scopes(session=session) + governed_boundaries = await cls.get_governed_boundaries(session=session) return cls( user_id=user.id, @@ -80,8 +83,28 @@ async def from_user( role_assignments=assignments, is_admin=bool(user.is_admin), default_scopes=default_scopes, + governed_boundaries=governed_boundaries, ) + @classmethod + async def get_governed_boundaries( + cls, + session: AsyncSession, + ) -> tuple[str, ...]: + """ + Load every retained governed namespace boundary. + + Deactivated boundaries stay enforced because restoration preserves their + roles and assignments. Hard deletion removes the boundary row entirely. + """ + statement = ( + select(NodeNamespace.namespace) + .where(NodeNamespace.is_governed_boundary.is_(True)) + .order_by(NodeNamespace.namespace) + ) + result = await session.execute(statement) + return tuple(result.scalars().all()) + @classmethod async def get_default_scopes( cls, diff --git a/datajunction-server/datajunction_server/internal/access/authorization/service.py b/datajunction-server/datajunction_server/internal/access/authorization/service.py index dcfce41d2f..871b588d7f 100644 --- a/datajunction-server/datajunction_server/internal/access/authorization/service.py +++ b/datajunction-server/datajunction_server/internal/access/authorization/service.py @@ -4,6 +4,7 @@ import logging from abc import ABC, abstractmethod +from collections.abc import Sequence from datetime import UTC, datetime from functools import cache from typing import TYPE_CHECKING, ClassVar @@ -21,6 +22,7 @@ ResourceRequest, ResourceType, RestrictiveScopeRule, + namespace_boundary_scope_targets, parse_restrictive_scope_rule, ) from datajunction_server.naming import parse_scope_pattern @@ -35,6 +37,27 @@ settings = get_settings() +def governed_boundary_rules( + boundaries: Sequence[str], +) -> tuple[RestrictiveScopeRule, ...]: + """Build mutation rules for persisted governed namespace boundaries.""" + actions = ( + ResourceAction.WRITE, + ResourceAction.DELETE, + ResourceAction.MANAGE, + ) + return tuple( + RestrictiveScopeRule( + action=action, + scope_type=scope_type, + scope_value=scope_value, + ) + for namespace in boundaries + for action in actions + for scope_type, scope_value in namespace_boundary_scope_targets(namespace) + ) + + class AuthorizationService(ABC): """ Abstract base class for authorization strategies. @@ -166,7 +189,7 @@ def authorize( for request in requests ] explicit_scopes = self.explicit_scopes(auth_context) - restrictive_rules = self.restrictive_rules() + restrictive_rules = self.restrictive_rules(auth_context.governed_boundaries) return [ self._make_decision( request, @@ -484,12 +507,16 @@ def _resource_in_scope( return False @classmethod - def restrictive_rules(cls) -> list[RestrictiveScopeRule]: - """Parse configured restrictive policy rules.""" - return [ + def restrictive_rules( + cls, + governed_boundaries: Sequence[str] = (), + ) -> list[RestrictiveScopeRule]: + """Combine configured policy with database-backed boundary rules.""" + configured_rules = [ parse_restrictive_scope_rule(value) for value in getattr(settings, "restrictive_scopes", []) or [] ] + return [*configured_rules, *governed_boundary_rules(governed_boundaries)] @classmethod def _matching_restrictive_rule( diff --git a/datajunction-server/datajunction_server/internal/custom_metadata.py b/datajunction-server/datajunction_server/internal/custom_metadata.py index 66bb38994d..154d7b3261 100644 --- a/datajunction-server/datajunction_server/internal/custom_metadata.py +++ b/datajunction-server/datajunction_server/internal/custom_metadata.py @@ -1,5 +1,6 @@ """Resolution, validation, and filter translation for custom_metadata schemas.""" +import datetime import re import jsonschema @@ -8,11 +9,15 @@ from sqlalchemy.ext.asyncio import AsyncSession from datajunction_server.database.custom_metadata_schema import CustomMetadataSchema -from datajunction_server.errors import DJInvalidInputException +from datajunction_server.errors import ( + DJAlreadyExistsException, + DJInvalidInputException, +) from datajunction_server.models.custom_metadata import ( CustomMetadataFilter, CustomMetadataOp, ) +from datajunction_server.models.deployment import CustomMetadataSchemaSpec from datajunction_server.models.node_type import NodeType @@ -181,6 +186,195 @@ def _at_path(jsonb, path: list[str]): return jsonb[path[0]] if len(path) == 1 else jsonb[tuple(path)] +def value_kind(json_schema: dict) -> str | None: + """The single string ``type`` a JSON Schema declares, or None.""" + declared = json_schema.get("type") + return declared if isinstance(declared, str) else None + + +def check_json_schema(key: str, json_schema: dict) -> None: + """Raise if *json_schema* is not itself a valid JSON Schema.""" + try: + jsonschema.Draft202012Validator.check_schema(json_schema) + except jsonschema.exceptions.SchemaError as exc: + raise DJInvalidInputException( + message=( + f"Invalid JSON Schema for custom_metadata key '{key}': {exc.message}" + ), + ) from exc + + +def scope_clause(key: str, namespace: str | None, node_type: str | None): + """Where-clauses selecting the one row that owns (key, namespace, node_type). + + Deliberately does not filter on ``deactivated_at``: the unique index spans + soft-deleted rows, so a caller that hides them would insert into a collision. + """ + return [ + CustomMetadataSchema.key == key, + CustomMetadataSchema.namespace.is_(None) + if namespace is None + else CustomMetadataSchema.namespace == namespace, + CustomMetadataSchema.node_type.is_(None) + if node_type is None + else CustomMetadataSchema.node_type == node_type, + ] + + +async def assert_not_reserved_globally( + session: AsyncSession, + key: str, + namespace: str | None, +) -> None: + """Refuse a namespace registration of a key an admin reserved globally.""" + if namespace is None: + return + reserved = ( + await session.execute( + select(CustomMetadataSchema).where( + CustomMetadataSchema.key == key, + CustomMetadataSchema.namespace.is_(None), + CustomMetadataSchema.reserved.is_(True), + CustomMetadataSchema.deactivated_at.is_(None), + ), + ) + ).scalar_one_or_none() + if reserved is not None: + raise DJAlreadyExistsException( + message=( + f"Key '{key}' is reserved globally and cannot be registered at " + "namespace scope." + ), + ) + + +async def upsert_schema_row( + session: AsyncSession, + *, + key: str, + namespace: str | None, + node_type: str | None, + json_schema: dict, + filterable: bool, + description: str | None, + reserved: bool, + current_user_id: int, +) -> CustomMetadataSchema: + """Create, update, or revive the single row owning this scope. + + Reviving rather than inserting alongside: the unique index counts + soft-deleted rows while every read hides them, so an insert beside a + tombstone violates the constraint. The row keeps its id and created_at, so + a retired key that comes back is the same registration, not a new one. + + Does not commit -- the caller owns the transaction, which is what lets a + dry-run deployment roll the registration back. + """ + row = ( + await session.execute( + select(CustomMetadataSchema).where( + *scope_clause(key, namespace, node_type), + ), + ) + ).scalar_one_or_none() + if row is None: + row = CustomMetadataSchema( + key=key, + namespace=namespace, + node_type=node_type, + created_by_id=current_user_id, + ) + session.add(row) + row.json_schema = json_schema + row.value_kind = value_kind(json_schema) + row.filterable = filterable + row.description = description + row.reserved = reserved + row.updated_by_id = current_user_id + row.deactivated_at = None + return row + + +async def upsert_schema_specs( + session: AsyncSession, + namespace: str, + specs: list[CustomMetadataSchemaSpec], + *, + current_user_id: int, + build_indexes: bool = True, +) -> None: + """Reconcile schema rows to exactly *specs*. + + Declared keys are upserted; rows in scope the specs no longer declare are + soft-deleted. Global rows are never touched -- no namespace owns them, so no + deployment can retire one. + + A spec carries its own namespace, defaulted to the deployment's by + `DeploymentSpec.set_namespaces` and constrained there to that namespace or one + beneath it. Reconciliation covers the deploying namespace plus whatever + sub-namespaces the specs name, and nothing else: declaring a schema for + `shared.conformed` must not retire rows for `shared.finance`, which a different + deployment owns, while an empty spec list still retires the deploying + namespace's own rows. + + Every schema is checked before anything is written, so a malformed spec fails + the deployment rather than half-applying it. ``build_indexes=False`` skips + index DDL for a dry run. + """ + for spec in specs: + check_json_schema(spec.key, spec.json_schema) + await assert_not_reserved_globally( + session, + spec.key, + spec.namespace or namespace, + ) + + declared: set[tuple[str, str | None, str]] = set() + for spec in specs: + node_type_val = spec.node_type.value if spec.node_type is not None else None + scope = spec.namespace or namespace + declared.add((spec.key, node_type_val, scope)) + await upsert_schema_row( + session, + key=spec.key, + namespace=scope, + node_type=node_type_val, + json_schema=spec.json_schema, + filterable=spec.filterable, + description=spec.description, + reserved=False, + current_user_id=current_user_id, + ) + + scopes = {namespace} | {scope for _, _, scope in declared} + existing = ( + ( + await session.execute( + select(CustomMetadataSchema).where( + CustomMetadataSchema.namespace.in_(scopes), + CustomMetadataSchema.deactivated_at.is_(None), + ), + ) + ) + .scalars() + .all() + ) + now = datetime.datetime.now(datetime.UTC) + for row in existing: + if (row.key, row.node_type, row.namespace) not in declared: + row.deactivated_at = now + + if build_indexes: + await session.flush() + for spec in specs: + if spec.filterable: + await ensure_expression_index( + session, + spec.key, + value_kind(spec.json_schema), + ) + + def custom_metadata_clause(col, f: CustomMetadataFilter): """Translate one CustomMetadataFilter into a SQLAlchemy boolean over a JSONB column.""" jsonb = type_coerce(col, JSONB) diff --git a/datajunction-server/datajunction_server/internal/deployment/fingerprints.py b/datajunction-server/datajunction_server/internal/deployment/fingerprints.py new file mode 100644 index 0000000000..5af5acf35d --- /dev/null +++ b/datajunction-server/datajunction_server/internal/deployment/fingerprints.py @@ -0,0 +1,565 @@ +import logging +from collections.abc import Callable, Iterable +from heapq import heappop, heappush + +from sqlalchemy.ext.asyncio import AsyncSession +from sqlalchemy.orm import joinedload, selectinload + +from datajunction_server.database import Node, NodeRevision +from datajunction_server.internal.deployment.utils import ( + extract_dimension_refs_from_filters, + extract_node_graph, +) +from datajunction_server.models.deployment import ( + CubeSpec, + DimensionSpec, + LinkableNodeSpec, + MetricSpec, + NodeSpec, + SourceSpec, + TransformSpec, +) +from datajunction_server.models.node import NodeType +from datajunction_server.models.semantic_fingerprint import ( + LATEST_SEMANTIC_FINGERPRINT_VERSION, + UNKNOWN_SEMANTIC_FINGERPRINT, + SemanticFingerprint, + SemanticFingerprintValue, +) +from datajunction_server.semantic_fingerprints.engine import ( + compose_node_fingerprint, + local_node_fingerprint, +) +from datajunction_server.semantic_fingerprints.merkle import ( + cycle_component_fingerprint, + strongly_connected_components, +) +from datajunction_server.sql.parsing.backends.exceptions import DJParseException +from datajunction_server.utils import SEPARATOR + +FingerprintMap = dict[str, SemanticFingerprintValue] +ParentOptions = tuple[str, ...] +ParentReferences = tuple[ParentOptions, ...] +ParentCandidates = tuple[frozenset[str], ParentReferences] +ParentCandidateCache = dict[int, ParentCandidates] +ParentResolver = Callable[[NodeSpec], ParentReferences] +logger = logging.getLogger(__name__) + + +def _exact_parent_references(names: Iterable[str]) -> ParentReferences: + return tuple((name,) for name in sorted(set(names))) + + +def _dimension_parent_options(reference: str) -> ParentOptions: + parts = reference.split(SEPARATOR) + return tuple(SEPARATOR.join(parts[:end]) for end in range(len(parts) - 1, 1, -1)) + + +def _dimension_parent_references(references: Iterable[str]) -> ParentReferences: + resolved = [] + for reference in sorted(set(references)): + options = _dimension_parent_options(reference) + if options: + resolved.append(options) + return tuple(resolved) + + +def _linkable_parent_candidates(spec: NodeSpec) -> ParentReferences: + linkable = spec + if not isinstance(linkable, LinkableNodeSpec): # pragma: no cover + raise TypeError(f"Expected linkable spec, got {type(spec).__name__}") + return _exact_parent_references( + link.rendered_dimension_node for link in linkable.dimension_links + ) + + +def _metric_parent_candidates(spec: NodeSpec) -> ParentReferences: + metric = spec + if not isinstance(metric, MetricSpec): # pragma: no cover + raise TypeError(f"Expected metric spec, got {type(spec).__name__}") + return _dimension_parent_references(metric.rendered_required_dimensions) + + +def _cube_parent_candidates(spec: NodeSpec) -> ParentReferences: + cube = spec + if not isinstance(cube, CubeSpec): # pragma: no cover + raise TypeError(f"Expected cube spec, got {type(spec).__name__}") + filter_references = { + f"{node_name}{SEPARATOR}{column_name}" + for node_name, column_name in extract_dimension_refs_from_filters( + cube.rendered_filters, + ) + } + return _dimension_parent_references( + [*cube.rendered_dimensions, *filter_references], + ) + + +SEMANTIC_PARENT_RESOLVERS: dict[type[NodeSpec], ParentResolver] = { + SourceSpec: _linkable_parent_candidates, + TransformSpec: _linkable_parent_candidates, + DimensionSpec: _linkable_parent_candidates, + MetricSpec: _metric_parent_candidates, + CubeSpec: _cube_parent_candidates, +} + + +def _candidate_parts( + spec: NodeSpec, + cache: ParentCandidateCache, +) -> ParentCandidates: + key = id(spec) + if key not in cache: + resolver = SEMANTIC_PARENT_RESOLVERS.get(type(spec)) + if resolver is None: + raise TypeError( + f"No semantic parent resolver for {type(spec).__name__}", + ) + query = ( + spec.rendered_metrics + if isinstance(spec, CubeSpec) + else extract_node_graph([spec]).get(spec.rendered_name, []) + ) + cache[key] = (frozenset(query), resolver(spec)) + return cache[key] + + +def _parent_candidates( + spec: NodeSpec, + cache: ParentCandidateCache | None = None, +) -> set[str]: + query, extra = _candidate_parts(spec, cache if cache is not None else {}) + return set(query) | { + candidate for parent_options in extra for candidate in parent_options + } + + +def _is_derived_metric(spec: NodeSpec) -> bool: + return ( + isinstance(spec, MetricSpec) + and spec.query_ast is not None + and spec.query_ast.select.from_ is None + ) + + +def _direct_query_parents( + spec: NodeSpec, + specs: dict[str, NodeSpec], + candidates: Iterable[str], +) -> set[str]: + candidate_names = set(candidates) + if _is_derived_metric(spec): + return { + name + for name in candidate_names + if name in specs and specs[name].node_type == NodeType.METRIC + } + return candidate_names & specs.keys() + + +def _resolve_parent_references( + references: ParentReferences, + specs: dict[str, NodeSpec], +) -> tuple[set[str], set[str]]: + resolved = set() + unresolved = set() + for options in references: + parent = next((candidate for candidate in options if candidate in specs), None) + if parent is not None: + resolved.add(parent) + else: + unresolved.add(options[0]) + return resolved, unresolved + + +def _resolved_parent_names( + spec: NodeSpec, + specs: dict[str, NodeSpec], + cache: ParentCandidateCache, +) -> tuple[list[str], list[str]]: + query, extra = _candidate_parts(spec, cache) + query_candidates = set(query) + query_parents = _direct_query_parents(spec, specs, query_candidates) + unresolved_query = ( + set() if _is_derived_metric(spec) else query_candidates - specs.keys() + ) + extra_parents, unresolved_extra = _resolve_parent_references( + extra, + specs, + ) + resolved = query_parents | extra_parents + unresolved = unresolved_query | unresolved_extra + return sorted(resolved), sorted(unresolved) + + +def _spec_with_normalized_required_dimensions( + spec: NodeSpec, + specs: dict[str, NodeSpec], + cache: ParentCandidateCache, +) -> NodeSpec: + if not isinstance(spec, MetricSpec) or not spec.required_dimensions: + return spec + + query_candidates, _ = _candidate_parts(spec, cache) + direct_parents = _direct_query_parents(spec, specs, query_candidates) + + required_dimensions = [] + for dimension in spec.rendered_required_dimensions: + direct_parent = next( + ( + parent + for parent in sorted( + direct_parents, + key=lambda name: (-len(name), name), + ) + if dimension.startswith(f"{parent}{SEPARATOR}") + ), + None, + ) + if direct_parent is not None: + dimension = dimension[len(direct_parent) + 1 :] + required_dimensions.append(dimension) + return spec.model_copy(update={"required_dimensions": required_dimensions}) + + +async def _load_external_specs( + session: AsyncSession, + seed_specs: Iterable[NodeSpec], + ignored_parse_errors: set[str], + parent_cache: ParentCandidateCache, +) -> dict[str, NodeSpec]: + seeds = list(seed_specs) + known_names = {spec.rendered_name for spec in seeds} + external_specs: dict[str, NodeSpec] = {} + pending = seeds + while pending: + candidates: set[str] = set() + for spec in pending: + try: + candidates.update(_parent_candidates(spec, parent_cache)) + except (DJParseException, TypeError, ValueError) as exc: + if spec.rendered_name not in ignored_parse_errors: + logger.warning( + "Semantic parent extraction failed for %s: %s", + spec.rendered_name, + exc, + ) + frontier = sorted(candidates - known_names) + if not frontier: + break + known_names.update(frontier) + nodes = await Node.get_by_names( + session, + frontier, + options=[ + joinedload(Node.current).options( + *NodeRevision.export_load_options(), + ), + selectinload(Node.tags), + selectinload(Node.owners), + ], + ) + pending = [await node.to_spec(session) for node in nodes] + external_specs.update({spec.rendered_name: spec for spec in pending}) + return external_specs + + +def _resolved_proposed_specs( + existing_specs: dict[str, NodeSpec], + proposed_specs: Iterable[NodeSpec], + deleted_names: set[str], +) -> dict[str, NodeSpec]: + specs = { + name: spec for name, spec in existing_specs.items() if name not in deleted_names + } + for proposed in proposed_specs: + existing = existing_specs.get(proposed.rendered_name) + if ( + isinstance(proposed, SourceSpec) + and not proposed.columns + and isinstance(existing, SourceSpec) + ): + proposed = proposed.model_copy( + deep=True, + update={"columns": existing.columns}, + ) + specs[proposed.rendered_name] = proposed + return specs + + +def _compute_merkle_fingerprints( + specs: dict[str, NodeSpec], + ignored_parse_errors: set[str], + parent_cache: ParentCandidateCache | None = None, + *, + version: int = LATEST_SEMANTIC_FINGERPRINT_VERSION, +) -> FingerprintMap: + parent_cache = parent_cache if parent_cache is not None else {} + graph: dict[str, list[str]] = {} + failed_names: set[str] = set() + fingerprint_specs: dict[str, NodeSpec] = {} + for name in sorted(specs): + spec = specs[name] + try: + graph[name], unresolved = _resolved_parent_names( + spec, + specs, + parent_cache, + ) + if unresolved: + failed_names.add(name) + if name not in ignored_parse_errors: + logger.warning( + "Fingerprint unavailable for %s; unresolved parents: %s", + name, + ", ".join(unresolved), + ) + except (DJParseException, TypeError, ValueError) as exc: + graph[name] = [] + failed_names.add(name) + if name not in ignored_parse_errors: + logger.warning("Fingerprint unavailable for %s: %s", name, exc) + try: + fingerprint_specs[name] = _spec_with_normalized_required_dimensions( + spec, + specs, + parent_cache, + ) + except (DJParseException, TypeError, ValueError) as exc: + failed_names.add(name) + if name not in ignored_parse_errors: + logger.warning("Fingerprint unavailable for %s: %s", name, exc) + + components = strongly_connected_components(graph) + component_by_name = { + name: component_index + for component_index, members in enumerate(components) + for name in members + } + component_results: dict[int, FingerprintMap] = {} + component_parents: dict[int, set[int]] = { + component_index: { + component_by_name[parent] + for member in members + for parent in graph[member] + if component_by_name[parent] != component_index + } + for component_index, members in enumerate(components) + } + component_children: dict[int, set[int]] = { + component_index: set() for component_index in range(len(components)) + } + for child, parents in component_parents.items(): + for parent_component in parents: + component_children[parent_component].add(child) + + remaining_parents = { + component_index: len(parents) + for component_index, parents in component_parents.items() + } + ready: list[int] = [] + for component_index, remaining in remaining_parents.items(): + if remaining == 0: + heappush(ready, component_index) + + processed = 0 + while ready: + component_index = heappop(ready) + processed += 1 + members = components[component_index] + unavailable: FingerprintMap = { + name: UNKNOWN_SEMANTIC_FINGERPRINT for name in members + } + if any(name in failed_names for name in members): + component_results[component_index] = unavailable + else: + external_edges: list[tuple[str, str, SemanticFingerprint]] = [] + for member in members: + for parent_name in graph[member]: + parent_component = component_by_name[parent_name] + if parent_component == component_index: + continue + parent_fingerprint = component_results[parent_component][ + parent_name + ] + if parent_fingerprint == UNKNOWN_SEMANTIC_FINGERPRINT: + break + external_edges.append( + (member, parent_name, parent_fingerprint), + ) + else: + continue + break + else: + is_cycle = len(members) > 1 or members[0] in graph[members[0]] + try: + if not is_cycle: + component_results[component_index] = { + members[0]: compose_node_fingerprint( + fingerprint_specs[members[0]], + version, + parent_fingerprints=[ + edge[2] for edge in external_edges + ], + ), + } + else: + local_fingerprints = { + name: local_node_fingerprint( + fingerprint_specs[name], + version, + ) + for name in members + } + internal_edges = [ + (member, parent) + for member in members + for parent in graph[member] + if component_by_name[parent] == component_index + ] + component_fingerprint = cycle_component_fingerprint( + members, + local_fingerprints, + internal_edges, + external_edges, + ) + component_results[component_index] = { + name: compose_node_fingerprint( + fingerprint_specs[name], + version, + parent_fingerprints=[component_fingerprint], + ) + for name in members + } + except (DJParseException, TypeError, ValueError) as exc: + component_results[component_index] = unavailable + if not all(name in ignored_parse_errors for name in members): + logger.warning( + "Fingerprint unavailable for component %s: %s", + members, + exc, + ) + if component_index not in component_results: + component_results[component_index] = unavailable + + for child in component_children[component_index]: + remaining_parents[child] -= 1 + if remaining_parents[child] == 0: + heappush(ready, child) + + if processed != len(components): # pragma: no cover + raise RuntimeError("SCC condensation graph contains a cycle") + + return {name: component_results[component_by_name[name]][name] for name in specs} + + +class SemanticFingerprintGraph: + """Semantic fingerprints evaluated within one graph snapshot.""" + + def __init__( + self, + specs: dict[str, NodeSpec], + *, + ignored_parse_errors: set[str] | None = None, + parent_cache: ParentCandidateCache | None = None, + version: int = LATEST_SEMANTIC_FINGERPRINT_VERSION, + ): + self._specs = dict(specs) + self._ignored_parse_errors = set(ignored_parse_errors or ()) + self._parent_cache = parent_cache if parent_cache is not None else {} + self._version = version + self._fingerprints: FingerprintMap | None = None + + def fingerprint(self, name: str) -> SemanticFingerprintValue: + """Return one node's fingerprint in this graph snapshot.""" + return self._evaluate()[name] + + def fingerprints( + self, + names: Iterable[str] | None = None, + ) -> FingerprintMap: + """Return fingerprints for selected nodes in this graph snapshot.""" + target_names = list(self._specs if names is None else names) + if not target_names: + return {} + fingerprints = self._evaluate() + return { + name: fingerprints[name] for name in target_names if name in fingerprints + } + + def _evaluate(self) -> FingerprintMap: + if self._fingerprints is None: + self._fingerprints = _compute_merkle_fingerprints( + self._specs, + self._ignored_parse_errors, + self._parent_cache, + version=self._version, + ) + return self._fingerprints + + +async def build_deployment_fingerprints( + session: AsyncSession, + existing_specs: dict[str, NodeSpec], + proposed_specs: Iterable[NodeSpec], + deleted_specs: Iterable[NodeSpec], + *, + additional_target_names: Iterable[str] = (), + version: int = LATEST_SEMANTIC_FINGERPRINT_VERSION, +) -> tuple[FingerprintMap, FingerprintMap]: + proposed_specs = list(proposed_specs) + additional_target_names = set(additional_target_names) + deleted_names = {spec.rendered_name for spec in deleted_specs} + proposed = _resolved_proposed_specs( + existing_specs, + proposed_specs, + deleted_names, + ) + submitted_names = {spec.rendered_name for spec in proposed_specs} + target_specs: dict[str, NodeSpec] = {} + target_names_to_load = ( + additional_target_names - existing_specs.keys() - submitted_names + ) + if target_names_to_load: + target_nodes = await Node.get_by_names( + session, + sorted(target_names_to_load), + options=[ + joinedload(Node.current).options( + *NodeRevision.export_load_options(), + ), + selectinload(Node.tags), + selectinload(Node.owners), + ], + ) + target_specs = { + spec.rendered_name: spec + for spec in [await node.to_spec(session) for node in target_nodes] + } + + parent_cache: ParentCandidateCache = {} + external = await _load_external_specs( + session, + [*existing_specs.values(), *proposed.values(), *target_specs.values()], + ignored_parse_errors=deleted_names, + parent_cache=parent_cache, + ) + external.update(target_specs) + current_graph = SemanticFingerprintGraph( + {**external, **existing_specs}, + ignored_parse_errors=deleted_names, + parent_cache=parent_cache, + version=version, + ) + proposed_graph = SemanticFingerprintGraph( + {**external, **proposed}, + parent_cache=parent_cache, + version=version, + ) + current = current_graph.fingerprints( + deleted_names | additional_target_names, + ) + proposed_hashes = proposed_graph.fingerprints( + submitted_names | additional_target_names, + ) + return current, proposed_hashes diff --git a/datajunction-server/datajunction_server/internal/deployment/orchestrator.py b/datajunction-server/datajunction_server/internal/deployment/orchestrator.py index 3504a25602..711d985c4b 100644 --- a/datajunction-server/datajunction_server/internal/deployment/orchestrator.py +++ b/datajunction-server/datajunction_server/internal/deployment/orchestrator.py @@ -47,12 +47,19 @@ AccessDenialMode, ) from datajunction_server.internal.access.authorization.context import AuthContext +from datajunction_server.internal.custom_metadata import upsert_schema_specs from datajunction_server.internal.deployment.dimension_reachability import ( DimensionReachability, ) +from datajunction_server.internal.deployment.fingerprints import ( + FingerprintMap, + build_deployment_fingerprints, +) from datajunction_server.internal.deployment.utils import ( DeploymentContext, classify_parents, + creates_cycle, + extract_dimension_refs_from_filters, extract_node_graph, topological_levels, ) @@ -102,6 +109,7 @@ SourceSpec, TagSpec, bump_version, + change_tier_name, declared_materialization_blocks, eq_or_fallback, render_prefixes, @@ -109,6 +117,8 @@ from datajunction_server.models.dimensionlink import ( JoinLinkInput, LinkType, + misplaced_node_column_message, + missing_join_on_message, ) from datajunction_server.models.hierarchy import HierarchyLevelInput from datajunction_server.models.history import ActivityType @@ -129,9 +139,11 @@ structured_to_legacy_unit, ) from datajunction_server.sql.dag import get_metric_parents_map +from datajunction_server.sql.parsing.backends.exceptions import DJParseException from datajunction_server.typing import UTCDatetime from datajunction_server.utils import ( SEPARATOR, + Version, get_namespace_from_name, get_settings, ) @@ -139,34 +151,10 @@ logger = logging.getLogger(__name__) -def _extract_dimension_refs_from_filters( - filters: list[str], -) -> list[tuple[str, str]]: - """Extract (node_name, column_name) pairs from filter expressions. - - Parses all filters as a single WHERE clause (joined with AND) and - collects namespaced column references. For example, - ``ns.hard_hat.state = 'CA'`` yields ``('ns.hard_hat', 'state')``. - - Returns a list of (node_name, column_name) tuples. Dimension node - names are identified by having at least one SEPARATOR in the namespace. - """ - from datajunction_server.sql.parsing.backends.antlr4 import ast, parse - - if not filters: - return [] - combined = " AND ".join(f"({f})" for f in filters) - try: - tree = parse(f"SELECT 1 WHERE {combined}") - except Exception: - return [] # Unparseable — skip, will be caught at query time - refs: list[tuple[str, str]] = [] - for col in tree.find_all(ast.Column): - if col.namespace and len(col.namespace) >= 1: - node_name = SEPARATOR.join(n.name for n in col.namespace) - if SEPARATOR in node_name: # pragma: no branch - refs.append((node_name, col.name.name)) - return refs +def _version_key(version: str) -> tuple[int, int]: + """A version as a sortable pair, so v10.0 outranks v9.0.""" + parsed = Version.parse(version) + return parsed.major, parsed.minor def _diff_column_metadata( @@ -338,12 +326,21 @@ class DeploymentPlan: node_graph: dict[str, list[str]] external_deps: set[str] to_delete_namespaces: list[str] = field(default_factory=list) + delete_references: dict[str, list[str]] = field(default_factory=dict) def is_empty(self) -> bool: return ( not self.to_deploy and not self.to_delete and not self.to_delete_namespaces ) + @property + def deletable_specs(self) -> list[NodeSpec]: + return [ + spec + for spec in self.to_delete + if spec.rendered_name not in self.delete_references + ] + @property def linked_dimension_nodes(self) -> set[str]: return { @@ -385,6 +382,14 @@ def __init__( # Cubes rebuilt onto a new, empty datasource. self._rebuilt_cubes: set[str] = set() self._branch_deploy: bool | None = None + # Cube name -> the changed upstreams that pulled it into this deploy. + self._cubes_bumped_by_upstream: dict[str, list[str]] = {} + # Node name -> the tier its change earned, for those cubes to inherit. + self._change_tiers: dict[str, ChangeTier] = {} + # Unchanged nodes re-deployed only to retry a pre-existing failure. + self._revalidation_only: set[str] = set() + self._current_semantic_fingerprints: FingerprintMap = {} + self._proposed_semantic_fingerprints: FingerprintMap = {} @property def _history_user(self) -> str: @@ -402,6 +407,38 @@ def _history_user(self) -> str: return author return self.context.current_user.username + def _warn_about_unmatched_cube_columns(self) -> None: + """ + Warn about `columns:` entries naming no column of the cube. The comparison + ignores them; without this the partition the author declared would just + silently do nothing. + """ + for spec in self.deployment_spec.nodes: + if not isinstance(spec, CubeSpec): + continue + for unmatched in spec.unmatched_column_names: + message = ( + f"Cube '{spec.rendered_name}' declares column '{unmatched}', " + f"which is not one of its columns, so the settings on " + f"it have no effect. A cube's columns are its metrics " + f"and dimensions, named exactly as they appear there." + ) + self.warnings.append( + DJError( + code=ErrorCode.INVALID_ARGUMENTS_TO_FUNCTION, + message=message, + ), + ) + self.deployed_results.append( + DeploymentResult( + name=spec.rendered_name, + deploy_type=DeploymentResult.Type.NODE, + status=DeploymentResult.Status.WARNING, + operation=DeploymentResult.Operation.NOOP, + message=message, + ), + ) + async def execute(self) -> DeploymentExecuteResult: """ Validate and deploy all resources and nodes into the specified namespace. @@ -427,6 +464,8 @@ async def execute(self) -> DeploymentExecuteResult: self.deployment_spec.namespace, ) + self._warn_about_unmatched_cube_columns() + result = DeploymentExecuteResult(results=[], downstream_impacts=[]) try: async with self.session.begin_nested(): @@ -525,6 +564,11 @@ async def _plan_and_execute(self) -> DeploymentExecuteResult: isinstance(spec, CubeSpec) for spec in self.deployment_spec.nodes ) if not needs_preagg_reconcile and not declares_cube: + with self._timer.phase("build semantic fingerprints"): + await self._build_and_apply_semantic_fingerprints( + deployment_plan, + [], + ) return DeploymentExecuteResult( results=await self._handle_no_changes(), downstream_impacts=[], @@ -594,6 +638,19 @@ async def _setup_deployment_resources(self): self.registry.add_owners(await self._setup_owners()) self.registry.add_catalogs(await self._setup_catalogs()) self.registry.add_attributes(await self._setup_attributes()) + # `is not None`, not truthiness: an empty list is a manifest that manages + # schemas and declares none, which retires them. Omitting the section + # leaves them alone. + if self.deployment_spec.custom_metadata_schemas is not None: + await upsert_schema_specs( + self.session, + self.deployment_spec.namespace, + self.deployment_spec.custom_metadata_schemas, + current_user_id=self.context.current_user.id, + # Index DDL is the one part a rolled-back SAVEPOINT would do for + # nothing, and a dry run needs no index to report impact. + build_indexes=not self.dry_run, + ) logger.info( "Set up deployment resources: %d namespaces, %d tags, %d owners, %d catalogs, %d attributes", len(self.registry.namespaces), @@ -828,6 +885,40 @@ async def _handle_no_changes(self) -> list[DeploymentResult]: await self._update_deployment_status() return self.deployed_results + def _apply_semantic_fingerprints(self) -> None: + for result in self.deployed_results: + if result.deploy_type != DeploymentResult.Type.NODE: + continue + fingerprints = ( + self._current_semantic_fingerprints + if result.operation == DeploymentResult.Operation.DELETE + else self._proposed_semantic_fingerprints + ) + result.semantic_fingerprint = fingerprints.get(result.name) + + async def _build_and_apply_semantic_fingerprints( + self, + plan: DeploymentPlan, + downstream: list, + ) -> None: + target_names = {impact.name for impact in downstream} | { + spec.rendered_name for spec in plan.to_delete + } + current, proposed = await build_deployment_fingerprints( + self.session, + plan.existing_specs, + self.deployment_spec.nodes, + plan.deletable_specs, + additional_target_names=target_names, + ) + self._current_semantic_fingerprints = current + self._proposed_semantic_fingerprints = proposed + self._apply_semantic_fingerprints() + for impact in downstream: + proposed_fingerprint = proposed.get(impact.name) + if current.get(impact.name) != proposed_fingerprint: + impact.semantic_fingerprint = proposed_fingerprint + async def _find_namespaces_to_create(self) -> set[str]: """ Identify all namespaces that need to be created based on nodes in the deployment. @@ -1521,6 +1612,21 @@ async def _build_copy_plan(self) -> DeploymentPlan: external_deps=external_dep_names, ) + def _extract_plan_node_graph( + self, + nodes: list[NodeSpec], + ) -> dict[str, list[str]]: + if not self.dry_run: + return extract_node_graph(nodes) + + node_graph = {} + for node in nodes: + try: + node_graph.update(extract_node_graph([node])) + except DJParseException: + node_graph[node.rendered_name] = [] + return node_graph + async def _create_deployment_plan( self, ) -> tuple[DeploymentPlan, list[DeploymentResult]]: @@ -1573,6 +1679,7 @@ async def _create_deployment_plan( else DeploymentResult.Status.SKIPPED, operation=DeploymentResult.Operation.NOOP, message="Unchanged, still INVALID" if is_invalid else "Unchanged", + change_tier=change_tier_name(ChangeTier.NONE), ), ) @@ -1581,7 +1688,7 @@ async def _create_deployment_plan( external_deps: set[str] = set() if to_deploy or to_delete: with self._timer.phase(" plan: extract node graph") as p: - node_graph = extract_node_graph( + node_graph = self._extract_plan_node_graph( [node for node in to_deploy if not isinstance(node, CubeSpec)], ) p.append(f"{len(node_graph)} nodes in graph") @@ -1702,7 +1809,7 @@ async def _create_deployment_plan( existing_node.name, ) - node_graph = extract_node_graph( + node_graph = self._extract_plan_node_graph( [node for node in to_deploy if not isinstance(node, CubeSpec)], ) @@ -1811,23 +1918,29 @@ async def _execute_deployment_plan(self, plan: DeploymentPlan) -> list: if r.deploy_type == DeploymentResult.Type.LINK and r.status != DeploymentResult.Status.SKIPPED } + plan.delete_references = await self._validate_node_deletion(plan.to_delete) with timer.phase("propagate impact") as p: downstream = await propagate_impact( session=self.session, namespace=self.deployment_spec.namespace, changed_node_names=changed_names, deleted_node_names=frozenset( - spec.rendered_name for spec in plan.to_delete + spec.rendered_name for spec in plan.deletable_specs ), changed_link_node_names=changed_link_names, ) p.append(f"{len(downstream)} downstream") + with timer.phase("build downstream semantic fingerprints"): + await self._build_and_apply_semantic_fingerprints(plan, downstream) # Hard-delete after impact propagation (cascade-deletes # NodeRelationship rows that were needed for the BFS above). if plan.to_delete: with timer.phase("delete nodes") as p: - delete_results = await self._delete_nodes(plan.to_delete) + delete_results = await self._delete_nodes( + plan.to_delete, + references=plan.delete_references, + ) p.append(f"{len(delete_results)} deleted") self.deployed_results.extend(delete_results) await self._update_deployment_status() @@ -1926,6 +2039,24 @@ async def _deploy_nodes( if dim_node not in deps: deps.append(dim_node) + # Each linked dimension is an ordering edge too, so a link onto a column + # this same push adds to an existing dimension validates against the new + # column set. Links can be cyclic where query lineage cannot (self-joins + # with a role, mutually linked dimensions), so an edge that would close a + # cycle is dropped and those nodes order by lineage alone. + for node_spec in plan.to_deploy: + if isinstance(node_spec, LinkableNodeSpec): + for link in node_spec.dimension_links: + dim_node = link.rendered_dimension_node + deps = ordering_graph.setdefault(node_spec.rendered_name, []) + if dim_node in deps or creates_cycle( + ordering_graph, + node_spec.rendered_name, + dim_node, + ): + continue + deps.append(dim_node) + # Order nodes topologically based on dependencies levels = topological_levels(ordering_graph, ascending=False) logger.info( @@ -1935,10 +2066,13 @@ async def _deploy_nodes( # Load all dependencies once upfront (not per-level). # The registry is checked first, so only external deps hit the DB. + # Uses ordering_graph, not plan.node_graph, so a required-dimension node + # is preloaded here too -- otherwise apply_metric_spec's lookup misses it + # and silently resolves required_dimensions to empty. t = time.perf_counter() is_copy = all(s._skip_validation for s in plan.to_deploy) dependency_nodes = await self.get_dependencies( - plan.node_graph, + ordering_graph, skip_type_reparsing=is_copy, ) timer.record( @@ -2117,6 +2251,20 @@ async def _process_node_dimension_link( link_spec.rendered_dimension_node, ) + if link_spec.type == LinkType.JOIN: + problems = self._join_link_problems( + cast(DimensionJoinLinkSpec, link_spec), + node_spec.rendered_name, + ) + if problems: + return DeploymentResult( + name=link_name, + deploy_type=DeploymentResult.Type.LINK, + status=DeploymentResult.Status.FAILED, + operation=DeploymentResult.Operation.CREATE, + message="\n".join(problems), + ) + if node.current and node.current.status == NodeStatus.INVALID: # Node is INVALID (no columns / bad SQL). Write the link aspirationally # so it's already present once the node is fixed. @@ -2149,6 +2297,22 @@ async def _process_node_dimension_link( dimension_node=dimension_node, ) + @staticmethod + def _join_link_problems( + link_spec: DimensionJoinLinkSpec, + node_name: str, + ) -> list[str]: + """List reasons a join link cannot be stored.""" + dimension_node = link_spec.rendered_dimension_node + problems = [] + if link_spec.node_column: + problems.append( + misplaced_node_column_message(node_name, dimension_node), + ) + if not link_spec.rendered_join_on and link_spec.join_type != JoinType.CROSS: + problems.append(missing_join_on_message(node_name, dimension_node)) + return problems + def _create_missing_node_link_result( self, link_name: str, @@ -2176,7 +2340,8 @@ async def _create_or_update_dimension_link( dimension_node=join_link.rendered_dimension_node, join_type=join_link.join_type, join_cardinality=join_link.join_cardinality, - join_on=join_link.rendered_join_on, + # A CROSS join has no ON clause, but join_sql is NOT NULL. + join_on=join_link.rendered_join_on or "", role=join_link.role, default_value=join_link.default_value, spark_hints=join_link.spark_hints, @@ -2360,16 +2525,22 @@ async def _swap_cube_materializations( new_revision, access_checker=access_checker, current_user=self.context.current_user, - previous_table_usable=not await is_non_trivial_cube_change( - self.session, - old_revision, - new_revision, + # An upstream change can leave the cube's shape identical while + # every row it serves differs, so the old table is not adoptable. + previous_table_usable=( + new_revision.name not in self._cubes_bumped_by_upstream + and not await is_non_trivial_cube_change( + self.session, + old_revision, + new_revision, + ) ), declared=declared, ) if swap: if swap.rebuilt_names: self._rebuilt_cubes.add(new_revision.name) + swap.is_branch_deploy = await self._is_branch_deploy() self._cube_materialization_swaps.append(swap) def _declared_materializations(self) -> dict[str, list[MaterializationSpec]]: @@ -2678,6 +2849,8 @@ async def _declare_cube_materialization( # take the replacements down with it if those had already been # scheduled. self._cube_materialization_swaps.append( + # No `rebuilt_names`, so this swap never reaches + # `.schedule()` -- it only stops the superseded workflow. CubeMaterializationSwap( cube_name=revision.name, previous_version=revision.version, @@ -2697,6 +2870,7 @@ async def _declare_cube_materialization( new_version=revision.version, rebuilt_names=[entry.materialization.name for entry in changed], superseded=[], + is_branch_deploy=await self._is_branch_deploy(), ), ) for entry in reconciled: @@ -2781,6 +2955,8 @@ async def _plan_coverage_backfill( if await backfill_recorded(self.session, materialization, span[0]): return self._cube_materialization_swaps.append( + # No `rebuilt_names`; this call already returned above whenever + # `_is_branch_deploy()` is true, so this swap is always main. CubeMaterializationSwap( cube_name=revision.name, previous_version=revision.version, @@ -2858,6 +3034,8 @@ def _remove_cube_materialization( ), ) self._cube_materialization_swaps.append( + # No `rebuilt_names`, so this swap never reaches `.schedule()` -- + # a teardown only stops workflows, it does not start one. CubeMaterializationSwap( cube_name=revision.name, previous_version=revision.version, @@ -3210,7 +3388,7 @@ async def _bulk_validate_cubes( if cube.rendered_filters: all_dim_node_names |= { node_name - for node_name, _ in _extract_dimension_refs_from_filters( + for node_name, _ in extract_dimension_refs_from_filters( cube.rendered_filters, ) } @@ -3489,7 +3667,7 @@ def _validate_single_cube( # reachable only under a role can deploy green and flip on revalidation, # the same gap just closed above for cube dimensions. if cube_spec.rendered_filters and cube_parent_rev_ids: - filter_refs = _extract_dimension_refs_from_filters( + filter_refs = extract_dimension_refs_from_filters( cube_spec.rendered_filters, ) filter_dim_nodes = {node_name for node_name, _ in filter_refs} @@ -3685,6 +3863,13 @@ async def _create_cubes_from_validation( """Create cube nodes and revisions from validation results without re-validation""" nodes, revisions = [], [] deployment_results = [] + await self._lock_versions( + [ + node + for result in validation_results + if (node := self.registry.nodes.get(result.spec.rendered_name)) + ], + ) for result in validation_results: cube_spec = cast(CubeSpec, result.spec) @@ -3701,6 +3886,12 @@ async def _create_cubes_from_validation( changelog, changed_fields, change_tier = await self._generate_changelog( result, ) + # A cube dragged in by an upstream change has an identical spec, so its + # own diff earns nothing; the upstream's tier is the whole bump. + change_tier = max( + change_tier, + self._inherited_change_tier(cube_spec.rendered_name), + ) if existing: new_node = existing new_node.current_version = self._deployed_version( @@ -3804,6 +3995,13 @@ async def _create_cubes_from_validation( + ("\n".join([""] + changelog)) + invalid_note, changed_fields=changed_fields, + change_tier=change_tier_name( + change_tier if existing else ChangeTier.MAJOR, + ), + semantic_fingerprint=self._proposed_semantic_fingerprints.get( + cube_spec.rendered_name, + ), + revalidation_only=cube_spec.rendered_name in self._revalidation_only, ) deployment_results.append(deployment_result) @@ -3945,7 +4143,12 @@ async def _validate_node_deletion( references: dict[str, list[str]] = {} # Query just IDs and names of nodes being deleted (more efficient than loading full objects) - stmt = select(Node.id, Node.name).where(Node.name.in_(list(nodes_to_delete))) + stmt = select(Node.id, Node.name).where( + Node.name.in_(list(nodes_to_delete)), + ) + if not self.dry_run: + # Keep new foreign-key references from racing the delete. + stmt = stmt.with_for_update() result = await self.session.execute(stmt) id_to_name = {node_id: node_name for node_id, node_name in result} deleted_node_ids = set(id_to_name.keys()) @@ -4029,11 +4232,16 @@ async def _validate_node_deletion( return references - async def _delete_nodes(self, to_delete: list[NodeSpec]) -> list[DeploymentResult]: + async def _delete_nodes( + self, + to_delete: list[NodeSpec], + references: dict[str, list[str]] | None = None, + ) -> list[DeploymentResult]: logger.info("Starting deletion of %d nodes", len(to_delete)) # Check which nodes have references that would prevent deletion - references = await self._validate_node_deletion(to_delete) + if references is None: + references = await self._validate_node_deletion(to_delete) # Bulk-delete every deletable node via the shared ``hard_delete_nodes`` # helper (the same machinery ``hard_delete_namespace`` uses): it @@ -4081,6 +4289,7 @@ async def _delete_nodes(self, to_delete: list[NodeSpec]) -> list[DeploymentResul results = [] for node_spec in to_delete: node_name = node_spec.rendered_name + semantic_fingerprint = self._current_semantic_fingerprints.get(node_name) if node_name in references: # Node has references - skip deletion and return FAILED result referencing_nodes = references[node_name] @@ -4096,6 +4305,8 @@ async def _delete_nodes(self, to_delete: list[NodeSpec]) -> list[DeploymentResul status=DeploymentResult.Status.FAILED, operation=DeploymentResult.Operation.DELETE, message=error_msg, + change_tier=change_tier_name(ChangeTier.MAJOR), + semantic_fingerprint=semantic_fingerprint, ), ) elif node_name in deleted_names: @@ -4106,6 +4317,8 @@ async def _delete_nodes(self, to_delete: list[NodeSpec]) -> list[DeploymentResul status=DeploymentResult.Status.SUCCESS, operation=DeploymentResult.Operation.DELETE, message=f"Node {node_name} has been removed.", + change_tier=change_tier_name(ChangeTier.MAJOR), + semantic_fingerprint=semantic_fingerprint, ), ) else: @@ -4117,6 +4330,8 @@ async def _delete_nodes(self, to_delete: list[NodeSpec]) -> list[DeploymentResul status=DeploymentResult.Status.FAILED, operation=DeploymentResult.Operation.DELETE, message=f"Node {node_name} not found.", + change_tier=change_tier_name(ChangeTier.MAJOR), + semantic_fingerprint=semantic_fingerprint, ), ) @@ -4202,18 +4417,48 @@ def filter_nodes_to_deploy( -- `change_tier` decides that and `_deployed_version` turns it into a version. So `force` and the INVALID re-deploy below can re-process a node without that implying anything about what changed. + + Those re-deployed only for that retry are recorded in `_revalidation_only` + and marked on their `DeploymentResult`. + + A cube whose own spec is unchanged is still processed when something + upstream of it is changing, matching what `_propagate_update_downstream` + does for a `PATCH`: the cube names the same metrics and dimensions, but + they now resolve against new upstream revisions, so its materialized table + was computed against a definition that no longer exists. """ to_create: list[NodeSpec] = [] to_update: list[NodeSpec] = [] to_skip: list[NodeSpec] = [] + revalidation_only: set[str] = set() force = self.deployment_spec.force for node_spec in self.deployment_spec.nodes: existing_spec = existing_nodes_map.get(node_spec.rendered_name) if not existing_spec: to_create.append(node_spec) - elif force or node_spec != existing_spec: - to_update.append(node_spec) else: + if force: + to_update.append(node_spec) + continue + existing_spec.namespace = node_spec.namespace + resolved_columns = None + proposed_columns = None + if isinstance(existing_spec, SourceSpec) and isinstance( + node_spec, + SourceSpec, + ): + if not existing_spec.columns: + resolved_columns = node_spec.columns + if not node_spec.columns: + proposed_columns = existing_spec.columns + changed_fields, reordered_fields = existing_spec.semantic_diff( + node_spec, + resolved_columns=resolved_columns, + other_resolved_columns=proposed_columns, + ) + if changed_fields or reordered_fields: + to_update.append(node_spec) + continue # Re-deploy unchanged nodes that are stuck in INVALID state so # they get revalidated (e.g. after an upstream fix). existing_node = self.registry.nodes.get(node_spec.rendered_name) @@ -4223,9 +4468,28 @@ def filter_nodes_to_deploy( and existing_node.current.status == NodeStatus.INVALID ): to_update.append(node_spec) + revalidation_only.add(node_spec.rendered_name) else: to_skip.append(node_spec) + self._revalidation_only = revalidation_only + changed_names = {spec.rendered_name for spec in to_create + to_update} + self._cubes_bumped_by_upstream = self._cubes_below_changed_nodes( + to_skip, + changed_names, + ) + if self._cubes_bumped_by_upstream: + to_update.extend( + spec + for spec in to_skip + if spec.rendered_name in self._cubes_bumped_by_upstream + ) + to_skip = [ + spec + for spec in to_skip + if spec.rendered_name not in self._cubes_bumped_by_upstream + ] + desired_node_names = {n.rendered_name for n in self.deployment_spec.nodes} to_delete = [ existing @@ -4235,6 +4499,61 @@ def filter_nodes_to_deploy( return to_create + to_update, to_skip, to_delete + def _cubes_below_changed_nodes( + self, + candidates: list[NodeSpec], + changed_names: set[str], + ) -> dict[str, list[str]]: + """Which unchanged cubes sit above a node this deploy is changing. + + Maps each such cube to the changed nodes above it, which is what + `_inherited_change_tier` reads to give the cube a bump as significant as + the change that caused it. + + The walk uses the parents already loaded with the namespace's nodes, so it + costs no queries and no parsing. It is transitive: a transform three levels + under a cube still reaches it, and no node between them has to have changed + for that to count -- an unchanged metric over an edited transform serves + different rows all the same. + """ + parents = { + name: [parent.name for parent in node.current.parents] + for name, node in self.registry.nodes.items() + if node.current + } + bumped: dict[str, list[str]] = {} + for spec in candidates: + if not isinstance(spec, CubeSpec): + continue + causes, seen = [], set() + queue = list(parents.get(spec.rendered_name, [])) + while queue: + ancestor = queue.pop() + if ancestor in seen: + continue + seen.add(ancestor) + if ancestor in changed_names: + causes.append(ancestor) + queue.extend(parents.get(ancestor, [])) + if causes: + bumped[spec.rendered_name] = sorted(causes) + return bumped + + def _inherited_change_tier(self, cube_name: str) -> ChangeTier: + """The tier a cube inherits from the upstream changes that pulled it in. + + The most significant of them: over-rebuilding a cube costs compute, while + under-rebuilding serves wrong numbers. An upstream that turned out not to + change anything contributes nothing. + """ + return max( + ( + self._change_tiers.get(upstream, ChangeTier.NONE) + for upstream in self._cubes_bumped_by_upstream.get(cube_name, []) + ), + default=ChangeTier.NONE, + ) + def _guard_against_accidental_wipe(self, plan: "DeploymentPlan") -> None: """Refuse to wipe a populated namespace from a *fully empty* spec. @@ -4647,6 +4966,13 @@ async def create_nodes_from_validation( ) -> tuple[list[Node], list[NodeRevision], list[DeploymentResult]]: nodes, revisions = [], [] deployment_results = [] + await self._lock_versions( + [ + node + for result in validation_results + if (node := self.registry.nodes.get(result.spec.rendered_name)) + ], + ) # Use no_autoflush to prevent premature flushing mid-loop (columns without # node_revision_id, nodes without IDs). The caller (bulk_deploy_nodes_in_level) # does a single session.flush() after collecting all objects. @@ -4721,6 +5047,9 @@ async def _process_valid_node_deploy( else DeploymentResult.Operation.CREATE ) changelog, changed_fields, change_tier = await self._generate_changelog(result) + # Read back by `_inherited_change_tier` when the cubes above this node are + # deployed, which happens after every non-cube node. + self._change_tiers[result.spec.rendered_name] = change_tier new_node = self._create_or_update_node(result.spec, existing, change_tier) new_revision = await self._create_node_revision( new_node, @@ -4763,6 +5092,13 @@ async def _process_valid_node_deploy( + ("\n".join([""] + changelog)) + invalid_note, changed_fields=changed_fields, + change_tier=change_tier_name( + change_tier if existing else ChangeTier.MAJOR, + ), + semantic_fingerprint=self._proposed_semantic_fingerprints.get( + result.spec.rendered_name, + ), + revalidation_only=result.spec.rendered_name in self._revalidation_only, ) return deployment_result, new_node, new_revision @@ -4807,63 +5143,44 @@ async def _generate_changelog( f"└─ Set properties for {sum(changed_count)} columns", ) - # Track changes to other node fields + # Classify changes from the same normalized values used by fingerprints. existing_node_spec = await existing.to_spec(self.session) - changed_fields = existing_node_spec.diff(result.spec) if existing else [] - - # Check if query changed (diff() ignores it, but we want to surface it) - if hasattr( - existing_node_spec, - "rendered_query", - ) and hasattr( # pragma: no branch + # to_spec() never sets namespace; semantic_diff() needs it to render ${prefix}. + existing_node_spec.namespace = result.spec.namespace + existing_columns: list[ColumnSpec] | None = None + proposed_columns: list[ColumnSpec] | None = result.inferred_columns + if isinstance(existing_node_spec, SourceSpec) and isinstance( result.spec, - "rendered_query", + SourceSpec, ): - old_query = existing_node_spec.rendered_query - new_query = result.spec.rendered_spec().rendered_query - if old_query != new_query: - changed_fields = ["query"] + changed_fields + if not existing_node_spec.columns: + existing_columns = proposed_columns + if not proposed_columns: + proposed_columns = existing_node_spec.columns + changed_fields, reordered_fields = existing_node_spec.semantic_diff( + result.spec, + resolved_columns=existing_columns, + other_resolved_columns=proposed_columns, + ) - # Check if column metadata changed (diff() ignores columns) + # Keep detailed column notes for the human-readable changelog. from datajunction_server.models.deployment import LinkableNodeSpec as LNS - if isinstance(result.spec, LNS) and isinstance(existing_node_spec, LNS): + if ( + "columns" in changed_fields + and isinstance(result.spec, LNS) + and isinstance(existing_node_spec, LNS) + ): col_change_notes = _diff_column_metadata( result.spec.rendered_spec().columns, existing_node_spec.columns, ) - if col_change_notes: - changed_fields = changed_fields + ["columns"] - for note in col_change_notes: - changelog.append(f"└─ {note}") - - # A cube's columns are derived from its metrics and dimensions; the only - # user-authored thing on them is partition config, which is exactly what - # CubeSpec.__eq__ compares. So a partition-only edit reaches the update path - # and has to be visible here too, or it would earn no version at all. - if isinstance(result.spec, CubeSpec) and isinstance( - existing_node_spec, - CubeSpec, - ): - incoming_partitions = { - col.name: col.partition - for col in result.spec.rendered_columns - if col.partition - } - existing_partitions = { - col.name: col.partition - for col in existing_node_spec.rendered_columns - if col.partition - } - if incoming_partitions != existing_partitions: - changed_fields = changed_fields + ["columns"] + for note in col_change_notes: + changelog.append(f"└─ {note}") if changed_fields: changelog.append("└─ Updated " + ", ".join(changed_fields)) - # Fields whose contents are unchanged but whose ordering moved. diff() - # compares list fields as sets and so cannot see these on its own. - reordered_fields = existing_node_spec.order_diff(result.spec) if reordered_fields: changelog.append("└─ Reordered " + ", ".join(reordered_fields)) @@ -4892,6 +5209,50 @@ def _deployed_version(current_version: str, change_tier: ChangeTier) -> str: """ return bump_version(current_version, max(change_tier, ChangeTier.MINOR)) + async def _lock_versions(self, existing: list[Node]) -> None: + """Refresh each node's `current_version` from committed state, under a lock. + + A deployment plans against a snapshot, so one that commits in between can + take the versions this one planned and the revision insert then fails on + `uq_noderevision_version`. Holding the row locks means a concurrent writer + has either already committed, or waits behind this deployment. + + We take the highest of `current_version` and the node's revisions, so a + node whose `current_version` lags its revisions still earns one of its own. + `_version_key` does the comparing: as strings, v10.0 sorts below v9.0. + """ + node_ids = [node.id for node in existing] + if not node_ids or self.dry_run: + return + # no_autoflush: the caller is mid-build on revisions whose columns have no + # node_revision_id yet, and a query would otherwise flush them. + with self.session.no_autoflush: + highest = dict( + ( + await self.session.execute( + select(Node.id, Node.current_version) + .where(Node.id.in_(node_ids)) + .order_by(Node.id) # a stable lock order between deploys + .with_for_update(), + ) + ).all(), + ) + revisions = ( + await self.session.execute( + select(NodeRevision.node_id, NodeRevision.version).where( + NodeRevision.node_id.in_(node_ids), + ), + ) + ).all() + for node_id, version in revisions: + # A node deleted between planning and the lock has no row to raise, + # though its revisions can still come back from the query above. + locked = highest.get(node_id) + if locked is not None and _version_key(version) > _version_key(locked): + highest[node_id] = version + for node in existing: + node.current_version = highest.get(node.id, node.current_version) + def _create_or_update_node( self, node_spec: NodeSpec, @@ -5100,13 +5461,18 @@ async def _create_node_revision( ) new_revision.schema_ = schema new_revision.table = table + source_columns = source_spec.columns + if not source_columns and new_node.current: + source_columns = [ + column.to_spec() for column in new_node.current.columns + ] new_revision.columns = [ self._create_column_from_spec( col, pk_columns, order=col.order if col.order is not None else idx, ) - for idx, col in enumerate(result.spec.columns) + for idx, col in enumerate(source_columns or []) ] if result.spec.node_type == NodeType.METRIC: diff --git a/datajunction-server/datajunction_server/internal/deployment/utils.py b/datajunction-server/datajunction_server/internal/deployment/utils.py index ba3887f72d..e360c386ac 100644 --- a/datajunction-server/datajunction_server/internal/deployment/utils.py +++ b/datajunction-server/datajunction_server/internal/deployment/utils.py @@ -21,6 +21,7 @@ from datajunction_server.sql.parsing import ast from datajunction_server.sql.parsing.ast import fast_parse_mode from datajunction_server.sql.parsing.backends.antlr4 import parse +from datajunction_server.sql.parsing.backends.exceptions import DJParseException from datajunction_server.utils import SEPARATOR logger = logging.getLogger(__name__) @@ -60,6 +61,26 @@ def extract_upstream_candidates( return tables +def extract_dimension_refs_from_filters( + filters: list[str], +) -> list[tuple[str, str]]: + """Extract namespaced dimension columns referenced by filter expressions.""" + if not filters: + return [] + combined = " AND ".join(f"({filter_})" for filter_ in filters) + try: + tree = parse(f"SELECT 1 WHERE {combined}") + except DJParseException: + return [] + refs = [] + for column in tree.find_all(ast.Column): + if column.namespace: + node_name = SEPARATOR.join(name.name for name in column.namespace) + if SEPARATOR in node_name: + refs.append((node_name, column.name.name)) + return refs + + def classify_parents( is_derived_metric: bool, dep_names: Iterable[str], @@ -137,6 +158,23 @@ def _find_upstreams_for_node(node: NodeSpec) -> tuple[str, list[str], ast.Query return node.rendered_name, [], None +def creates_cycle(graph: dict[str, list[str]], source: str, target: str) -> bool: + """ + Whether adding ``source -> target`` to ``graph`` would close a cycle. + + True when ``source`` is already reachable from ``target``, which includes + a self-edge. Assumes ``graph`` is currently acyclic. + """ + seen: set[str] = set() + frontier = {target} + while frontier: + if source in frontier: + return True + seen |= frontier + frontier = {dep for node in frontier for dep in graph.get(node, [])} - seen + return False + + def topological_levels( graph: dict[str, list[str]], ascending: bool = True, diff --git a/datajunction-server/datajunction_server/internal/deployment/validation.py b/datajunction-server/datajunction_server/internal/deployment/validation.py index d6d65f3bf5..daaf3156f8 100644 --- a/datajunction-server/datajunction_server/internal/deployment/validation.py +++ b/datajunction-server/datajunction_server/internal/deployment/validation.py @@ -27,7 +27,11 @@ LinkableNodeSpec, NodeSpec, ) -from datajunction_server.models.dimensionlink import JoinType +from datajunction_server.models.dimensionlink import ( + JoinType, + misplaced_node_column_message, + missing_join_on_message, +) from datajunction_server.models.node import NodeStatus, NodeType from datajunction_server.sql.dag import get_dimensions from datajunction_server.sql.parsing.ast import fast_parse_mode @@ -209,11 +213,9 @@ def _validate_dimension_link_specs( result.errors.append( DJError( code=ErrorCode.INVALID_COLUMN, - message=( - f"Dimension link from {node_name} to " - f"{link.rendered_dimension_node} sets node_column, " - "which only applies to reference links. Express the " - "join in join_on instead." + message=misplaced_node_column_message( + node_name, + link.rendered_dimension_node, ), ), ) @@ -224,12 +226,9 @@ def _validate_dimension_link_specs( result.errors.append( DJError( code=ErrorCode.INVALID_COLUMN, - message=( - f"Dimension link from {node_name} to " - f"{link.rendered_dimension_node} has no join_on " - "clause. Set join_on to the equality between this " - "node's foreign key column(s) and the dimension's " - "primary key." + message=missing_join_on_message( + node_name, + link.rendered_dimension_node, ), ), ) @@ -493,6 +492,10 @@ def validate_query_node( err for err in [ self._check_inferred_columns(inferred_columns), + self._check_declared_columns_exist( + spec, + validation.output_columns, + ), self._check_primary_key(inferred_columns, spec), self._check_metric_query(spec, spec.query_ast), ] @@ -598,6 +601,36 @@ def _check_inferred_columns(self, columns: list[ColumnSpec]) -> DJError | None: ) return None + @staticmethod + def _check_declared_columns_exist( + spec: NodeSpec, + output_columns: list, + ) -> DJError | None: + """ + Check that every declared column in the spec actually appears in the + query's output. A declared column that doesn't match any output column + is silently dropped (its metadata is never applied), so this is + surfaced as an error instead. + """ + declared_names = { + col.name + for col in ( + spec.columns if hasattr(spec, "columns") and spec.columns else [] + ) + } + output_names = {name for name, _ in output_columns} + unmatched = sorted(declared_names - output_names) + if unmatched: + return DJError( + code=ErrorCode.INVALID_COLUMN, + message=( + f"Declared column(s) {unmatched} on node {spec.rendered_name} " + "do not match any column produced by the query. Check for a " + "missing or mismatched column alias." + ), + ) + return None + def _check_primary_key( self, inferred_columns: list[ColumnSpec], diff --git a/datajunction-server/datajunction_server/internal/impact.py b/datajunction-server/datajunction_server/internal/impact.py index 389fba1e66..ff93eb7ef1 100644 --- a/datajunction-server/datajunction_server/internal/impact.py +++ b/datajunction-server/datajunction_server/internal/impact.py @@ -16,10 +16,17 @@ from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from sqlalchemy.orm import joinedload, selectinload +from sqlalchemy.orm import aliased, joinedload, selectinload from sqlalchemy.sql.operators import is_ -from datajunction_server.database.node import Node, NodeRelationship, NodeRevision +from datajunction_server.database.column import Column as DBColumn +from datajunction_server.database.dimensionlink import DimensionLink +from datajunction_server.database.node import ( + BoundDimensionsRelationship, + Node, + NodeRelationship, + NodeRevision, +) from datajunction_server.database.user import User from datajunction_server.instrumentation.provider import get_metrics_provider from datajunction_server.internal.deployment.dimension_reachability import ( @@ -191,11 +198,37 @@ async def _build_propagation_context( # --------------------------------------------------------------------------- +async def _cube_filter_children_by_parent( + session: AsyncSession, +) -> dict[str, set[int]]: + from datajunction_server.internal.deployment.utils import ( + extract_dimension_refs_from_filters, + ) + + rows = ( + await session.execute( + select(Node.id, NodeRevision.cube_filters) + .join( + NodeRevision, + (NodeRevision.node_id == Node.id) + & (NodeRevision.version == Node.current_version), + ) + .where(Node.type == NodeType.CUBE) + .where(is_(Node.deactivated_at, None)), + ) + ).all() + children_by_parent: dict[str, set[int]] = defaultdict(set) + for node_id, filters in rows: + for parent_name, _ in extract_dimension_refs_from_filters(filters or []): + children_by_parent[parent_name].add(node_id) + return children_by_parent + + async def _propagate_via_parent_graph( session: AsyncSession, ctx: PropagationContext, ) -> list[DownstreamImpact]: - """BFS through NodeRelationship to find all downstream nodes. + """BFS through persisted semantic relationships to find downstream nodes. Returns impacts without mutating DB state — Phase 3 determines the actual impact type via revalidation. @@ -207,19 +240,68 @@ async def _propagate_via_parent_graph( visited_node_ids = set(frontier_ids) results: list[DownstreamImpact] = [] depth = 1 + cube_filter_children = await _cube_filter_children_by_parent(session) while frontier_ids: - rows = ( + relationship_rows = ( await session.execute( select(NodeRevision.node_id, NodeRelationship.parent_id) .join(NodeRelationship, NodeRelationship.child_id == NodeRevision.id) - .where(NodeRelationship.parent_id.in_(frontier_ids)), + .join(Node, Node.id == NodeRevision.node_id) + .where(NodeRelationship.parent_id.in_(frontier_ids)) + .where(Node.current_version == NodeRevision.version), + ) + ).all() + dimension_link_rows = ( + await session.execute( + select(NodeRevision.node_id, DimensionLink.dimension_id) + .join( + DimensionLink, + DimensionLink.node_revision_id == NodeRevision.id, + ) + .join(Node, Node.id == NodeRevision.node_id) + .where(DimensionLink.dimension_id.in_(frontier_ids)) + .where(Node.current_version == NodeRevision.version), + ) + ).all() + + parent_revision = aliased(NodeRevision) + metric_revision = aliased(NodeRevision) + required_dimension_rows = ( + await session.execute( + select(metric_revision.node_id, parent_revision.node_id) + .select_from(BoundDimensionsRelationship) + .join( + DBColumn, + DBColumn.id == BoundDimensionsRelationship.bound_dimension_id, + ) + .join( + parent_revision, + parent_revision.id == DBColumn.node_revision_id, + ) + .join( + metric_revision, + metric_revision.id == BoundDimensionsRelationship.metric_id, + ) + .join(Node, Node.id == metric_revision.node_id) + .where(parent_revision.node_id.in_(frontier_ids)) + .where(Node.current_version == metric_revision.version), ) ).all() child_to_parents: dict[int, set[int]] = {} - for child_node_id, parent_id in rows: + for child_node_id, parent_id in [ + *relationship_rows, + *dimension_link_rows, + *required_dimension_rows, + ]: child_to_parents.setdefault(child_node_id, set()).add(parent_id) + for parent_id in frontier_ids: + parent_node = ctx.visited_nodes_by_id.get(parent_id) + if parent_node is None: # pragma: no cover + continue + for child_node_id in cube_filter_children.get(parent_node.name, set()): + child_to_parents.setdefault(child_node_id, set()).add(parent_id) unvisited = [nid for nid in child_to_parents if nid not in visited_node_ids] if not unvisited: diff --git a/datajunction-server/datajunction_server/internal/materializations.py b/datajunction-server/datajunction_server/internal/materializations.py index bf07e0aafd..eafc574400 100644 --- a/datajunction-server/datajunction_server/internal/materializations.py +++ b/datajunction-server/datajunction_server/internal/materializations.py @@ -418,6 +418,10 @@ class CubeMaterializationSwap: # cube's declared coverage against what its datasource already holds. backfill: CoverageBackfill | None = None + # Whether this swap belongs to a branch-preview deploy, so the query service + # can be told at schedule time rather than re-deriving it. + is_branch_deploy: bool = False + @dataclass class CubeMaterializationSwapOutcome: @@ -782,6 +786,7 @@ async def apply_cube_materialization_swap( materialization_names=swap.rebuilt_names, query_service_client=query_service_client, request_headers=request_headers, + is_branch_deploy=swap.is_branch_deploy, ) except Exception as exc: _logger.warning( @@ -1238,6 +1243,7 @@ async def schedule_materialization_jobs( materialization_names: list[str], query_service_client: QueryServiceClient, request_headers: dict[str, str] | None = None, + is_branch_deploy: bool = False, ) -> dict[str, MaterializationInfo]: """ Schedule recurring materialization jobs @@ -1261,6 +1267,7 @@ async def schedule_materialization_jobs( materialization, query_service_client, request_headers=request_headers, + is_branch_deploy=is_branch_deploy, ) await record_workflow_names(session, materializations, materialization_to_output) return materialization_to_output diff --git a/datajunction-server/datajunction_server/internal/namespaces.py b/datajunction-server/datajunction_server/internal/namespaces.py index d3eb5f10b9..e5a226c4d0 100644 --- a/datajunction-server/datajunction_server/internal/namespaces.py +++ b/datajunction-server/datajunction_server/internal/namespaces.py @@ -42,7 +42,10 @@ lock_namespace_boundary_lifecycle, ) from datajunction_server.internal.nodes import get_single_cube_revision_metadata -from datajunction_server.models.access import ResourceAction, ResourceType +from datajunction_server.models.access import ( + ResourceAction, + namespace_boundary_scope_targets, +) from datajunction_server.models.deployment import ( CubeSpec, DeploymentSourceType, @@ -136,8 +139,9 @@ async def list_namespaces_in_hierarchy( """ statement = select(NodeNamespace).where( or_( - NodeNamespace.namespace.like( - f"{namespace}.%", + NodeNamespace.namespace.startswith( + f"{namespace}.", + autoescape=True, ), NodeNamespace.namespace == namespace, ), @@ -435,17 +439,6 @@ async def create_namespace( return parents -def namespace_boundary_scope_targets( - namespace: str, -) -> list[tuple[ResourceType, str]]: - """Return every scope governed by a namespace boundary.""" - return [ - (ResourceType.NAMESPACE, namespace), - (ResourceType.NAMESPACE, f"{namespace}.*"), - (ResourceType.NODE, f"{namespace}.*"), - ] - - def _namespace_boundary_scopes( namespace: str, action: ResourceAction, @@ -1006,7 +999,7 @@ async def hard_delete_namespace( select(Node.id, Node.name, Node.type) .where( or_( - Node.namespace.like(f"{namespace}.%"), + Node.namespace.startswith(f"{namespace}.", autoescape=True), Node.namespace == namespace, ), ) diff --git a/datajunction-server/datajunction_server/internal/nodes.py b/datajunction-server/datajunction_server/internal/nodes.py index 1434b75756..cb07772034 100644 --- a/datajunction-server/datajunction_server/internal/nodes.py +++ b/datajunction-server/datajunction_server/internal/nodes.py @@ -100,11 +100,14 @@ CubeSpec, DeploymentResult, bump_version, + fold_change_tiers, + version_change_tier, ) from datajunction_server.models.dimensionlink import ( JoinLinkInput, JoinType, LinkDimensionIdentifier, + missing_join_on_message, ) from datajunction_server.models.history import status_change_history from datajunction_server.models.materialization import ( @@ -128,6 +131,7 @@ from datajunction_server.models.table_metadata import TableMetadata, TableOwner from datajunction_server.service_clients import QueryServiceClient from datajunction_server.sql.dag import ( + _node_output_options, get_downstream_nodes, get_nodes_with_common_dimensions, topological_sort, @@ -1528,6 +1532,11 @@ async def update_node_with_query( current_user=current_user, save_history=save_history, cache=cache, + # Downstream cubes inherit this tier: a cube's own shape can be identical + # across an upstream change that alters every row it serves. + change_tier=version_change_tier(old_revision.version, node.current_version), # type: ignore + query_service_client=query_service_client, + request_headers=request_headers, ) await session.refresh(node, ["current"]) await session.refresh(node.current, ["materializations"]) # type: ignore @@ -2025,6 +2034,59 @@ async def update_cube_node( ) return None + return await save_new_cube_revision( + session, + node_revision, + create_cube, + change_tier, + request_headers=request_headers, + query_service_client=query_service_client, + current_user=current_user, + access_checker=access_checker, + save_history=save_history, + ) + + +async def save_new_cube_revision( + session: AsyncSession, + node_revision: NodeRevision, + create_cube: CreateCubeNode, + change_tier: ChangeTier, + *, + request_headers: dict[str, str] | None, + query_service_client: QueryServiceClient | None, + current_user: User, + access_checker: AccessChecker, + save_history: Callable, + extra_history_details: dict | None = None, + previous_table_usable: bool | None = None, +) -> NodeRevision: + """ + Commit `create_cube` as the cube's next revision, at the version `change_tier` + earns, and move its materializations onto it. + + Everything a new cube revision needs beyond deciding *that* there should be one: + resolving the cube against the current metrics and dimensions, the version bump, + carrying partition columns forward, the audit event, and the materialization + swap. Both writers that start from a cube's *own* definition go through here -- + a user editing the cube (`update_cube_node`) and an upstream change propagating + into it -- so the two cannot produce differently-shaped revisions. Deploy builds + cube revisions on its own path and swaps them itself + (`orchestrator._swap_cube_materializations`). + + `extra_history_details` is merged into the UPDATE event's details, which is how + the propagation path records the upstream node and version that caused the bump. + + `previous_table_usable` is recorded on the swap's history event for an operator + deciding whether the rebuild can adopt the old table or needs a backfill. Left + unset it is derived from `is_non_trivial_cube_change`, which compares the cube's + own shape -- correct when the cube itself changed. An upstream change must pass + False: the shapes can be identical while every row differs, which is the same + reason propagation does not use that predicate to decide whether to rebuild. + """ + old_metrics = [m.name for m in node_revision.cube_metrics()] + old_dimensions = node_revision.cube_dimensions() + # Disable autoflush to prevent partial state from being persisted if an error # occurs during revision creation. This ensures that node.current_version and # the new NodeRevision are committed atomically - either both succeed or both @@ -2049,6 +2111,7 @@ async def update_cube_node( activity_type=ActivityType.UPDATE, details={ "version": new_cube_revision.version, # type: ignore + **(extra_history_details or {}), }, pre={ "metrics": old_metrics, @@ -2089,10 +2152,14 @@ async def update_cube_node( new_cube_revision, access_checker=access_checker, current_user=current_user, - previous_table_usable=not await is_non_trivial_cube_change( - session, - node_revision, - new_cube_revision, + previous_table_usable=( + previous_table_usable + if previous_table_usable is not None + else not await is_non_trivial_cube_change( + session, + node_revision, + new_cube_revision, + ) ), ) if swap: @@ -2121,9 +2188,16 @@ async def propagate_update_downstream( current_user: User, save_history: Callable, cache: Cache | None = None, + change_tier: ChangeTier = ChangeTier.MAJOR, + query_service_client: QueryServiceClient | None = None, + request_headers: dict[str, str] | None = None, ): """ Background task to propagate the updated node's changes to all of its downstream children. + + `change_tier` is how significant the change to `node` itself was, which is what + downstream cubes inherit as their own bump. It defaults to MAJOR because + over-rebuilding a cube costs compute while under-rebuilding serves wrong numbers. """ try: async with session_context() as session: @@ -2133,6 +2207,9 @@ async def propagate_update_downstream( current_user=current_user, save_history=save_history, cache=cache, + change_tier=change_tier, + query_service_client=query_service_client, + request_headers=request_headers, ) except Exception: _logger.exception( @@ -2141,12 +2218,111 @@ async def propagate_update_downstream( ) +async def _reload_nodes_in_order( + session: AsyncSession, + names: list[str], +) -> list[Node]: + """ + Reload the named nodes in one query, preserving the order they were given in. + + Used to recover the tail of a propagation walk after a rollback has expired the + instances it was holding. A node that has disappeared meanwhile is dropped + rather than resurrected, and an empty list of names is a select that matches + nothing rather than a case to special-case. + """ + reloaded = ( + ( + await session.execute( + select(Node) + .where(Node.name.in_(names)) + .options(*_node_output_options()), + ) + ) + .unique() + .scalars() + .all() + ) + position = {name: index for index, name in enumerate(names)} + return sorted(reloaded, key=lambda reloaded_node: position[reloaded_node.name]) + + +async def _rebuild_downstream_cube( + session: AsyncSession, + cube: Node, + upstream: Node, + change_tier: ChangeTier, + *, + current_user: User, + save_history: Callable, + query_service_client: QueryServiceClient | None, + request_headers: dict[str, str] | None, +) -> None: + """ + Bump a cube because something upstream of it changed. + + Nobody edited the cube, so there is no payload to reconstruct it from: the new + revision is resolved from the cube's *own* current metrics and dimensions, which + is exactly what `update_cube_node` falls back to for every field a PATCH omits. + Re-resolving them is the point -- the metrics and dimensions are the same names, + but they now resolve against new upstream revisions, so the cube's columns, + elements and parents recompile against what the upstream became. + + The cube inherits the upstream's tier rather than being classified on its own + shape. `is_non_trivial_cube_change` asks whether the *cube's* definition moved, + which is the wrong question here: widening a transform's WHERE clause changes no + metric, no dimension and no component identity, yet every row in the cube's + materialized table was computed under the old filter. Over-rebuilding costs + compute; under-rebuilding serves wrong numbers. + """ + cube_node = await Node.get_cube_by_name(session, cube.name) + node_revision = cube_node.current # type: ignore + create_cube = CreateCubeNode( + name=node_revision.name, + display_name=node_revision.display_name, + description=node_revision.description, + metrics=[metric.name for metric in node_revision.cube_metrics()], + dimensions=node_revision.cube_dimensions(), + mode=node_revision.mode, + filters=node_revision.cube_filters or [], + custom_metadata=node_revision.custom_metadata, + ) + # Propagation runs after the response returned, outside the request's + # AccessChecker, so the rebuild gets a fresh one. It does not gate the bump: + # a cube whose owner pointed it at this upstream is not the updater's call. + access_checker = AccessChecker(await AuthContext.from_user(session, current_user)) + await save_new_cube_revision( + session, + node_revision, + create_cube, + change_tier, + request_headers=request_headers, + query_service_client=query_service_client, + current_user=current_user, + access_checker=access_checker, + save_history=save_history, + extra_history_details={ + "upstream": { + "node": upstream.name, + "version": upstream.current_version, + }, + "reason": f"Caused by update of `{upstream.name}` to " + f"{upstream.current_version}", + }, + # The cube's own shape can be identical across an upstream change that + # altered every row, so the old table cannot be assumed adoptable. + previous_table_usable=False, + ) + + async def _propagate_update_downstream( session: AsyncSession, node: Node, current_user: User, save_history: Callable, cache: Cache | None = None, + change_tier: ChangeTier = ChangeTier.MAJOR, + query_service_client: QueryServiceClient | None = None, + request_headers: dict[str, str] | None = None, ): """ Propagate the updated node's changes to all of its downstream children. @@ -2164,15 +2340,24 @@ async def _propagate_update_downstream( graph asserting VALID for nodes that are now broken -- silently, since the caller above swallows exceptions. Pinned by tests/internal/nodes/background_authz_test.py. + + Cubes are included. They used to be filtered out, which meant a cube kept + serving a materialized table built against a definition that no longer existed + -- the only way to recompile it was a no-op edit to the cube itself. They also + can't go through `revalidate_node`, whose cube branch only refreshes status; a + cube's next revision comes from `save_new_cube_revision`, which is what + `_rebuild_downstream_cube` calls. """ _logger.info("Propagating update of node %s downstream", node.name) downstreams = await get_downstream_nodes( session, node.name, include_deactivated=False, - include_cubes=False, ) downstreams = topological_sort(downstreams) + # Kept separately because the rollback below expires every instance, and + # reading `.name` back off one would lazy-load from async code. + downstream_names = [downstream.name for downstream in downstreams] _logger.info( "Node %s updated — revalidating %s downstreams", node.name, @@ -2192,18 +2377,9 @@ async def _propagate_update_downstream( downstream.name, node.name, ) - node_validator = await revalidate_node( - downstream.name, - session, - current_user=current_user, - save_history=save_history, - # propagate_update_downstream writes its own richer history event - # below (with upstream context); skip the inner one to avoid - # duplicate audit rows for the same revision bump. - record_revision_bump_event=False, - ) - # Reset the upstreams DAG cache of any downstream nodes + # Before the per-type work, so a cube -- which returns early -- is + # invalidated too. if cache: upstream_cache_key = downstream.upstream_cache_key() results = cache.get(upstream_cache_key) @@ -2216,6 +2392,60 @@ async def _propagate_update_downstream( ) cache.delete(upstream_cache_key) + if downstream.type == NodeType.CUBE: + # Any tier rebuilds, and the churn is deliberate. Narrowing this by + # comparing the upstream's resolved columns was rejected: a query edit + # can move a filter, a join or a CASE threshold while leaving every + # column and type identical, and each changes every row the cube serves. + # Nothing short of reading the SQL tells those apart, so a changed query + # makes anything built from it suspect. Only NONE is skipped, the one + # case where DJ knows nothing material happened. + # + # A rebuild can fail, and one cube's failure must not cost the remaining + # downstreams theirs. + if change_tier is not ChangeTier.NONE: + try: + await _rebuild_downstream_cube( + session, + downstream, + node, + change_tier, + current_user=current_user, + save_history=save_history, + query_service_client=query_service_client, + request_headers=request_headers, + ) + except Exception: + _logger.exception( + "Error rebuilding downstream cube %s after update of node %s", + downstream.name, + node.name, + ) + # Discard partial writes, then reload the rest of the walk: + # the rollback expires every instance, so the next iteration + # would lazy-load from async code. Failure path only. + # + # `node` and `current_user` survive it because they belong to + # the request's session, not this one, so the rollback never + # touches them and reading `.name` below needs no IO. Load + # either one in this session and that stops being true. + await session.rollback() + downstreams[idx + 1 :] = await _reload_nodes_in_order( + session, + downstream_names[idx + 1 :], + ) + continue + node_validator = await revalidate_node( + downstream.name, + session, + current_user=current_user, + save_history=save_history, + # propagate_update_downstream writes its own richer history event + # below (with upstream context); skip the inner one to avoid + # duplicate audit rows for the same revision bump. + record_revision_bump_event=False, + ) + # Record history event if ( original_node_revision.version != downstream.current_version @@ -2946,6 +3176,10 @@ async def validate_complex_dimension_link( message=f"Cannot link dimension to a node of type {dimension_node.type}. " "Must be a dimension node.", ) + if not link_input.join_on and link_input.join_type != JoinType.CROSS: + raise DJInvalidInputException( + message=missing_join_on_message(node.name, link_input.dimension_node), # type: ignore + ) if ( dimension_node.current.catalog is not None # type: ignore @@ -3105,12 +3339,14 @@ async def upsert_complex_dimension_link( if link.dimension_id == dimension_node.id and link.role == link_input.role # type: ignore ] activity_type = ActivityType.CREATE + # A CROSS join has no ON clause, but join_sql is NOT NULL. + join_sql = link_input.join_on or "" if existing_link: # Update the existing dimension link activity_type = ActivityType.UPDATE dimension_link = existing_link[0] - dimension_link.join_sql = link_input.join_on + dimension_link.join_sql = join_sql dimension_link.join_type = DimensionLink.parse_join_type( join_relation.join_type, ) @@ -3122,7 +3358,7 @@ async def upsert_complex_dimension_link( dimension_link = DimensionLink( node_revision_id=new_revision.id, # type: ignore dimension_id=dimension_node.id, # type: ignore - join_sql=link_input.join_on, + join_sql=join_sql, join_type=DimensionLink.parse_join_type(join_relation.join_type), join_cardinality=link_input.join_cardinality, role=link_input.role, @@ -4007,42 +4243,62 @@ async def revalidate_node( # columns remain a faithful snapshot of what was committed at that # version. Track *why* the validator decided a column changed so the # history event can explain the bump. - type_changes: list[dict] = [] - order_fixed: list[str] = [] - added_columns: list[str] = [] - for col in node_validator.columns: - existing_col = existing_columns.get(col.name) - if existing_col is None: - added_columns.append(col.name) - continue - if existing_col.type != col.type: - type_changes.append( - { - "column": col.name, - "from": str(existing_col.type), - "to": str(col.type), - }, - ) - if existing_col.order is None: - order_fixed.append(col.name) - updated_columns = bool(type_changes or order_fixed or added_columns) + # + # Uses the shared comparison so removals are seen at all: walking the + # validator's columns can only find what the validator produced, so a column + # the revision still stores but the query no longer selects went unnoticed. + column_changes = describe_column_changes( + node.current.columns, # type: ignore + node_validator.columns, + ) + type_changes: list[dict] = column_changes.get("type_changes", []) + added_columns: list[str] = column_changes.get("added_columns", []) + removed_columns: list[str] = column_changes.get("removed_columns", []) + # Not a column change, and so not part of the shared comparison: `order` is + # DJ's own bookkeeping rather than anything the query says. + order_fixed: list[str] = [ + col.name + for col in node_validator.columns + if col.name in existing_columns and existing_columns[col.name].order is None + ] + # A tier rather than a version, so propagation can hand the same value to + # `bump_version` for downstream cubes. + # + # Any column change is major. An addition looks harmless -- nothing could + # already reference a column that did not exist -- but the query produced it, + # and a query edit can move a filter or a join while leaving the rest of the + # projection identical. Demoting additions to MINOR buys nothing anyway: the + # cube rebuild below skips only NONE, so a minor bump rebuilds all the same. + # + # `order_fixed` earns no tier. DJ filling in a missing projection index is its + # own bookkeeping, not a change to the node, so a revision would describe + # nothing and any tier above NONE would rebuild every cube below. It is applied + # to the current revision in place instead, below. + change_tier = fold_change_tiers( + [ + ChangeTier.MAJOR + if (type_changes or removed_columns or added_columns) + else ChangeTier.NONE, + ], + ) + updated_columns = change_tier is not ChangeTier.NONE _logger.info( - "Columns updated: %s for node %s (current version: %s) — " - "type_changes=%s, order_fixed=%s, added_columns=%s", + "Columns updated: %s (tier %s) for node %s (current version: %s) — " + "type_changes=%s, order_fixed=%s, added_columns=%s, removed_columns=%s", updated_columns, + change_tier.name, node.name, node.current.version, type_changes, order_fixed, added_columns, + removed_columns, ) # Only create a new revision if the columns have been updated if updated_columns: # type: ignore new_revision = copy_existing_node_revision(node.current, current_user) # type: ignore - new_revision.version = str( - Version.parse(node.current.version).next_major_version(), # type: ignore - ) + new_revision.version = bump_version(node.current.version, change_tier) # type: ignore new_revision.status = node_validator.status # Snapshot pending m2m state before compile — autoflush during @@ -4111,19 +4367,16 @@ async def revalidate_node( # Record the revision bump so the audit trail reflects revalidate- # driven version changes, not just deploy-driven ones, and explains # *which* validator-detected differences triggered it (type changes, - # missing column orders, new columns). Skip when the caller will - # write its own (richer) event for the same bump. + # new columns, dropped columns, missing column orders). Skip when the + # caller will write its own (richer) event for the same bump. if record_revision_bump_event: history_details: dict = { "version": new_revision.version, "reason": "revalidate", + **column_changes, } - if type_changes: - history_details["type_changes"] = type_changes if order_fixed: history_details["order_fixed"] = order_fixed - if added_columns: - history_details["added_columns"] = added_columns await save_history( event=History( entity_type=EntityType.NODE, @@ -4135,6 +4388,38 @@ async def revalidate_node( ), session=session, ) + elif order_fixed: + # No new revision was earned, so the backfill lands on the current one. + # Leaving it unset is not free: readers sort by `order` with None last, so + # an unordered column drifts to the end of the projection. Only missing + # values are filled, so an order set deliberately is never overwritten. + for idx, validator_col in enumerate(node_validator.columns): + stored_col = existing_columns[validator_col.name] + if stored_col.order is None: + stored_col.order = idx + session.add(node.current) # type: ignore + + # The audit trail records that DJ touched the row even though the node's + # definition did not change, so a column that changes position has an + # explanation. Not gated on ``record_revision_bump_event``: this is not a + # revision bump, and no caller writes a richer event in its place. + await save_history( + event=History( + entity_type=EntityType.NODE, + entity_name=node.name, # type: ignore + node=node.name, # type: ignore + activity_type=ActivityType.UPDATE, + details={ + # The version the node still has -- this event explains a + # metadata fix, not a bump. + "version": node.current.version, # type: ignore + "reason": "column order backfill", + "order_fixed": order_fixed, + }, + user=current_user.username, + ), + session=session, + ) await session.commit() await session.refresh(node.current) # type: ignore await session.refresh(node, ["current"]) diff --git a/datajunction-server/datajunction_server/internal/sql.py b/datajunction-server/datajunction_server/internal/sql.py index 26689d09e0..3b9efa7232 100644 --- a/datajunction-server/datajunction_server/internal/sql.py +++ b/datajunction-server/datajunction_server/internal/sql.py @@ -188,6 +188,8 @@ async def generate_metrics_sql( dialect: Dialect | None = None, query_parameters: dict[str, Any] | None = None, endpoint: str = "/sql/metrics/v3/", + populate_cube_metrics: bool = True, + query_type: str | None = None, ) -> "BuildV3GeneratedSQL": """ Shared core for the "generate SQL for specific metrics" flow, used by both the @@ -198,8 +200,8 @@ async def generate_metrics_sql( provided, load the cube directly (pins it so ``find_matching_cube`` can't pick a different / differently-filtered materialization). - If a cube revision is in play, prepend its stored ``cube_filters`` to the - request filters (and fall back to the cube's full metric/dimension set for - a bare cube query with no explicit metrics/dimensions). + request filters. Metrics callers also fall back to the cube's metrics when + none are explicit; dimensions callers disable that behavior. - If ``dialect`` is None, auto-resolve it via ``resolve_dialect_and_engine_for_metrics``; adopt that resolver's cube only when no cube was otherwise provided (mirrors the canonical endpoint). @@ -227,7 +229,7 @@ async def generate_metrics_sql( if matched_cube is not None: if matched_cube.cube_filters: merged_filters = list(matched_cube.cube_filters) + merged_filters - if not metrics: + if not metrics and populate_cube_metrics: metrics = matched_cube.cube_node_metrics if not dimensions: dimensions = matched_cube.cube_node_dimensions @@ -236,7 +238,7 @@ async def generate_metrics_sql( # cube will actually back the build (materialized, Druid or auto-dialect), # verify it covers every filtered dimension so we fail loud here instead # of emitting Druid SQL that references a column the cube table lacks. - if use_materialized and dialect in (None, Dialect.DRUID): + if metrics and use_materialized and dialect in (None, Dialect.DRUID): await validate_pinned_cube_covers_filters( session, matched_cube, @@ -252,7 +254,7 @@ async def generate_metrics_sql( metrics=metrics, dimensions=dimensions, use_materialized=use_materialized, - matched_cube=matched_cube, + matched_cube=matched_cube if metrics else None, filters=merged_filters, ) resolved_dialect = execution_ctx.dialect @@ -274,7 +276,8 @@ async def generate_metrics_sql( ) elapsed_ms = (time.monotonic() - _t0) * 1000 - _tags = {"query_type": "metrics", "query_version": "v3"} + query_type = query_type or ("metrics" if metrics else "dimensions") + _tags = {"query_type": query_type, "query_version": "v3"} provider = get_metrics_provider() provider.timer("dj.sql.build_latency_ms", elapsed_ms, _tags) provider.counter("dj.sql.requests", tags=_tags) @@ -289,7 +292,7 @@ async def generate_metrics_sql( elapsed_ms, extra={ "endpoint": endpoint, - "query_type": "metrics", + "query_type": query_type, "query_version": "v3", "metrics": metrics, "dimensions": dimensions, @@ -300,6 +303,72 @@ async def generate_metrics_sql( return result +async def generate_dimensions_sql( + session: AsyncSession, + *, + dimensions: list[str], + filters: list[str] | None = None, + cube: str | None = None, + matched_cube: NodeRevision | None = None, + orderby: list[str] | None = None, + limit: int | None = None, + use_materialized: bool = True, + dialect: Dialect | None = None, + query_parameters: dict[str, Any] | None = None, + endpoint: str = "/sql/dimensions/v3/", +) -> "BuildV3GeneratedSQL": + """Generate direct-domain or cube-scoped distinct dimension values.""" + if cube and matched_cube is None: + cube_node = await Node.get_cube_by_name(session, cube) + if cube_node: + matched_cube = cube_node.current + + if matched_cube is None: + return await generate_metrics_sql( + session, + metrics=[], + dimensions=dimensions, + filters=filters, + orderby=orderby, + limit=limit, + use_materialized=use_materialized, + dialect=dialect, + query_parameters=query_parameters, + endpoint=endpoint, + populate_cube_metrics=False, + query_type="dimensions", + ) + + from datajunction_server.construction.build_v3.node_query import ( + project_dimension_values_sql, + ) + from datajunction_server.construction.build_v3.utils import ( + extract_filter_dimension_refs, + ) + + cube_filters = list(matched_cube.cube_filters or []) + merged_filters = cube_filters + list(filters or []) + required_dimensions = set(dimensions) | set( + extract_filter_dimension_refs(merged_filters), + ) + cube_covers_query = required_dimensions.issubset( + set(matched_cube.cube_node_dimensions), + ) + generated = await generate_metrics_sql( + session, + metrics=list(matched_cube.cube_node_metrics), + dimensions=dimensions, + filters=filters if cube_covers_query else merged_filters, + matched_cube=matched_cube if cube_covers_query else None, + use_materialized=use_materialized, + dialect=dialect, + query_parameters=query_parameters, + endpoint=endpoint, + query_type="dimensions", + ) + return project_dimension_values_sql(generated, dimensions, orderby, limit) + + async def build_sql_for_multiple_metrics( session: AsyncSession, metrics: list[str], diff --git a/datajunction-server/datajunction_server/materialization/jobs/cube_materialization.py b/datajunction-server/datajunction_server/materialization/jobs/cube_materialization.py index 8acd992f2c..f4e2e840b8 100644 --- a/datajunction-server/datajunction_server/materialization/jobs/cube_materialization.py +++ b/datajunction-server/datajunction_server/materialization/jobs/cube_materialization.py @@ -41,6 +41,8 @@ def schedule( self, materialization: Materialization, query_service_client: QueryServiceClient, + request_headers: dict[str, str] | None = None, + is_branch_deploy: bool = False, ): """ Since this is a settings-only dummy job, we do nothing in this stage. @@ -60,6 +62,7 @@ def schedule( materialization: Materialization, query_service_client: QueryServiceClient, request_headers: dict[str, str] | None = None, + is_branch_deploy: bool = False, ) -> MaterializationInfo: """ Use the query service to kick off the materialization setup. @@ -134,6 +137,7 @@ def schedule( materialization: Materialization, query_service_client: QueryServiceClient, request_headers: dict[str, str] | None = None, + is_branch_deploy: bool = False, ) -> MaterializationInfo: """ Use the query service to kick off the materialization setup. @@ -164,6 +168,7 @@ def schedule( platform=cube_config.platform, measures_materializations=cube_config.measures_materializations, combiners=cube_config.combiners, + is_branch_deploy=is_branch_deploy, ), request_headers=request_headers, ) diff --git a/datajunction-server/datajunction_server/materialization/jobs/materialization_job.py b/datajunction-server/datajunction_server/materialization/jobs/materialization_job.py index 4f9cdd7b3b..fcf061dff9 100644 --- a/datajunction-server/datajunction_server/materialization/jobs/materialization_job.py +++ b/datajunction-server/datajunction_server/materialization/jobs/materialization_job.py @@ -59,6 +59,8 @@ def schedule( self, materialization: Materialization, query_service_client: QueryServiceClient, + request_headers: dict[str, str] | None = None, + is_branch_deploy: bool = False, ) -> MaterializationInfo: """ Schedules the materialization job, typically done by calling a separate service @@ -80,6 +82,7 @@ def schedule( materialization: Materialization, query_service_client: QueryServiceClient, request_headers: dict[str, str] | None = None, + is_branch_deploy: bool = False, ) -> MaterializationInfo: """ Placeholder for the actual implementation. diff --git a/datajunction-server/datajunction_server/mcp/transport.py b/datajunction-server/datajunction_server/mcp/transport.py index f3bc56e005..d4f4f52fd7 100644 --- a/datajunction-server/datajunction_server/mcp/transport.py +++ b/datajunction-server/datajunction_server/mcp/transport.py @@ -13,6 +13,8 @@ contexts where the lifespan never fires (e.g. ASGITransport-based tests without ``LifespanManager``), the manager is started lazily on the first request. +- On lifespan exit the mount stops taking requests and waits up to + ``drain_timeout`` seconds for in-flight ones to respond. - Tools call ``get_mcp_session()`` to read the per-request ``AsyncSession``. """ @@ -30,6 +32,7 @@ from fastapi import FastAPI from mcp.server.streamable_http_manager import StreamableHTTPSessionManager +from starlette.responses import PlainTextResponse from starlette.routing import Mount from starlette.types import Receive, Scope, Send @@ -39,6 +42,9 @@ logger = logging.getLogger(__name__) +# Well under gunicorn's 30s graceful timeout. +DRAIN_TIMEOUT = 5.0 + # Re-export for convenience — tools and tests import from this module. __all__ = ["get_mcp_session", "mount_mcp"] @@ -48,6 +54,7 @@ def mount_mcp( path: str = "/mcp", *, request_context: Callable[[Scope], AbstractContextManager[object]] | None = None, + drain_timeout: float = DRAIN_TIMEOUT, ) -> None: """Mount the MCP HTTP transport on ``app`` at ``path``. @@ -61,6 +68,9 @@ def mount_mcp( query-service-client provider. The contextmanager is entered before the MCP request is handled and exited (even on error) afterward. + ``drain_timeout``: seconds to wait on lifespan exit for in-flight requests + to respond before the session manager tears their streams down. + Must be called once during app construction. """ session_manager = StreamableHTTPSessionManager( @@ -89,7 +99,13 @@ async def ensure_started() -> None: lazy_started = True logger.info("MCP session manager started lazily (no lifespan)") - async def asgi_handler(scope: Scope, receive: Receive, send: Send) -> None: + # In-flight request tracking, so shutdown can wait them out. + in_flight = 0 + idle = asyncio.Event() + idle.set() + draining = False + + async def handle_scope(scope: Scope, receive: Receive, send: Send) -> None: """Wrap MCP request handling with a DB session bound to a ContextVar.""" if scope["type"] != "http": # pragma: no cover await ensure_started() @@ -111,6 +127,35 @@ async def asgi_handler(scope: Scope, receive: Receive, send: Send) -> None: finally: _session_var.reset(token) + async def asgi_handler(scope: Scope, receive: Receive, send: Send) -> None: + """Count the request, or turn it away while draining.""" + nonlocal in_flight + if draining: + response = PlainTextResponse("Server shutting down", status_code=503) + await response(scope, receive, send) + return + + in_flight += 1 + idle.clear() + try: + await handle_scope(scope, receive, send) + finally: + in_flight -= 1 + if in_flight == 0: + idle.set() + + async def drain_requests() -> None: + """Wait for in-flight MCP requests to respond.""" + nonlocal draining + draining = True + if idle.is_set(): + return + logger.info("Draining %d MCP requests", in_flight) + try: + await asyncio.wait_for(idle.wait(), drain_timeout) + except TimeoutError: + logger.warning("Dropping %d MCP requests still in flight", in_flight) + app.router.routes.append(Mount(path, app=asgi_handler)) # Compose the MCP manager's task-group lifespan with whatever the app @@ -124,7 +169,10 @@ async def combined_lifespan(_app: FastAPI): started_via_lifespan = True try: async with original_lifespan(_app): - yield + try: + yield + finally: + await drain_requests() finally: started_via_lifespan = False diff --git a/datajunction-server/datajunction_server/models/access.py b/datajunction-server/datajunction_server/models/access.py index cde1ce3a68..4a836f3285 100644 --- a/datajunction-server/datajunction_server/models/access.py +++ b/datajunction-server/datajunction_server/models/access.py @@ -30,6 +30,17 @@ class ResourceAction(StrEnum): MANAGE = "manage" # Grant/revoke permissions (RBAC-specific) +def namespace_boundary_scope_targets( + namespace: str, +) -> tuple[tuple[ResourceType, str], ...]: + """Return every scope governed by a namespace boundary.""" + return ( + (ResourceType.NAMESPACE, namespace), + (ResourceType.NAMESPACE, f"{namespace}.*"), + (ResourceType.NODE, f"{namespace}.*"), + ) + + @dataclass(frozen=True) class RestrictiveScopeRule: """A configured action and resource scope that requires an explicit grant.""" diff --git a/datajunction-server/datajunction_server/models/cube_materialization.py b/datajunction-server/datajunction_server/models/cube_materialization.py index 780afacc76..1e12ebb40f 100644 --- a/datajunction-server/datajunction_server/models/cube_materialization.py +++ b/datajunction-server/datajunction_server/models/cube_materialization.py @@ -571,6 +571,11 @@ class DruidCubeMaterializationInput(BaseModel): # possible for metrics at different levels. combiners: list[CombineMaterialization] + # True when scheduled against a branch-preview deployment rather than main, + # so the query service can decide how to target its run (e.g. avoid + # gap-closing backfills against today's date). + is_branch_deploy: bool = False + # ============================================================================= # V2: Pre-agg based cube materialization diff --git a/datajunction-server/datajunction_server/models/custom_metadata.py b/datajunction-server/datajunction_server/models/custom_metadata.py index 8f9ace1846..8cc940ba1b 100644 --- a/datajunction-server/datajunction_server/models/custom_metadata.py +++ b/datajunction-server/datajunction_server/models/custom_metadata.py @@ -33,7 +33,6 @@ class CustomMetadataSchemaCreate(BaseModel): json_schema: dict filterable: bool = True description: str | None = None - owner: str | None = None reserved: bool = False @@ -46,7 +45,6 @@ class CustomMetadataSchemaOutput(BaseModel): value_kind: str | None = None filterable: bool description: str | None = None - owner: str | None = None reserved: bool = False created_by_id: int | None = None updated_by_id: int | None = None diff --git a/datajunction-server/datajunction_server/models/deployment.py b/datajunction-server/datajunction_server/models/deployment.py index 75329e5429..bfc3545192 100644 --- a/datajunction-server/datajunction_server/models/deployment.py +++ b/datajunction-server/datajunction_server/models/deployment.py @@ -19,7 +19,6 @@ DJInvalidDeploymentConfig, DJInvalidInputException, ) -from datajunction_server.models.base import labelize from datajunction_server.models.dimensionlink import ( JoinCardinality, JoinType, @@ -41,6 +40,7 @@ NodeType, ) from datajunction_server.models.partition import Granularity, PartitionType +from datajunction_server.models.semantic_fingerprint import SemanticFingerprintValue from datajunction_server.models.unit import ( Unit, legacy_unit_to_structured, @@ -68,6 +68,19 @@ class ChangeTier(IntEnum): MAJOR = 20 +ChangeTierName = Literal["none", "minor", "major"] + + +def change_tier_name(tier: ChangeTier) -> ChangeTierName: + """Return the stable API representation of a change tier.""" + names: dict[ChangeTier, ChangeTierName] = { + ChangeTier.NONE: "none", + ChangeTier.MINOR: "minor", + ChangeTier.MAJOR: "major", + } + return names[tier] + + def fold_change_tiers(tiers: Iterable[ChangeTier]) -> ChangeTier: """ Reduce the tiers of several individual changes to the tier of the change as a @@ -90,6 +103,26 @@ def bump_version(version: str, tier: ChangeTier) -> str: return str(parsed) +def version_change_tier(old_version: str, new_version: str) -> ChangeTier: + """ + The tier that turned `old_version` into `new_version` -- `bump_version` read + backwards. + + Callers that only see the two version strings, notably downstream propagation + (which runs after the upstream revision has already been committed and so never + sees the classifier's answer), recover the tier here rather than plumbing it + through every update path. A major bump reads as MAJOR even when the minor part + also moved, since a major bump resets the minor to zero. + """ + old = Version.parse(old_version) + new = Version.parse(new_version) + if new.major != old.major: + return ChangeTier.MAJOR + if new.minor != old.minor: + return ChangeTier.MINOR + return ChangeTier.NONE + + class DeploymentStatus(str, Enum): PENDING = "pending" RUNNING = "running" @@ -473,10 +506,13 @@ class DimensionLinkSpec(BaseModel): role: str | None = None namespace: str | None = Field(default=None, exclude=True) + def _comparison_key(self) -> tuple[Any, ...]: + return (self.type, self.role) + def __eq__(self, other: object) -> bool: if not isinstance(other, DimensionLinkSpec): return False # pragma: no cover - return self.type == other.type and self.role == other.role + return self._comparison_key() == other._comparison_key() class DimensionJoinLinkSpec(DimensionLinkSpec): @@ -517,30 +553,17 @@ def rendered_join_on(self) -> str | None: ) def __hash__(self) -> int: - return hash( - ( - self.type, - self.role, - self.rendered_dimension_node, - self.join_type, - self.join_cardinality, - self.rendered_join_on, - self.node_column, - self.default_value, - ), - ) + return hash(self._comparison_key()) - def __eq__(self, other: object) -> bool: - if not isinstance(other, DimensionJoinLinkSpec): - return False # pragma: no cover + def _comparison_key(self) -> tuple[Any, ...]: return ( - super().__eq__(other) - and self.rendered_dimension_node == other.rendered_dimension_node - and self.join_type == other.join_type - and self.join_cardinality == other.join_cardinality - and self.rendered_join_on == other.rendered_join_on - and self.node_column == other.node_column - and self.default_value == other.default_value + *super()._comparison_key(), + self.rendered_dimension_node, + self.join_type, + self.join_cardinality, + self.rendered_join_on, + self.node_column, + self.default_value, ) @@ -570,24 +593,14 @@ def dimension_attribute(self) -> str: return self.dimension.rsplit(".", 1)[-1] def __hash__(self) -> int: - return hash( - ( - self.type, - self.role, - self.rendered_dimension_node, - self.dimension_attribute, - self.node_column, - ), - ) + return hash(self._comparison_key()) - def __eq__(self, other: object) -> bool: - if not isinstance(other, DimensionReferenceLinkSpec): - return False + def _comparison_key(self) -> tuple[Any, ...]: return ( - super().__eq__(other) - and self.rendered_dimension_node == other.rendered_dimension_node - and self.dimension_attribute == other.dimension_attribute - and self.node_column == other.node_column + *super()._comparison_key(), + self.rendered_dimension_node, + self.dimension_attribute, + self.node_column, ) @@ -642,8 +655,10 @@ class NodeSpec(NamespacedSpec): # anything. Fields absent here are not order-sensitive, so reordering them is # not a change at all. `diff()` compares list fields as sets and cannot see a # reorder on its own, which is why `order_diff()` exists alongside it. - FIELD_ORDER_CHANGE_TIERS: ClassVar[dict[str, ChangeTier]] = {} - + FIELD_ORDER_CHANGE_TIERS: ClassVar[dict[str, ChangeTier]] = { + "owners": ChangeTier.NONE, + "tags": ChangeTier.NONE, + } _query_ast: Any | None = PrivateAttr(default=None) # Internal: marks specs from already-validated sources (e.g., branch copies) # that can skip expensive SQL parsing and validation @@ -716,14 +731,34 @@ def rendered_spec(self) -> "NodeSpec": rendered_json = json.dumps(raw).replace("${prefix}", prefix) return self.__class__.model_validate_json(rendered_json) + def semantic_diff( + self, + other: "NodeSpec", + *, + resolved_columns: list[ColumnSpec] | None = None, + other_resolved_columns: list[ColumnSpec] | None = None, + ) -> tuple[list[str], list[str]]: + """Compare two specs using the same normalized values as fingerprints.""" + from datajunction_server.semantic_fingerprints import ( + semantic_diff as compare_semantics, + ) + + return compare_semantics( + self, + other, + resolved_columns=resolved_columns, + other_resolved_columns=other_resolved_columns, + ) + def diff(self, other: "NodeSpec") -> list[str]: """ Return a list of fields that differ between this and another NodeSpec. - Renders ${prefix} placeholders in `other` before comparing so that - specs with unresolved prefixes don't produce false positives. + Renders ${prefix} on both sides -- rendering only `other` would leave + `self` compared against a rendered value it can never match, since + `description`/`custom_metadata` store `${prefix}` verbatim. """ return diff( - self, + self.rendered_spec(), other.rendered_spec(), ignore_fields=["name", "namespace", "query", "columns"], ) @@ -768,6 +803,11 @@ def has_explicit_change_tier(cls, field: str) -> bool: """Whether some class in the MRO classifies `field`.""" return cls._declared_tier("FIELD_CHANGE_TIERS", field) is not None + @classmethod + def has_explicit_order_change_tier(cls, field: str) -> bool: + """Whether some class in the MRO classifies reordering `field`.""" + return cls._declared_tier("FIELD_ORDER_CHANGE_TIERS", field) is not None + @classmethod def unclassified_fields(cls) -> list[str]: """Fields on this spec class that nobody classified. Should always be empty.""" @@ -777,13 +817,31 @@ def unclassified_fields(cls) -> list[str]: if not cls.has_explicit_change_tier(field) ] + @classmethod + def unclassified_list_order_fields(cls) -> list[str]: + """List fields without an explicit order classification.""" + from datajunction_server.semantic_fingerprints.normalization import ( + annotation_contains_list, + ) + + return [ + field + for field, field_info in cls.model_fields.items() + if annotation_contains_list(field_info.annotation) + and not cls.has_explicit_order_change_tier(field) + ] + @classmethod def order_sensitive_fields(cls) -> list[str]: - """Fields for which some class in the MRO classifies a reorder.""" + """Fields whose declared reorder tier is not NONE.""" fields: dict[str, None] = {} for klass in cls.__mro__: - for field in klass.__dict__.get("FIELD_ORDER_CHANGE_TIERS", {}): - fields.setdefault(field, None) + for field, tier in klass.__dict__.get( + "FIELD_ORDER_CHANGE_TIERS", + {}, + ).items(): + if tier != ChangeTier.NONE: + fields.setdefault(field, None) return list(fields) @classmethod @@ -842,6 +900,11 @@ class LinkableNodeSpec(NodeSpec): "dimension_links": ChangeTier.MAJOR, "primary_key": ChangeTier.MAJOR, } + FIELD_ORDER_CHANGE_TIERS: ClassVar[dict[str, ChangeTier]] = { + "columns": ChangeTier.NONE, + "dimension_links": ChangeTier.NONE, + "primary_key": ChangeTier.NONE, + } @model_validator(mode="after") def set_namespaces(self): @@ -864,21 +927,25 @@ def links_mapping(self) -> dict[tuple[str, str | None], DimensionLinkSpec]: def __eq__(self, other: object) -> bool: if not isinstance(other, LinkableNodeSpec): return False # pragma: no cover - dimension_links_equal = sorted( - self.dimension_links or [], - key=lambda link: (link.rendered_dimension_node, link.role or ""), - ) == sorted( - other.dimension_links or [], - key=lambda link: (link.rendered_dimension_node, link.role or ""), + from datajunction_server.semantic_fingerprints.normalization import ( + normalize_dimension_links, ) + return ( super().__eq__(other) and eq_columns( self.columns, other.columns, - compare_types=True if self.node_type == NodeType.SOURCE else False, + compare_types=self.node_type == NodeType.SOURCE, + ) + and normalize_dimension_links( + self.dimension_links, + preserve_order=False, + ) + == normalize_dimension_links( + other.dimension_links, + preserve_order=False, ) - and dimension_links_equal and set(self.primary_key or []) == set(other.primary_key or []) ) @@ -983,7 +1050,7 @@ class MetricSpec(NodeSpec): FIELD_CHANGE_TIERS: ClassVar[dict[str, ChangeTier]] = { "query": ChangeTier.MAJOR, - "columns": ChangeTier.MAJOR, + "columns": ChangeTier.NONE, # Required dimensions constrain which queries the metric can answer. "required_dimensions": ChangeTier.MAJOR, # Everything below is presentation metadata on the metric's single output @@ -996,6 +1063,10 @@ class MetricSpec(NodeSpec): "min_decimal_exponent": ChangeTier.MINOR, "max_decimal_exponent": ChangeTier.MINOR, } + FIELD_ORDER_CHANGE_TIERS: ClassVar[dict[str, ChangeTier]] = { + "columns": ChangeTier.NONE, + "required_dimensions": ChangeTier.NONE, + } # Class-level adapter used by __init__ to eagerly validate structured # unit input. `ClassVar` keeps Pydantic from treating it as a field. @@ -1037,7 +1108,7 @@ def __init__(self, **data: Any): @property def unit(self) -> str | dict | None: """ - Return the canonical metric unit value for serialization. + Return the normalized metric unit value for serialization. Returns: - `None` if no unit is set. @@ -1049,7 +1120,7 @@ def unit(self) -> str | dict | None: shape should read `column.unit` on the metric's output column. """ if self.unit_structured is not None: - # Canonical dict shape (JSON-friendly, no None values). + # Normalized dict shape (JSON-friendly, no None values). return unit_to_dict(self.unit_structured) if self.unit_enum is None or self.unit_enum == MetricUnit.UNKNOWN: return None @@ -1071,19 +1142,67 @@ def rendered_required_dimensions(self) -> list[str]: for required_dim in (self.required_dimensions or []) ] + @property + def canonical_required_dimensions(self) -> list[str]: + """ + Required dimensions rewritten so the two ways of naming the same column + compare equal. + + ns.orders_fact.currency_code -> currency_code (a query parent) + currency_code -> currency_code (already canonical) + ns.date_dim.dateint -> ns.date_dim.dateint (not a query parent) + + Parents come from the spec's own query, so this needs no session. + """ + required_dims = self.rendered_required_dimensions + if not any(SEPARATOR in required_dim for required_dim in required_dims): + return required_dims + + from datajunction_server.internal.deployment.utils import ( + extract_upstream_candidates, + ) + + parents = ( + extract_upstream_candidates(self.query_ast, is_metric=True) + if self.query_ast is not None + else set() + ) + return [ + required_dim.rsplit(SEPARATOR, 1)[1] + if SEPARATOR in required_dim + and required_dim.rsplit(SEPARATOR, 1)[0] in parents + else required_dim + for required_dim in required_dims + ] + + def diff(self, other: "NodeSpec") -> list[str]: + """ + Diffs `required_dimensions` on its canonical form, so the two spellings + of a parent column don't read as a change and inflate the change tier. + """ + changed = super().diff(other) + if ( + "required_dimensions" in changed + and isinstance(other, MetricSpec) + and set(self.canonical_required_dimensions) + == set(other.canonical_required_dimensions) + ): + changed.remove("required_dimensions") + return changed + def model_dump(self, **kwargs): # pragma: no cover base = super().model_dump(**kwargs) base["unit"] = self.unit return base - def _canonical_unit(self) -> "Unit | None": + def _normalized_unit(self) -> "Unit | None": """ - Reduce both legacy and structured inputs to the same canonical Unit + Reduce both legacy and structured inputs to the same normalized Unit instance for equality comparisons. Returns None when the metric has no unit (or only the UNKNOWN sentinel). Two specs that author the same conceptual unit via different input shapes (`unit: dollar` vs `unit: {kind: currency, code: USD}`) produce equal frozen Unit - instances — so __eq__ doesn't falsely report drift between YAML and + instances, so __eq__ doesn't falsely report drift between YAML and DB-roundtripped specs. """ if self.unit_structured is not None: @@ -1095,12 +1214,23 @@ def _canonical_unit(self) -> "Unit | None": def __eq__(self, other: object) -> bool: if not isinstance(other, MetricSpec): return False + from datajunction_server.semantic_fingerprints.normalization import ( + normalize_sequence, + ) + return ( super().__eq__(other) and self.query_ast.compare(other.query_ast) - and (self.required_dimensions or []) == (other.required_dimensions or []) + and normalize_sequence( + self.canonical_required_dimensions, + preserve_order=False, + ) + == normalize_sequence( + other.canonical_required_dimensions, + preserve_order=False, + ) and eq_or_fallback(self.direction, other.direction, MetricDirection.NEUTRAL) - and self._canonical_unit() == other._canonical_unit() + and self._normalized_unit() == other._normalized_unit() and self.significant_digits == other.significant_digits and self.min_decimal_exponent == other.min_decimal_exponent and self.max_decimal_exponent == other.max_decimal_exponent @@ -1177,6 +1307,8 @@ class CubeSpec(NodeSpec): # Filters are ANDed together, so their ordering carries no meaning at all # and reordering them is genuinely a no-op. "filters": ChangeTier.NONE, + "columns": ChangeTier.NONE, + "materialization": ChangeTier.NONE, } @field_validator("materialization", mode="before") @@ -1272,6 +1404,27 @@ def rendered_columns(self) -> list[ColumnSpec]: rendered.append(rendered_col) return rendered + @property + def matched_rendered_columns(self) -> list["ColumnSpec"]: + """ + The ``columns:`` entries naming a column this cube has, keyed the same way + the deploy keys them: `Column.cube_element_name`, so a metric or dimension + as written, with its `[role]` suffix. Anything else is never persisted, so + comparing it would report a change on every deploy. + """ + cube_columns = set(self.rendered_metrics) | set(self.rendered_dimensions) + return [col for col in self.rendered_columns if col.name in cube_columns] + + @property + def unmatched_column_names(self) -> list[str]: + """ + Names in ``columns:`` that are not columns of this cube, and so are ignored. + """ + cube_columns = set(self.rendered_metrics) | set(self.rendered_dimensions) + return [ + col.name for col in self.rendered_columns if col.name not in cube_columns + ] + def __eq__(self, other: object) -> bool: if not isinstance(other, CubeSpec): return False @@ -1298,15 +1451,13 @@ def __eq__(self, other: object) -> bool: # Compare only partition config for user-specified columns. # Cube element columns (types, order, attributes) are auto-derived and ignored. - incoming_partitions = { - col.name: col.partition for col in self.rendered_columns if col.partition - } - existing_partitions = { - col.name: col.partition - for col in (other.rendered_columns or []) - if col.partition - } - return incoming_partitions == existing_partitions + from datajunction_server.semantic_fingerprints.normalization import ( + normalize_cube_columns, + ) + + return normalize_cube_columns( + self.matched_rendered_columns, + ) == normalize_cube_columns(other.matched_rendered_columns) NodeUnion = Annotated[ @@ -1366,7 +1517,7 @@ def diff( """ return [ field - for field in one.model_fields.keys() + for field in one.model_fields if field not in (ignore_fields or []) and hasattr(one, field) and hasattr(two, field) @@ -1374,6 +1525,25 @@ def diff( ] +class CustomMetadataSchemaSpec(BaseModel): + """ + Specification for a custom_metadata JSON Schema to register for a namespace. + + The namespace defaults to the enclosing DeploymentSpec's. Naming one narrows the + schema to a sub-namespace, which is how a vocabulary can be rolled out to part of + a repo's graph -- conformed dimensions first, say -- before it governs all of it. + A namespace outside the deploying one is rejected: a manifest may scope narrower + than itself, never wider. + """ + + key: str + node_type: NodeType | None = None + namespace: str | None = None + json_schema: dict + filterable: bool = True + description: str | None = None + + class GitDeploymentSource(BaseModel): """ Deployment from a tracked git repository. @@ -1473,6 +1643,11 @@ class DeploymentSpec(BaseModel): tags: list[TagSpec] = Field(default_factory=list) hierarchies: list[HierarchySpec] = Field(default_factory=list) preaggregations: list[PreAggSpec] = Field(default_factory=list) + # None and [] mean different things: None is "this manifest does not manage + # schemas", [] is "it manages them and declares none", which retires the + # namespace's rows. A list default would make every deployment that omits + # the section look like the latter. + custom_metadata_schemas: list[CustomMetadataSchemaSpec] | None = None source: DeploymentSource | None = None # CI/CD provenance tracking git_config: NamespaceGitConfig | None = None # Git branch management config force: bool = Field( @@ -1542,6 +1717,22 @@ def set_namespaces(self): for preagg in self.preaggregations: if not preagg.namespace: preagg.namespace = self.namespace + for schema in self.custom_metadata_schemas or []: + if not schema.namespace: + schema.namespace = self.namespace + elif schema.namespace != self.namespace and not schema.namespace.startswith( + f"{self.namespace}.", + ): + # Narrower than the deploying namespace is a rollout choice; + # wider, or sideways, would let one repo govern another's nodes. + raise DJInvalidDeploymentConfig( + message=( + f"custom_metadata schema '{schema.key}' declares namespace " + f"'{schema.namespace}', which is not '{self.namespace}' or " + "beneath it. A deployment may scope a schema to its own " + "namespace or a sub-namespace, never to another." + ), + ) return self @@ -1590,6 +1781,11 @@ class Type(str, Enum): operation: Operation message: str = "" changed_fields: list[str] = Field(default_factory=list) + change_tier: ChangeTierName | None = None + semantic_fingerprint: SemanticFingerprintValue | None = None + # True when the node was re-deployed only to retry a pre-existing failure. + # Failure reasons are not compared. Nullable for older persisted rows. + revalidation_only: bool | None = None class DeploymentInfo(BaseModel): @@ -1627,6 +1823,8 @@ def eq_columns( - If a column is missing display_name or description, it's treated as empty string. If the compare_types flag is False, the column types will not be compared. """ + from datajunction_server.semantic_fingerprints.normalization import normalize_column + a_map = {col.name: col for col in a or []} b_map = {col.name: col for col in b or []} # For source nodes (compare_types=True), column additions and removals from @@ -1635,39 +1833,19 @@ def eq_columns( if compare_types and a and b and set(a_map.keys()) != set(b_map.keys()): return False a_cols, b_cols = [], [] - for col_name in set(a_map.keys()).union(set(b_map.keys())): - a_col = a_map.get(col_name).model_copy() if a_map.get(col_name) else None # type: ignore - b_col = b_map.get(col_name).model_copy() if b_map.get(col_name) else None # type: ignore - if not a_col: - a_col = ColumnSpec( - name=col_name, - display_name=labelize(col_name), - type=b_col.type if b_col else "", - attributes=[], - ) - if not a_col.display_name: - a_col.display_name = labelize(col_name) - if not a_col.description: - a_col.description = "" - if not b_col: - b_col = ColumnSpec( # pragma: no cover - name=col_name, - display_name=labelize(col_name), - type=a_col.type if a_col else "", - attributes=[], - ) - if not b_col.display_name: - b_col.display_name = labelize(col_name) - if not b_col.description: # pragma: no cover - b_col.description = "" - if not compare_types: - a_col.type = "" - b_col.type = "" - # Remove primary_key from copies for comparison - if "primary_key" in a_col.attributes: - a_col.attributes = list(set(a_col.attributes) - {"primary_key"}) - if "primary_key" in b_col.attributes: - b_col.attributes = list(set(b_col.attributes) - {"primary_key"}) + for col_name in sorted(set(a_map).union(b_map)): + a_col = normalize_column( + a_map.get(col_name), + col_name, + b_map[col_name].type if col_name in b_map else "", + compare_types, + ) + b_col = normalize_column( + b_map.get(col_name), + col_name, + a_map[col_name].type if col_name in a_map else "", + compare_types, + ) a_cols.append(a_col) b_cols.append(b_col) return a_cols == b_cols diff --git a/datajunction-server/datajunction_server/models/dimensionlink.py b/datajunction-server/datajunction_server/models/dimensionlink.py index 76d5657e2d..9c50fa6e3a 100644 --- a/datajunction-server/datajunction_server/models/dimensionlink.py +++ b/datajunction-server/datajunction_server/models/dimensionlink.py @@ -49,6 +49,23 @@ class LinkType(StrEnum): REFERENCE = "reference" +def missing_join_on_message(node_name: str, dimension_node: str) -> str: + """Error text for a join link with no join_on.""" + return ( + f"Dimension link from {node_name} to {dimension_node} has no join_on " + "clause. Set join_on to the equality between this node's foreign key " + "column(s) and the dimension's primary key." + ) + + +def misplaced_node_column_message(node_name: str, dimension_node: str) -> str: + """Error text for node_column on a join link.""" + return ( + f"Dimension link from {node_name} to {dimension_node} sets node_column, " + "which only applies to reference links. Express the join in join_on instead." + ) + + class LinkDimensionIdentifier(BaseModel): """ Input for linking a dimension to a node diff --git a/datajunction-server/datajunction_server/models/impact.py b/datajunction-server/datajunction_server/models/impact.py index c0b67cdf1d..866764904c 100644 --- a/datajunction-server/datajunction_server/models/impact.py +++ b/datajunction-server/datajunction_server/models/impact.py @@ -7,6 +7,7 @@ from pydantic import BaseModel, Field from datajunction_server.models.node import NodeStatus, NodeType +from datajunction_server.models.semantic_fingerprint import SemanticFingerprintValue class ImpactType(str, Enum): @@ -34,3 +35,4 @@ class DownstreamImpact(BaseModel): # Defaulted so that deployment rows persisted before this field existed can # still be rehydrated from JSON. owners: list[str] = Field(default_factory=list) + semantic_fingerprint: SemanticFingerprintValue | None = None diff --git a/datajunction-server/datajunction_server/models/materialization.py b/datajunction-server/datajunction_server/models/materialization.py index 6bb2329c43..421e6269ca 100644 --- a/datajunction-server/datajunction_server/models/materialization.py +++ b/datajunction-server/datajunction_server/models/materialization.py @@ -288,6 +288,12 @@ class MaterializationConfigInfoUnified( Materialization config + info """ + # The node revision's own version string, not derivable from `config` for every + # job type (e.g. DruidCubeMaterializationJob's config has no `cube` field), so a + # caller listing materializations `include_all_revisions=True` can tell which + # revision each one belongs to without a separate lookup. + node_version: str + class SparkConf(RootModel): """Spark configuration""" diff --git a/datajunction-server/datajunction_server/models/semantic_fingerprint.py b/datajunction-server/datajunction_server/models/semantic_fingerprint.py new file mode 100644 index 0000000000..fc42e51317 --- /dev/null +++ b/datajunction-server/datajunction_server/models/semantic_fingerprint.py @@ -0,0 +1,33 @@ +"""Models and version constants for semantic fingerprints.""" + +from typing import Literal, TypeAlias + +from pydantic import BaseModel, Field, field_validator + + +LATEST_SEMANTIC_FINGERPRINT_VERSION = 1 +SUPPORTED_SEMANTIC_FINGERPRINT_VERSIONS = frozenset( + {LATEST_SEMANTIC_FINGERPRINT_VERSION}, +) +UNKNOWN_SEMANTIC_FINGERPRINT: Literal["unknown"] = "unknown" + + +class SemanticFingerprint(BaseModel): + """A versioned digest of a node's semantic definition.""" + + version: int = LATEST_SEMANTIC_FINGERPRINT_VERSION + digest: str = Field( + min_length=64, + max_length=64, + pattern=r"^[0-9a-f]+$", + ) + + @field_validator("version") + @classmethod + def validate_version(cls, version: int) -> int: + if version not in SUPPORTED_SEMANTIC_FINGERPRINT_VERSIONS: + raise ValueError(f"Unsupported semantic fingerprint version: {version}") + return version + + +SemanticFingerprintValue: TypeAlias = SemanticFingerprint | Literal["unknown"] diff --git a/datajunction-server/datajunction_server/semantic_fingerprints/__init__.py b/datajunction-server/datajunction_server/semantic_fingerprints/__init__.py new file mode 100644 index 0000000000..05ca21edf4 --- /dev/null +++ b/datajunction-server/datajunction_server/semantic_fingerprints/__init__.py @@ -0,0 +1,5 @@ +"""Semantic fingerprint construction and comparison.""" + +from datajunction_server.semantic_fingerprints.engine import semantic_diff + +__all__ = ["semantic_diff"] diff --git a/datajunction-server/datajunction_server/semantic_fingerprints/engine.py b/datajunction-server/datajunction_server/semantic_fingerprints/engine.py new file mode 100644 index 0000000000..5da8beeb2b --- /dev/null +++ b/datajunction-server/datajunction_server/semantic_fingerprints/engine.py @@ -0,0 +1,72 @@ +"""Version dispatch for semantic fingerprints.""" + +from __future__ import annotations + +from collections.abc import Callable, Iterable +from typing import TYPE_CHECKING + +from datajunction_server.models.semantic_fingerprint import ( + LATEST_SEMANTIC_FINGERPRINT_VERSION, + SemanticFingerprint, +) +from datajunction_server.semantic_fingerprints.normalization import ( + semantic_diff as compare_semantics, +) +from datajunction_server.semantic_fingerprints.v1 import build_fingerprint + +if TYPE_CHECKING: + from datajunction_server.models.deployment import ColumnSpec, NodeSpec + + +_BUILDERS: dict[int, Callable[..., SemanticFingerprint]] = { + 1: build_fingerprint, +} + + +def compose_node_fingerprint( + spec: NodeSpec, + version: int = LATEST_SEMANTIC_FINGERPRINT_VERSION, + *, + parent_fingerprints: Iterable[SemanticFingerprint], + resolved_columns: list[ColumnSpec] | None = None, +) -> SemanticFingerprint: + """Compose a node fingerprint from its definition and parent fingerprints.""" + builder = _BUILDERS.get(version) + if builder is None: + raise ValueError(f"Unsupported semantic fingerprint version: {version}") + return builder( + spec, + parent_fingerprints, + resolved_columns=resolved_columns, + ) + + +def local_node_fingerprint( + spec: NodeSpec, + version: int = LATEST_SEMANTIC_FINGERPRINT_VERSION, + *, + resolved_columns: list[ColumnSpec] | None = None, +) -> SemanticFingerprint: + """Fingerprint a node definition without graph parents.""" + return compose_node_fingerprint( + spec, + version, + parent_fingerprints=(), + resolved_columns=resolved_columns, + ) + + +def semantic_diff( + one: NodeSpec, + two: NodeSpec, + *, + resolved_columns: list[ColumnSpec] | None = None, + other_resolved_columns: list[ColumnSpec] | None = None, +) -> tuple[list[str], list[str]]: + """Compare two specs using the same normalized values as fingerprints.""" + return compare_semantics( + one, + two, + resolved_columns=resolved_columns, + other_resolved_columns=other_resolved_columns, + ) diff --git a/datajunction-server/datajunction_server/semantic_fingerprints/merkle.py b/datajunction-server/datajunction_server/semantic_fingerprints/merkle.py new file mode 100644 index 0000000000..121b90f84c --- /dev/null +++ b/datajunction-server/datajunction_server/semantic_fingerprints/merkle.py @@ -0,0 +1,96 @@ +"""Pure graph and component hashing for semantic fingerprints.""" + +import hashlib + +from datajunction_server.models.semantic_fingerprint import SemanticFingerprint +from datajunction_server.semantic_fingerprints.normalization import canonical_json + + +def strongly_connected_components( + graph: dict[str, list[str]], +) -> list[tuple[str, ...]]: + """Find graph components iteratively in deterministic order.""" + visited: set[str] = set() + finish_order: list[str] = [] + for start in sorted(graph): + if start in visited: + continue + visited.add(start) + dfs_stack = [(start, 0)] + while dfs_stack: + name, parent_index = dfs_stack[-1] + parents = graph[name] + if parent_index < len(parents): + parent = parents[parent_index] + dfs_stack[-1] = (name, parent_index + 1) + if parent not in visited: + visited.add(parent) + dfs_stack.append((parent, 0)) + continue + dfs_stack.pop() + finish_order.append(name) + + reverse_graph: dict[str, list[str]] = {name: [] for name in graph} + for child, parents in graph.items(): + for parent in parents: + reverse_graph[parent].append(child) + for children in reverse_graph.values(): + children.sort() + + components: list[tuple[str, ...]] = [] + visited.clear() + for start in reversed(finish_order): + if start in visited: + continue + visited.add(start) + members = [] + component_stack = [start] + while component_stack: + name = component_stack.pop() + members.append(name) + for child in reversed(reverse_graph[name]): + if child not in visited: + visited.add(child) + component_stack.append(child) + components.append(tuple(sorted(members))) + return components + + +def cycle_component_fingerprint( + members: tuple[str, ...], + local_fingerprints: dict[str, SemanticFingerprint], + internal_edges: list[tuple[str, str]], + external_edges: list[tuple[str, str, SemanticFingerprint]], +) -> SemanticFingerprint: + """Hash the members and edge structure of one cyclic component.""" + version = local_fingerprints[members[0]].version + payload = { + "domain": "datajunction/node-semantic-scc", + "version": version, + "members": [ + { + "name": name, + "fingerprint": local_fingerprints[name].model_dump(mode="json"), + } + for name in members + ], + "internal_edges": [ + {"child": child, "parent": parent} + for child, parent in sorted(internal_edges) + ], + "external_edges": [ + { + "child": child, + "parent": parent, + "fingerprint": fingerprint.model_dump(mode="json"), + } + for child, parent, fingerprint in sorted( + external_edges, + key=lambda edge: (edge[0], edge[1]), + ) + ], + } + return SemanticFingerprint( + version=version, + digest=hashlib.sha256(canonical_json(payload).encode("utf-8")).hexdigest(), + ) diff --git a/datajunction-server/datajunction_server/semantic_fingerprints/normalization.py b/datajunction-server/datajunction_server/semantic_fingerprints/normalization.py new file mode 100644 index 0000000000..b00d0c148f --- /dev/null +++ b/datajunction-server/datajunction_server/semantic_fingerprints/normalization.py @@ -0,0 +1,283 @@ +"""Semantic value normalization shared by comparison and fingerprinting.""" + +import json +import math +from collections.abc import Iterable +from decimal import Decimal +from enum import Enum +from typing import Any, get_args, get_origin + +from pydantic import BaseModel + +from datajunction_server.models.base import labelize +from datajunction_server.models.deployment import ( + ChangeTier, + ColumnSpec, + CubeSpec, + DimensionJoinLinkSpec, + DimensionReferenceLinkSpec, + LinkableNodeSpec, + MetricSpec, + NodeSpec, + SourceSpec, +) +from datajunction_server.models.node import MetricDirection +from datajunction_server.sql.parsing.backends.exceptions import DJParseException + + +def canonical_json(value: Any) -> str: + """Serialize a fingerprint value deterministically.""" + return json.dumps( + value, + allow_nan=False, + ensure_ascii=False, + separators=(",", ":"), + sort_keys=True, + ) + + +def normalize_value(value: Any) -> Any: + """Convert supported values to deterministic JSON-compatible values.""" + if isinstance(value, Enum): + return normalize_value(value.value) + if isinstance(value, BaseModel): + return normalize_value(value.model_dump(mode="python")) + if isinstance(value, dict): + if any(not isinstance(key, str) for key in value): + raise TypeError("Semantic fingerprint mappings require string keys") + return {key: normalize_value(item) for key, item in value.items()} + if isinstance(value, (list, tuple)): + return [normalize_value(item) for item in value] + if isinstance(value, bool): + return value + if isinstance(value, float): + if not math.isfinite(value): + raise ValueError("Semantic fingerprint values must be finite") + if value.is_integer(): + return int(value) + return value + if isinstance(value, Decimal): + if not value.is_finite(): + raise ValueError("Semantic fingerprint values must be finite") + if value == value.to_integral_value(): + return int(value) + return {"decimal": format(value.normalize(), "f")} + if value is None or isinstance(value, (str, int)): + return value + raise TypeError( + f"Unsupported semantic fingerprint value: {type(value).__name__}", + ) + + +def normalize_sequence( + values: Iterable[Any], + *, + preserve_order: bool, +) -> list[Any]: + """Normalize a sequence while removing duplicate semantic values.""" + unique: dict[str, Any] = {} + for value in values: + normalized = normalize_value(value) + unique.setdefault(canonical_json(normalized), normalized) + return ( + list(unique.values()) + if preserve_order + else [unique[key] for key in sorted(unique)] + ) + + +def annotation_contains_list(annotation: Any) -> bool: + """Return whether an annotation contains a list type.""" + return get_origin(annotation) is list or any( + annotation_contains_list(argument) for argument in get_args(annotation) + ) + + +def normalize_column( + column: ColumnSpec | None, + name: str, + fallback_type: str | None, + compare_types: bool, +) -> ColumnSpec: + """Fill equivalent column defaults before comparison or hashing.""" + normalized = ( + column.model_copy() + if column + else ColumnSpec( + name=name, + display_name=labelize(name), + type=fallback_type or "", + attributes=[], + ) + ) + normalized.display_name = normalized.display_name or labelize(name) + normalized.description = normalized.description or "" + normalized.attributes = sorted(set(normalized.attributes) - {"primary_key"}) + if not compare_types: + normalized.type = "" + return normalized + + +def normalize_columns( + columns: list[ColumnSpec] | None, + *, + compare_types: bool, +) -> list[Any]: + """Normalize authored and resolved node columns.""" + column_map = {column.name: column for column in columns or []} + normalized_columns = [] + for name in sorted(column_map): + column = column_map[name] + normalized = normalize_column(column, name, column.type, compare_types) + if not compare_types: + default = normalize_column(None, name, column.type, compare_types) + if normalized == default: + continue + normalized_columns.append(normalize_value(normalized)) + return normalized_columns + + +def normalize_dimension_links( + links: list[DimensionJoinLinkSpec | DimensionReferenceLinkSpec] | None, + *, + preserve_order: bool, +) -> list[Any]: + """Normalize dimension links by their semantic comparison key.""" + return normalize_sequence( + (link._comparison_key() for link in links or []), + preserve_order=preserve_order, + ) + + +def normalize_cube_columns(columns: list[ColumnSpec] | None) -> dict[str, Any]: + """Normalize the authored partition configuration of cube columns.""" + return { + column.name: normalize_value(column.partition) + for column in columns or [] + if column.partition + } + + +def normalize_field( + spec: NodeSpec, + field: str, + *, + resolved_columns: list[ColumnSpec] | None = None, + preserve_order: bool = False, + structural_version: int | None = None, +) -> Any: + """Return the normalized semantic value of one node field.""" + value = getattr(spec, field) + if field == "query": + from datajunction_server.sql.parsing.structural import serialize_ast + + return ( + serialize_ast(spec.query_ast, version=structural_version) + if spec.query_ast is not None + else spec.rendered_query + ) + if field == "columns": + if isinstance(spec, CubeSpec): + return normalize_cube_columns(spec.matched_rendered_columns) + return normalize_columns( + resolved_columns + if isinstance(spec, SourceSpec) and resolved_columns is not None + else value, + compare_types=isinstance(spec, SourceSpec), + ) + if field == "required_dimensions" and isinstance(spec, MetricSpec): + try: + value = spec.canonical_required_dimensions + except DJParseException: + value = spec.rendered_required_dimensions + if field == "dimension_links" and isinstance(spec, LinkableNodeSpec): + return normalize_dimension_links( + spec.dimension_links, + preserve_order=preserve_order, + ) + if field == "unit_enum" and isinstance(spec, MetricSpec): + return normalize_value(spec._normalized_unit()) + if field == "direction" and isinstance(spec, MetricSpec): + value = value or MetricDirection.NEUTRAL + if field == "description": + value = value or None + if field == "custom_metadata": + value = value or {} + if value is None and annotation_contains_list( + type(spec).model_fields[field].annotation, + ): + value = [] + + normalized = normalize_value(value) + return ( + normalize_sequence(normalized, preserve_order=preserve_order) + if isinstance(normalized, list) + else normalized + ) + + +def semantic_diff( + one: NodeSpec, + two: NodeSpec, + *, + resolved_columns: list[ColumnSpec] | None, + other_resolved_columns: list[ColumnSpec] | None, +) -> tuple[list[str], list[str]]: + """Compare two specs using their normalized semantic values.""" + if one.node_type != two.node_type: + return ["node_type"], [] + + rendered_one = one.rendered_spec() + rendered_two = two.rendered_spec() + changed_fields = [] + reordered_fields = [] + for field, field_info in type(rendered_two).model_fields.items(): + if field in {"name", "namespace", "node_type"}: + continue + if isinstance(rendered_two, MetricSpec) and field == "unit_structured": + continue + if field_info.exclude is True and field != "unit_enum": + continue + if type(rendered_two).field_change_tier(field) == ChangeTier.NONE: + continue + if field == "display_name" and getattr(rendered_two, field) is None: + continue + + try: + left = normalize_field( + rendered_one, + field, + resolved_columns=resolved_columns, + ) + right = normalize_field( + rendered_two, + field, + resolved_columns=other_resolved_columns, + ) + except DJParseException: + if field != "query": # pragma: no cover + raise + left = rendered_one.rendered_query + right = rendered_two.rendered_query + if left != right: + changed_fields.append(field) + continue + + if type(rendered_two).field_order_change_tier(field) == ChangeTier.NONE: + continue + left_ordered = normalize_field( + rendered_one, + field, + resolved_columns=resolved_columns, + preserve_order=True, + ) + right_ordered = normalize_field( + rendered_two, + field, + resolved_columns=other_resolved_columns, + preserve_order=True, + ) + if left_ordered != right_ordered: + reordered_fields.append(field) + + return changed_fields, reordered_fields diff --git a/datajunction-server/datajunction_server/semantic_fingerprints/v1.py b/datajunction-server/datajunction_server/semantic_fingerprints/v1.py new file mode 100644 index 0000000000..08cc05e83c --- /dev/null +++ b/datajunction-server/datajunction_server/semantic_fingerprints/v1.py @@ -0,0 +1,99 @@ +"""Frozen semantic fingerprint version 1.""" + +from __future__ import annotations + +import hashlib +from collections.abc import Iterable +from typing import TYPE_CHECKING + +from datajunction_server.models.deployment import ( + CubeSpec, + DimensionSpec, + MetricSpec, + SourceSpec, + TransformSpec, +) +from datajunction_server.models.semantic_fingerprint import SemanticFingerprint +from datajunction_server.semantic_fingerprints.normalization import ( + canonical_json, + normalize_field, + normalize_value, +) + +if TYPE_CHECKING: + from datajunction_server.models.deployment import ColumnSpec, NodeSpec + + +_FIELDS_BY_SPEC_TYPE: dict[type[NodeSpec], tuple[str, ...]] = { + SourceSpec: ( + "columns", + "dimension_links", + "primary_key", + "catalog", + "schema_", + "table", + ), + TransformSpec: ( + "columns", + "dimension_links", + "primary_key", + "query", + ), + DimensionSpec: ( + "columns", + "dimension_links", + "primary_key", + "query", + ), + MetricSpec: ("query", "required_dimensions"), + CubeSpec: ("metrics", "dimensions", "filters", "columns"), +} + + +def semantic_fields(spec_type: type[NodeSpec]) -> tuple[str, ...]: + """Return the frozen field projection for a concrete node type.""" + try: + return _FIELDS_BY_SPEC_TYPE[spec_type] + except KeyError as exc: + raise TypeError( + f"No semantic fingerprint fields for {spec_type.__name__}", + ) from exc + + +def build_fingerprint( + spec: NodeSpec, + parent_fingerprints: Iterable[SemanticFingerprint], + *, + resolved_columns: list[ColumnSpec] | None, +) -> SemanticFingerprint: + """Build a version 1 semantic fingerprint.""" + fingerprint_fields = semantic_fields(type(spec)) + rendered = spec.rendered_spec() + fields = { + field: normalize_field( + rendered, + field, + resolved_columns=resolved_columns, + structural_version=1, + ) + for field in fingerprint_fields + } + node_payload = { + "domain": "datajunction/node-semantic", + "node_type": normalize_value(rendered.node_type), + "fields": fields, + } + node_digest = hashlib.sha256( + canonical_json(node_payload).encode("utf-8"), + ).hexdigest() + parents = list(parent_fingerprints) + if any(parent.version != 1 for parent in parents): + raise ValueError("Parent fingerprint version does not match node version") + payload = { + "domain": "datajunction/node-semantic-merkle", + "version": 1, + "node": node_digest, + "parents": sorted({parent.digest for parent in parents}), + } + digest = hashlib.sha256(canonical_json(payload).encode("utf-8")).hexdigest() + return SemanticFingerprint(version=1, digest=digest) diff --git a/datajunction-server/datajunction_server/service_clients.py b/datajunction-server/datajunction_server/service_clients.py index e55b72efb3..b6cc5c3ea6 100644 --- a/datajunction-server/datajunction_server/service_clients.py +++ b/datajunction-server/datajunction_server/service_clients.py @@ -405,12 +405,23 @@ async def submit_query( request_headers: dict[str, str] | None = None, ) -> QueryWithResults: """Submit a query to the query service.""" - # ``request_headers`` intentionally not forwarded — see - # ``get_columns_for_table`` for the reason. + # Request credentials must not be forwarded. Cache-Control is deliberately + # preserved because DJQS uses it to choose result-cache retention. + headers = {"accept": "application/json"} + cache_control = next( + ( + value + for name, value in (request_headers or {}).items() + if name.lower() == "cache-control" + ), + None, + ) + if cache_control: + headers["cache-control"] = cache_control response = await self._arequest( "POST", "/queries/", - headers={"accept": "application/json"}, + headers=headers, json=query_create.model_dump(), ) if response.status_code not in (200, 201): diff --git a/datajunction-server/datajunction_server/sql/parsing/ast.py b/datajunction-server/datajunction_server/sql/parsing/ast.py index 26f15b5a0b..324bb2abbc 100644 --- a/datajunction-server/datajunction_server/sql/parsing/ast.py +++ b/datajunction-server/datajunction_server/sql/parsing/ast.py @@ -18,6 +18,7 @@ from functools import reduce from itertools import chain, zip_longest import re +from sqlglot import exp as sqlglot_exp from typing import ( TYPE_CHECKING, Any, @@ -63,24 +64,34 @@ from datajunction_server.sql.parsing.backends.exceptions import DJParseException from datajunction_server.sql.parsing.types import ( BigIntType, + BinaryType, BooleanType, ColumnType, DateTimeBase, + DateType, DayTimeIntervalType, DecimalType, DoubleType, + FixedType, FloatType, IntegerBase, IntegerType, ListType, + LongType, MapType, NestedField, NullType, + SmallIntType, StringBase, StringType, StructType, + TimeType, + TinyIntType, TimestampType, TimestamptzType, + UUIDType, + UnknownType, + VarcharType, WildcardType, YearMonthIntervalType, ) @@ -107,6 +118,108 @@ def get_render_dialect() -> Dialect | None: return _render_dialect.get() +def _sqlglot_type(column_type: ColumnType) -> sqlglot_exp.DataType: + """Convert a DJ column type to a sqlglot type.""" + type_ = sqlglot_exp.DataType.Type + + if isinstance(column_type, NullType): + return sqlglot_exp.DataType.build(type_.NULL) + if isinstance(column_type, FixedType): + return sqlglot_exp.DataType.build(f"BINARY({column_type.length})") + if isinstance(column_type, DecimalType): + return sqlglot_exp.DataType.build( + f"DECIMAL({column_type.precision}, {column_type.scale})", + ) + if isinstance(column_type, StructType): + fields = [] + for nested_field in column_type.fields: + constraints = ( + [ + sqlglot_exp.ColumnConstraint( + kind=sqlglot_exp.NotNullColumnConstraint(), + ), + ] + if nested_field.is_required + else None + ) + fields.append( + sqlglot_exp.ColumnDef( + this=sqlglot_exp.to_identifier(nested_field.name.name), + kind=_sqlglot_type(nested_field.type), + constraints=constraints, + ), + ) + return sqlglot_exp.DataType(this=type_.STRUCT, expressions=fields, nested=True) + if isinstance(column_type, ListType): + return sqlglot_exp.DataType( + this=type_.ARRAY, + expressions=[_sqlglot_type(column_type.element.type)], + nested=True, + ) + if isinstance(column_type, MapType): + return sqlglot_exp.DataType( + this=type_.MAP, + expressions=[ + _sqlglot_type(column_type.key.type), + _sqlglot_type(column_type.value.type), + ], + nested=True, + ) + + primitive_types = ( + (BooleanType, type_.BOOLEAN), + (TinyIntType, type_.TINYINT), + (SmallIntType, type_.SMALLINT), + (IntegerType, type_.INT), + (LongType, type_.BIGINT), + (BigIntType, type_.BIGINT), + (FloatType, type_.FLOAT), + (DoubleType, type_.DOUBLE), + (DateType, type_.DATE), + (TimeType, type_.TIME), + (TimestampType, type_.TIMESTAMP), + (TimestamptzType, type_.TIMESTAMPTZ), + (StringType, type_.TEXT), + (UUIDType, type_.UUID), + (BinaryType, type_.BINARY), + ) + for dj_type, sqlglot_type in primitive_types: + if isinstance(column_type, dj_type): + return sqlglot_exp.DataType.build(sqlglot_type) + + if isinstance(column_type, VarcharType): + return sqlglot_exp.DataType.build(str(column_type)) + if isinstance(column_type, (DayTimeIntervalType, YearMonthIntervalType)): + return sqlglot_exp.DataType.build(str(column_type)) + if isinstance(column_type, (UnknownType, WildcardType)): + return sqlglot_exp.DataType.build(type_.UNKNOWN) + + return sqlglot_exp.DataType.build(type_.UNKNOWN) + + +def _sqlglot_schema(query: Query) -> dict[str, Any]: + """Build sqlglot schema from DJ metadata attached to compiled tables.""" + schema: dict[str, Any] = {} + for table in query.find_all(Table): + if not table.dj_node: + continue + + table_schema: dict[str, sqlglot_exp.DataType] = { + column.name: _sqlglot_type(column.type) + for column in table.dj_node.columns + if column.type is not None + } + if not table_schema: + continue + + current = schema + table_parts = table.identifier(quotes=False).split(".") + for part in table_parts[:-1]: + current = current.setdefault(part, {}) + current[table_parts[-1]] = table_schema + return schema + + @contextmanager def render_for_dialect(dialect: Dialect): """ @@ -146,7 +259,7 @@ def to_sql(query: Query, dialect: Dialect | None = None) -> str: with render_for_dialect(dialect): rendered = str(query) try: - return transpile_sql(rendered, dialect) + return transpile_sql(rendered, dialect, schema=_sqlglot_schema(query)) except Exception: # pragma: no cover - fall back to native render logger.warning( "Transpilation to %s failed; falling back to native render", diff --git a/datajunction-server/datajunction_server/sql/parsing/structural.py b/datajunction-server/datajunction_server/sql/parsing/structural.py new file mode 100644 index 0000000000..fd00945cd9 --- /dev/null +++ b/datajunction-server/datajunction_server/sql/parsing/structural.py @@ -0,0 +1,136 @@ +"""Versioned structural serialization for parsed SQL.""" + +import math +from collections.abc import Callable +from decimal import Decimal +from enum import Enum +from typing import Any + +from datajunction_server.sql.parsing.ast import Node +from datajunction_server.sql.parsing.types import ColumnType + +_AST_NODE_TAGS_V1 = frozenset( + { + "Alias", + "ArithmeticUnaryOp", + "Between", + "BinaryOp", + "Boolean", + "Case", + "Cast", + "Column", + "DefaultName", + "Frame", + "FrameBound", + "From", + "Function", + "FunctionTable", + "FunctionTableExpression", + "Hint", + "In", + "InlineTable", + "Interval", + "IntervalUnit", + "IsBoolean", + "IsDistinctFrom", + "IsNull", + "Join", + "JoinCriteria", + "Lambda", + "LateralView", + "Like", + "Name", + "Null", + "Number", + "Organization", + "Over", + "Query", + "QueryParameter", + "Relation", + "Rlike", + "Select", + "SelectExpression", + "SetOp", + "SortItem", + "String", + "Struct", + "Subscript", + "Table", + "UnaryOp", + "UnNamed", + "Wildcard", + }, +) + + +def _serialize_number_v1(value: float | Decimal) -> int | float | dict[str, str]: + if isinstance(value, bool): + raise TypeError("Boolean values are not SQL numbers") + if isinstance(value, float): + if not math.isfinite(value): + raise ValueError("Structural SQL numbers must be finite") + if value.is_integer(): + return int(value) + return value + if isinstance(value, Decimal): + if not value.is_finite(): + raise ValueError("Structural SQL numbers must be finite") + if value == value.to_integral_value(): + return int(value) + return {"decimal": format(value.normalize(), "f")} + return value + + +def _serialize_ast_v1(query_ast: Node) -> Any: + """Serialize an AST without SQL rendering or Python module-qualified names.""" + + def serialize(value: Any) -> Any: + if isinstance(value, Node): + tag = type(value).__name__ + if tag not in _AST_NODE_TAGS_V1: + raise TypeError(f"Unsupported structural SQL node: {tag}") + return { + "type": tag, + "fields": { + name: serialize(field_value) + for name, field_value in value.fields( + flat=False, + nodes_only=False, + obfuscated=False, + nones=True, + named=True, + ) + }, + } + if isinstance(value, ColumnType): + return {"type": "column_type", "value": str(value)} + if isinstance(value, Enum): + return serialize(value.value) + if isinstance(value, Decimal): + return _serialize_number_v1(value) + if isinstance(value, float): + return _serialize_number_v1(value) + if isinstance(value, (list, tuple)): + return [serialize(item) for item in value] + if value is None or isinstance(value, (str, int, bool)): + return value + raise TypeError(f"Unsupported structural SQL value: {type(value).__name__}") + + return serialize(query_ast) + + +_LATEST_VERSION = 1 +_SERIALIZERS: dict[int, Callable[[Node], Any]] = { + 1: _serialize_ast_v1, +} + + +def serialize_ast(query_ast: Node, *, version: int | None = None) -> Any: + """Serialize an AST using a stable structural format.""" + selected_version = _LATEST_VERSION if version is None else version + serializer = _SERIALIZERS.get(selected_version) + if serializer is None: + raise ValueError( + f"Unsupported structural SQL serialization version: {selected_version}", + ) + return serializer(query_ast) diff --git a/datajunction-server/datajunction_server/transpilation.py b/datajunction-server/datajunction_server/transpilation.py index 7b87e30592..6ce2968090 100644 --- a/datajunction-server/datajunction_server/transpilation.py +++ b/datajunction-server/datajunction_server/transpilation.py @@ -1,8 +1,11 @@ """SQL transpilation plugins manager.""" import logging +from typing import Any import sqlglot +from sqlglot.optimizer.annotate_types import annotate_types +from sqlglot.optimizer.qualify import qualify from datajunction_server.models.dialect import DialectRegistry, dialect_plugin from datajunction_server.models.engine import Dialect @@ -60,6 +63,7 @@ def transpile_sql( *, input_dialect: Dialect | None = None, output_dialect: Dialect | None = None, + schema: dict[str, Any] | None = None, ) -> str: """ Transpile a given SQL query using the specific library. @@ -70,19 +74,22 @@ def transpile_sql( and output_dialect and output_dialect.name in dir(sqlglot.dialects.Dialects) ): - value = sqlglot.transpile( - query, - read=str(input_dialect.name.lower()), # type: ignore - write=str(output_dialect.name.lower()), # type: ignore + input_name = str(input_dialect.name.lower()) + expression = sqlglot.parse_one(query, read=input_name) + if schema: + expression = qualify(expression, schema=schema, dialect=input_name) + annotate_types(expression, schema=schema, dialect=input_name) + return expression.sql( + dialect=str(output_dialect.name.lower()), pretty=True, - )[0] - return value + ) return query def transpile_sql( sql: str, dialect: Dialect | None = None, + schema: dict[str, Any] | None = None, ) -> str: """ Transpile SQL to a target dialect. @@ -94,6 +101,7 @@ def transpile_sql( Args: sql: The SQL string to transpile dialect: The target SQL dialect (if None, returns SQL unchanged) + schema: Optional SQLGlot schema mapping used to annotate AST types Returns: The transpiled SQL string @@ -104,9 +112,16 @@ def transpile_sql( dialect.name.lower(), ): plugin = plugin_class() + kwargs: dict[str, Any] = { + "input_dialect": Dialect.SPARK, + "output_dialect": dialect, + } + # Schema annotation is a SQLGlot capability. Keeping it out of calls + # to other plugins preserves compatibility with their existing API. + if schema is not None and isinstance(plugin, SQLGlotTranspilationPlugin): + kwargs["schema"] = schema return plugin.transpile_sql( sql, - input_dialect=Dialect.SPARK, - output_dialect=dialect, + **kwargs, ) return sql diff --git a/datajunction-server/pyproject.toml b/datajunction-server/pyproject.toml index 03f55af646..5b555a5483 100644 --- a/datajunction-server/pyproject.toml +++ b/datajunction-server/pyproject.toml @@ -142,6 +142,11 @@ Repository = "https://github.com/DataJunction/dj" [tool.coverage.run] source = ['datajunction_server/'] concurrency = ["thread,greenlet"] +# Record paths relative to the project root. Coverage data produced by +# different CI shards is combined later, and each runner has its own absolute +# checkout path -- without this, combine treats the same file from two shards +# as two files and the merged total is quietly wrong rather than failing. +relative_files = true [tool.pytest.ini_options] asyncio_mode = "auto" diff --git a/datajunction-server/tests/api/deployment_impact_test.py b/datajunction-server/tests/api/deployment_impact_test.py index b5f88e8ab0..1ea0190613 100644 --- a/datajunction-server/tests/api/deployment_impact_test.py +++ b/datajunction-server/tests/api/deployment_impact_test.py @@ -7,10 +7,18 @@ import pytest +from datajunction_server.internal.deployment.fingerprints import ( + SemanticFingerprintGraph, + build_deployment_fingerprints, +) from datajunction_server.models.deployment import ( ColumnSpec, + CubeSpec, DeploymentInfo, DeploymentSpec, + DimensionJoinLinkSpec, + DimensionSpec, + MetricSpec, SourceSpec, TransformSpec, ) @@ -39,6 +47,79 @@ async def _wait_for_deployment(client, deployment_id: str, timeout: int = 30): return (await client.get(f"/deployments/{deployment_id}")).json() +def _source( + name: str, + *, + description: str | None = None, + dimension_links: list[DimensionJoinLinkSpec] | None = None, +) -> SourceSpec: + return SourceSpec( + name=name, + catalog="default", + schema_="test", + table=name, + columns=[ColumnSpec(name="id", type="int")], + description=description, + dimension_links=dimension_links or [], + ) + + +def _dimension_project(namespace: str) -> DeploymentSpec: + return DeploymentSpec( + namespace=namespace, + nodes=[ + _source("raw"), + DimensionSpec( + name="dimension", + query="SELECT id FROM ${prefix}raw", + primary_key=["id"], + ), + ], + ) + + +def _updated_dimension_project(project: DeploymentSpec) -> DeploymentSpec: + updated = project.model_copy(deep=True) + dimension = updated.nodes[1] + assert isinstance(dimension, DimensionSpec) + dimension.query = "SELECT id FROM ${prefix}raw WHERE id IS NOT NULL" + return updated + + +def _fingerprint_graph(*projects: DeploymentSpec) -> SemanticFingerprintGraph: + return SemanticFingerprintGraph( + {node.rendered_name: node for project in projects for node in project.nodes}, + ) + + +async def _deploy(client, *specs: DeploymentSpec): + for spec in specs: + response = await client.post( + "/deployments", + json=spec.model_dump(by_alias=True), + ) + assert response.status_code == 200 + await _wait_for_deployment(client, response.json()["uuid"]) + + +async def _impact(client, spec: DeploymentSpec): + response = await client.post( + "/deployments/impact", + json=spec.model_dump(by_alias=True), + ) + assert response.status_code == 200 + return response.json() + + +async def _impact_nodes(client, spec): + data = await _impact(client, spec) + return { + result["name"]: result + for result in data["results"] + if result["deploy_type"] == "node" + } + + class TestDeploymentImpactEndpoint: """Tests for POST /deployments/impact (orchestrator dry-run).""" @@ -76,10 +157,31 @@ async def test_impact_create_new_nodes(self, client_with_roads): assert len(node_results) == 1 assert node_results[0].name == "impact_create_test.orders" assert node_results[0].operation == "create" + assert node_results[0].change_tier == "major" + expected = _fingerprint_graph(spec).fingerprint( + spec.nodes[0].rendered_name, + ) + assert node_results[0].semantic_fingerprint == expected + + @pytest.mark.asyncio + async def test_impact_builds_fingerprints_once(self, client_with_roads): + spec = DeploymentSpec( + namespace="impact_single_fingerprint_build", + nodes=[_source("orders")], + ) + + with mock.patch( + "datajunction_server.internal.deployment.orchestrator." + "build_deployment_fingerprints", + wraps=build_deployment_fingerprints, + ) as build_fingerprints: + await _impact(client_with_roads, spec) + + assert build_fingerprints.await_count == 1 @pytest.mark.asyncio async def test_impact_detects_updates(self, client_with_roads): - """After deploying a node, a dry-run with a changed query shows UPDATE.""" + """A source change updates its own result and descendant fingerprint.""" initial_spec = DeploymentSpec( namespace="impact_update_test", nodes=[ @@ -109,13 +211,13 @@ async def test_impact_detects_updates(self, client_with_roads): nodes=[ TransformSpec( name="orders_summary", - query="SELECT 1 AS order_id, 'updated' AS status FROM ${prefix}raw", + query="SELECT 1 AS order_id FROM ${prefix}raw", ), SourceSpec( name="raw", catalog="default", schema_="test", - table="raw", + table="raw_v2", columns=[ColumnSpec(name="order_id", type="int")], ), ], @@ -135,8 +237,36 @@ async def test_impact_detects_updates(self, client_with_roads): ] skip_results = [r for r in data["results"] if r["operation"] == "noop"] assert len(update_results) == 1 - assert "orders_summary" in update_results[0]["name"] + assert update_results[0]["name"] == "impact_update_test.raw" assert len(skip_results) >= 1 + assert update_results[0]["change_tier"] == "major" + assert ( + update_results[0]["semantic_fingerprint"] + == _fingerprint_graph(updated_spec) + .fingerprint(updated_spec.nodes[1].rendered_name) + .model_dump() + ) + updated_fingerprint = _fingerprint_graph(updated_spec).fingerprint( + updated_spec.nodes[0].rendered_name, + ) + initial_fingerprint = _fingerprint_graph(initial_spec).fingerprint( + initial_spec.nodes[0].rendered_name, + ) + unchanged_nodes = [ + result for result in skip_results if result["deploy_type"] == "node" + ] + assert all(result["change_tier"] == "none" for result in unchanged_nodes) + transform_result = next( + result + for result in unchanged_nodes + if result["name"] == "impact_update_test.orders_summary" + ) + assert ( + transform_result["semantic_fingerprint"] == updated_fingerprint.model_dump() + ) + assert ( + transform_result["semantic_fingerprint"] != initial_fingerprint.model_dump() + ) @pytest.mark.asyncio async def test_impact_detects_deletions(self, client_with_roads): @@ -192,6 +322,62 @@ async def test_impact_detects_deletions(self, client_with_roads): delete_results = [r for r in data["results"] if r["operation"] == "delete"] assert len(delete_results) == 1 assert "to_delete" in delete_results[0]["name"] + assert delete_results[0]["change_tier"] == "major" + assert ( + delete_results[0]["semantic_fingerprint"] + == _fingerprint_graph(initial_spec) + .fingerprint(initial_spec.nodes[1].rendered_name) + .model_dump() + ) + + @pytest.mark.asyncio + async def test_impact_minor_full_noop_and_forced_revalidation( + self, + client_with_roads, + ): + initial = DeploymentSpec( + namespace="impact_tiers_test", + nodes=[ + _source("one", description="Before"), + _source("two"), + ], + ) + await _deploy(client_with_roads, initial) + initial_graph = _fingerprint_graph(initial) + + minor = initial.model_copy(deep=True) + minor.nodes[0].description = "After" + by_name = await _impact_nodes(client_with_roads, minor) + assert by_name["impact_tiers_test.one"]["change_tier"] == "minor" + assert ( + by_name["impact_tiers_test.one"]["semantic_fingerprint"] + == initial_graph.fingerprint(initial.nodes[0].rendered_name).model_dump() + ) + assert by_name["impact_tiers_test.two"]["change_tier"] == "none" + + equivalent = initial.model_copy(deep=True) + equivalent.nodes[1].columns = None + noop_nodes = await _impact_nodes(client_with_roads, equivalent) + assert set(noop_nodes) == { + "impact_tiers_test.one", + "impact_tiers_test.two", + } + assert all(result["change_tier"] == "none" for result in noop_nodes.values()) + assert all(result["semantic_fingerprint"] for result in noop_nodes.values()) + assert ( + noop_nodes["impact_tiers_test.two"]["semantic_fingerprint"] + == initial_graph.fingerprint(initial.nodes[1].rendered_name).model_dump() + ) + + forced = equivalent.model_copy(update={"force": True}) + forced_nodes = await _impact_nodes(client_with_roads, forced) + assert all(result["operation"] == "update" for result in forced_nodes.values()) + assert all(result["change_tier"] == "none" for result in forced_nodes.values()) + assert all(result["semantic_fingerprint"] for result in forced_nodes.values()) + assert ( + forced_nodes["impact_tiers_test.two"]["semantic_fingerprint"] + == initial_graph.fingerprint(initial.nodes[1].rendered_name).model_dump() + ) @pytest.mark.asyncio async def test_dry_run_does_not_mutate_db(self, client_with_roads): @@ -317,5 +503,214 @@ async def test_downstream_impacts_returned_for_invalid_parent( "caused_by": ["impact_downstream_test.base"], "is_external": False, "owners": ["dj"], + "semantic_fingerprint": _fingerprint_graph(modified_spec) + .fingerprint( + modified_spec.nodes[1].rendered_name, + ) + .model_dump(), }, ] + + @pytest.mark.asyncio + async def test_external_downstream_impacts_include_changed_fingerprint( + self, + client_with_roads, + ): + parent = DeploymentSpec( + namespace="impact_external_parent", + nodes=[_source("base")], + ) + consumer = DeploymentSpec( + namespace="impact_external_consumer", + nodes=[ + TransformSpec( + name="derived", + query="SELECT id FROM impact_external_parent.base", + ), + ], + ) + await _deploy(client_with_roads, parent, consumer) + + updated_parent = parent.model_copy(deep=True) + assert isinstance(updated_parent.nodes[0], SourceSpec) + updated_parent.nodes[0].table = "base_v2" + data = await _impact(client_with_roads, updated_parent) + + external = next( + impact + for impact in data["downstream_impacts"] + if impact["name"] == "impact_external_consumer.derived" + ) + expected = _fingerprint_graph(updated_parent, consumer).fingerprint( + consumer.nodes[0].rendered_name, + ) + assert external["is_external"] is True + assert external["semantic_fingerprint"] == expected.model_dump() + + metadata_only = parent.model_copy(deep=True) + metadata_only.nodes[0].description = "Updated description" + data = await _impact(client_with_roads, metadata_only) + external = next( + impact + for impact in data["downstream_impacts"] + if impact["name"] == "impact_external_consumer.derived" + ) + assert external["semantic_fingerprint"] is None + + @pytest.mark.asyncio + async def test_unparseable_node_and_dependents_return_unknown_fingerprints( + self, + client_with_roads, + ): + initial = DeploymentSpec( + namespace="impact_unknown", + nodes=[ + _source("base"), + TransformSpec( + name="broken", + query="SELECT id FROM ${prefix}base", + ), + TransformSpec( + name="dependent", + query="SELECT id FROM ${prefix}broken", + ), + ], + ) + await _deploy(client_with_roads, initial) + + proposed = DeploymentSpec( + namespace="impact_unknown", + nodes=[ + _source("base"), + TransformSpec(name="broken", query="SELECT ("), + TransformSpec( + name="dependent", + query="SELECT id FROM ${prefix}broken", + ), + ], + ) + data = await _impact(client_with_roads, proposed) + results = { + result["name"]: result + for result in data["results"] + if result["deploy_type"] == "node" + } + + assert results["impact_unknown.broken"]["status"] == "invalid" + assert "[invalid]" in results["impact_unknown.broken"]["message"] + assert results["impact_unknown.broken"]["semantic_fingerprint"] == "unknown" + assert results["impact_unknown.dependent"]["semantic_fingerprint"] == "unknown" + + @pytest.mark.asyncio + async def test_blocked_delete_keeps_current_fingerprint_state( + self, + client_with_roads, + ): + parent = DeploymentSpec( + namespace="impact_blocked_delete", + nodes=[_source("keep"), _source("base")], + ) + consumer = DeploymentSpec( + namespace="impact_blocked_consumer", + nodes=[ + TransformSpec( + name="derived", + query="SELECT id FROM impact_blocked_delete.base", + ), + ], + ) + await _deploy(client_with_roads, parent, consumer) + + without_parent = parent.model_copy( + deep=True, + update={"nodes": [parent.nodes[0]]}, + ) + data = await _impact(client_with_roads, without_parent) + + delete_result = next( + result + for result in data["results"] + if result["name"] == "impact_blocked_delete.base" + ) + assert delete_result["status"] == "failed" + assert ( + delete_result["semantic_fingerprint"] + == _fingerprint_graph(parent) + .fingerprint(parent.nodes[1].rendered_name) + .model_dump() + ) + assert data["downstream_impacts"] == [] + + @pytest.mark.asyncio + async def test_dimension_link_descendants_are_discovered( + self, + client_with_roads, + ): + parent = _dimension_project("impact_link_parent") + consumer = DeploymentSpec( + namespace="impact_link_consumer", + nodes=[ + _source( + "fact", + dimension_links=[ + DimensionJoinLinkSpec( + dimension_node="impact_link_parent.dimension", + join_on=( + "impact_link_consumer.fact.id = " + "impact_link_parent.dimension.id" + ), + ), + ], + ), + ], + ) + await _deploy(client_with_roads, parent, consumer) + + updated_parent = _updated_dimension_project(parent) + data = await _impact(client_with_roads, updated_parent) + + external = next( + impact + for impact in data["downstream_impacts"] + if impact["name"] == "impact_link_consumer.fact" + ) + expected = _fingerprint_graph(updated_parent, consumer).fingerprint( + consumer.nodes[0].rendered_name, + ) + assert external["semantic_fingerprint"] == expected.model_dump() + + @pytest.mark.asyncio + async def test_cube_filter_descendants_are_discovered( + self, + client_with_roads, + ): + parent = _dimension_project("impact_filter_parent") + consumer = DeploymentSpec( + namespace="impact_filter_consumer", + nodes=[ + _source("fact"), + MetricSpec( + name="metric", + query="SELECT COUNT(*) FROM ${prefix}fact", + ), + CubeSpec( + name="cube", + metrics=["${prefix}metric"], + filters=["impact_filter_parent.dimension.id > 0"], + ), + ], + ) + await _deploy(client_with_roads, parent, consumer) + + updated_parent = _updated_dimension_project(parent) + data = await _impact(client_with_roads, updated_parent) + + external = next( + impact + for impact in data["downstream_impacts"] + if impact["name"] == "impact_filter_consumer.cube" + ) + expected = _fingerprint_graph(updated_parent, consumer).fingerprint( + consumer.nodes[2].rendered_name, + ) + assert external["semantic_fingerprint"] == expected.model_dump() diff --git a/datajunction-server/tests/api/deployments_test.py b/datajunction-server/tests/api/deployments_test.py index a2f5b161b5..ca250163d8 100644 --- a/datajunction-server/tests/api/deployments_test.py +++ b/datajunction-server/tests/api/deployments_test.py @@ -8,7 +8,7 @@ from unittest.mock import AsyncMock, MagicMock, patch import pytest -from sqlalchemy import select +from sqlalchemy import select, text from sqlalchemy.orm import selectinload import datajunction_server.internal.materializations @@ -17,10 +17,12 @@ _normalize_repo_path, ) from datajunction_server.database.availabilitystate import AvailabilityState +from datajunction_server.database.column import Column as DBColumn from datajunction_server.database.materialization import Materialization -from datajunction_server.database.node import Node, NodeRelationship +from datajunction_server.database.node import Node, NodeRelationship, NodeRevision from datajunction_server.database.tag import Tag from datajunction_server.errors import DJInvalidInputException +from datajunction_server.internal.deployment.orchestrator import DeploymentOrchestrator from datajunction_server.internal.git.github_service import GitHubServiceError from datajunction_server.models import access from datajunction_server.models.deployment import ( @@ -1329,7 +1331,16 @@ def deployment_payload(deployment_spec: DeploymentSpec) -> dict: return deployment_spec.model_dump() -async def deploy_and_wait(client, deployment_spec: DeploymentSpec): +# Additive `DeploymentResult` fields, stripped rather than asserted below. +ADDITIVE_RESULT_FIELDS = ( + "change_tier", + "semantic_fingerprint", + "revalidation_only", +) + + +async def deploy_and_poll(client, deployment_spec: DeploymentSpec): + """Deploy and wait, keeping every field the API returned.""" response = await client.post( "/deployments", json=deployment_payload(deployment_spec), @@ -1346,6 +1357,15 @@ async def deploy_and_wait(client, deployment_spec: DeploymentSpec): return data +async def deploy_and_wait(client, deployment_spec: DeploymentSpec): + data = await deploy_and_poll(client, deployment_spec) + for result in data.get("results", []): + for additive in ADDITIVE_RESULT_FIELDS: + assert additive in result + result.pop(additive) + return data + + @pytest.mark.xdist_group(name="deployments") class TestDeploymentAuthorization: @pytest.mark.asyncio @@ -1428,6 +1448,69 @@ async def test_deploy_failed_on_non_existent_upstream_deps( assert f"{namespace}.default.us_state" in link_result["name"] assert link_result["status"] == "failed" + @pytest.mark.asyncio + async def test_revalidation_only_marks_pre_existing_failures(self, client): + """ + An unchanged node re-deployed only to retry a pre-existing failure is + marked `revalidation_only`, so a caller can tell the failures this + deployment caused from the ones it inherited -- and still sees the + recovery when a later deploy fixes the node's upstream. + """ + namespace = "revalidation_only" + transform = TransformSpec( + name="${prefix}default.repair_totals", + query="SELECT repair_order_id, price FROM ${prefix}default.repairs", + owners=["dj"], + ) + source = SourceSpec( + name="${prefix}default.repairs", + table="repairs", + catalog="default", + schema_="roads", + columns=[ + ColumnSpec(name="repair_order_id", type="int"), + ColumnSpec(name="price", type="float"), + ], + owners=["dj"], + ) + + # The transform is created broken: its source is not in the deployment. + data = await deploy_and_poll( + client, + DeploymentSpec(namespace=namespace, nodes=[transform]), + ) + created = next(r for r in data["results"] if r["deploy_type"] == "node") + assert created["status"] == "invalid" + assert created["operation"] == "create" + assert created["revalidation_only"] is False + + # Re-deploying the same spec retries the node, which is still broken. + data = await deploy_and_poll( + client, + DeploymentSpec(namespace=namespace, nodes=[transform]), + ) + retried = next(r for r in data["results"] if r["deploy_type"] == "node") + assert retried["status"] == "invalid" + assert retried["operation"] == "update" + assert retried["revalidation_only"] is True + + # Adding the missing source fixes the node, which reports as a success. + data = await deploy_and_poll( + client, + DeploymentSpec(namespace=namespace, nodes=[transform, source]), + ) + recovered = next( + r + for r in data["results"] + if r["name"] == f"{namespace}.default.repair_totals" + ) + assert recovered["status"] == "success" + assert recovered["revalidation_only"] is True + added_source = next( + r for r in data["results"] if r["name"] == f"{namespace}.default.repairs" + ) + assert added_source["revalidation_only"] is False + @pytest.mark.asyncio async def test_deploy_failed_on_non_existent_link_deps( self, @@ -1772,6 +1855,167 @@ async def test_deploy_with_reference_dimension_link( for r in data["results"] ) + @pytest.mark.asyncio + async def test_redeploy_is_noop_for_role_qualified_reference_link( + self, + client, + default_hard_hats, + default_us_states, + default_us_state, + ): + """ + A reference link with a role must redeploy as a noop, since nothing + about it changed. `Column.dimension_column` stores the role baked + into a "[role]" suffix, and to_spec() must split that back out into + the link's own `role` field rather than leaving it in the exported + `dimension` string -- otherwise the exported spec never compares + equal to the one it was authored from. + """ + namespace = "reference_link_role_noop" + dim_spec = DimensionSpec( + name="default.hard_hat", + description="Hard hat dimension", + query=""" + SELECT + hard_hat_id, + state + FROM ${prefix}default.hard_hats + """, + primary_key=["hard_hat_id"], + owners=["dj"], + dimension_links=[ + DimensionReferenceLinkSpec( + node_column="state", + dimension="${prefix}default.us_state.state_short", + role="home_state", + ), + ], + ) + nodes_list = [dim_spec, default_hard_hats, default_us_states, default_us_state] + link_name = ( + "reference_link_role_noop.default.hard_hat -> " + "reference_link_role_noop.default.us_state[home_state]" + ) + + data = await deploy_and_wait( + client, + DeploymentSpec(namespace=namespace, nodes=nodes_list), + ) + assert data["status"] == "success", data + assert [ + result for result in data["results"] if result["name"] == link_name + ] == [ + { + "deploy_type": "link", + "message": "Reference link successfully deployed", + "name": link_name, + "operation": "create", + "changed_fields": [], + "status": "success", + }, + ] + + data = await deploy_and_wait( + client, + DeploymentSpec(namespace=namespace, nodes=nodes_list), + ) + assert data["status"] == "success", data + assert [ + result + for result in data["results"] + if result["name"] + in (link_name, "reference_link_role_noop.default.hard_hat") + ] == [ + { + "deploy_type": "node", + "message": "Unchanged", + "name": "reference_link_role_noop.default.hard_hat", + "operation": "noop", + "changed_fields": [], + "status": "skipped", + }, + ] + + @pytest.mark.asyncio + async def test_redeploy_is_noop_for_join_link_with_default_value( + self, + client, + default_hard_hats, + default_us_states, + default_us_state, + ): + """ + A join link with a `default_value` must redeploy as a noop, since + nothing about it changed. `DimensionLink.to_spec()` must include + `default_value` -- otherwise the exported spec always reports it as + None and never compares equal to the one it was authored from. + """ + namespace = "join_link_default_value_noop" + dim_spec = DimensionSpec( + name="default.hard_hat", + description="Hard hat dimension", + query=""" + SELECT + hard_hat_id, + state + FROM ${prefix}default.hard_hats + """, + primary_key=["hard_hat_id"], + owners=["dj"], + dimension_links=[ + DimensionJoinLinkSpec( + dimension_node="${prefix}default.us_state", + join_type="left", + join_on="${prefix}default.hard_hat.state = ${prefix}default.us_state.state_short", + default_value="Unknown", + ), + ], + ) + nodes_list = [dim_spec, default_hard_hats, default_us_states, default_us_state] + link_name = ( + "join_link_default_value_noop.default.hard_hat -> " + "join_link_default_value_noop.default.us_state" + ) + + data = await deploy_and_wait( + client, + DeploymentSpec(namespace=namespace, nodes=nodes_list), + ) + assert data["status"] == "success", data + assert [ + result for result in data["results"] if result["name"] == link_name + ] == [ + { + "deploy_type": "link", + "message": "Join link successfully deployed", + "name": link_name, + "operation": "create", + "changed_fields": [], + "status": "success", + }, + ] + + data = await deploy_and_wait( + client, + DeploymentSpec(namespace=namespace, nodes=nodes_list), + ) + assert data["status"] == "success", data + assert [ + result + for result in data["results"] + if result["name"] + in (link_name, "join_link_default_value_noop.default.hard_hat") + ] == [ + { + "deploy_type": "node", + "message": "Unchanged", + "name": "join_link_default_value_noop.default.hard_hat", + "operation": "noop", + "changed_fields": [], + "status": "skipped", + }, + ] + @pytest.mark.asyncio async def test_required_dimension_from_linked_dimension_roundtrips( self, @@ -1867,6 +2111,89 @@ async def test_required_dimension_from_linked_dimension_roundtrips( "${prefix}default.us_state.state_region", ] + @pytest.mark.asyncio + async def test_full_redeploy_is_noop( + self, + client, + default_hard_hats, + default_us_states, + default_us_state, + ): + """ + A namespace covering source/dimension/transform/metric/cube, join and + reference links (one with a role, one with a default_value), a + required dimension pulled from a linked dimension, and a description + mentioning `${prefix}` as prose -- deployed twice with no changes -- + must come back fully noop the second time. Regression test for the + combination of export round-trip bugs found in required_dimensions, + reference link roles, join link default_value, and description/ + custom_metadata rendering. + """ + hard_hat = DimensionSpec( + name="default.hard_hat", + description="Hard hat dimension. See also ${prefix}default.us_state.", + query="SELECT hard_hat_id, state FROM ${prefix}default.hard_hats", + primary_key=["hard_hat_id"], + owners=["dj"], + custom_metadata={"see_also": "${prefix}default.us_state"}, + dimension_links=[ + DimensionJoinLinkSpec( + dimension_node="${prefix}default.us_state", + join_type="left", + join_on=( + "${prefix}default.hard_hat.state = " + "${prefix}default.us_state.state_short" + ), + default_value="Unknown", + ), + DimensionReferenceLinkSpec( + node_column="state", + dimension="${prefix}default.us_state.state_short", + role="home_state", + ), + ], + ) + num_hard_hats = MetricSpec( + name="default.num_hard_hats", + node_type=NodeType.METRIC, + query="SELECT COUNT(*) FROM ${prefix}default.hard_hat", + required_dimensions=["${prefix}default.us_state.state_name"], + owners=["dj"], + ) + repairs_cube = CubeSpec( + name="default.hard_hat_cube", + description="See also ${prefix}default.num_hard_hats.", + dimensions=["${prefix}default.us_state.state_name"], + metrics=["${prefix}default.num_hard_hats"], + owners=["dj"], + ) + nodes = [ + default_hard_hats, + default_us_states, + default_us_state, + hard_hat, + num_hard_hats, + repairs_cube, + ] + + data = await deploy_and_wait( + client, + DeploymentSpec(namespace="full_redeploy_noop", nodes=nodes), + ) + assert data["status"] == "success", data["results"] + + data = await deploy_and_wait( + client, + DeploymentSpec(namespace="full_redeploy_noop", nodes=nodes), + ) + assert data["status"] == "success", data["results"] + assert all(result["operation"] == "noop" for result in data["results"]), data[ + "results" + ] + assert all(result["changed_fields"] == [] for result in data["results"]), data[ + "results" + ] + @pytest.mark.asyncio async def test_deploy_reconciles_external_preaggregation( self, @@ -2214,8 +2541,8 @@ async def test_deploy_dimension_with_update( "name": f"{namespace}.default.hard_hat", "status": "success", "operation": "update", - "changed_fields": ["query", "columns"], - "message": "Updated dimension (v2.0)\n└─ Column removed: hard_hat_id, state\n└─ Updated query, columns", + "changed_fields": ["query"], + "message": "Updated dimension (v2.0)\n└─ Updated query", } update_us_state = next( res @@ -2274,12 +2601,12 @@ async def test_deploy_metric_with_update( ) assert metric_result == { "deploy_type": "node", - "message": "Updated metric (v2.0)\n└─ Updated query, display_name\n" + "message": "Updated metric (v2.0)\n└─ Updated query\n" "[invalid] Metric metric_update.default.avg_length_of_employment has an invalid " "query, should have an aggregate expression", "name": "metric_update.default.avg_length_of_employment", "operation": "update", - "changed_fields": ["query", "display_name"], + "changed_fields": ["query"], "status": "invalid", } @@ -2299,10 +2626,10 @@ async def test_deploy_metric_with_update( ) assert metric_result == { "deploy_type": "node", - "message": "Updated metric (v3.0)\n└─ Updated query, display_name", + "message": "Updated metric (v3.0)\n└─ Updated query", "name": "metric_update.default.avg_length_of_employment", "operation": "update", - "changed_fields": ["query", "display_name"], + "changed_fields": ["query"], "status": "success", } @@ -2649,9 +2976,7 @@ async def test_deploy_cube_dimension_reorder( assert data["status"] == "success" assert data["results"][-1] == { "deploy_type": "node", - # The filter reorder is reported for the reader's benefit but earns no - # version of its own: v1.1 comes from the dimension reorder alone. - "message": "Updated cube (v1.1)\n└─ Reordered dimensions, filters", + "message": "Updated cube (v1.1)\n└─ Reordered dimensions", "name": f"{namespace}.default.repairs_cube", "operation": "update", "changed_fields": [], @@ -2665,7 +2990,7 @@ async def test_deploy_cube_dimension_reorder( ] @pytest.mark.asyncio - async def test_patch_and_deployment_agree_on_version( + async def test_patch_and_deployment_agree_on_an_upstream_change( self, client, default_hard_hats, @@ -2675,58 +3000,230 @@ async def test_patch_and_deployment_agree_on_version( default_avg_length_of_employment, ): """ - The same edit, applied once through `PATCH /nodes/{name}` and once through a - deployment, must land on the same version. Both paths classify significance - through the one shared classifier, and this is the test that keeps that a - fact rather than an aspiration. + A change to a node the cube sits on must bump the cube the same on both paths. + + The sibling of `test_patch_and_deployment_agree_on_version`, which only covers + edits to the cube itself. Here the cube's own spec is byte-identical across the + two deploys -- it still names the same metric and dimensions -- and only the + metric underneath it changes. That is the case a deploy currently cannot see: + an unchanged spec is skipped, so the cube keeps a revision compiled against the + old definition, while the PATCH path propagates into it. """ upstreams = [ default_hard_hats, default_hard_hat, default_us_states, default_us_state, - default_avg_length_of_employment, ] - def build_cube(**overrides) -> CubeSpec: - fields: dict = { - "name": "default.repairs_cube", - "display_name": "Repairs Cube", - "description": "Cube for analyzing repair orders", - "dimensions": ["${prefix}default.hard_hat.state"], - "metrics": ["${prefix}default.avg_length_of_employment"], - "filters": ["${prefix}default.hard_hat.state='AZ'"], - "owners": ["dj"], - } - return CubeSpec(**{**fields, **overrides}) + def build_metric(query: str) -> MetricSpec: + return default_avg_length_of_employment.model_copy( + deep=True, + update={"query": query}, + ) - async def deploy(namespace: str, cube: CubeSpec) -> None: + original = default_avg_length_of_employment.query + # Same shape, different rows: the cube's elements are untouched, so nothing + # about the cube's own spec records that this happened. + edited = ( + "SELECT avg(IF(state = 'AZ', CAST(NOW() AS DATE) - hire_date, NULL)) " + "FROM ${prefix}default.hard_hat" + ) + + cube = CubeSpec( + name="default.repairs_cube", + display_name="Repairs Cube", + description="Cube for analyzing repair orders", + dimensions=["${prefix}default.hard_hat.state"], + metrics=["${prefix}default.avg_length_of_employment"], + owners=["dj"], + ) + + async def deploy(namespace: str, metric_query: str) -> None: data = await deploy_and_wait( client, DeploymentSpec( namespace=namespace, - nodes=[spec.model_copy(deep=True) for spec in upstreams] + [cube], + nodes=( + [spec.model_copy(deep=True) for spec in upstreams] + + [build_metric(metric_query), cube.model_copy(deep=True)] + ), ), ) assert data["status"] == "success", data - async def version_of(namespace: str) -> str: - response = await client.get(f"/nodes/{namespace}.default.repairs_cube/") + async def version_of(namespace: str, node: str) -> str: + response = await client.get(f"/nodes/{namespace}.default.{node}/") + assert response.status_code == 200, response.text return response.json()["version"] - patch_ns, deploy_ns = "cube_equivalence_patch", "cube_equivalence_deploy" - await deploy(patch_ns, build_cube()) - await deploy(deploy_ns, build_cube()) - assert await version_of(patch_ns) == "v1.0" - assert await version_of(deploy_ns) == "v1.0" + patch_ns, deploy_ns = ( + "upstream_equivalence_patch", + "upstream_equivalence_deploy", + ) + await deploy(patch_ns, original) + await deploy(deploy_ns, original) + assert await version_of(patch_ns, "repairs_cube") == "v1.0" + assert await version_of(deploy_ns, "repairs_cube") == "v1.0" - # A metadata-only edit is minor on both paths. + # The same upstream edit, once through PATCH and once through a deployment. response = await client.patch( - f"/nodes/{patch_ns}.default.repairs_cube", - json={"description": "Cube for analyzing repair orders, revised"}, + f"/nodes/{patch_ns}.default.avg_length_of_employment", + json={"query": edited.replace("${prefix}", f"{patch_ns}.")}, ) assert response.status_code == 200, response.json() - await deploy( + await deploy(deploy_ns, edited) + + # The metric moved on both paths -- that part already agreed. + assert await version_of( + patch_ns, + "avg_length_of_employment", + ) == await version_of( + deploy_ns, + "avg_length_of_employment", + ) + # And so must the cube above it. + assert await version_of(patch_ns, "repairs_cube") == await version_of( + deploy_ns, + "repairs_cube", + ) + + @pytest.mark.asyncio + async def test_deployment_bumps_a_cube_over_a_changed_source( + self, + client, + default_hard_hats, + default_hard_hat, + default_us_states, + default_us_state, + default_avg_length_of_employment, + ): + """ + A cube bumps for a change anywhere above it, not just in its own parents. + + Here the edited node is the source two levels down: neither the metric nor + the dimension the cube names changes, so nothing the cube points at directly + moved, and only walking the whole way up finds the edit. The cube lands on + the tier the source earned rather than one of its own. + """ + cube = CubeSpec( + name="default.repairs_cube", + display_name="Repairs Cube", + description="Cube for analyzing repair orders", + dimensions=["${prefix}default.hard_hat.state"], + metrics=["${prefix}default.avg_length_of_employment"], + owners=["dj"], + ) + + async def deploy(source: SourceSpec) -> None: + data = await deploy_and_wait( + client, + DeploymentSpec( + namespace="upstream_source_deploy", + nodes=[ + spec.model_copy(deep=True) + for spec in ( + source, + default_hard_hat, + default_us_states, + default_us_state, + default_avg_length_of_employment, + cube, + ) + ], + ), + ) + assert data["status"] == "success", data + + async def version_of(node: str) -> str: + response = await client.get( + f"/nodes/upstream_source_deploy.default.{node}/", + ) + assert response.status_code == 200, response.text + return response.json()["version"] + + await deploy(default_hard_hats) + assert await version_of("repairs_cube") == "v1.0" + + widened = default_hard_hats.model_copy( + deep=True, + update={ + "columns": default_hard_hats.columns + + [ColumnSpec(name="badge_number", type="int")], + }, + ) + await deploy(widened) + + assert await version_of("hard_hats") == "v2.0" + # Untouched, so they keep the revisions they had. + assert await version_of("hard_hat") == "v1.0" + assert await version_of("avg_length_of_employment") == "v1.0" + # The cube alone follows the source, at the tier the source earned. + assert await version_of("repairs_cube") == "v2.0" + + @pytest.mark.asyncio + async def test_patch_and_deployment_agree_on_version( + self, + client, + default_hard_hats, + default_hard_hat, + default_us_states, + default_us_state, + default_avg_length_of_employment, + ): + """ + The same edit, applied once through `PATCH /nodes/{name}` and once through a + deployment, must land on the same version. Both paths classify significance + through the one shared classifier, and this is the test that keeps that a + fact rather than an aspiration. + """ + upstreams = [ + default_hard_hats, + default_hard_hat, + default_us_states, + default_us_state, + default_avg_length_of_employment, + ] + + def build_cube(**overrides) -> CubeSpec: + fields: dict = { + "name": "default.repairs_cube", + "display_name": "Repairs Cube", + "description": "Cube for analyzing repair orders", + "dimensions": ["${prefix}default.hard_hat.state"], + "metrics": ["${prefix}default.avg_length_of_employment"], + "filters": ["${prefix}default.hard_hat.state='AZ'"], + "owners": ["dj"], + } + return CubeSpec(**{**fields, **overrides}) + + async def deploy(namespace: str, cube: CubeSpec) -> None: + data = await deploy_and_wait( + client, + DeploymentSpec( + namespace=namespace, + nodes=[spec.model_copy(deep=True) for spec in upstreams] + [cube], + ), + ) + assert data["status"] == "success", data + + async def version_of(namespace: str) -> str: + response = await client.get(f"/nodes/{namespace}.default.repairs_cube/") + return response.json()["version"] + + patch_ns, deploy_ns = "cube_equivalence_patch", "cube_equivalence_deploy" + await deploy(patch_ns, build_cube()) + await deploy(deploy_ns, build_cube()) + assert await version_of(patch_ns) == "v1.0" + assert await version_of(deploy_ns) == "v1.0" + + # A metadata-only edit is minor on both paths. + response = await client.patch( + f"/nodes/{patch_ns}.default.repairs_cube", + json={"description": "Cube for analyzing repair orders, revised"}, + ) + assert response.status_code == 200, response.json() + await deploy( deploy_ns, build_cube(description="Cube for analyzing repair orders, revised"), ) @@ -2796,7 +3293,7 @@ async def deploy(**overrides) -> None: await deploy(description="Hard hats, revised") response = await client.get(f"/nodes/{name}/") assert response.status_code == 200, response.json() - assert response.json()["version"] == "v2.0" + assert response.json()["version"] == "v1.1" assert response.json()["description"] == "Hard hats, revised" @pytest.mark.asyncio @@ -3882,7 +4379,7 @@ async def test_deploy_failed_with_bad_node_spec_links( { "deploy_type": "node", "message": "Created source (v1.0)", - "name": f"{namespace}.default.hard_hats", + "name": f"{namespace}.default.us_states", "status": "success", "operation": "create", "changed_fields": [], @@ -3890,7 +4387,15 @@ async def test_deploy_failed_with_bad_node_spec_links( { "deploy_type": "node", "message": "Created source (v1.0)", - "name": f"{namespace}.default.us_states", + "name": f"{namespace}.default.hard_hats", + "status": "success", + "operation": "create", + "changed_fields": [], + }, + { + "deploy_type": "node", + "message": "Created dimension (v1.0)", + "name": f"{namespace}.default.us_state", "status": "success", "operation": "create", "changed_fields": [], @@ -3906,14 +4411,6 @@ async def test_deploy_failed_with_bad_node_spec_links( "operation": "create", "changed_fields": [], }, - { - "deploy_type": "node", - "message": "Created dimension (v1.0)", - "name": f"{namespace}.default.us_state", - "status": "success", - "operation": "create", - "changed_fields": [], - }, { "deploy_type": "link", "message": "Join link successfully deployed\n" @@ -3963,7 +4460,7 @@ async def test_deploy_succeeds_with_existing_deps( { "deploy_type": "node", "message": "Created source (v1.0)", - "name": f"{namespace}.default.hard_hats", + "name": f"{namespace}.default.us_states", "status": "success", "operation": "create", "changed_fields": [], @@ -3971,7 +4468,7 @@ async def test_deploy_succeeds_with_existing_deps( { "deploy_type": "node", "message": "Created source (v1.0)", - "name": f"{namespace}.default.us_states", + "name": f"{namespace}.default.hard_hats", "status": "success", "operation": "create", "changed_fields": [], @@ -3979,7 +4476,7 @@ async def test_deploy_succeeds_with_existing_deps( { "deploy_type": "node", "message": "Created dimension (v1.0)", - "name": f"{namespace}.default.hard_hat", + "name": f"{namespace}.default.us_state", "status": "success", "operation": "create", "changed_fields": [], @@ -3987,7 +4484,7 @@ async def test_deploy_succeeds_with_existing_deps( { "deploy_type": "node", "message": "Created dimension (v1.0)", - "name": f"{namespace}.default.us_state", + "name": f"{namespace}.default.hard_hat", "status": "success", "operation": "create", "changed_fields": [], @@ -4251,10 +4748,10 @@ async def test_deploy_tags( ) assert data["results"][-1] == { "deploy_type": "node", - "message": "Updated dimension (v2.0)\n└─ Column removed: state_id, state_name, state_region, state_short\n└─ Updated tags, columns", + "message": "Updated dimension (v1.1)\n└─ Updated tags", "name": "node_update.default.us_state", "operation": "update", - "changed_fields": ["tags", "columns"], + "changed_fields": ["tags"], "status": "success", } node = await Node.get_by_name(session, f"{namespace}.default.us_state") @@ -4423,7 +4920,7 @@ async def test_roads_deployment(self, session, client, roads_nodes): { "deploy_type": "node", "message": "Created source (v1.0)", - "name": f"{namespace}.default.contractors", + "name": f"{namespace}.default.dispatchers", "status": "success", "operation": "create", "changed_fields": [], @@ -4431,15 +4928,15 @@ async def test_roads_deployment(self, session, client, roads_nodes): { "deploy_type": "node", "message": "Created source (v1.0)", - "name": f"{namespace}.default.hard_hats", + "name": f"{namespace}.default.us_states", "status": "success", "operation": "create", "changed_fields": [], }, { "deploy_type": "node", - "message": "Created source (v1.0)", - "name": f"{namespace}.default.municipality", + "message": "Created dimension (v1.0)", + "name": f"{namespace}.default.dispatcher", "status": "success", "operation": "create", "changed_fields": [], @@ -4447,7 +4944,7 @@ async def test_roads_deployment(self, session, client, roads_nodes): { "deploy_type": "node", "message": "Created source (v1.0)", - "name": f"{namespace}.default.repair_order_details", + "name": f"{namespace}.default.hard_hats", "status": "success", "operation": "create", "changed_fields": [], @@ -4455,7 +4952,7 @@ async def test_roads_deployment(self, session, client, roads_nodes): { "deploy_type": "node", "message": "Created source (v1.0)", - "name": f"{namespace}.default.repair_orders", + "name": f"{namespace}.default.municipality", "status": "success", "operation": "create", "changed_fields": [], @@ -4463,7 +4960,7 @@ async def test_roads_deployment(self, session, client, roads_nodes): { "deploy_type": "node", "message": "Created source (v1.0)", - "name": f"{namespace}.default.repair_type", + "name": f"{namespace}.default.municipality_municipality_type", "status": "success", "operation": "create", "changed_fields": [], @@ -4471,15 +4968,15 @@ async def test_roads_deployment(self, session, client, roads_nodes): { "deploy_type": "node", "message": "Created source (v1.0)", - "name": f"{namespace}.default.us_region", + "name": f"{namespace}.default.municipality_type", "status": "success", "operation": "create", "changed_fields": [], }, { "deploy_type": "node", - "message": "Created source (v1.0)", - "name": f"{namespace}.default.us_states", + "message": "Created dimension (v1.0)", + "name": f"{namespace}.default.us_state", "status": "success", "operation": "create", "changed_fields": [], @@ -4487,7 +4984,7 @@ async def test_roads_deployment(self, session, client, roads_nodes): { "deploy_type": "node", "message": "Created source (v1.0)", - "name": f"{namespace}.default.dispatchers", + "name": f"{namespace}.default.contractors", "status": "success", "operation": "create", "changed_fields": [], @@ -4502,8 +4999,8 @@ async def test_roads_deployment(self, session, client, roads_nodes): }, { "deploy_type": "node", - "message": "Created source (v1.0)", - "name": f"{namespace}.default.municipality_municipality_type", + "message": "Created dimension (v1.0)", + "name": f"{namespace}.default.municipality_dim", "status": "success", "operation": "create", "changed_fields": [], @@ -4511,71 +5008,71 @@ async def test_roads_deployment(self, session, client, roads_nodes): { "deploy_type": "node", "message": "Created source (v1.0)", - "name": f"{namespace}.default.municipality_type", + "name": f"{namespace}.default.repair_orders", "status": "success", "operation": "create", "changed_fields": [], }, { "deploy_type": "node", - "message": "Created transform (v1.0)", - "name": f"{namespace}.default.national_level_agg", + "message": "Created dimension (v1.0)", + "name": f"{namespace}.default.contractor", "status": "success", "operation": "create", "changed_fields": [], }, { "deploy_type": "node", - "message": "Created transform (v1.0)", - "name": f"{namespace}.default.regional_level_agg", + "message": "Created dimension (v1.0)", + "name": f"{namespace}.default.repair_order", "status": "success", "operation": "create", "changed_fields": [], }, { "deploy_type": "node", - "message": "Created transform (v1.0)", - "name": f"{namespace}.default.repair_orders_fact", + "message": "Created source (v1.0)", + "name": f"{namespace}.default.repair_order_details", "status": "success", "operation": "create", "changed_fields": [], }, { "deploy_type": "node", - "message": "Created metric (v1.0)", - "name": f"{namespace}.default.avg_length_of_employment", + "message": "Created source (v1.0)", + "name": f"{namespace}.default.repair_type", "status": "success", "operation": "create", "changed_fields": [], }, { "deploy_type": "node", - "message": "Created metric (v1.0)", - "name": f"{namespace}.default.avg_repair_order_discounts", + "message": "Created source (v1.0)", + "name": f"{namespace}.default.us_region", "status": "success", "operation": "create", "changed_fields": [], }, { "deploy_type": "node", - "message": "Created metric (v1.0)", - "name": f"{namespace}.default.avg_repair_price", + "message": "Created transform (v1.0)", + "name": f"{namespace}.default.national_level_agg", "status": "success", "operation": "create", "changed_fields": [], }, { "deploy_type": "node", - "message": "Created metric (v1.0)", - "name": f"{namespace}.default.avg_time_to_dispatch", + "message": "Created transform (v1.0)", + "name": f"{namespace}.default.regional_level_agg", "status": "success", "operation": "create", "changed_fields": [], }, { "deploy_type": "node", - "message": "Created dimension (v1.0)", - "name": f"{namespace}.default.contractor", + "message": "Created transform (v1.0)", + "name": f"{namespace}.default.repair_orders_fact", "status": "success", "operation": "create", "changed_fields": [], @@ -4583,31 +5080,31 @@ async def test_roads_deployment(self, session, client, roads_nodes): { "deploy_type": "node", "message": "Created metric (v1.0)", - "name": f"{namespace}.default.discounted_orders_rate", + "name": f"{namespace}.default.avg_length_of_employment", "status": "success", "operation": "create", "changed_fields": [], }, { "deploy_type": "node", - "message": "Created dimension (v1.0)", - "name": f"{namespace}.default.dispatcher", + "message": "Created metric (v1.0)", + "name": f"{namespace}.default.avg_repair_order_discounts", "status": "success", "operation": "create", "changed_fields": [], }, { "deploy_type": "node", - "message": "Created source (v1.0)", - "name": f"{namespace}.default.hard_hat_state", + "message": "Created metric (v1.0)", + "name": f"{namespace}.default.avg_repair_price", "status": "success", "operation": "create", "changed_fields": [], }, { "deploy_type": "node", - "message": "Created dimension (v1.0)", - "name": f"{namespace}.default.municipality_dim", + "message": "Created metric (v1.0)", + "name": f"{namespace}.default.avg_time_to_dispatch", "status": "success", "operation": "create", "changed_fields": [], @@ -4615,39 +5112,39 @@ async def test_roads_deployment(self, session, client, roads_nodes): { "deploy_type": "node", "message": "Created metric (v1.0)", - "name": f"{namespace}.default.num_repair_orders", + "name": f"{namespace}.default.discounted_orders_rate", "status": "success", "operation": "create", "changed_fields": [], }, { "deploy_type": "node", - "message": "Created metric (v1.0)", - "name": f"{namespace}.default.regional_repair_efficiency", + "message": "Created source (v1.0)", + "name": f"{namespace}.default.hard_hat_state", "status": "success", "operation": "create", "changed_fields": [], }, { "deploy_type": "node", - "message": "Created dimension (v1.0)", - "name": f"{namespace}.default.repair_order", + "message": "Created metric (v1.0)", + "name": f"{namespace}.default.num_repair_orders", "status": "success", "operation": "create", "changed_fields": [], }, { "deploy_type": "node", - "message": "Created source (v1.0)", - "name": f"{namespace}.default.repair_orders_view", + "message": "Created metric (v1.0)", + "name": f"{namespace}.default.regional_repair_efficiency", "status": "success", "operation": "create", "changed_fields": [], }, { "deploy_type": "node", - "message": "Created metric (v1.0)", - "name": f"{namespace}.default.total_repair_cost", + "message": "Created source (v1.0)", + "name": f"{namespace}.default.repair_orders_view", "status": "success", "operation": "create", "changed_fields": [], @@ -4655,15 +5152,15 @@ async def test_roads_deployment(self, session, client, roads_nodes): { "deploy_type": "node", "message": "Created metric (v1.0)", - "name": f"{namespace}.default.total_repair_order_discounts", + "name": f"{namespace}.default.total_repair_cost", "status": "success", "operation": "create", "changed_fields": [], }, { "deploy_type": "node", - "message": "Created dimension (v1.0)", - "name": f"{namespace}.default.us_state", + "message": "Created metric (v1.0)", + "name": f"{namespace}.default.total_repair_order_discounts", "status": "success", "operation": "create", "changed_fields": [], @@ -8046,23 +8543,204 @@ async def test_service_account_used_when_no_git_author( @pytest.mark.xdist_group(name="deployments") -class TestDeploymentColumnOrdering: - """Tests for column ordering in deployments""" +class TestCubeRedeployIdempotence: + """A cube spec deployed twice must not churn a new version.""" @pytest.mark.asyncio - async def test_deployment_preserves_column_order(self, client): - """ - Test that column order is preserved for both source specs and - inferred columns from transform queries. + async def test_partitioned_cube_redeploy_is_a_noop( + self, + client, + default_hard_hats, + default_hard_hat, + default_us_states, + default_us_state, + default_avg_length_of_employment, + ): """ - namespace = "column_order_test" + Deploying an unchanged cube that declares a column partition is a noop. - # Create a source with columns in non-alphabetical order - source_spec = SourceSpec( - name="test_source", - description="Test source", - catalog="default", - schema="test_schema", + The cube branch of the deployment diff compares the partitions on the + incoming spec's columns against the partitions on the existing spec's. + If a declared partition never lands on the stored cube column, the two + sides can never agree, so every deploy reports ``columns`` changed -- + and ``columns`` is unclassified, so it bumps a MAJOR version. The cube + then gains a version on every deploy of the namespace forever, with + nothing about it having changed. + """ + namespace = "cube_redeploy_idempotence" + cube = CubeSpec( + name="${prefix}default.repairs_cube", + display_name="Repairs Cube", + description="Cube for analyzing repair orders", + dimensions=[ + "${prefix}default.hard_hat.state", + "${prefix}default.hard_hat.hire_date", + ], + metrics=["${prefix}default.avg_length_of_employment"], + owners=["dj"], + columns=[ + ColumnSpec( + name="${prefix}default.hard_hat.hire_date", + partition=PartitionSpec( + type=PartitionType.TEMPORAL, + granularity=Granularity.DAY, + format="yyyyMMdd", + ), + ), + ], + ) + upstreams = [ + default_hard_hats, + default_hard_hat, + default_us_states, + default_us_state, + default_avg_length_of_employment, + ] + + def deployment() -> DeploymentSpec: + return DeploymentSpec( + namespace=namespace, + nodes=[spec.model_copy(deep=True) for spec in upstreams] + + [cube.model_copy(deep=True)], + ) + + first = await deploy_and_wait(client, deployment()) + assert first["status"] == "success", first["results"] + + cube_name = f"{namespace}.default.repairs_cube" + version_after_first = (await client.get(f"/nodes/{cube_name}/")).json()[ + "version" + ] + + second = await deploy_and_wait(client, deployment()) + assert second["status"] == "success", second + cube_result = next( + result for result in second["results"] if result["name"] == cube_name + ) + assert cube_result["changed_fields"] == [], cube_result["message"] + assert cube_result["operation"] == "noop" + + version_after_second = (await client.get(f"/nodes/{cube_name}/")).json()[ + "version" + ] + assert version_after_second == version_after_first + + @pytest.mark.asyncio + async def test_partition_on_a_non_cube_column_does_not_churn( + self, + client, + default_hard_hats, + default_hard_hat, + default_us_states, + default_us_state, + default_avg_length_of_employment, + ): + """ + A partition declared on a name that is not one of the cube's columns must + not make every redeploy look like a change. + + ``rendered_columns`` keeps whatever the spec declared, but only names that + match a real cube column can be persisted. A declaration that matches + nothing therefore sits in ``incoming_partitions`` and never appears in + ``existing_partitions``, so the two can never agree: each deploy reports + ``columns`` changed, and because ``columns`` is unclassified it takes a + MAJOR version. The cube then gains a version on every deploy forever. + """ + namespace = "cube_redeploy_phantom_partition" + cube = CubeSpec( + name="${prefix}default.repairs_cube", + display_name="Repairs Cube", + description="Cube for analyzing repair orders", + dimensions=[ + "${prefix}default.hard_hat.state", + "${prefix}default.hard_hat.hire_date", + ], + metrics=["${prefix}default.avg_length_of_employment"], + owners=["dj"], + columns=[ + ColumnSpec( + # Deliberately not a cube column: the cube's column is + # ``default.hard_hat.hire_date``, fully qualified. + name="hire_date", + partition=PartitionSpec( + type=PartitionType.TEMPORAL, + granularity=Granularity.DAY, + format="yyyyMMdd", + ), + ), + ], + ) + upstreams = [ + default_hard_hats, + default_hard_hat, + default_us_states, + default_us_state, + default_avg_length_of_employment, + ] + + def deployment() -> DeploymentSpec: + return DeploymentSpec( + namespace=namespace, + nodes=[spec.model_copy(deep=True) for spec in upstreams] + + [cube.model_copy(deep=True)], + ) + + first = await deploy_and_wait(client, deployment()) + assert first["status"] == "success", first["results"] + + cube_name = f"{namespace}.default.repairs_cube" + version_after_first = (await client.get(f"/nodes/{cube_name}/")).json()[ + "version" + ] + + second = await deploy_and_wait(client, deployment()) + cube_results = [ + result for result in second["results"] if result["name"] == cube_name + ] + cube_result = next( + result for result in cube_results if result["status"] != "warning" + ) + assert cube_result["changed_fields"] == [], cube_result["message"] + + version_after_second = (await client.get(f"/nodes/{cube_name}/")).json()[ + "version" + ] + assert version_after_second == version_after_first + + # The declaration is ignored, but the author is told it is. + assert any( + "declares column 'hire_date'" in warning["message"] + for warning in second["warnings"] + ), second["warnings"] + + # The same warning is also reported structurally, against the cube's + # rendered name (not the raw `${prefix}...` spec name), so a consumer + # can attach it to that node without parsing free text. + cube_warning_result = next( + result for result in cube_results if result["status"] == "warning" + ) + assert cube_warning_result["operation"] == "noop" + assert "declares column 'hire_date'" in cube_warning_result["message"] + assert "${prefix}" not in cube_warning_result["message"] + + +class TestDeploymentColumnOrdering: + """Tests for column ordering in deployments""" + + @pytest.mark.asyncio + async def test_deployment_preserves_column_order(self, client): + """ + Test that column order is preserved for both source specs and + inferred columns from transform queries. + """ + namespace = "column_order_test" + + # Create a source with columns in non-alphabetical order + source_spec = SourceSpec( + name="test_source", + description="Test source", + catalog="default", + schema="test_schema", table="test_table", columns=[ ColumnSpec(name="z_column", type="string"), @@ -10179,3 +10857,1282 @@ async def test_cross_parent_ratio_shared_dim_via_pending_links_is_valid( response = await client.get(f"/nodes/{namespace}.fd_ratio_cross/") assert response.status_code == 200, response.json() assert response.json()["status"] == "valid" + + +def _clone_column_list(model, **overrides: str) -> tuple[str, str]: + """Quoted column names and the SELECT list that clones a row of `model`.""" + names = [c.name for c in model.__table__.columns if c.name != "id"] + return ( + ", ".join(f'"{name}"' for name in names), + ", ".join(overrides.get(name, f'r."{name}"') for name in names), + ) + + +async def commit_competing_revision( + session_factory, + node_name: str, + version: str, + advance_current: bool = True, +): + """Commit a revision at `version` for `node_name` from another session. + + Stands in for the deployment that wins a race: it clones the node's current + revision, and its columns, at the version an in-flight deployment has already + planned, and moves `Node.current_version` onto it. Raw SQL so nothing lands in + the deploying session's identity map. `advance_current` off leaves + `Node.current_version` behind the revision it just wrote. + """ + session = await session_factory() + current_revision = ( + "SELECT rev.id FROM noderevision rev JOIN node n ON n.id = rev.node_id " + "WHERE n.name = :name AND rev.version = n.current_version" + ) + names, values = _clone_column_list(NodeRevision, version=f"'{version}'") + new_id = ( + await session.execute( + text( + f"INSERT INTO noderevision ({names}) SELECT {values} " # noqa: S608 + f"FROM noderevision r WHERE r.id = ({current_revision}) RETURNING id", + ), + {"name": node_name}, + ) + ).scalar_one() + names, values = _clone_column_list(DBColumn, node_revision_id=str(new_id)) + await session.execute( + text( + f'INSERT INTO "column" ({names}) SELECT {values} FROM "column" r ' # noqa: S608 + f"WHERE r.node_revision_id = ({current_revision})", + ), + {"name": node_name}, + ) + if advance_current: + await session.execute( + text("UPDATE node SET current_version = :version WHERE name = :name"), + {"version": version, "name": node_name}, + ) + await session.commit() + await session.close() + + +@pytest.mark.xdist_group(name="deployments") +class TestConcurrentDeploymentVersionBump: + """Two deployments to one namespace that overlap in time. + + A deployment plans against a snapshot of `Node.current_version`. When another + deployment commits between that snapshot and the revision insert, every version + the loser planned is already taken and the bulk insert dies on + uq_noderevision_version, failing the whole deploy. The version written has to + come from committed state at write time, not from the plan-time snapshot. + """ + + @pytest.fixture + def cube(self): + return CubeSpec( + name="default.repairs_cube", + display_name="Repairs Cube", + description="Cube for analyzing repair orders", + dimensions=["${prefix}default.hard_hat.state"], + metrics=["${prefix}default.avg_length_of_employment"], + owners=["dj"], + ) + + @pytest.fixture + def nodes_list( + self, + cube, + default_hard_hats, + default_hard_hat, + default_us_states, + default_us_state, + default_avg_length_of_employment, + ): + return [ + default_hard_hats, + default_hard_hat, + default_us_states, + default_us_state, + default_avg_length_of_employment, + cube, + ] + + @pytest.mark.asyncio + async def test_deploy_bumps_from_committed_version_not_snapshot( + self, + client, + session_factory, + cube, + nodes_list, + default_avg_length_of_employment, + ): + """ + Both deployments carry a description-only edit off v1.0, so both compute + v1.1. The loser must land v1.2 -- the version after the one the winner + committed -- for the regular node and the cube alike, rather than failing + the user's deploy on uq_noderevision_version. + """ + namespace = "deploy_version_race" + metric_name = f"{namespace}.default.avg_length_of_employment" + cube_name = f"{namespace}.default.repairs_cube" + data = await deploy_and_wait( + client, + DeploymentSpec(namespace=namespace, nodes=nodes_list), + ) + assert data["status"] == "success", data + + # The winner commits v1.1 for both nodes after the deployment below has + # planned, but before it writes any revision. + original = DeploymentOrchestrator._execute_deployment_plan + + async def winning_deploy_commits_first(orchestrator, plan): + await commit_competing_revision(session_factory, metric_name, "v1.1") + await commit_competing_revision(session_factory, cube_name, "v1.1") + return await original(orchestrator, plan) + + default_avg_length_of_employment.description = "Average length of employment!" + cube.description = "Cube for analyzing repair orders, revised" + with patch.object( + DeploymentOrchestrator, + "_execute_deployment_plan", + winning_deploy_commits_first, + ): + data = await deploy_and_wait( + client, + DeploymentSpec(namespace=namespace, nodes=nodes_list), + ) + assert data["status"] == "success", data + + assert (await client.get(f"/nodes/{metric_name}/")).json()["version"] == "v1.2" + assert (await client.get(f"/nodes/{cube_name}/")).json()["version"] == "v1.2" + + @pytest.mark.asyncio + async def test_deploy_bumps_past_highest_revision( + self, + client, + session_factory, + cube, + nodes_list, + default_avg_length_of_employment, + ): + """ + A node whose `current_version` lags its highest revision must still earn a + free version. v10.0 exists while `current_version` says v9.0, so the deploy + has to bump past v10.0 -- which also means comparing versions semantically, + since v9.0 sorts above v10.0 as a string. The cloned revisions carry no + required-dimension or cube-element rows, so the re-deploy reads as major. + """ + namespace = "deploy_version_strand" + metric_name = f"{namespace}.default.avg_length_of_employment" + cube_name = f"{namespace}.default.repairs_cube" + data = await deploy_and_wait( + client, + DeploymentSpec(namespace=namespace, nodes=nodes_list), + ) + assert data["status"] == "success", data + + for name in (metric_name, cube_name): + await commit_competing_revision(session_factory, name, "v9.0") + await commit_competing_revision( + session_factory, + name, + "v10.0", + advance_current=False, + ) + + default_avg_length_of_employment.description = "Average length of employment!" + cube.description = "Cube for analyzing repair orders, revised" + data = await deploy_and_wait( + client, + DeploymentSpec(namespace=namespace, nodes=nodes_list), + ) + assert data["status"] == "success", data + + assert (await client.get(f"/nodes/{metric_name}/")).json()["version"] == "v11.0" + assert (await client.get(f"/nodes/{cube_name}/")).json()["version"] == "v11.0" + + +@pytest.mark.xdist_group(name="deployments") +class TestRequiredDimensionsRedeployIdempotence: + """A metric whose `required_dimensions` point at a column on its own parent + re-deploys as a noop, whichever of the two accepted spellings it was + authored in. The bare case is the control: it passed before the fix too, + since the bare name is what the export already emits.""" + + def _nodes(self, required_dimensions): + return [ + SourceSpec( + name="rd_orders_raw", + description="Raw orders", + catalog="default", + schema="roads", + table="rd_orders_raw", + columns=[ + ColumnSpec(name="order_id", type="bigint"), + ColumnSpec(name="currency_code", type="string"), + ], + dimension_links=[], + owners=["dj"], + ), + TransformSpec( + name="rd_orders_fact", + description="Orders fact", + query="SELECT order_id, currency_code FROM ${prefix}rd_orders_raw", + dimension_links=[], + owners=["dj"], + ), + MetricSpec( + name="rd_num_orders", + # Named so an inferred display_name doesn't turn up in `changed_fields`. + display_name="Rd Num Orders", + description="Number of orders", + query="SELECT count(order_id) FROM ${prefix}rd_orders_fact", + required_dimensions=required_dimensions, + owners=["dj"], + ), + ] + + @pytest.mark.parametrize( + "namespace, required_dimensions", + [ + ("rd_bare", ["currency_code"]), + ("rd_qualified", ["${prefix}rd_orders_fact.currency_code"]), + ], + ) + @pytest.mark.asyncio + async def test_redeploy_is_noop(self, client, namespace, required_dimensions): + nodes = self._nodes(required_dimensions) + metric_name = f"{namespace}.rd_num_orders" + + data = await deploy_and_wait( + client, + DeploymentSpec(namespace=namespace, nodes=nodes), + ) + assert data["status"] == "success", data + assert [ + result for result in data["results"] if result["name"] == metric_name + ] == [ + { + "deploy_type": "node", + "message": "Created metric (v1.0)", + "name": metric_name, + "operation": "create", + "changed_fields": [], + "status": "success", + }, + ] + + response = await client.get(f"/nodes/{metric_name}/") + assert response.status_code == 200, response.json() + assert response.json()["version"] == "v1.0" + + data = await deploy_and_wait( + client, + DeploymentSpec(namespace=namespace, nodes=self._nodes(required_dimensions)), + ) + assert data["status"] == "success", data + assert [ + result for result in data["results"] if result["name"] == metric_name + ] == [ + { + "deploy_type": "node", + "message": "Unchanged", + "name": metric_name, + "operation": "noop", + "changed_fields": [], + "status": "skipped", + }, + ] + + response = await client.get(f"/nodes/{metric_name}/") + assert response.status_code == 200, response.json() + assert response.json()["version"] == "v1.0" + + @pytest.mark.asyncio + async def test_redeploy_is_noop_for_linked_dimension_column_multi_segment_namespace( + self, + client, + ): + """Same shape as `test_redeploy_is_noop_for_linked_dimension_column`, but + under a dotted (git-branch-shaped) namespace and with the parent having + several dimension_links, reproducing the real semantic-shared churn on + `can_stream_accounts_28d` / `dt_date_d_v2.dateint`.""" + namespace = "rd_multi.linked_dim_branch" + + def _nodes(): + return [ + SourceSpec( + name="rd_date_raw", + description="Raw date", + catalog="default", + schema="roads", + table="rd_date_raw", + columns=[ColumnSpec(name="dateint", type="int")], + dimension_links=[], + owners=["dj"], + ), + DimensionSpec( + name="rd_date_dim", + description="Date dimension", + query="SELECT dateint FROM ${prefix}rd_date_raw", + primary_key=["dateint"], + dimension_links=[], + owners=["dj"], + ), + SourceSpec( + name="rd_geo_raw", + description="Raw geo", + catalog="default", + schema="roads", + table="rd_geo_raw", + columns=[ColumnSpec(name="country_iso_code", type="string")], + dimension_links=[], + owners=["dj"], + ), + DimensionSpec( + name="rd_geo_dim", + description="Geo dimension", + query="SELECT country_iso_code FROM ${prefix}rd_geo_raw", + primary_key=["country_iso_code"], + dimension_links=[], + owners=["dj"], + ), + SourceSpec( + name="rd_orders_raw", + description="Raw orders", + catalog="default", + schema="roads", + table="rd_orders_raw", + columns=[ + ColumnSpec(name="order_id", type="bigint"), + ColumnSpec(name="account_id", type="bigint"), + ColumnSpec(name="dateint", type="int"), + ColumnSpec(name="country_iso_code", type="string"), + ], + dimension_links=[], + owners=["dj"], + ), + TransformSpec( + name="rd_orders_fact", + description="Orders fact", + query=( + "SELECT order_id, account_id, dateint, country_iso_code " + "FROM ${prefix}rd_orders_raw" + ), + dimension_links=[ + DimensionJoinLinkSpec( + dimension_node="${prefix}rd_date_dim", + join_type="left", + join_on=( + "${prefix}rd_orders_fact.dateint = " + "${prefix}rd_date_dim.dateint" + ), + ), + DimensionJoinLinkSpec( + dimension_node="${prefix}rd_date_dim", + join_type="left", + join_on=( + "${prefix}rd_orders_fact.account_id = " + "${prefix}rd_date_dim.dateint" + ), + role="rd_account_signup_date", + ), + DimensionJoinLinkSpec( + dimension_node="${prefix}rd_geo_dim", + join_type="left", + join_on=( + "${prefix}rd_orders_fact.country_iso_code = " + "${prefix}rd_geo_dim.country_iso_code" + ), + role="rd_signup_country", + ), + ], + owners=["dj"], + ), + MetricSpec( + name="rd_num_orders", + display_name="Rd Num Orders", + description="Number of orders", + query="SELECT count(order_id) FROM ${prefix}rd_orders_fact", + required_dimensions=["${prefix}rd_date_dim.dateint"], + owners=["dj"], + ), + ] + + metric_name = f"{namespace}.rd_num_orders" + + data = await deploy_and_wait( + client, + DeploymentSpec(namespace=namespace, nodes=_nodes()), + ) + assert data["status"] == "success", data + response = await client.get(f"/nodes/{metric_name}/") + assert response.status_code == 200, response.json() + assert response.json()["version"] == "v1.0" + + data = await deploy_and_wait( + client, + DeploymentSpec(namespace=namespace, nodes=_nodes()), + ) + assert data["status"] == "success", data + assert [ + result for result in data["results"] if result["name"] == metric_name + ] == [ + { + "deploy_type": "node", + "message": "Unchanged", + "name": metric_name, + "operation": "noop", + "changed_fields": [], + "status": "skipped", + }, + ], data["results"] + + response = await client.get(f"/nodes/{metric_name}/") + assert response.status_code == 200, response.json() + assert response.json()["version"] == "v1.0" + + @pytest.mark.asyncio + async def test_unrelated_edit_preserves_linked_dimension_column(self, client): + """Updating a metric for an unrelated reason (not a redeploy noop) must + not silently drop a required dimension reached via a dimension_link, + when the linked dimension node itself isn't part of this update batch.""" + namespace = "rd_linked_dim_unrelated_edit" + + def _nodes(description): + return [ + SourceSpec( + name="rd_date_raw", + description="Raw date", + catalog="default", + schema="roads", + table="rd_date_raw", + columns=[ColumnSpec(name="dateint", type="int")], + dimension_links=[], + owners=["dj"], + ), + DimensionSpec( + name="rd_date_dim", + description="Date dimension", + query="SELECT dateint FROM ${prefix}rd_date_raw", + primary_key=["dateint"], + dimension_links=[], + owners=["dj"], + ), + SourceSpec( + name="rd_orders_raw", + description="Raw orders", + catalog="default", + schema="roads", + table="rd_orders_raw", + columns=[ + ColumnSpec(name="order_id", type="bigint"), + ColumnSpec(name="dateint", type="int"), + ], + dimension_links=[], + owners=["dj"], + ), + TransformSpec( + name="rd_orders_fact", + description="Orders fact", + query="SELECT order_id, dateint FROM ${prefix}rd_orders_raw", + dimension_links=[ + DimensionJoinLinkSpec( + dimension_node="${prefix}rd_date_dim", + join_type="inner", + join_on=( + "${prefix}rd_orders_fact.dateint = " + "${prefix}rd_date_dim.dateint" + ), + ), + ], + owners=["dj"], + ), + MetricSpec( + name="rd_num_orders", + display_name="Rd Num Orders", + description=description, + query="SELECT count(order_id) FROM ${prefix}rd_orders_fact", + required_dimensions=["${prefix}rd_date_dim.dateint"], + owners=["dj"], + ), + ] + + metric_name = f"{namespace}.rd_num_orders" + + data = await deploy_and_wait( + client, + DeploymentSpec(namespace=namespace, nodes=_nodes("Number of orders")), + ) + assert data["status"] == "success", data + response = await client.get(f"/metrics/{metric_name}/") + assert response.status_code == 200, response.json() + assert response.json()["required_dimensions"] == ["dateint"] + + # Only the metric's description changes here, so `rd_date_dim` and + # `rd_orders_fact` are unchanged and are not part of this update's + # deploy-ordering graph load -- required_dimensions must still resolve. + data = await deploy_and_wait( + client, + DeploymentSpec( + namespace=namespace, + nodes=_nodes("Number of orders, revised"), + ), + ) + assert data["status"] == "success", data + assert [ + result for result in data["results"] if result["name"] == metric_name + ] == [ + { + "deploy_type": "node", + "message": "Updated metric (v1.1)\n└─ Updated description", + "name": metric_name, + "operation": "update", + "changed_fields": ["description"], + "status": "success", + }, + ], data["results"] + + response = await client.get(f"/metrics/{metric_name}/") + assert response.status_code == 200, response.json() + assert response.json()["required_dimensions"] == ["dateint"] + + @pytest.mark.asyncio + async def test_metadata_edit_on_qualified_metric_is_minor(self, client): + """The change tier is folded from `changed_fields`, so a qualified + required dimension tagging along there drags a metadata-only edit up to + MAJOR.""" + namespace = "rd_qualified_minor" + nodes = self._nodes(["${prefix}rd_orders_fact.currency_code"]) + metric_name = f"{namespace}.rd_num_orders" + + data = await deploy_and_wait( + client, + DeploymentSpec(namespace=namespace, nodes=nodes), + ) + assert data["status"] == "success", data + + nodes[-1].description = "Number of orders, revised" + data = await deploy_and_wait( + client, + DeploymentSpec(namespace=namespace, nodes=nodes), + ) + assert data["status"] == "success", data + assert [ + result for result in data["results"] if result["name"] == metric_name + ] == [ + { + "deploy_type": "node", + "message": "Updated metric (v1.1)\n└─ Updated description", + "name": metric_name, + "operation": "update", + "changed_fields": ["description"], + "status": "success", + }, + ] + + response = await client.get(f"/nodes/{metric_name}/") + assert response.status_code == 200, response.json() + assert response.json()["version"] == "v1.1" + + @pytest.mark.asyncio + async def test_redeploy_is_noop_for_linked_dimension_column(self, client): + """A required dimension one hop out via a dimension_link (not a direct + query parent) should also redeploy as a noop. Reproduces the churn seen + with `${prefix}dt_date_d_v2.dateint`-shaped required dimensions.""" + namespace = "rd_linked_dim" + + def _nodes(): + return [ + SourceSpec( + name="rd_date_raw", + description="Raw date", + catalog="default", + schema="roads", + table="rd_date_raw", + columns=[ColumnSpec(name="dateint", type="int")], + dimension_links=[], + owners=["dj"], + ), + DimensionSpec( + name="rd_date_dim", + description="Date dimension", + query="SELECT dateint FROM ${prefix}rd_date_raw", + primary_key=["dateint"], + dimension_links=[], + owners=["dj"], + ), + SourceSpec( + name="rd_orders_raw", + description="Raw orders", + catalog="default", + schema="roads", + table="rd_orders_raw", + columns=[ + ColumnSpec(name="order_id", type="bigint"), + ColumnSpec(name="dateint", type="int"), + ], + dimension_links=[], + owners=["dj"], + ), + TransformSpec( + name="rd_orders_fact", + description="Orders fact", + query="SELECT order_id, dateint FROM ${prefix}rd_orders_raw", + dimension_links=[ + DimensionJoinLinkSpec( + dimension_node="${prefix}rd_date_dim", + join_type="inner", + join_on=( + "${prefix}rd_orders_fact.dateint = " + "${prefix}rd_date_dim.dateint" + ), + ), + ], + owners=["dj"], + ), + MetricSpec( + name="rd_num_orders", + display_name="Rd Num Orders", + description="Number of orders", + query="SELECT count(order_id) FROM ${prefix}rd_orders_fact", + required_dimensions=["${prefix}rd_date_dim.dateint"], + owners=["dj"], + ), + ] + + metric_name = f"{namespace}.rd_num_orders" + + data = await deploy_and_wait( + client, + DeploymentSpec(namespace=namespace, nodes=_nodes()), + ) + assert data["status"] == "success", data + response = await client.get(f"/nodes/{metric_name}/") + assert response.status_code == 200, response.json() + assert response.json()["version"] == "v1.0" + + data = await deploy_and_wait( + client, + DeploymentSpec(namespace=namespace, nodes=_nodes()), + ) + assert data["status"] == "success", data + assert [ + result for result in data["results"] if result["name"] == metric_name + ] == [ + { + "deploy_type": "node", + "message": "Unchanged", + "name": metric_name, + "operation": "noop", + "changed_fields": [], + "status": "skipped", + }, + ], data["results"] + + response = await client.get(f"/nodes/{metric_name}/") + assert response.status_code == 200, response.json() + assert response.json()["version"] == "v1.0" + + +def _us_state_dim(*, with_abbr: bool) -> DimensionSpec: + """The us_state dimension, optionally carrying a new `state_abbr` column.""" + columns = ["state_id", "state_name"] + (["state_abbr"] if with_abbr else []) + return DimensionSpec( + name="default.us_state", + description="US state dimension", + query=f"SELECT {', '.join(columns)} FROM ${{prefix}}default.us_states", + primary_key=["state_id"], + owners=["dj"], + ) + + +def _hard_hat_dim_with_reference_link() -> DimensionSpec: + """Hard hat dimension whose reference link points at `us_state.state_abbr`.""" + return DimensionSpec( + name="default.hard_hat", + description="Hard hat dimension", + query="SELECT hard_hat_id, state FROM ${prefix}default.hard_hats", + primary_key=["hard_hat_id"], + owners=["dj"], + dimension_links=[ + DimensionReferenceLinkSpec( + node_column="state", + dimension="${prefix}default.us_state.state_abbr", + ), + ], + ) + + +@pytest.mark.xdist_group(name="deployments") +class TestDimensionAttributeAddedInSamePush: + """ + One push that both adds an attribute to an existing dimension and adds a node + linking to that attribute. + + Link validation resolves the attribute against the dimension's *persisted* + columns, so a linked dimension has to deploy in an earlier topological level + than the node linking to it, whether or not there is query lineage between + them. + """ + + @pytest.mark.asyncio + async def test_reference_link_to_attribute_added_in_same_push( + self, + client, + default_us_states, + default_hard_hats, + ): + namespace = "dim_attr_same_push_ref" + + first = await deploy_and_wait( + client, + DeploymentSpec( + namespace=namespace, + nodes=[ + default_us_states, + default_hard_hats, + _us_state_dim(with_abbr=False), + ], + ), + ) + assert first["status"] == "success", first + + nodes = [ + default_us_states, + default_hard_hats, + _us_state_dim(with_abbr=True), + _hard_hat_dim_with_reference_link(), + ] + second = await deploy_and_wait( + client, + DeploymentSpec(namespace=namespace, nodes=nodes), + ) + assert second["status"] == "success", second["results"] + assert second["results"] == [ + { + "name": f"{namespace}.default.us_states", + "deploy_type": "node", + "status": "skipped", + "operation": "noop", + "message": "Unchanged", + "changed_fields": [], + }, + { + "name": f"{namespace}.default.hard_hats", + "deploy_type": "node", + "status": "skipped", + "operation": "noop", + "message": "Unchanged", + "changed_fields": [], + }, + { + "name": f"{namespace}.default.us_state", + "deploy_type": "node", + "status": "success", + "operation": "update", + "message": "Updated dimension (v2.0)\n\u2514\u2500 Updated query", + "changed_fields": ["query"], + }, + { + "name": f"{namespace}.default.hard_hat", + "deploy_type": "node", + "status": "success", + "operation": "create", + "message": "Created dimension (v1.0)", + "changed_fields": [], + }, + { + "name": ( + f"{namespace}.default.hard_hat -> {namespace}.default.us_state" + ), + "deploy_type": "link", + "status": "success", + "operation": "create", + "message": "Reference link successfully deployed", + "changed_fields": [], + }, + ] + + # The identical push a second time is a noop for every node. + third = await deploy_and_wait( + client, + DeploymentSpec(namespace=namespace, nodes=nodes), + ) + assert third["status"] == "success", third["results"] + assert third["results"] == [ + { + "name": f"{namespace}.default.us_states", + "deploy_type": "node", + "status": "skipped", + "operation": "noop", + "message": "Unchanged", + "changed_fields": [], + }, + { + "name": f"{namespace}.default.hard_hats", + "deploy_type": "node", + "status": "skipped", + "operation": "noop", + "message": "Unchanged", + "changed_fields": [], + }, + { + "name": f"{namespace}.default.us_state", + "deploy_type": "node", + "status": "skipped", + "operation": "noop", + "message": "Unchanged", + "changed_fields": [], + }, + { + "name": f"{namespace}.default.hard_hat", + "deploy_type": "node", + "status": "skipped", + "operation": "noop", + "message": "Unchanged", + "changed_fields": [], + }, + ] + + @pytest.mark.asyncio + async def test_join_link_to_attribute_added_in_same_push( + self, + client, + default_us_states, + default_hard_hats, + ): + """ + The join-link form of the same push. The join_on clause names the new + dimension column, so the dimension has to deploy first here too. + """ + namespace = "dim_attr_same_push_join" + + first = await deploy_and_wait( + client, + DeploymentSpec( + namespace=namespace, + nodes=[ + default_us_states, + default_hard_hats, + _us_state_dim(with_abbr=False), + ], + ), + ) + assert first["status"] == "success", first + + second = await deploy_and_wait( + client, + DeploymentSpec( + namespace=namespace, + nodes=[ + default_us_states, + default_hard_hats, + _us_state_dim(with_abbr=True), + TransformSpec( + name="default.hard_hats_fact", + description="Hard hats fact", + query=( + "SELECT hard_hat_id, state FROM ${prefix}default.hard_hats" + ), + owners=["dj"], + dimension_links=[ + DimensionJoinLinkSpec( + dimension_node="${prefix}default.us_state", + join_type="inner", + join_on=( + "${prefix}default.hard_hats_fact.state" + " = ${prefix}default.us_state.state_abbr" + ), + ), + ], + ), + ], + ), + ) + assert second["status"] == "success", second["results"] + assert [ + result + for result in second["results"] + if result["deploy_type"] == "link" + or result["name"] == f"{namespace}.default.hard_hats_fact" + ] == [ + { + "name": f"{namespace}.default.hard_hats_fact", + "deploy_type": "node", + "status": "success", + "operation": "create", + "message": "Created transform (v1.0)", + "changed_fields": [], + }, + { + "name": ( + f"{namespace}.default.hard_hats_fact ->" + f" {namespace}.default.us_state" + ), + "deploy_type": "link", + "status": "success", + "operation": "create", + "message": "Join link successfully deployed", + "changed_fields": [], + }, + ] + + @pytest.mark.asyncio + async def test_query_lineage_on_the_dimension_avoids_the_failure( + self, + client, + default_us_states, + default_hard_hats, + ): + """ + The same push, except the linking node's query also reads the dimension. + The dimension is already a query parent, so the link adds no new ordering + edge and the deploy order is the same either way. + """ + namespace = "dim_attr_same_push_lineage" + + first = await deploy_and_wait( + client, + DeploymentSpec( + namespace=namespace, + nodes=[ + default_us_states, + default_hard_hats, + _us_state_dim(with_abbr=False), + ], + ), + ) + assert first["status"] == "success", first + + second = await deploy_and_wait( + client, + DeploymentSpec( + namespace=namespace, + nodes=[ + default_us_states, + default_hard_hats, + _us_state_dim(with_abbr=True), + TransformSpec( + name="default.hard_hats_fact", + description="Hard hats fact", + query=( + "SELECT h.hard_hat_id, h.state, s.state_abbr AS abbr" + " FROM ${prefix}default.hard_hats h" + " LEFT JOIN ${prefix}default.us_state s" + " ON h.state = s.state_abbr" + ), + owners=["dj"], + dimension_links=[ + DimensionJoinLinkSpec( + dimension_node="${prefix}default.us_state", + join_type="inner", + join_on=( + "${prefix}default.hard_hats_fact.state" + " = ${prefix}default.us_state.state_abbr" + ), + ), + ], + ), + ], + ), + ) + assert second["status"] == "success", second["results"] + assert second["results"] == [ + { + "name": f"{namespace}.default.us_states", + "deploy_type": "node", + "status": "skipped", + "operation": "noop", + "message": "Unchanged", + "changed_fields": [], + }, + { + "name": f"{namespace}.default.hard_hats", + "deploy_type": "node", + "status": "skipped", + "operation": "noop", + "message": "Unchanged", + "changed_fields": [], + }, + { + "name": f"{namespace}.default.us_state", + "deploy_type": "node", + "status": "success", + "operation": "update", + "message": "Updated dimension (v2.0)\n\u2514\u2500 Updated query", + "changed_fields": ["query"], + }, + { + "name": f"{namespace}.default.hard_hats_fact", + "deploy_type": "node", + "status": "success", + "operation": "create", + "message": "Created transform (v1.0)", + "changed_fields": [], + }, + { + "name": ( + f"{namespace}.default.hard_hats_fact ->" + f" {namespace}.default.us_state" + ), + "deploy_type": "link", + "status": "success", + "operation": "create", + "message": "Join link successfully deployed", + "changed_fields": [], + }, + ] + + @pytest.mark.asyncio + async def test_brand_new_dimension_and_link_in_one_push( + self, + client, + default_us_states, + default_hard_hats, + ): + """ + Both halves in one push into an empty namespace. The link orders the + dimension ahead of the node linking to it, and the push succeeds. + """ + namespace = "dim_attr_fresh_ns" + + data = await deploy_and_wait( + client, + DeploymentSpec( + namespace=namespace, + nodes=[ + default_us_states, + default_hard_hats, + _us_state_dim(with_abbr=True), + _hard_hat_dim_with_reference_link(), + ], + ), + ) + assert data["status"] == "success", data["results"] + assert data["results"] == [ + { + "name": f"{namespace}.default.us_states", + "deploy_type": "node", + "status": "success", + "operation": "create", + "message": "Created source (v1.0)", + "changed_fields": [], + }, + { + "name": f"{namespace}.default.hard_hats", + "deploy_type": "node", + "status": "success", + "operation": "create", + "message": "Created source (v1.0)", + "changed_fields": [], + }, + { + "name": f"{namespace}.default.us_state", + "deploy_type": "node", + "status": "success", + "operation": "create", + "message": "Created dimension (v1.0)", + "changed_fields": [], + }, + { + "name": f"{namespace}.default.hard_hat", + "deploy_type": "node", + "status": "success", + "operation": "create", + "message": "Created dimension (v1.0)", + "changed_fields": [], + }, + { + "name": ( + f"{namespace}.default.hard_hat -> {namespace}.default.us_state" + ), + "deploy_type": "link", + "status": "success", + "operation": "create", + "message": "Reference link successfully deployed", + "changed_fields": [], + }, + ] + + @pytest.mark.asyncio + async def test_self_link_still_deploys(self, client, default_us_states): + """ + A dimension that joins to itself with a role. The link cannot be an + ordering edge, so it is dropped from the deploy order. + """ + namespace = "dim_link_self_join" + link_name = ( + f"{namespace}.default.us_state -> {namespace}.default.us_state[abbr]" + ) + + data = await deploy_and_wait( + client, + DeploymentSpec( + namespace=namespace, + nodes=[ + default_us_states, + DimensionSpec( + name="default.us_state", + description="US state dimension", + query=( + "SELECT state_id, state_name, state_abbr" + " FROM ${prefix}default.us_states" + ), + primary_key=["state_id"], + owners=["dj"], + dimension_links=[ + DimensionJoinLinkSpec( + dimension_node="${prefix}default.us_state", + role="abbr", + join_on=( + "${prefix}default.us_state.state_name" + " = ${prefix}default.us_state.state_abbr" + ), + ), + ], + ), + ], + ), + ) + assert data["status"] == "success", data["results"] + assert data["results"] == [ + { + "name": f"{namespace}.default.us_states", + "deploy_type": "node", + "status": "success", + "operation": "create", + "message": "Created source (v1.0)", + "changed_fields": [], + }, + { + "name": f"{namespace}.default.us_state", + "deploy_type": "node", + "status": "success", + "operation": "create", + "message": "Created dimension (v1.0)", + "changed_fields": [], + }, + { + "name": link_name, + "deploy_type": "link", + "status": "success", + "operation": "create", + "message": "Join link successfully deployed", + "changed_fields": [], + }, + ] + + @pytest.mark.asyncio + async def test_mutually_linked_dimensions_still_deploy( + self, + client, + default_us_states, + default_hard_hats, + ): + """ + Two dimensions that link to each other. One of the two link edges would + close a cycle, so it is dropped and both deploy in the same level. + """ + namespace = "dim_link_mutual" + + data = await deploy_and_wait( + client, + DeploymentSpec( + namespace=namespace, + nodes=[ + default_us_states, + default_hard_hats, + DimensionSpec( + name="default.us_state", + description="US state dimension", + query=( + "SELECT state_id, state_name, state_abbr" + " FROM ${prefix}default.us_states" + ), + primary_key=["state_id"], + owners=["dj"], + dimension_links=[ + DimensionJoinLinkSpec( + dimension_node="${prefix}default.hard_hat", + join_on=( + "${prefix}default.us_state.state_abbr" + " = ${prefix}default.hard_hat.state" + ), + ), + ], + ), + DimensionSpec( + name="default.hard_hat", + description="Hard hat dimension", + query=( + "SELECT hard_hat_id, state FROM ${prefix}default.hard_hats" + ), + primary_key=["hard_hat_id"], + owners=["dj"], + dimension_links=[ + DimensionJoinLinkSpec( + dimension_node="${prefix}default.us_state", + join_on=( + "${prefix}default.hard_hat.state" + " = ${prefix}default.us_state.state_abbr" + ), + ), + ], + ), + ], + ), + ) + assert data["status"] == "success", data["results"] + assert data["results"] == [ + { + "name": f"{namespace}.default.hard_hats", + "deploy_type": "node", + "status": "success", + "operation": "create", + "message": "Created source (v1.0)", + "changed_fields": [], + }, + { + "name": f"{namespace}.default.hard_hat", + "deploy_type": "node", + "status": "success", + "operation": "create", + "message": "Created dimension (v1.0)", + "changed_fields": [], + }, + { + "name": f"{namespace}.default.us_states", + "deploy_type": "node", + "status": "success", + "operation": "create", + "message": "Created source (v1.0)", + "changed_fields": [], + }, + { + "name": f"{namespace}.default.us_state", + "deploy_type": "node", + "status": "success", + "operation": "create", + "message": "Created dimension (v1.0)", + "changed_fields": [], + }, + { + "name": ( + f"{namespace}.default.us_state -> {namespace}.default.hard_hat" + ), + "deploy_type": "link", + "status": "success", + "operation": "create", + "message": "Join link successfully deployed", + "changed_fields": [], + }, + { + "name": ( + f"{namespace}.default.hard_hat -> {namespace}.default.us_state" + ), + "deploy_type": "link", + "status": "success", + "operation": "create", + "message": "Join link successfully deployed", + "changed_fields": [], + }, + ] diff --git a/datajunction-server/tests/api/dimension_links_test.py b/datajunction-server/tests/api/dimension_links_test.py index 700612ddf6..2da84958d1 100644 --- a/datajunction-server/tests/api/dimension_links_test.py +++ b/datajunction-server/tests/api/dimension_links_test.py @@ -4,31 +4,96 @@ Each test gets its own isolated database with COMPLEX_DIMENSION_LINK data loaded fresh. """ +import os +import pathlib +import subprocess +import sys +from collections.abc import Generator + import pytest import pytest_asyncio from httpx import AsyncClient from requests import Response +from testcontainers.postgres import PostgresContainer from datajunction_server.sql.parsing.backends.antlr4 import parse -from tests.conftest import post_and_raise_if_error +from tests.conftest import ( + externally_managed_postgres, + require_shared_template, + cleanup_database_for_module, + create_database_for_module, +) from tests.construction.build_v3 import assert_sql_equal -from tests.examples import COMPLEX_DIMENSION_LINK, SERVICE_SETUP + + +DIM_LINKS_TEMPLATE_DB_NAME = "template_dimension_links" + + +@pytest.fixture(scope="session") +def dim_links_template_database( + postgres_container: PostgresContainer, +) -> Generator[str, None, None]: + """ + A template database holding just the COMPLEX_DIMENSION_LINK examples. + + These tests used ``isolated_client``, which builds an empty database per + test -- create_all for every table, then the default attribute types, + catalogs and user, then the examples over HTTP. That is ~2s of setup for + each of the 14 tests, all of it producing identical state. + + Build it once and let each test clone it instead. Cloning is ~90ms, and + every test still gets its own database, so the tests that mutate links stay + isolated. + """ + externally_managed = externally_managed_postgres() + if externally_managed: + require_shared_template( + postgres_container, + DIM_LINKS_TEMPLATE_DB_NAME, + f"psql -c 'CREATE DATABASE {DIM_LINKS_TEMPLATE_DB_NAME};' && python " + f"tests/helpers/populate_template.py " + f"/{DIM_LINKS_TEMPLATE_DB_NAME} COMPLEX_DIMENSION_LINK", + ) + yield DIM_LINKS_TEMPLATE_DB_NAME + return + + url = create_database_for_module(postgres_container, DIM_LINKS_TEMPLATE_DB_NAME) + script = pathlib.Path(__file__).parent.parent / "helpers" / "populate_template.py" + project_root = pathlib.Path(__file__).parent.parent.parent + env = { + **os.environ, + "PYTHONPATH": f"{project_root}{os.pathsep}{os.environ.get('PYTHONPATH', '')}", + } + result = subprocess.run( + [sys.executable, str(script), url, "COMPLEX_DIMENSION_LINK"], + capture_output=True, + text=True, + cwd=str(project_root), + env=env, + ) + if result.returncode != 0: + raise RuntimeError( + f"Failed to populate dimension links template:\n" + f"{result.stdout}\n{result.stderr}", + ) + yield DIM_LINKS_TEMPLATE_DB_NAME + if not externally_managed: + cleanup_database_for_module(postgres_container, DIM_LINKS_TEMPLATE_DB_NAME) + + +@pytest.fixture +def isolated_client_template(dim_links_template_database: str) -> str: + """Have ``isolated_client`` clone the dimension-links template.""" + return dim_links_template_database @pytest_asyncio.fixture async def dimensions_link_client(isolated_client: AsyncClient) -> AsyncClient: """ - Function-scoped fixture that provides a client with COMPLEX_DIMENSION_LINK data. + Client whose database already has the COMPLEX_DIMENSION_LINK examples. - Uses isolated_client for complete isolation - each test gets its own fresh - database with the dimension link examples loaded. + They arrive with the template clone, so there is nothing to load here. """ - for endpoint, json in SERVICE_SETUP + COMPLEX_DIMENSION_LINK: - await post_and_raise_if_error( - client=isolated_client, - endpoint=endpoint, - json=json, # type: ignore - ) return isolated_client @@ -117,6 +182,49 @@ async def test_link_dimension_with_errors( ) +@pytest.mark.asyncio +async def test_link_dimension_without_join_on( + dimensions_link_client: AsyncClient, +): + """ + A join link with no join_on cannot be stored, so it is rejected up front. + """ + response = await dimensions_link_client.post( + "/nodes/default.events/link", + json={ + "dimension_node": "default.users", + "join_cardinality": "many_to_one", + }, + ) + assert response.status_code == 422 + assert response.json()["message"] == ( + "Dimension link from default.events to default.users has no join_on " + "clause. Set join_on to the equality between this node's foreign key " + "column(s) and the dimension's primary key." + ) + + +@pytest.mark.asyncio +async def test_link_dimension_with_cross_join( + dimensions_link_client: AsyncClient, +): + """ + A CROSS join has no ON clause, so it stores an empty join_sql. + """ + response = await dimensions_link_client.post( + "/nodes/default.events/link", + json={ + "dimension_node": "default.users", + "join_type": "cross", + }, + ) + assert response.status_code == 201 + assert response.json()["message"] == ( + "Dimension node default.users has been successfully linked to node " + "default.events." + ) + + @pytest.fixture def link_events_to_users_without_role( dimensions_link_client: AsyncClient, diff --git a/datajunction-server/tests/api/dimensions_test.py b/datajunction-server/tests/api/dimensions_test.py index 6d6d191444..23b8ee9f69 100644 --- a/datajunction-server/tests/api/dimensions_test.py +++ b/datajunction-server/tests/api/dimensions_test.py @@ -2,6 +2,8 @@ Tests for the dimensions API. """ +import re + import pytest from httpx import AsyncClient from sqlalchemy.ext.asyncio import AsyncSession @@ -160,7 +162,7 @@ async def test_list_dimension_query_count( node_queries = [ query for query in capture_queries - if "FROM node" in query or "FROM dimensionlink" in query + if re.search(r"\bFROM (?:node|dimensionlink)\b", query) ] assert len(node_queries) == 3, "\n\n".join(node_queries) diff --git a/datajunction-server/tests/api/djql_test.py b/datajunction-server/tests/api/djql_test.py index 69fdc54531..7a43e3f9c0 100644 --- a/datajunction-server/tests/api/djql_test.py +++ b/datajunction-server/tests/api/djql_test.py @@ -302,9 +302,8 @@ async def test_get_djsql_metric_table_exception( "/djsql/data/", params={"query": query}, ) - assert ( - response.json()["message"] == "DJ SQL queries must SELECT FROM metrics. " - "Example: SELECT metric1, dim1 FROM metrics GROUP BY dim1" + assert response.json()["message"] == ( + "DJ SQL queries must SELECT FROM metrics or dimensions." ) @@ -509,6 +508,103 @@ async def test_get_djsql_with_orderby_and_limit( ] +@pytest.mark.asyncio +async def test_get_djsql_role_qualified_dimension( + module__client_with_examples: AsyncClient, +) -> None: + """ + A role-qualified dimension attribute is allowed in the projection and is + classified as a dimension (not a metric) when it's also in the GROUP BY. + """ + query = """ + SELECT + default.special_country_dim.name[birth_country], + default.avg_user_age + FROM metrics + GROUP BY default.special_country_dim.name[birth_country] + """ + + response = await module__client_with_examples.get( + "/djsql/", + params={"query": query}, + ) + assert response.status_code == 200, response.json() + + semantics = { + (col["semantic_name"], col["semantic_type"]) + for col in response.json()["columns"] + } + assert semantics == { + ("default.special_country_dim.name[birth_country]", "dimension"), + ("default.avg_user_age", "metric"), + } + + +@pytest.mark.asyncio +async def test_get_djsql_role_path_dimension( + module__client_with_examples: AsyncClient, +) -> None: + """ + A multi-hop role path dimension attribute is allowed in the projection and + is classified as a dimension (not a metric) when it's also in the GROUP BY. + """ + query = """ + SELECT + default.date_dim.dateint[birth_country->formation_date], + default.avg_user_age + FROM metrics + GROUP BY default.date_dim.dateint[birth_country->formation_date] + """ + + response = await module__client_with_examples.get( + "/djsql/", + params={"query": query}, + ) + assert response.status_code == 200, response.json() + + semantics = { + (col["semantic_name"], col["semantic_type"]) + for col in response.json()["columns"] + } + assert semantics == { + ("default.date_dim.dateint[birth_country->formation_date]", "dimension"), + ("default.avg_user_age", "metric"), + } + + +@pytest.mark.asyncio +async def test_get_djsql_role_qualified_dimension_with_alias( + module__client_with_examples: AsyncClient, +) -> None: + """ + A role-qualified attribute may carry an alias. Unlike a plain column, which + absorbs its own alias, it is wrapped in an Alias node, so the alias has to be + unwrapped before the reference is classified. + """ + query = """ + SELECT + default.special_country_dim.name[birth_country] AS birth_country_name, + default.avg_user_age + FROM metrics + GROUP BY default.special_country_dim.name[birth_country] + """ + + response = await module__client_with_examples.get( + "/djsql/", + params={"query": query}, + ) + assert response.status_code == 200, response.json() + + semantics = { + (col["semantic_name"], col["semantic_type"]) + for col in response.json()["columns"] + } + assert semantics == { + ("default.special_country_dim.name[birth_country]", "dimension"), + ("default.avg_user_age", "metric"), + } + + @pytest.mark.asyncio async def test_get_djsql_no_nodes( module__client_with_roads: AsyncClient, diff --git a/datajunction-server/tests/api/files/materializations_test/spark_sql.full.materializations.json b/datajunction-server/tests/api/files/materializations_test/spark_sql.full.materializations.json index ed539cc15f..29c96979e8 100644 --- a/datajunction-server/tests/api/files/materializations_test/spark_sql.full.materializations.json +++ b/datajunction-server/tests/api/files/materializations_test/spark_sql.full.materializations.json @@ -131,5 +131,6 @@ "urls":[ "http://fake.url/job" ], - "workflow_names":[] + "workflow_names":[], + "node_version":"v1.1" } diff --git a/datajunction-server/tests/api/files/materializations_test/spark_sql.full.partition.materializations.json b/datajunction-server/tests/api/files/materializations_test/spark_sql.full.partition.materializations.json index 9861f48ebf..540ada1855 100644 --- a/datajunction-server/tests/api/files/materializations_test/spark_sql.full.partition.materializations.json +++ b/datajunction-server/tests/api/files/materializations_test/spark_sql.full.partition.materializations.json @@ -131,5 +131,6 @@ "urls":[ "http://fake.url/job" ], - "workflow_names":[] + "workflow_names":[], + "node_version":"v1.1" } diff --git a/datajunction-server/tests/api/git_test.py b/datajunction-server/tests/api/git_test.py index 58c7222706..02a2b07a6f 100644 --- a/datajunction-server/tests/api/git_test.py +++ b/datajunction-server/tests/api/git_test.py @@ -4402,7 +4402,32 @@ async def test_branch_creation_copies_nodes( assert response.status_code == HTTPStatus.CREATED data = response.json() assert data["branch"]["namespace"] == "copy_test.feature_copy" - assert data["deployment_results"] == [ + assert all( + result["change_tier"] == "major" + for result in data["deployment_results"] + ) + assert all( + result["semantic_fingerprint"]["version"] == 1 + for result in data["deployment_results"] + ) + assert all( + result["revalidation_only"] is False + for result in data["deployment_results"] + ) + deployment_results = [ + { + key: value + for key, value in result.items() + if key + not in { + "change_tier", + "semantic_fingerprint", + "revalidation_only", + } + } + for result in data["deployment_results"] + ] + assert deployment_results == [ { "deploy_type": "node", "message": "Created source (v1.0)", diff --git a/datajunction-server/tests/api/graphql/tags_test.py b/datajunction-server/tests/api/graphql/tags_test.py index 7d867adb49..080b11197f 100644 --- a/datajunction-server/tests/api/graphql/tags_test.py +++ b/datajunction-server/tests/api/graphql/tags_test.py @@ -7,14 +7,14 @@ from httpx import AsyncClient -@pytest_asyncio.fixture +@pytest_asyncio.fixture(scope="module") async def client_with_tags( - client_with_roads: AsyncClient, + module__client_with_roads: AsyncClient, ) -> AsyncClient: """ Provides a DJ client fixture seeded with tags """ - await client_with_roads.post( + await module__client_with_roads.post( "/tags/", json={ "name": "sales_report", @@ -24,7 +24,7 @@ async def client_with_tags( "tag_metadata": {}, }, ) - await client_with_roads.post( + await module__client_with_roads.post( "/tags/", json={ "name": "other_report", @@ -34,7 +34,7 @@ async def client_with_tags( "tag_metadata": {}, }, ) - await client_with_roads.post( + await module__client_with_roads.post( "/tags/", json={ "name": "coffee", @@ -44,7 +44,7 @@ async def client_with_tags( "tag_metadata": {}, }, ) - await client_with_roads.post( + await module__client_with_roads.post( "/tags/", json={ "name": "tea", @@ -55,16 +55,16 @@ async def client_with_tags( }, ) - await client_with_roads.post( + await module__client_with_roads.post( "/nodes/default.total_repair_cost/tags/?tag_names=sales_report", ) - await client_with_roads.post( + await module__client_with_roads.post( "/nodes/default.avg_repair_price/tags/?tag_names=sales_report", ) - await client_with_roads.post( + await module__client_with_roads.post( "/nodes/default.num_repair_orders/tags/?tag_names=other_report", ) - return client_with_roads + return module__client_with_roads @pytest.mark.asyncio diff --git a/datajunction-server/tests/api/helpers_test.py b/datajunction-server/tests/api/helpers_test.py index 5e86b61633..37c47e005c 100644 --- a/datajunction-server/tests/api/helpers_test.py +++ b/datajunction-server/tests/api/helpers_test.py @@ -11,6 +11,7 @@ from datajunction_server.api import helpers from datajunction_server.api.helpers import ( + _resolve_required_dimensions, dedupe_cube_elements, find_required_dimensions, ) @@ -302,3 +303,21 @@ async def test_find_required_dimensions_full_path_match( assert len(matched_cols) == 1 assert matched_cols[0].name == "month" + + +def test_resolve_required_dimensions_short_name_ambiguous_across_parents(): + """ + A short name that matches a column on more than one direct parent must be + flagged invalid rather than silently resolved to one of them. + """ + orders_currency = Column(name="currency_code") + refunds_currency = Column(name="currency_code") + + invalid_dims, matched_cols = _resolve_required_dimensions( + required_dimensions=["currency_code"], + parent_columns=[orders_currency, refunds_currency], + dim_nodes={}, + ) + + assert invalid_dims == {"currency_code"} + assert matched_cols == [] diff --git a/datajunction-server/tests/api/namespaces_test.py b/datajunction-server/tests/api/namespaces_test.py index 36e8d8dded..bf412ef4ba 100644 --- a/datajunction-server/tests/api/namespaces_test.py +++ b/datajunction-server/tests/api/namespaces_test.py @@ -3,18 +3,28 @@ """ import asyncio +from datetime import UTC, datetime, timedelta from http import HTTPStatus from unittest import mock import pytest from httpx import AsyncClient +from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession -from datajunction_server.api.namespaces import provision_node_namespace +from datajunction_server.api import namespaces as namespace_api +from datajunction_server.api.namespaces import ( + hard_delete_node_namespace, + provision_node_namespace, +) from datajunction_server.database.namespace import NodeNamespace +from datajunction_server.database.node import Node +from datajunction_server.database.rbac import Role, RoleAssignment, RoleScope from datajunction_server.database.user import OAuthProvider, PrincipalKind, User +from datajunction_server.internal.access.authentication.tokens import create_token from datajunction_server.internal.access.authorization import ( AuthorizationService, + RBACAuthorizationService, ) from datajunction_server.internal.namespaces import ( _merge_columns_preserving_comments, @@ -96,6 +106,347 @@ async def test_provision_namespace_boundary( access_checker.check.assert_awaited_once() +@pytest.mark.parametrize( + ("is_governed_boundary", "expected_action"), + [ + (True, ResourceAction.MANAGE), + (False, ResourceAction.DELETE), + ], +) +async def test_hard_delete_boundary_uses_policy_lifecycle_action( + is_governed_boundary, + expected_action, + mocker, +): + mocker.patch.object( + NodeNamespace, + "get", + new=mocker.AsyncMock( + return_value=mocker.Mock( + is_governed_boundary=is_governed_boundary, + ), + ), + ) + access_checker = mocker.MagicMock() + access_checker.check = mocker.AsyncMock( + side_effect=RuntimeError("stop after authorization"), + ) + + with pytest.raises(RuntimeError, match="stop after authorization"): + await hard_delete_node_namespace( + "policy_boundary", + session=mocker.MagicMock(), + current_user=mocker.MagicMock(), + save_history=mocker.AsyncMock(), + access_checker=access_checker, + query_service_client=mocker.MagicMock(), + request=mocker.MagicMock(), + ) + + access_checker.add_namespace.assert_called_once_with( + "policy_boundary", + expected_action, + ) + + +async def test_provisioned_boundary_enforces_rbac_without_restrictive_config( + client: AsyncClient, + session: AsyncSession, + current_user: User, + settings_no_qs, + mocker, +): + service_settings = mocker.patch( + "datajunction_server.internal.access.authorization.service.settings", + ) + service_settings.default_access_policy = "permissive" + service_settings.restrictive_scopes = [] + context_settings = mocker.patch( + "datajunction_server.internal.access.authorization.context.settings", + ) + context_settings.default_access_role = None + mocker.patch(VALIDATOR_AUTH_SERVICE, lambda: RBACAuthorizationService()) + + current_user.is_admin = True + await session.commit() + + owner_username = "api-boundary-owner" + outsider_username = "api-boundary-outsider" + owner_group = "api-boundary-owners" + for username in (owner_username, outsider_username): + response = await client.post( + "/basic/user/", + data={ + "email": f"{username}@example.com", + "username": username, + "password": "test-password", + }, + ) + assert response.status_code == HTTPStatus.CREATED + + response = await client.post("/groups/", params={"username": owner_group}) + assert response.status_code == HTTPStatus.CREATED + response = await client.post( + f"/groups/{owner_group}/members/", + params={"member_username": owner_username}, + ) + assert response.status_code == HTTPStatus.CREATED + + response = await client.post( + "/service-accounts", + json={"name": "API boundary deployer"}, + ) + assert response.status_code == HTTPStatus.OK + deployer = response.json() + response = await client.post( + "/service-accounts/token", + data={ + "client_id": deployer["client_id"], + "client_secret": deployer["client_secret"], + }, + ) + assert response.status_code == HTTPStatus.OK + deployer_token = response.json()["token"] + + response = await client.post( + "/namespaces/example.metrics/provision", + json={ + "owner_group": owner_group, + "deployer_service_accounts": [deployer["client_id"]], + }, + ) + assert response.status_code == HTTPStatus.CREATED + boundary = await session.get(NodeNamespace, "example.metrics") + assert boundary is not None + assert boundary.is_governed_boundary is True + + def user_token(username: str) -> str: + return create_token( + {"username": username}, + secret=settings_no_qs.secret, + iss=settings_no_qs.url, + expires_delta=timedelta(hours=1), + ) + + client.headers["Authorization"] = f"Bearer {user_token(owner_username)}" + response = await client.post( + "/roles/", + json={ + "name": "api-boundary-writer", + "scopes": [ + { + "action": "write", + "scope_type": "node", + "scope_value": "example.metrics.*", + }, + ], + }, + ) + assert response.status_code == HTTPStatus.CREATED + + client.headers["Authorization"] = f"Bearer {deployer_token}" + response = await client.delete("/namespaces/example.metrics/hard/") + assert response.status_code == HTTPStatus.FORBIDDEN + response = await client.post("/namespaces/example.metrics.deployed/") + assert response.status_code == HTTPStatus.CREATED + + client.headers["Authorization"] = f"Bearer {user_token(outsider_username)}" + response = await client.post("/namespaces/example.metrics.denied/") + assert response.status_code == HTTPStatus.FORBIDDEN + response = await client.get("/namespaces/example.metrics/") + assert response.status_code == HTTPStatus.OK + response = await client.post("/namespaces/example.open/") + assert response.status_code == HTTPStatus.CREATED + + +@pytest.mark.parametrize("cascade", [False, True]) +@pytest.mark.parametrize("grant", ["none", "delete", "partial", "manage", "stale"]) +async def test_hard_delete_ancestor_requires_manage_on_every_boundary( + client: AsyncClient, + session: AsyncSession, + current_user: User, + mocker, + cascade: bool, + grant: str, +): + """An ancestor delete cannot remove another owner's governance boundary.""" + root = "deleteparent" + boundaries = [f"{root}.first", f"{root}.second"] + unrelated = f"{root}other.governed" + session.add_all( + NodeNamespace(namespace=name) for name in [root, *boundaries, unrelated] + ) + await session.commit() + node_name = f"{boundaries[0]}.source" + if cascade: + response = await client.post( + "/nodes/source/", + json={ + "name": node_name, + "catalog": "default", + "schema_": "public", + "table": "example", + "columns": [{"name": "id", "type": "int"}], + }, + ) + assert response.status_code == HTTPStatus.OK, response.json() + + for boundary in [*boundaries, unrelated]: + item = await session.get(NodeNamespace, boundary) + item.is_governed_boundary = True + # Deactivation retains the boundary policy until hard deletion. + item = await session.get(NodeNamespace, boundaries[1]) + item.deactivated_at = datetime.now(UTC) + + role = Role(name="ancestor-deleter", created_by_id=current_user.id) + session.add(role) + await session.flush() + session.add( + RoleAssignment( + principal_id=current_user.id, + role_id=role.id, + granted_by_id=current_user.id, + ), + ) + # An explicit DELETE grant on the ancestor alone must not remove children. + scopes = [(root, ResourceAction.DELETE)] + if grant == "delete": + scopes.extend((name, ResourceAction.DELETE) for name in boundaries) + elif grant == "partial": + scopes.append((boundaries[0], ResourceAction.MANAGE)) + elif grant == "manage": + scopes.extend((name, ResourceAction.MANAGE) for name in boundaries) + session.add_all( + RoleScope( + role_id=role.id, + action=action, + scope_type=ResourceType.NAMESPACE, + scope_value=name, + ) + for name, action in scopes + ) + await session.commit() + + service_settings = mocker.patch( + "datajunction_server.internal.access.authorization.service.settings", + ) + service_settings.default_access_policy = "permissive" + service_settings.restrictive_scopes = [] + context_settings = mocker.patch( + "datajunction_server.internal.access.authorization.context.settings", + ) + context_settings.default_access_role = None + if grant == "stale": + # Provisioning can commit after the request's auth context is loaded. + mocker.patch( + "datajunction_server.internal.access.authorization.context." + "AuthContext.get_governed_boundaries", + new=mocker.AsyncMock(return_value=()), + ) + mocker.patch(VALIDATOR_AUTH_SERVICE, lambda: RBACAuthorizationService()) + delete_spy = mocker.spy( + namespace_api, + "hard_delete_namespace", + ) + + response = await client.delete( + f"/namespaces/{root}/hard/", + params={"cascade": cascade}, + ) + remaining = set((await session.execute(select(NodeNamespace.namespace))).scalars()) + assert unrelated in remaining + if grant == "manage": + assert response.status_code == HTTPStatus.OK, response.json() + assert set(response.json()["impact"]["deleted_namespaces"]) == { + root, + *boundaries, + } + assert not ({root, *boundaries} & remaining) + if cascade: + assert response.json()["impact"]["deleted_nodes"] == [node_name] + delete_spy.assert_awaited_once() + else: + assert response.status_code == HTTPStatus.FORBIDDEN, response.json() + assert {root, *boundaries} <= remaining + delete_spy.assert_not_awaited() + if cascade: + assert await session.scalar(select(Node.name).where(Node.name == node_name)) + + +@pytest.mark.parametrize("outside_governed", [False, True]) +async def test_hard_delete_namespace_preserves_underscore_collision( + client: AsyncClient, + session: AsyncSession, + current_user: User, + mocker, + outside_governed: bool, +): + """A literal namespace prefix must govern both authorization and deletion.""" + root = "lunch.taco_truck" + child = f"{root}.child" + outside = "lunch.tacoXtruck.child" + session.add_all( + NodeNamespace(namespace=name) for name in ["lunch", root, child, outside] + ) + await session.commit() + for namespace in [child, outside]: + response = await client.post( + "/nodes/source/", + json={ + "name": f"{namespace}.source", + "catalog": "default", + "schema_": "public", + "table": "example", + "columns": [{"name": "id", "type": "int"}], + }, + ) + assert response.status_code == HTTPStatus.OK, response.json() + + boundary = await session.get(NodeNamespace, root) + boundary.is_governed_boundary = True + outside_boundary = await session.get(NodeNamespace, outside) + outside_boundary.is_governed_boundary = outside_governed + role = Role(name="literal-boundary-owner", created_by_id=current_user.id) + session.add(role) + await session.flush() + session.add_all( + [ + RoleScope( + role_id=role.id, + action=ResourceAction.MANAGE, + scope_type=ResourceType.NAMESPACE, + scope_value=root, + ), + RoleAssignment( + principal_id=current_user.id, + role_id=role.id, + granted_by_id=current_user.id, + ), + ], + ) + await session.commit() + service_settings = mocker.patch( + "datajunction_server.internal.access.authorization.service.settings", + ) + service_settings.default_access_policy = "permissive" + service_settings.restrictive_scopes = [] + mocker.patch(VALIDATOR_AUTH_SERVICE, lambda: RBACAuthorizationService()) + + response = await client.delete(f"/namespaces/{root}/hard/?cascade=true") + assert response.status_code == HTTPStatus.OK, response.json() + assert set(response.json()["impact"]["deleted_namespaces"]) == {root, child} + assert response.json()["impact"]["deleted_nodes"] == [f"{child}.source"] + remaining = set((await session.execute(select(NodeNamespace.namespace))).scalars()) + assert {"lunch", outside} <= remaining + assert not {root, child} & remaining + assert ( + await session.scalar( + select(Node.name).where(Node.name == f"{outside}.source"), + ) + == f"{outside}.source" + ) + + @pytest.mark.asyncio async def test_list_all_namespaces( module__client_with_all_examples: AsyncClient, diff --git a/datajunction-server/tests/api/nodes_test.py b/datajunction-server/tests/api/nodes_test.py index 79cf36067b..94c974d1a4 100644 --- a/datajunction-server/tests/api/nodes_test.py +++ b/datajunction-server/tests/api/nodes_test.py @@ -6418,15 +6418,15 @@ async def test_revalidate_preserves_column_metadata( """, ), ) - # Also clear the order so revalidate detects a change and creates a - # new revision (matching the existing test_revalidate_sets_column_order - # trigger). + # Also move a stored column's type away from what the query produces, so + # revalidate detects a real change and forks a new revision. (A cleared + # column order would not: that is backfilled in place without a bump.) await session.execute( text( """ UPDATE "column" - SET "order" = NULL - WHERE node_revision_id IN ( + SET type = 'string' + WHERE name = 'dispatcher_id' AND node_revision_id IN ( SELECT id FROM noderevision WHERE name = 'default.test_meta_preserved' ) """, @@ -6548,13 +6548,14 @@ async def test_revalidate_preserves_column_metadata_post_deployment( }, f"Deploy did not land descriptions: {post_deploy_descriptions}" # Force revalidate_node to detect a column change so it creates a new - # revision (same trigger as test_revalidate_sets_column_order_when_missing). + # revision. A stored type that disagrees with the query is a major change; + # a cleared column order is not, since that is backfilled in place. await session.execute( text( """ UPDATE "column" - SET "order" = NULL - WHERE node_revision_id IN ( + SET type = 'string' + WHERE name = 'dispatcher_id' AND node_revision_id IN ( SELECT id FROM noderevision WHERE name = 'meta_preserve_test.test_deploy_meta' ) """, @@ -6587,10 +6588,15 @@ async def test_revalidate_writes_history_event( client_with_roads: AsyncClient, session: AsyncSession, ): - """revalidate_node must record a history event when it creates a new - revision. Without it, /history?node= only shows deploy-driven - updates and silently masks revalidate-driven version bumps (UI - validate, downstream propagation, manual /validate calls). + """revalidate_node must record a history event when it changes a node's + stored columns. Without it, /history?node= only shows deploy-driven + updates and silently masks what revalidation did (UI validate, downstream + propagation, manual /validate calls). + + The column-order backfill exercised here no longer bumps the version -- + filling in DJ's own bookkeeping is not a change to the node -- but it does + still rewrite the stored row, so the audit trail must still explain it. + Losing the bump must not mean losing the record. """ response = await client_with_roads.post( "/nodes/transform/", @@ -6603,8 +6609,7 @@ async def test_revalidate_writes_history_event( ) assert response.status_code == 201 - # Force revalidate_node to create a new revision (same trigger as the - # column-order-cleared tests above). + # Clear the stored order so revalidate_node has a backfill to do. await session.execute( text( """ @@ -6637,32 +6642,31 @@ async def test_revalidate_writes_history_event( ) ).json() - # An additional history event was emitted by revalidate_node - # specifically (reason: revalidate) when it bumped the revision. + # An additional history event was emitted by revalidate_node, naming the + # columns whose missing order it backfilled and the version it left the + # node on. Without this the audit trail says nothing at all: no version + # changed, so nothing else records that the row was rewritten. revalidate_events = [ - e + e["details"] for e in history_after if e.get("activity_type") == "update" - and (e.get("details") or {}).get("reason") == "revalidate" + and (e.get("details") or {}).get("reason") == "column order backfill" ] - assert len(revalidate_events) >= 1, ( - f"Expected a revalidate-reason history event after revision bump; " - f"got events: {history_after}" - ) - # The event details explain WHY the validator bumped the revision — - # in this case, by recording which columns had their missing order - # backfilled. Without this the audit trail says "revalidate happened" - # but not "this is what it changed". - details = revalidate_events[0]["details"] - assert details.get("order_fixed"), ( - f"Expected order_fixed in revalidate event details; got {details}" - ) - assert set(details["order_fixed"]) == {"repair_order_id"}, ( - f"Expected order_fixed to list the column whose order was cleared; " - f"got {details}" - ) + assert revalidate_events == [ + { + "version": "v1.0", + "reason": "column order backfill", + "order_fixed": ["repair_order_id"], + }, + ], f"Expected one backfill event; got events: {history_after}" assert len(history_after) > len(history_before) + # And the node itself did not turn over. + node_after = ( + await client_with_roads.get("/nodes/default.test_revalidate_history") + ).json() + assert node_after["version"] == "v1.0" + @pytest.mark.asyncio async def test_revalidate_via_propagation_writes_one_history_event( self, @@ -6696,12 +6700,15 @@ async def capture_history(event, session): # type: ignore ) assert response.status_code == 201 - # Force the revalidate code path to create a new revision. + # Force the revalidate code path to create a new revision. It has to be a + # real change -- a stored type the query disagrees with -- because the + # cheaper triggers no longer fork a revision, and a test that suppresses an + # event which was never going to be written proves nothing. await session.execute( text( """ UPDATE "column" - SET "order" = NULL + SET type = 'string' WHERE node_revision_id IN ( SELECT id FROM noderevision WHERE name = 'default.test_one_event_only' ) diff --git a/datajunction-server/tests/api/preaggregations_test.py b/datajunction-server/tests/api/preaggregations_test.py index db1dd0bcba..72a8b9804e 100644 --- a/datajunction-server/tests/api/preaggregations_test.py +++ b/datajunction-server/tests/api/preaggregations_test.py @@ -2,11 +2,22 @@ from unittest.mock import MagicMock +import json +import os +import pathlib +import subprocess +import sys +from collections.abc import Generator + +from urllib.parse import urlparse + import pytest import pytest_asyncio from httpx import AsyncClient from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncSession +from psycopg import connect +from testcontainers.postgres import PostgresContainer from sqlalchemy.orm import joinedload from datajunction_server.database.node import Node, NodeRevision @@ -20,6 +31,13 @@ from datajunction_server.construction.build_v3.builder import build_measures_sql from datajunction_server.models.access import ResourceAction from tests.authz import VALIDATOR_AUTH_SERVICE, deny +from tests.conftest import ( + FuncPostgresContainer, + cleanup_database_for_module, + clone_database_from_template, + externally_managed_postgres, + require_shared_template, +) @pytest.fixture @@ -127,132 +145,161 @@ async def _plan_preagg( return response.json()["preaggs"][0] -@pytest_asyncio.fixture -async def client_with_preaggs( - client_with_build_v3: AsyncClient, -): - """ - Creates pre-aggregations for testing using BUILD_V3 examples. +PREAGGS_TEMPLATE_DB_NAME = "template_preaggs" + + +def _preagg_ids_from(postgres_container, dbname: str) -> list[int]: + """Read preagg ids out of an already-built template, in creation order.""" + url = urlparse(postgres_container.get_connection_url()) + with connect( + host=url.hostname, + port=url.port, + dbname=dbname, + user=url.username, + password=url.password, + autocommit=True, + ) as conn: + return [ + row[0] + for row in conn.execute( + "SELECT id FROM pre_aggregation ORDER BY id", + ).fetchall() + ] - Uses /preaggs/plan API to create preaggs, which is more realistic - and ensures consistency with the actual API behavior. - NOTE: Gets session from client's dependency override to ensure we use - the SAME session that the client uses, avoiding event loop binding issues - with pytest-xdist in Python 3.11. +@pytest.fixture(scope="session") +def preaggs_template_database( + postgres_container: PostgresContainer, + template_database: str, +) -> Generator[tuple[str, list[int]], None, None]: """ - client = client_with_build_v3 - - # Get session from the client's dependency override - this ensures we use - # the same session that the API handlers use, avoiding event loop issues - from datajunction_server.utils import get_session + A template database that already holds the ten planned pre-aggregations. - session = client.app.dependency_overrides[get_session]() - - # preagg1: Basic preagg with FULL strategy, single grain - # total_revenue + total_quantity by status - preagg1_data = await _plan_preagg( - client, - metrics=["v3.total_revenue", "v3.total_quantity"], - dimensions=["v3.order_details.status"], - strategy="full", - schedule="0 0 * * *", - ) + Planning them costs ten ``/preaggs/plan`` round trips (~1.2s). Doing that + once and letting every test clone the result (~90ms) keeps each test fully + isolated -- which the 67 mutating tests here need -- without re-planning. - # preagg2: Multi-grain preagg (status + category) - # total_revenue + avg_unit_price by status and category - preagg2_data = await _plan_preagg( - client, - metrics=["v3.total_revenue", "v3.avg_unit_price"], - dimensions=["v3.order_details.status", "v3.product.category"], - strategy="full", - schedule="0 * * * *", - ) + Returns the template name and the preagg ids, in preagg1..preagg10 order. + """ + externally_managed = externally_managed_postgres() + if externally_managed: + # Planned once outside pytest; read the ids back off the template. + require_shared_template( + postgres_container, + PREAGGS_TEMPLATE_DB_NAME, + f"psql -c 'CREATE DATABASE {PREAGGS_TEMPLATE_DB_NAME} " + f"TEMPLATE template_all_examples;' && python " + f"tests/helpers/populate_preaggs_template.py " + f"/{PREAGGS_TEMPLATE_DB_NAME}", + ) + yield ( + PREAGGS_TEMPLATE_DB_NAME, + _preagg_ids_from( + postgres_container, + PREAGGS_TEMPLATE_DB_NAME, + ), + ) + return - # preagg3: Same grain as preagg1 but different metrics (for grain group hash testing) - # max_unit_price by status - preagg3_data = await _plan_preagg( - client, - metrics=["v3.max_unit_price"], - dimensions=["v3.order_details.status"], - strategy="full", + clone_database_from_template( + postgres_container, + template_name=template_database, + target_name=PREAGGS_TEMPLATE_DB_NAME, ) - - # preagg4: No strategy set (for testing "requires strategy" validation) - # total_revenue by category - preagg4_data = await _plan_preagg( - client, - metrics=["v3.total_revenue"], - dimensions=["v3.product.category"], + url = ( + postgres_container.get_connection_url().rsplit("/", 1)[0] + + f"/{PREAGGS_TEMPLATE_DB_NAME}" ) - # preagg5-10: Additional preaggs for tests that modify state - # These use different dimension combinations to avoid grain_group_hash conflicts - preagg5_data = await _plan_preagg( - client, - metrics=["v3.order_count"], - dimensions=["v3.order_details.status"], - strategy="full", - schedule="0 0 * * *", - ) - preagg6_data = await _plan_preagg( - client, - metrics=["v3.min_unit_price"], - dimensions=["v3.order_details.status"], - strategy="full", - schedule="0 0 * * *", + script = ( + pathlib.Path(__file__).parent.parent + / "helpers" + / "populate_preaggs_template.py" ) - preagg7_data = await _plan_preagg( - client, - metrics=["v3.total_revenue"], - dimensions=["v3.customer.customer_id"], - ) - preagg8_data = await _plan_preagg( - client, - metrics=["v3.page_view_count"], - dimensions=["v3.product.category"], - strategy="full", - schedule="0 0 * * *", + project_root = pathlib.Path(__file__).parent.parent.parent + env = { + **os.environ, + "PYTHONPATH": f"{project_root}{os.pathsep}{os.environ.get('PYTHONPATH', '')}", + } + result = subprocess.run( + [sys.executable, str(script), url], + capture_output=True, + text=True, + cwd=str(project_root), + env=env, ) - preagg9_data = await _plan_preagg( - client, - metrics=["v3.session_count"], - dimensions=["v3.product.category"], - strategy="full", - schedule="0 0 * * *", + if result.returncode != 0: + raise RuntimeError( + f"Failed to populate preaggs template:\n{result.stdout}\n{result.stderr}", + ) + ids = next( + json.loads(line.split(" ", 1)[1]) + for line in result.stdout.splitlines() + if line.startswith("PREAGG_IDS ") ) - preagg10_data = await _plan_preagg( - client, - metrics=["v3.visitor_count"], - dimensions=["v3.product.category"], + yield PREAGGS_TEMPLATE_DB_NAME, ids + if not externally_managed: + # A shared template outlives this process; dropping it would pull the + # rug from under the other workers. + cleanup_database_for_module(postgres_container, PREAGGS_TEMPLATE_DB_NAME) + + +@pytest.fixture +def func__postgres_container( + request, + postgres_container: PostgresContainer, + template_database: str, + preaggs_template_database: tuple[str, list[int]], +): + """ + Clone from the preaggs template, but only for tests that ask for preaggs. + + Overrides the conftest fixture for this module. Tests here that drive + ``client_with_build_v3`` directly plan their own preaggs and assert on the + result, so they must start from the base template -- handing them the + pre-seeded one makes those assertions see rows they did not create. + """ + wants_preaggs = "client_with_preaggs" in request.fixturenames + template_name = preaggs_template_database[0] if wants_preaggs else template_database + dbname = f"test_preagg_{abs(hash(request.node.name)) % 10000000}_{id(request)}" + db_url = clone_database_from_template( + postgres_container, + template_name=template_name, + target_name=dbname, ) + yield FuncPostgresContainer(postgres_container, db_url, dbname) + cleanup_database_for_module(postgres_container, dbname) + + +@pytest_asyncio.fixture +async def client_with_preaggs( + client_with_build_v3: AsyncClient, + preaggs_template_database: tuple[str, list[int]], +): + """ + Client whose database already contains the ten pre-aggregations. + + The preaggs come from the template clone, so nothing is planned here -- we + only load the ORM objects the tests reference. Each test still gets its own + database, so the ones that mutate stay isolated. + """ + client = client_with_build_v3 - # Fetch actual PreAggregation objects from DB for tests that need them + from datajunction_server.utils import get_session + + session = client.app.dependency_overrides[get_session]() + + _, preagg_ids = preaggs_template_database _opts = [joinedload(PreAggregation.node_revision)] - preagg1 = await session.get(PreAggregation, preagg1_data["id"], options=_opts) - preagg2 = await session.get(PreAggregation, preagg2_data["id"], options=_opts) - preagg3 = await session.get(PreAggregation, preagg3_data["id"], options=_opts) - preagg4 = await session.get(PreAggregation, preagg4_data["id"], options=_opts) - preagg5 = await session.get(PreAggregation, preagg5_data["id"], options=_opts) - preagg6 = await session.get(PreAggregation, preagg6_data["id"], options=_opts) - preagg7 = await session.get(PreAggregation, preagg7_data["id"], options=_opts) - preagg8 = await session.get(PreAggregation, preagg8_data["id"], options=_opts) - preagg9 = await session.get(PreAggregation, preagg9_data["id"], options=_opts) - preagg10 = await session.get(PreAggregation, preagg10_data["id"], options=_opts) + preaggs = [ + await session.get(PreAggregation, preagg_id, options=_opts) + for preagg_id in preagg_ids + ] yield { "client": client, "session": session, - "preagg1": preagg1, - "preagg2": preagg2, - "preagg3": preagg3, - "preagg4": preagg4, - "preagg5": preagg5, - "preagg6": preagg6, - "preagg7": preagg7, - "preagg8": preagg8, - "preagg9": preagg9, - "preagg10": preagg10, + **{f"preagg{index}": preagg for index, preagg in enumerate(preaggs, start=1)}, } diff --git a/datajunction-server/tests/api/semantic_layer_test.py b/datajunction-server/tests/api/semantic_layer_test.py index aa3925330c..b0dba2401e 100644 --- a/datajunction-server/tests/api/semantic_layer_test.py +++ b/datajunction-server/tests/api/semantic_layer_test.py @@ -18,6 +18,7 @@ _arrow_type_name, _dimensions_payload, _filter_to_sql, + _generated_column_arrow_type_name, _metrics_payload, _quote_value, ) @@ -83,14 +84,17 @@ def test_metric_and_dimension_payloads_use_cube_column_types(self): SimpleNamespace( cube_element_name="sem.total_amount", type="decimal(18,2)", + display_name="Total amount", ), SimpleNamespace( cube_element_name="sem.region.region_id", type="bigint", + display_name="Region ID", ), SimpleNamespace( cube_element_name="sem.region.region_name[home]", type="varchar(255)", + display_name="Home region name", ), ], cube_node_metrics=["sem.total_amount"], @@ -103,11 +107,44 @@ def test_metric_and_dimension_payloads_use_cube_column_types(self): metrics = _metrics_payload(cube) # type: ignore[arg-type] dimensions = _dimensions_payload(cube) # type: ignore[arg-type] - assert metrics[0].type == "decimal" - assert {dimension.id: dimension.type for dimension in dimensions} == { - "sem.region.region_id": "int", - "sem.region.region_name[home]": "utf8", - } + assert [metric.model_dump() for metric in metrics] == [ + { + "id": "sem.total_amount", + "name": "total_amount", + "type": "decimal", + "definition": "sem.total_amount", + "description": None, + "aggregation": "OTHER", + "metadata": { + "display_name": "Total amount", + }, + }, + ] + + assert [dim.model_dump() for dim in dimensions] == [ + { + "id": "sem.region.region_id", + "name": "region_id", + "type": "int", + "definition": "sem.region.region_id", + "description": None, + "grain": None, + "metadata": { + "display_name": "Region ID", + }, + }, + { + "id": "sem.region.region_name[home]", + "name": "region_name[home]", + "type": "utf8", + "definition": "sem.region.region_name[home]", + "description": None, + "grain": None, + "metadata": { + "display_name": "Home region name", + }, + }, + ] def test_metric_and_dimension_payloads_fallback_when_type_is_unknown(self): cube = SimpleNamespace( @@ -115,10 +152,12 @@ def test_metric_and_dimension_payloads_fallback_when_type_is_unknown(self): SimpleNamespace( cube_element_name="sem.total_amount", type=None, + display_name="Total amount", ), SimpleNamespace( cube_element_name="sem.region.region_name", type="unknown_type", + display_name="Region name", ), ], cube_node_metrics=["sem.total_amount"], @@ -144,6 +183,21 @@ def test_metric_and_dimension_payloads_fallback_when_column_is_missing(self): assert metrics[0].type == "floating" assert dimensions[0].type == "utf8" + def test_generated_sql_column_types_use_standard_names(self): + column = SimpleNamespace(type="varchar(255)", semantic_type="dimension") + + assert _generated_column_arrow_type_name(column) == "utf8" + + def test_generated_sql_dimension_column_type_fallback(self): + column = SimpleNamespace(type="unknown_type", semantic_type="dimension") + + assert _generated_column_arrow_type_name(column) == "utf8" + + def test_generated_sql_metric_column_type_fallback(self): + column = SimpleNamespace(type=None, semantic_type="metric") + + assert _generated_column_arrow_type_name(column) == "floating" + # --------------------------------------------------------------------------- # DB-backed integration tests (require the testcontainers Postgres harness) @@ -334,12 +388,20 @@ async def test_semantic_endpoints_end_to_end(client: AsyncClient): ) assert resp.status_code == 400, resp.text - # Dimension-only queries are rejected at the boundary (400). - resp = await client.post( - f"/semantic/views/{view}/sql", - json={"query": {"metrics": [], "dimensions": ["sem.region.region_name"]}}, + # Dimension-only queries return the dimension's distinct values. + resp = await _expect( + await client.post( + f"/semantic/views/{view}/sql", + json={ + "query": { + "metrics": [], + "dimensions": ["sem.region.region_name"], + }, + }, + ), + 200, ) - assert resp.status_code == 400, resp.text + assert "DISTINCT" in resp.json()["sql"] # Unknown view -> 404. resp = await client.post( @@ -414,6 +476,16 @@ async def test_generate_sql_rejects_limit_over_max(client: AsyncClient): assert str(MAX_ROW_LIMIT) in resp.json()["detail"] +@pytest.mark.asyncio +async def test_generate_sql_rejects_empty_query(client: AsyncClient): + resp = await client.post( + "/semantic/views/any_view/sql", + json={"query": {}}, + ) + assert resp.status_code == 400, resp.text + assert "at least one metric or dimension" in resp.json()["detail"] + + # --------------------------------------------------------------------------- # get_view unknown-view 404 (the ``cube_node is None`` branch) # --------------------------------------------------------------------------- diff --git a/datajunction-server/tests/api/sql_test.py b/datajunction-server/tests/api/sql_test.py index 6bc3d721b7..8dea866be1 100644 --- a/datajunction-server/tests/api/sql_test.py +++ b/datajunction-server/tests/api/sql_test.py @@ -1583,8 +1583,7 @@ async def test_metric_with_node_level_and_nth_order_filters( ), default_repair_orders_fact AS ( SELECT repair_orders.repair_order_id, - repair_orders.hard_hat_id, - repair_orders.dispatcher_id + repair_orders.hard_hat_id FROM default.roads.repair_orders repair_orders JOIN default.roads.repair_order_details repair_order_details ON repair_orders.repair_order_id = repair_order_details.repair_order_id WHERE repair_orders.dispatcher_id = 1 OR repair_orders.dispatcher_id IS NOT NULL ), @@ -1676,8 +1675,7 @@ async def test_metric_with_nth_order_dimensions_filters( SELECT repair_orders.repair_order_id, repair_orders.municipality_id, repair_orders.hard_hat_id, - repair_orders.dispatcher_id, - repair_orders.order_date + repair_orders.dispatcher_id FROM default.roads.repair_orders repair_orders JOIN default.roads.repair_order_details repair_order_details ON repair_orders.repair_order_id = repair_order_details.repair_order_id WHERE repair_orders.dispatcher_id = 1 AND repair_orders.order_date >= '2020-01-01' ), diff --git a/datajunction-server/tests/api/tags_test.py b/datajunction-server/tests/api/tags_test.py index c99680e31b..51393627fa 100644 --- a/datajunction-server/tests/api/tags_test.py +++ b/datajunction-server/tests/api/tags_test.py @@ -186,6 +186,66 @@ async def test_update_tag(self, module__client: AsyncClient) -> None: (activity["activity_type"], activity["entity_type"]) for activity in history ] == [("update", "tag"), ("update", "tag"), ("create", "tag")] + @pytest.mark.asyncio + async def test_delete_tag(self, module__client: AsyncClient) -> None: + """ + Tests ``DELETE /tags/{name}/`` + """ + response = await self.create_tag(module__client) + assert response.status_code == 201 + + response = await module__client.delete("/tags/sales_report/") + assert response.status_code == 204 + + response = await module__client.get("/tags/sales_report/") + assert response.status_code == 404 + assert ( + response.json()["message"] + == "A tag with name `sales_report` does not exist." + ) + + # Check history + response = await module__client.get("/history/tag/sales_report/") + assert [ + (activity["activity_type"], activity["entity_type"]) + for activity in response.json() + ] == [("delete", "tag"), ("create", "tag")] + + @pytest.mark.asyncio + async def test_delete_nonexistent_tag(self, module__client: AsyncClient) -> None: + """ + Tests ``DELETE /tags/{name}/`` for a tag that doesn't exist + """ + response = await module__client.delete("/tags/does_not_exist/") + assert response.status_code == 404 + assert ( + response.json()["message"] + == "A tag with name `does_not_exist` does not exist." + ) + + @pytest.mark.asyncio + async def test_delete_tag_with_nodes(self, client_with_dbt: AsyncClient) -> None: + """ + Tests that ``DELETE /tags/{name}/`` refuses a tag that still has nodes + """ + await self.create_tag(client_with_dbt) + response = await client_with_dbt.post( + "/nodes/default.items_sold_count/tags/?tag_names=sales_report", + ) + assert response.status_code == 200 + + response = await client_with_dbt.delete("/tags/sales_report/") + assert response.status_code == 409 + assert response.json()["message"] == ( + "Cannot delete tag `sales_report` as it is still attached to 1 node(s). " + "Remove the tag from these nodes first." + ) + + # Deactivated nodes don't block the delete + await client_with_dbt.delete("/nodes/default.items_sold_count") + response = await client_with_dbt.delete("/tags/sales_report/") + assert response.status_code == 204 + @pytest.mark.asyncio async def test_list_tags(self, module__client: AsyncClient) -> None: """ diff --git a/datajunction-server/tests/api/test_custom_metadata_api.py b/datajunction-server/tests/api/test_custom_metadata_api.py index 679bc46d09..8791169a95 100644 --- a/datajunction-server/tests/api/test_custom_metadata_api.py +++ b/datajunction-server/tests/api/test_custom_metadata_api.py @@ -596,52 +596,6 @@ async def test_created_by_id_and_updated_by_id_populated( assert body["updated_by_id"] == admin_id -@pytest.mark.asyncio -async def test_owner_round_trips( - client: AsyncClient, - session: AsyncSession, -) -> None: - """owner field is persisted and returned in output.""" - from datajunction_server.database.user import User - from datajunction_server.models.user import OAuthProvider - from datajunction_server.internal.access.authentication.tokens import create_token - from datetime import timedelta - import httpx - from datajunction_server.api.main import app - - admin_user = User( - username="admin_owner", - email=None, - name=None, - oauth_provider=OAuthProvider.BASIC, - is_admin=True, - ) - session.add(admin_user) - await session.commit() - admin_token = create_token( - {"username": "admin_owner"}, - secret="a-fake-secretkey", - iss="http://localhost:8000/", - expires_delta=timedelta(hours=24), - ) - async with AsyncClient( - transport=httpx.ASGITransport(app=app), - base_url="http://test", - headers={"Authorization": f"Bearer {admin_token}"}, - ) as admin_client: - resp = await admin_client.post( - "/metadata-schemas/", - json={ - "key": "owner_key", - "json_schema": {"type": "string"}, - "owner": "team-data-eng", - }, - ) - assert resp.status_code in (200, 201) - body = resp.json() - assert body["owner"] == "team-data-eng" - - # --------------------------------------------------------------------------- # Repo-managed namespaces — the API is not a second writer # --------------------------------------------------------------------------- diff --git a/datajunction-server/tests/conftest.py b/datajunction-server/tests/conftest.py index bd96ffe340..fd4cf0afee 100644 --- a/datajunction-server/tests/conftest.py +++ b/datajunction-server/tests/conftest.py @@ -38,6 +38,7 @@ from fastapi_cache.backends.inmemory import InMemoryBackend from httpx import AsyncClient from psycopg import connect +from psycopg.errors import UndefinedTable from pytest_mock import MockerFixture from sqlalchemy import event, text from sqlalchemy.dialects.postgresql import insert @@ -259,7 +260,9 @@ def func__postgres_container( """ # Create a unique database name for this test test_name = request.node.name - dbname = f"test_func_{abs(hash(test_name)) % 10000000}_{id(request)}" + dbname = ( + f"test_func_{worker_suffix()}_{abs(hash(test_name)) % 10000000}_{id(request)}" + ) # Clone from template db_url = clone_database_from_template( @@ -286,7 +289,9 @@ def func__clean_postgres_container( """ # Create a unique database name for this test test_name = request.node.name - dbname = f"test_clean_{abs(hash(test_name)) % 10000000}_{id(request)}" + dbname = ( + f"test_clean_{worker_suffix()}_{abs(hash(test_name)) % 10000000}_{id(request)}" + ) # Create a fresh empty database (no template) db_url = create_database_for_module(postgres_container, dbname) @@ -371,6 +376,135 @@ def duckdb_conn() -> duckdb.DuckDBPyConnection: yield conn +class ExternalPostgres: + """ + Stands in for ``PostgresContainer`` when tests run against a server this + process did not start, so nothing tears it down at the end of a session. + """ + + def __init__(self, url: str): + self._url = url + + def get_connection_url(self) -> str: + return self._url + + +def externally_managed_postgres() -> bool: + """True when the Postgres server (and its templates) outlive this process.""" + return bool(os.environ.get("DJ_TEST_POSTGRES_URL")) + + +def database_exists(postgres, dbname: str) -> bool: + """Whether ``dbname`` is already present on the server.""" + url = urlparse(postgres.get_connection_url()) + with connect( + host=url.hostname, + port=url.port, + dbname=url.path.lstrip("/"), + user=url.username, + password=url.password, + autocommit=True, + ) as conn: + row = conn.execute( + "SELECT 1 FROM pg_database WHERE datname = %s", + (dbname,), + ).fetchone() + return row is not None + + +def template_is_populated(postgres, dbname: str) -> bool: + """ + Whether `dbname` exists and finished building, per the marker row + `mark_template_populated` writes on the base database. + + Checked via the marker instead of connecting to `dbname` directly, + since `clone_database_from_template` calls `pg_terminate_backend` + against connections to the template before cloning it. + """ + if not database_exists(postgres, dbname): + return False + url = urlparse(postgres.get_connection_url()) + with connect( + host=url.hostname, + port=url.port, + dbname=url.path.lstrip("/"), + user=url.username, + password=url.password, + autocommit=True, + ) as conn: + try: + row = conn.execute( + "SELECT 1 FROM test_template_status WHERE template_name = %s", + (dbname,), + ).fetchone() + except UndefinedTable: + return False + return row is not None + + +def require_shared_template(postgres, dbname: str, how_to_build: str) -> None: + """ + Insist that a shared template is present and populated. + + On a server this process does not own, templates are built once before + pytest runs. Building them here instead would race between workers and, on + failure, leave a half-made database behind, so refuse loudly and say what + is missing. + """ + if template_is_populated(postgres, dbname): + return + state = "exists but is empty" if database_exists(postgres, dbname) else "is missing" + raise RuntimeError( + f"Shared template database `{dbname}` {state}.\n" + f"DJ_TEST_POSTGRES_URL is set, so templates must be built before pytest " + f"starts. Build it with:\n\n {how_to_build}\n\n" + f"If it exists but is empty, drop it first: " + f'psql -c "DROP DATABASE {dbname};"', + ) + + +def require_shared_readonly_role(postgres) -> None: + """ + Insist the ``readonly_user`` role exists on a shared server. + + For containers it starts itself the suite creates this role, but on a server + it does not own it cannot. The reader database URL depends on it, so a + missing role surfaces as an authentication error deep inside an unrelated + test rather than as a setup problem. + """ + url = urlparse(postgres.get_connection_url()) + with connect( + host=url.hostname, + port=url.port, + dbname=url.path.lstrip("/"), + user=url.username, + password=url.password, + autocommit=True, + ) as conn: + row = conn.execute( + "SELECT 1 FROM pg_roles WHERE rolname = 'readonly_user'", + ).fetchone() + if row is None: + raise RuntimeError( + "Shared Postgres is missing the `readonly_user` role, which the " + "reader database URL needs.\nDJ_TEST_POSTGRES_URL is set, so the " + "role has to be created before pytest starts:\n\n" + ' psql -c "CREATE ROLE readonly_user WITH LOGIN ' + "PASSWORD 'readonly'\"", + ) + + +def worker_suffix() -> str: + """ + Identifier unique to this xdist worker. + + Generated database names have to include it once workers share one server: + the old names leaned on ``id(request)``, which is only unique within a + process. + """ + return os.environ.get("PYTEST_XDIST_WORKER", "gw0") + + @pytest.fixture(scope="session") def postgres_container() -> PostgresContainer: """ @@ -380,7 +514,17 @@ def postgres_container() -> PostgresContainer: 1. The 'dj' database (default) 2. The template database with all examples pre-loaded 3. Per-module databases cloned from the template - """ + Under pytest-xdist "session" means *per worker process*, so without + ``DJ_TEST_POSTGRES_URL`` every worker starts its own container and builds its + own template -- N times the same ~50s of work, all at once at startup. Set + that variable to a running Postgres and the workers share it, cloning + templates somebody else already built. + """ + external_url = os.environ.get("DJ_TEST_POSTGRES_URL") + if external_url: + yield ExternalPostgres(external_url) # type: ignore[misc] + return + postgres = PostgresContainer( image="postgres:latest", username="dj", @@ -571,7 +715,9 @@ async def clean_client( # Create a unique database for this test test_name = request.node.name - dbname = f"test_clean_{abs(hash(test_name)) % 10000000}_{id(request)}" + dbname = ( + f"test_clean_{worker_suffix()}_{abs(hash(test_name)) % 10000000}_{id(request)}" + ) db_url = create_database_for_module(postgres_container, dbname) # Create settings for this clean database @@ -1744,7 +1890,7 @@ async def module__clean_client( """ # Create a unique database for this module module_name = request.module.__name__ - dbname = f"test_mod_clean_{abs(hash(module_name)) % 10000000}" + dbname = f"test_mod_clean_{worker_suffix()}_{abs(hash(module_name)) % 10000000}" db_url = create_database_for_module(postgres_container, dbname) # Create settings for this clean database @@ -1860,12 +2006,27 @@ async def wrapped_request(method, url, *args, **kwargs): cleanup_database_for_module(postgres_container, dbname) +@pytest.fixture +def isolated_client_template() -> str | None: + """ + Template database for ``isolated_client`` to clone, if any. + + Defaults to none, so ``isolated_client`` builds an empty database and the + caller loads what it needs. A module whose tests all want the same data can + override this with a template name: creating the schema and loading examples + costs seconds per test, cloning a template costs ~90ms, and each test still + gets its own database. + """ + return None + + @pytest_asyncio.fixture async def isolated_client( request, postgres_container: PostgresContainer, mocker: MockerFixture, background_tasks, + isolated_client_template: str | None, ) -> AsyncGenerator[AsyncClient, None]: """ Function-scoped client with a CLEAN database (no template, no pre-loaded examples). @@ -1880,8 +2041,16 @@ async def isolated_client( # Create a unique database for this test function test_name = request.node.name - dbname = f"test_isolated_{abs(hash(test_name)) % 10000000}_{id(request)}" - db_url = create_database_for_module(postgres_container, dbname) + dbname = f"test_isolated_{worker_suffix()}_{abs(hash(test_name)) % 10000000}_{id(request)}" + db_url = ( + clone_database_from_template( + postgres_container, + template_name=isolated_client_template, + target_name=dbname, + ) + if isolated_client_template + else create_database_for_module(postgres_container, dbname) + ) # Create settings for this clean database writer_db = DatabaseConfig(uri=db_url) @@ -1918,10 +2087,11 @@ async def isolated_client( poolclass=NullPool, # Avoids lock binding issues across event loops ) - # Create tables in the clean database - async with engine.begin() as conn: - await conn.execute(text("CREATE EXTENSION IF NOT EXISTS pg_trgm;")) - await conn.run_sync(Base.metadata.create_all) + # Create tables in the clean database. A clone already has them. + if not isolated_client_template: + async with engine.begin() as conn: + await conn.execute(text("CREATE EXTENSION IF NOT EXISTS pg_trgm;")) + await conn.run_sync(Base.metadata.create_all) async_session_factory = async_sessionmaker( bind=engine, @@ -1932,14 +2102,16 @@ async def isolated_client( async with async_session_factory() as session: session.remove = AsyncMock(return_value=None) - # Initialize the empty database with required seed data - from datajunction_server.api.attributes import default_attribute_types - from datajunction_server.internal.seed import seed_default_catalogs + # Initialize the empty database with required seed data. A clone of a + # template already carries it. + if not isolated_client_template: + from datajunction_server.api.attributes import default_attribute_types + from datajunction_server.internal.seed import seed_default_catalogs - await default_attribute_types(session) - await seed_default_catalogs(session) - await create_default_user(session) - await session.commit() + await default_attribute_types(session) + await seed_default_catalogs(session) + await create_default_user(session) + await session.commit() def get_session_override() -> AsyncSession: return session @@ -2181,6 +2353,16 @@ def template_database(postgres_container: PostgresContainer) -> str: Session-scoped fixture that creates a template database with ALL examples. This runs ONCE per test session and then each module clones from it. """ + if externally_managed_postgres(): + # Built once outside pytest; every worker just clones it. + require_shared_readonly_role(postgres_container) + require_shared_template( + postgres_container, + TEMPLATE_DB_NAME, + f"python tests/helpers/populate_template.py " + f"/{TEMPLATE_DB_NAME}", + ) + return TEMPLATE_DB_NAME template_url = create_database_for_module(postgres_container, TEMPLATE_DB_NAME) _populate_template_via_subprocess(template_url) return TEMPLATE_DB_NAME @@ -2197,7 +2379,7 @@ def module__postgres_container( Each module gets its own database cloned from the template with all examples. """ path = pathlib.Path(request.module.__file__).resolve() - dbname = f"test_mod_{abs(hash(path)) % 10000000}" + dbname = f"test_mod_{worker_suffix()}_{abs(hash(path)) % 10000000}" module_db_url = clone_database_from_template( postgres_container, diff --git a/datajunction-server/tests/construction/build_v3/cte_transitive_consumer_test.py b/datajunction-server/tests/construction/build_v3/cte_transitive_consumer_test.py index 70a0fdc9e5..a2aea3baeb 100644 --- a/datajunction-server/tests/construction/build_v3/cte_transitive_consumer_test.py +++ b/datajunction-server/tests/construction/build_v3/cte_transitive_consumer_test.py @@ -5,6 +5,9 @@ import pytest +from tests.construction.build_v3 import assert_sql_equal +from tests.construction.build_v3.projection_invariant import unprojected_references + class TestTransitiveConsumerProjection: """ @@ -136,9 +139,31 @@ async def test_transitive_consumer_column_survives_pruning( assert resp.status_code == 200, resp.text sql = resp.json()["grain_groups"][0]["sql"] - item_cte = sql.split(f"{ns}_bundles")[0] - assert "channel" in item_cte, sql - assert "bundle_id" in item_cte, ( - f"{ns}.item CTE pruned bundle_id, but {ns}.bundles " - f"selects it from that CTE:\n\n{sql}" + # ``bundles`` is itself pruned to what ``fact`` reads from it, which + # narrows its own demand on ``item`` — so ``item`` need not keep + # ``bundle_id``. It does keep ``channel``, which the outer select asks + # for through a separate join. + assert_sql_equal( + sql, + """ + WITH ctetrans_item AS ( + SELECT item_id, + CASE WHEN kind_code = 'AUTO' THEN 'auto' ELSE 'manual' END AS channel + FROM default.ctetrans.src_item + ), + ctetrans_bundles AS ( + SELECT i.item_id FROM ctetrans_item AS i + ), + ctetrans_fact AS ( + SELECT e.item_id, e.amount + FROM default.ctetrans.src_event AS e + LEFT JOIN ctetrans_bundles AS b ON e.item_id = b.item_id + ) + SELECT t2.channel, SUM(t1.amount) AS amount_sum_3ade80bc + FROM ctetrans_fact t1 + LEFT OUTER JOIN ctetrans_item t2 ON t1.item_id = t2.item_id + GROUP BY t2.channel + """, + normalize_aliases=True, ) + assert unprojected_references(sql) == set(), sql diff --git a/datajunction-server/tests/construction/build_v3/cte_unqualified_columns_test.py b/datajunction-server/tests/construction/build_v3/cte_unqualified_columns_test.py new file mode 100644 index 0000000000..1857930b8b --- /dev/null +++ b/datajunction-server/tests/construction/build_v3/cte_unqualified_columns_test.py @@ -0,0 +1,404 @@ +""" +Columns a CTE reads without a table qualifier still hold its source open. + +The projection pruner attributes a column to a node by its qualifier, so a bare +``code_a`` used to belong to nobody and the CTE producing it dropped it. These +tests state the invariant directly: whatever a CTE reads from another CTE, that +other CTE projects. +""" + +import pytest + +from tests.construction.build_v3 import assert_sql_equal +from tests.construction.build_v3.projection_invariant import unprojected_references + + +class TestUnqualifiedColumnsSurvivePruning: + """ + ``child_dim`` unions two parents and reads ``parent_dim``'s columns bare, so + the pruner has to keep them even though nothing says where they come from. + """ + + async def _setup(self, client, ns: str): + resp = await client.post(f"/namespaces/{ns}/") + assert resp.status_code in (200, 201), resp.text + + resp = await client.post( + "/nodes/source/", + json={ + "name": f"{ns}.src_item", + "catalog": "default", + "schema_": ns, + "table": "src_item", + "columns": [ + {"name": "item_id", "type": "int"}, + {"name": "code_a", "type": "string"}, + {"name": "code_b", "type": "string"}, + {"name": "code_c", "type": "string"}, + {"name": "region", "type": "string"}, + ], + }, + ) + assert resp.status_code in (200, 201), resp.text + + resp = await client.post( + "/nodes/source/", + json={ + "name": f"{ns}.src_partner_item", + "catalog": "default", + "schema_": ns, + "table": "src_partner_item", + "columns": [ + {"name": "external_item_id", "type": "int"}, + {"name": "is_special", "type": "bool"}, + ], + }, + ) + assert resp.status_code in (200, 201), resp.text + + resp = await client.post( + "/nodes/source/", + json={ + "name": f"{ns}.src_event", + "catalog": "default", + "schema_": ns, + "table": "src_event", + "columns": [ + {"name": "event_id", "type": "int"}, + {"name": "item_id", "type": "int"}, + {"name": "amount", "type": "double"}, + ], + }, + ) + assert resp.status_code in (200, 201), resp.text + + resp = await client.post( + "/nodes/dimension/", + json={ + "name": f"{ns}.parent_dim", + "mode": "published", + "primary_key": ["item_id"], + "query": ( + f"SELECT item_id, code_a, code_b, code_c, region FROM {ns}.src_item" + ), + }, + ) + assert resp.status_code in (200, 201), resp.text + + resp = await client.post( + "/nodes/dimension/", + json={ + "name": f"{ns}.partner_dim", + "mode": "published", + "primary_key": ["external_item_id"], + "query": ( + f"SELECT external_item_id, is_special FROM {ns}.src_partner_item" + ), + }, + ) + assert resp.status_code in (200, 201), resp.text + + # Every code_* here is bare: no alias, no node qualifier. + resp = await client.post( + "/nodes/dimension/", + json={ + "name": f"{ns}.child_dim", + "mode": "published", + "primary_key": ["item_id"], + "query": ( + f"SELECT item_id, " + f"CASE WHEN code_a = 'PROMO' THEN 'promo' " + f"WHEN code_b = 'DIRECT' AND code_c = 'OPEN' THEN 'direct' " + f"ELSE 'other' END AS channel " + f"FROM {ns}.parent_dim " + f"UNION ALL " + f"SELECT external_item_id AS item_id, " + f"CASE WHEN is_special THEN 'special' ELSE 'other' END AS channel " + f"FROM {ns}.partner_dim" + ), + }, + ) + assert resp.status_code in (200, 201), resp.text + + resp = await client.post( + "/nodes/transform/", + json={ + "name": f"{ns}.fact", + "mode": "published", + "query": f"SELECT event_id, item_id, amount FROM {ns}.src_event", + }, + ) + assert resp.status_code in (200, 201), resp.text + + for dimension in ("parent_dim", "child_dim"): + resp = await client.post( + f"/nodes/{ns}.fact/link", + json={ + "dimension_node": f"{ns}.{dimension}", + "join_type": "left", + "join_on": f"{ns}.fact.item_id = {ns}.{dimension}.item_id", + }, + ) + assert resp.status_code in (200, 201), resp.text + + resp = await client.post( + "/nodes/metric/", + json={ + "name": f"{ns}.total_amount", + "mode": "published", + "query": f"SELECT SUM(amount) FROM {ns}.fact", + }, + ) + assert resp.status_code in (200, 201), resp.text + + @pytest.mark.asyncio + async def test_every_cte_projects_what_its_readers_ask_for( + self, + client_with_service_setup, + ): + client = client_with_service_setup + ns = "cteunqual" + await self._setup(client, ns) + + resp = await client.get( + "/sql/measures/v3/", + params={ + "metrics": [f"{ns}.total_amount"], + "dimensions": [f"{ns}.parent_dim.region", f"{ns}.child_dim.channel"], + }, + ) + assert resp.status_code == 200, resp.text + sql = resp.json()["grain_groups"][0]["sql"] + + # ``parent_dim`` keeps code_a, code_b and code_c: ``child_dim`` names + # them bare, and the outer select never asks for them. + assert_sql_equal( + sql, + """ + WITH cteunqual_fact AS ( + SELECT item_id, amount FROM default.cteunqual.src_event + ), + cteunqual_parent_dim AS ( + SELECT item_id, code_a, code_b, code_c, region + FROM default.cteunqual.src_item + ), + cteunqual_partner_dim AS ( + SELECT external_item_id, is_special + FROM default.cteunqual.src_partner_item + ), + cteunqual_child_dim AS ( + SELECT item_id, + CASE + WHEN code_a = 'PROMO' THEN 'promo' + WHEN code_b = 'DIRECT' AND code_c = 'OPEN' THEN 'direct' + ELSE 'other' + END AS channel + FROM cteunqual_parent_dim + UNION ALL + SELECT external_item_id AS item_id, + CASE WHEN is_special THEN 'special' ELSE 'other' END AS channel + FROM cteunqual_partner_dim + ) + SELECT t2.region, t3.channel, SUM(t1.amount) AS amount_sum_f9cdc501 + FROM cteunqual_fact t1 + LEFT OUTER JOIN cteunqual_parent_dim t2 ON t1.item_id = t2.item_id + LEFT OUTER JOIN cteunqual_child_dim t3 ON t1.item_id = t3.item_id + GROUP BY t2.region, t3.channel + """, + normalize_aliases=True, + ) + assert unprojected_references(sql) == set(), sql + + +class TestPassThroughColumnSurvivesPruning: + """ + ``parent_dim`` derives ``channel`` itself and ``child_dim`` passes it + through, naming it bare. The reference carries no qualifier, so pruning the + parent for an unrelated dimension used to drop the column out from under it. + """ + + async def _setup(self, client, ns: str): + resp = await client.post(f"/namespaces/{ns}/") + assert resp.status_code in (200, 201), resp.text + + resp = await client.post( + "/nodes/source/", + json={ + "name": f"{ns}.src_item", + "catalog": "default", + "schema_": ns, + "table": "src_item", + "columns": [ + {"name": "item_id", "type": "int"}, + {"name": "code_a", "type": "string"}, + {"name": "region", "type": "string"}, + ], + }, + ) + assert resp.status_code in (200, 201), resp.text + + resp = await client.post( + "/nodes/source/", + json={ + "name": f"{ns}.src_partner_item", + "catalog": "default", + "schema_": ns, + "table": "src_partner_item", + "columns": [ + {"name": "external_item_id", "type": "int"}, + {"name": "is_special", "type": "bool"}, + ], + }, + ) + assert resp.status_code in (200, 201), resp.text + + resp = await client.post( + "/nodes/source/", + json={ + "name": f"{ns}.src_event", + "catalog": "default", + "schema_": ns, + "table": "src_event", + "columns": [ + {"name": "event_id", "type": "int"}, + {"name": "item_id", "type": "int"}, + {"name": "amount", "type": "double"}, + ], + }, + ) + assert resp.status_code in (200, 201), resp.text + + # The parent derives `channel`, so the child has a column to pass through. + resp = await client.post( + "/nodes/dimension/", + json={ + "name": f"{ns}.parent_dim", + "mode": "published", + "primary_key": ["item_id"], + "query": ( + f"SELECT item_id, " + f"CASE WHEN code_a = 'PROMO' THEN 'promo' ELSE 'other' END AS channel, " + f"region FROM {ns}.src_item" + ), + }, + ) + assert resp.status_code in (200, 201), resp.text + + resp = await client.post( + "/nodes/dimension/", + json={ + "name": f"{ns}.partner_dim", + "mode": "published", + "primary_key": ["external_item_id"], + "query": ( + f"SELECT external_item_id, is_special FROM {ns}.src_partner_item" + ), + }, + ) + assert resp.status_code in (200, 201), resp.text + + # `channel` is named bare: no alias, no node qualifier. + resp = await client.post( + "/nodes/dimension/", + json={ + "name": f"{ns}.child_dim", + "mode": "published", + "primary_key": ["item_id"], + "query": ( + f"SELECT item_id, channel FROM {ns}.parent_dim " + f"UNION ALL " + f"SELECT external_item_id AS item_id, " + f"CASE WHEN is_special THEN 'special' ELSE 'other' END AS channel " + f"FROM {ns}.partner_dim" + ), + }, + ) + assert resp.status_code in (200, 201), resp.text + + resp = await client.post( + "/nodes/transform/", + json={ + "name": f"{ns}.fact", + "mode": "published", + "query": f"SELECT event_id, item_id, amount FROM {ns}.src_event", + }, + ) + assert resp.status_code in (200, 201), resp.text + + for dimension in ("parent_dim", "child_dim"): + resp = await client.post( + f"/nodes/{ns}.fact/link", + json={ + "dimension_node": f"{ns}.{dimension}", + "join_type": "left", + "join_on": f"{ns}.fact.item_id = {ns}.{dimension}.item_id", + }, + ) + assert resp.status_code in (200, 201), resp.text + + resp = await client.post( + "/nodes/metric/", + json={ + "name": f"{ns}.total_amount", + "mode": "published", + "query": f"SELECT SUM(amount) FROM {ns}.fact", + }, + ) + assert resp.status_code in (200, 201), resp.text + + @pytest.mark.asyncio + async def test_parent_keeps_the_column_its_child_passes_through( + self, + client_with_service_setup, + ): + client = client_with_service_setup + ns = "ctepassthru" + await self._setup(client, ns) + + # Asking for a dimension from the parent as well is what turns pruning + # on for it; `child_dim.channel` alone leaves the parent unpruned. + resp = await client.get( + "/sql/measures/v3/", + params={ + "metrics": [f"{ns}.total_amount"], + "dimensions": [f"{ns}.parent_dim.region", f"{ns}.child_dim.channel"], + }, + ) + assert resp.status_code == 200, resp.text + sql = resp.json()["grain_groups"][0]["sql"] + + # ``parent_dim`` derives ``channel`` and ``child_dim`` passes it + # through by bare name, so pruning the parent for ``region`` alone + # must not take ``channel`` with it. + assert_sql_equal( + sql, + """ + WITH ctepassthru_fact AS ( + SELECT item_id, amount FROM default.ctepassthru.src_event + ), + ctepassthru_parent_dim AS ( + SELECT item_id, + CASE WHEN code_a = 'PROMO' THEN 'promo' ELSE 'other' END AS channel, + region + FROM default.ctepassthru.src_item + ), + ctepassthru_partner_dim AS ( + SELECT external_item_id, is_special + FROM default.ctepassthru.src_partner_item + ), + ctepassthru_child_dim AS ( + SELECT item_id, channel FROM ctepassthru_parent_dim + UNION ALL + SELECT external_item_id AS item_id, + CASE WHEN is_special THEN 'special' ELSE 'other' END AS channel + FROM ctepassthru_partner_dim + ) + SELECT t2.region, t3.channel, SUM(t1.amount) AS amount_sum_d1ea4ab8 + FROM ctepassthru_fact t1 + LEFT OUTER JOIN ctepassthru_parent_dim t2 ON t1.item_id = t2.item_id + LEFT OUTER JOIN ctepassthru_child_dim t3 ON t1.item_id = t3.item_id + GROUP BY t2.region, t3.channel + """, + normalize_aliases=True, + ) + assert unprojected_references(sql) == set(), sql diff --git a/datajunction-server/tests/construction/build_v3/cube_matcher_test.py b/datajunction-server/tests/construction/build_v3/cube_matcher_test.py index 89a3cf6301..2a7b5e5a55 100644 --- a/datajunction-server/tests/construction/build_v3/cube_matcher_test.py +++ b/datajunction-server/tests/construction/build_v3/cube_matcher_test.py @@ -19,6 +19,7 @@ build_sql_from_cube, build_synthetic_grain_group, find_matching_cube, + resolve_dialect_and_engine_for_metrics, validate_pinned_cube_covers_filters, ) from datajunction_server.construction.build_v3.decomposition import ( @@ -79,6 +80,17 @@ def test_materialized_dimension_lookup_preserves_roles(): ) +@pytest.mark.asyncio +async def test_resolve_dialect_requires_metric_or_dimension(): + with pytest.raises(DJInvalidInputException, match="metric or dimension"): + await resolve_dialect_and_engine_for_metrics( + session=None, # type: ignore[arg-type] + metrics=[], + dimensions=[], + use_materialized=False, + ) + + class TestExtractFilterDimensionRefs: """Unit tests for the shared filter-dimension extraction helper. @@ -1211,6 +1223,18 @@ async def test_pinned_cube_non_druid_dialect_skips_validation( class TestBuildSqlFromCube: """Tests for build_sql_from_cube function.""" + @pytest.mark.asyncio + async def test_requires_metric(self): + with pytest.raises(DJInvalidInputException, match="At least one metric"): + await build_sql_from_cube( + session=None, # type: ignore[arg-type] + cube=None, # type: ignore[arg-type] + metrics=[], + dimensions=[], + filters=None, + dialect=Dialect.TRINO, + ) + @pytest.mark.asyncio async def test_builds_sql_from_cube_single_metric( self, diff --git a/datajunction-server/tests/construction/build_v3/djsql_test.py b/datajunction-server/tests/construction/build_v3/djsql_test.py index 2e42770568..ea03a4d386 100644 --- a/datajunction-server/tests/construction/build_v3/djsql_test.py +++ b/datajunction-server/tests/construction/build_v3/djsql_test.py @@ -7,6 +7,8 @@ import pytest +from datajunction_server.api.djsql import _build_djsql_query + from . import assert_sql_equal @@ -296,9 +298,7 @@ async def test_missing_from_metrics(self, client_with_build_v3): @pytest.mark.asyncio async def test_no_metrics_in_select(self, client_with_build_v3): - """ - Test that DJ SQL requires at least one metric. - """ + """Dimension-only queries use the dimensions pseudo-table.""" response = await client_with_build_v3.get( "/djsql/", params={ @@ -311,8 +311,84 @@ async def test_no_metrics_in_select(self, client_with_build_v3): }, ) - assert response.status_code == 422 # Validation error - no metrics - assert "metric" in response.json()["message"].lower() + assert response.status_code == 422 + assert "require at least one metric" in response.json()["message"] + + @pytest.mark.asyncio + @pytest.mark.parametrize("include_group_by", [False, True]) + async def test_dimensions_pseudo_table( + self, + client_with_build_v3, + include_group_by, + ): + group_by = "GROUP BY v3.customer.name" if include_group_by else "" + response = await client_with_build_v3.get( + "/djsql/", + params={ + "query": f""" + SELECT v3.customer.name + FROM dimensions + WHERE v3.customer.name != 'Unknown' + {group_by} + ORDER BY v3.customer.name DESC + LIMIT 5 + """, + "dialect": "spark", + }, + ) + + assert response.status_code == 200, response.json() + assert_sql_equal( + response.json()["sql"], + """ + WITH v3_customer AS ( + SELECT customer_id, name, email, registration_date, location_id + FROM default.v3.customers + ) + SELECT DISTINCT name + FROM v3_customer + WHERE name != 'Unknown' + ORDER BY name DESC + LIMIT 5 + """, + ) + + @pytest.mark.asyncio + async def test_dimensions_pseudo_table_rejects_mismatched_group_by( + self, + client_with_build_v3, + ): + response = await client_with_build_v3.get( + "/djsql/", + params={ + "query": """ + SELECT v3.customer.name + FROM dimensions + GROUP BY v3.customer.email + """, + }, + ) + + assert response.status_code == 422 + assert "must match" in response.json()["message"] + + @pytest.mark.asyncio + async def test_dimensions_pseudo_table_shared_data_path( + self, + client_with_build_v3, + session, + ): + generated, execution = await _build_djsql_query( + session=session, + query="SELECT v3.customer.name FROM dimensions", + use_materialized=True, + engine_name=None, + engine_version=None, + ) + + assert "DISTINCT" in generated.sql + assert generated.columns[0].semantic_name == "v3.customer.name" + assert execution.catalog_name == "default" @pytest.mark.asyncio async def test_column_not_in_group_by(self, client_with_build_v3): diff --git a/datajunction-server/tests/construction/build_v3/fanout_guard_test.py b/datajunction-server/tests/construction/build_v3/fanout_guard_test.py index bbad9c251d..97d8335e5c 100644 --- a/datajunction-server/tests/construction/build_v3/fanout_guard_test.py +++ b/datajunction-server/tests/construction/build_v3/fanout_guard_test.py @@ -14,6 +14,7 @@ """ import pytest +import pytest_asyncio def fanout_warnings(payload: dict) -> list[dict]: @@ -25,8 +26,8 @@ def fanout_warnings(payload: dict) -> list[dict]: ] -@pytest.fixture -async def setup_fanout_links(client_with_build_v3): +@pytest_asyncio.fixture(scope="module") +async def setup_fanout_links(module__client_with_build_v3): """ Add a dimension that fans out from ``v3.order_details``. @@ -38,7 +39,7 @@ async def setup_fanout_links(client_with_build_v3): Defined locally so only the fan-out tests pay the setup cost; the global BUILD_V3 fixture is unaffected. """ - response = await client_with_build_v3.post( + response = await module__client_with_build_v3.post( "/nodes/source/", json={ "name": "v3.src_order_promotions", @@ -57,7 +58,7 @@ async def setup_fanout_links(client_with_build_v3): ) assert response.status_code in (200, 201, 409), response.json() - response = await client_with_build_v3.post( + response = await module__client_with_build_v3.post( "/nodes/dimension/", json={ "name": "v3.order_promotion", @@ -73,7 +74,7 @@ async def setup_fanout_links(client_with_build_v3): assert response.status_code in (200, 201, 409), response.json() # order_details -> order_promotion: one order line maps to many promotions. - response = await client_with_build_v3.post( + response = await module__client_with_build_v3.post( "/nodes/v3.order_details/link", json={ "dimension_node": "v3.order_promotion", @@ -85,7 +86,7 @@ async def setup_fanout_links(client_with_build_v3): assert response.status_code in (200, 201, 409), response.json() # A second dimension linked many_to_many, to exercise that cardinality. - response = await client_with_build_v3.post( + response = await module__client_with_build_v3.post( "/nodes/source/", json={ "name": "v3.src_order_channels", @@ -103,7 +104,7 @@ async def setup_fanout_links(client_with_build_v3): ) assert response.status_code in (200, 201, 409), response.json() - response = await client_with_build_v3.post( + response = await module__client_with_build_v3.post( "/nodes/dimension/", json={ "name": "v3.order_channel", @@ -117,7 +118,7 @@ async def setup_fanout_links(client_with_build_v3): ) assert response.status_code in (200, 201, 409), response.json() - response = await client_with_build_v3.post( + response = await module__client_with_build_v3.post( "/nodes/v3.order_details/link", json={ "dimension_node": "v3.order_channel", @@ -130,7 +131,7 @@ async def setup_fanout_links(client_with_build_v3): # A MIN metric: its merge function is MIN (non-additive), so it's # duplication-immune and must NOT trip the guard even across a fan-out link. - response = await client_with_build_v3.post( + response = await module__client_with_build_v3.post( "/nodes/metric/", json={ "name": "v3.min_unit_price", @@ -144,7 +145,7 @@ async def setup_fanout_links(client_with_build_v3): # A COUNT_IF metric: merge=SUM (additive), so it inflates under fan-out exactly # like SUM/COUNT. Its phase-1 aggregation name is "COUNT_IF" (not SUM/COUNT/AVG), # so keying on the merge function — not the aggregation name — is what catches it. - response = await client_with_build_v3.post( + response = await module__client_with_build_v3.post( "/nodes/metric/", json={ "name": "v3.completed_order_count", @@ -158,7 +159,7 @@ async def setup_fanout_links(client_with_build_v3): # A MEDIAN metric: non-decomposable (holistic), so it builds via raw # passthrough and is aggregated over the duplicated rows. It depends on the # multiset of rows, so a fan-out distorts it and the guard MUST fire. - response = await client_with_build_v3.post( + response = await module__client_with_build_v3.post( "/nodes/metric/", json={ "name": "v3.median_unit_price", @@ -171,7 +172,7 @@ async def setup_fanout_links(client_with_build_v3): # A MAX_BY metric: non-decomposable too, but argmax reads a single extreme # row, so duplication can't change it. It must NOT trip the guard. - response = await client_with_build_v3.post( + response = await module__client_with_build_v3.post( "/nodes/metric/", json={ "name": "v3.unit_price_at_max_order", @@ -184,7 +185,7 @@ async def setup_fanout_links(client_with_build_v3): # Non-decomposable (MAX_BY) but embeds COUNT(DISTINCT). Both terms are immune, so # the metric is too — the embedded-DISTINCT case that must NOT trip the guard. - response = await client_with_build_v3.post( + response = await module__client_with_build_v3.post( "/nodes/metric/", json={ "name": "v3.price_at_max_order_per_distinct_order", @@ -200,7 +201,7 @@ async def setup_fanout_links(client_with_build_v3): # An APPROX_COUNT_DISTINCT metric: decomposable, but its component merges via # hll_union_agg (not SUM), so duplication can't inflate it. Must NOT warn. - response = await client_with_build_v3.post( + response = await module__client_with_build_v3.post( "/nodes/metric/", json={ "name": "v3.approx_order_count", @@ -214,7 +215,7 @@ async def setup_fanout_links(client_with_build_v3): # A metric that SUMs a dimension attribute reachable only through the fan-out # link. The join is emitted even when no promotion dimension is requested, so the # guard must see it — this is the metric-expression-dimension fan-out case. - response = await client_with_build_v3.post( + response = await module__client_with_build_v3.post( "/nodes/metric/", json={ "name": "v3.total_promo_discount", @@ -232,11 +233,11 @@ class TestFanoutGuardWarns: @pytest.mark.asyncio async def test_sum_over_one_to_many_warns( self, - client_with_build_v3, + module__client_with_build_v3, setup_fanout_links, ): """SUM across a one_to_many link inflates → 200 with an actionable warning.""" - response = await client_with_build_v3.get( + response = await module__client_with_build_v3.get( "/sql/measures/v3/", params={ "metrics": ["v3.total_revenue"], @@ -268,11 +269,11 @@ async def test_sum_over_one_to_many_warns( @pytest.mark.asyncio async def test_sum_over_many_to_many_warns( self, - client_with_build_v3, + module__client_with_build_v3, setup_fanout_links, ): """SUM across a many_to_many link inflates → 200 with a warning.""" - response = await client_with_build_v3.get( + response = await module__client_with_build_v3.get( "/sql/measures/v3/", params={ "metrics": ["v3.total_revenue"], @@ -287,11 +288,11 @@ async def test_sum_over_many_to_many_warns( @pytest.mark.asyncio async def test_avg_over_one_to_many_warns( self, - client_with_build_v3, + module__client_with_build_v3, setup_fanout_links, ): """AVG decomposes into SUM/COUNT components, both inflate → 200 + warning.""" - response = await client_with_build_v3.get( + response = await module__client_with_build_v3.get( "/sql/measures/v3/", params={ "metrics": ["v3.avg_unit_price"], @@ -304,7 +305,7 @@ async def test_avg_over_one_to_many_warns( @pytest.mark.asyncio async def test_count_if_over_one_to_many_warns( self, - client_with_build_v3, + module__client_with_build_v3, setup_fanout_links, ): """COUNT_IF has merge=SUM (additive) → inflates → 200 + warning. @@ -312,7 +313,7 @@ async def test_count_if_over_one_to_many_warns( COUNT_IF's phase-1 aggregation name is not in {SUM,COUNT,AVG}, so a name-based guard would miss it; keying on the merge function is what catches it. """ - response = await client_with_build_v3.get( + response = await module__client_with_build_v3.get( "/sql/measures/v3/", params={ "metrics": ["v3.completed_order_count"], @@ -325,7 +326,7 @@ async def test_count_if_over_one_to_many_warns( @pytest.mark.asyncio async def test_median_over_one_to_many_warns( self, - client_with_build_v3, + module__client_with_build_v3, setup_fanout_links, ): """A non-decomposable holistic metric (MEDIAN) inflates → 200 + warning. @@ -333,7 +334,7 @@ async def test_median_over_one_to_many_warns( MEDIAN has no components and no SUM merge, so the merge check alone can't see it; it's caught via the grain group's non-decomposable metrics. """ - response = await client_with_build_v3.get( + response = await module__client_with_build_v3.get( "/sql/measures/v3/", params={ "metrics": ["v3.median_unit_price"], @@ -348,7 +349,7 @@ async def test_median_over_one_to_many_warns( @pytest.mark.asyncio async def test_metric_expression_dimension_fanout_warns( self, - client_with_build_v3, + module__client_with_build_v3, setup_fanout_links, ): """A dimension attribute SUM'd inside the metric expression still fans out. @@ -357,7 +358,7 @@ async def test_metric_expression_dimension_fanout_warns( v3.order_promotion.discount, so the build emits the one_to_many join anyway. The guard must scan the emitted join paths, not just the requested dimensions. """ - response = await client_with_build_v3.get( + response = await module__client_with_build_v3.get( "/sql/measures/v3/", params={ "metrics": ["v3.total_promo_discount"], @@ -372,11 +373,11 @@ async def test_metric_expression_dimension_fanout_warns( @pytest.mark.asyncio async def test_metrics_endpoint_also_warns( self, - client_with_build_v3, + module__client_with_build_v3, setup_fanout_links, ): """The guard fires on /sql/metrics/v3/ as well, not just measures.""" - response = await client_with_build_v3.get( + response = await module__client_with_build_v3.get( "/sql/metrics/v3/", params={ "metrics": ["v3.total_revenue"], @@ -389,14 +390,14 @@ async def test_metrics_endpoint_also_warns( @pytest.mark.asyncio async def test_node_sql_endpoint_also_warns( self, - client_with_build_v3, + module__client_with_build_v3, setup_fanout_links, ): """ /sql/{node_name}/ builds a metric node through v3 as well, so its TranslatedSQL response has to carry the warning too. """ - response = await client_with_build_v3.get( + response = await module__client_with_build_v3.get( "/sql/v3.total_revenue/", params={"dimensions": ["v3.order_promotion.campaign"]}, ) @@ -406,7 +407,7 @@ async def test_node_sql_endpoint_also_warns( @pytest.mark.asyncio async def test_derived_metric_warning_names_the_requested_metric( self, - client_with_build_v3, + module__client_with_build_v3, setup_fanout_links, ): """ @@ -418,7 +419,7 @@ async def test_derived_metric_warning_names_the_requested_metric( v3.avg_order_value = v3.total_revenue / v3.order_count, and only the SUM in v3.total_revenue inflates -- v3.order_count is COUNT(DISTINCT ...). """ - response = await client_with_build_v3.get( + response = await module__client_with_build_v3.get( "/sql/measures/v3/", params={ "metrics": ["v3.avg_order_value"], @@ -437,7 +438,7 @@ async def test_derived_metric_warning_names_the_requested_metric( @pytest.mark.asyncio async def test_warning_names_only_the_inflated_metric( self, - client_with_build_v3, + module__client_with_build_v3, setup_fanout_links, ): """ @@ -447,7 +448,7 @@ async def test_warning_names_only_the_inflated_metric( so both land in the same grain group, but only the SUM inflates. Naming both would send the caller to fix a metric that is already correct. """ - response = await client_with_build_v3.get( + response = await module__client_with_build_v3.get( "/sql/measures/v3/", params={ "metrics": ["v3.total_revenue", "v3.order_count"], @@ -464,7 +465,7 @@ async def test_warning_names_only_the_inflated_metric( @pytest.mark.asyncio async def test_window_metric_over_one_to_many_warns( self, - client_with_build_v3, + module__client_with_build_v3, setup_fanout_links, ): """ @@ -477,7 +478,7 @@ async def test_window_metric_over_one_to_many_warns( grain, so the guard must fire on that grain group too (not just the base metric path). """ - response = await client_with_build_v3.get( + response = await module__client_with_build_v3.get( "/sql/measures/v3/", params={ "metrics": ["v3.wow_revenue_change"], @@ -502,11 +503,11 @@ class TestFanoutGuardAllows: @pytest.mark.asyncio async def test_count_distinct_over_one_to_many_ok( self, - client_with_build_v3, + module__client_with_build_v3, setup_fanout_links, ): """COUNT(DISTINCT) has no merge function → fan-out-immune → no warning.""" - response = await client_with_build_v3.get( + response = await module__client_with_build_v3.get( "/sql/measures/v3/", params={ "metrics": ["v3.order_count"], @@ -519,11 +520,11 @@ async def test_count_distinct_over_one_to_many_ok( @pytest.mark.asyncio async def test_min_over_one_to_many_ok( self, - client_with_build_v3, + module__client_with_build_v3, setup_fanout_links, ): """MIN's merge function is MIN, not additive → duplication-immune → no warning.""" - response = await client_with_build_v3.get( + response = await module__client_with_build_v3.get( "/sql/measures/v3/", params={ "metrics": ["v3.min_unit_price"], @@ -536,11 +537,11 @@ async def test_min_over_one_to_many_ok( @pytest.mark.asyncio async def test_max_by_over_one_to_many_ok( self, - client_with_build_v3, + module__client_with_build_v3, setup_fanout_links, ): """MAX_BY is non-decomposable but argmax reads one extreme row → immune → no warning.""" - response = await client_with_build_v3.get( + response = await module__client_with_build_v3.get( "/sql/measures/v3/", params={ "metrics": ["v3.unit_price_at_max_order"], @@ -553,11 +554,11 @@ async def test_max_by_over_one_to_many_ok( @pytest.mark.asyncio async def test_approx_count_distinct_over_one_to_many_ok( self, - client_with_build_v3, + module__client_with_build_v3, setup_fanout_links, ): """APPROX_COUNT_DISTINCT merges via hll_union_agg (not SUM) → immune → no warning.""" - response = await client_with_build_v3.get( + response = await module__client_with_build_v3.get( "/sql/measures/v3/", params={ "metrics": ["v3.approx_order_count"], @@ -570,7 +571,7 @@ async def test_approx_count_distinct_over_one_to_many_ok( @pytest.mark.asyncio async def test_embedded_count_distinct_over_one_to_many_ok( self, - client_with_build_v3, + module__client_with_build_v3, setup_fanout_links, ): """ @@ -578,7 +579,7 @@ async def test_embedded_count_distinct_over_one_to_many_ok( duplication-immune. Keying invariance on the Count class alone would flag it; reading the DISTINCT off the call keeps it quiet. """ - response = await client_with_build_v3.get( + response = await module__client_with_build_v3.get( "/sql/measures/v3/", params={ "metrics": ["v3.price_at_max_order_per_distinct_order"], @@ -591,11 +592,11 @@ async def test_embedded_count_distinct_over_one_to_many_ok( @pytest.mark.asyncio async def test_sum_over_safe_many_to_one_ok( self, - client_with_build_v3, + module__client_with_build_v3, setup_fanout_links, ): """SUM across the default many_to_one customer link does not fan out → no warning.""" - response = await client_with_build_v3.get( + response = await module__client_with_build_v3.get( "/sql/measures/v3/", params={ "metrics": ["v3.total_revenue"], @@ -608,14 +609,14 @@ async def test_sum_over_safe_many_to_one_ok( @pytest.mark.asyncio async def test_sum_without_fanout_dimension_ok( self, - client_with_build_v3, + module__client_with_build_v3, setup_fanout_links, ): """ The same SUM metric is fine as long as the fan-out dimension is not requested — the unsafe link is never traversed. """ - response = await client_with_build_v3.get( + response = await module__client_with_build_v3.get( "/sql/measures/v3/", params={ "metrics": ["v3.total_revenue"], @@ -634,11 +635,11 @@ class TestFanoutGuardCombinedEndpoint: @pytest.mark.asyncio async def test_combined_source_path_warns( self, - client_with_build_v3, + module__client_with_build_v3, setup_fanout_links, ): """SUM across a one_to_many link, combined-from-source → 200 + warning.""" - response = await client_with_build_v3.get( + response = await module__client_with_build_v3.get( "/sql/measures/v3/combined", params={ "metrics": ["v3.total_revenue"], @@ -651,7 +652,7 @@ async def test_combined_source_path_warns( @pytest.mark.asyncio async def test_combined_preagg_path_warns( self, - client_with_build_v3, + module__client_with_build_v3, setup_fanout_links, ): """ @@ -661,7 +662,7 @@ async def test_combined_preagg_path_warns( computes measures from source internally to derive the grain groups, so the fan-out risk is known and must be surfaced. """ - response = await client_with_build_v3.get( + response = await module__client_with_build_v3.get( "/sql/measures/v3/combined", params={ "use_preagg_tables": "true", @@ -678,11 +679,11 @@ async def test_combined_preagg_path_warns( @pytest.mark.asyncio async def test_combined_preagg_path_without_fanout_ok( self, - client_with_build_v3, + module__client_with_build_v3, setup_fanout_links, ): """The pre-agg path does not warn when no fan-out dimension is requested.""" - response = await client_with_build_v3.get( + response = await module__client_with_build_v3.get( "/sql/measures/v3/combined", params={ "use_preagg_tables": "true", @@ -703,11 +704,11 @@ class TestFanoutGuardPreaggPlan: @pytest.mark.asyncio async def test_preaggs_plan_over_one_to_many_warns( self, - client_with_build_v3, + module__client_with_build_v3, setup_fanout_links, ): """Planning a pre-agg for SUM across a one_to_many link → 201 + warning.""" - response = await client_with_build_v3.post( + response = await module__client_with_build_v3.post( "/preaggs/plan", json={ "metrics": ["v3.total_revenue"], @@ -723,11 +724,11 @@ async def test_preaggs_plan_over_one_to_many_warns( @pytest.mark.asyncio async def test_preaggs_plan_without_fanout_ok( self, - client_with_build_v3, + module__client_with_build_v3, setup_fanout_links, ): """Planning a pre-agg without a fan-out dimension does not warn.""" - response = await client_with_build_v3.post( + response = await module__client_with_build_v3.post( "/preaggs/plan", json={ "metrics": ["v3.total_revenue"], diff --git a/datajunction-server/tests/construction/build_v3/filter_pushdown_test.py b/datajunction-server/tests/construction/build_v3/filter_pushdown_test.py index e1ae28286c..cb7d5f2932 100644 --- a/datajunction-server/tests/construction/build_v3/filter_pushdown_test.py +++ b/datajunction-server/tests/construction/build_v3/filter_pushdown_test.py @@ -36,8 +36,7 @@ async def test_date_filter_pushed_to_parent_cte( """ WITH v3_order_details AS ( - SELECT o.order_date, - oi.product_id, + SELECT oi.product_id, oi.quantity * oi.unit_price AS line_total FROM default.v3.orders o JOIN default.v3.order_items oi ON o.order_id = oi.order_id @@ -129,8 +128,7 @@ async def test_multiple_filters_pushed_to_different_ctes( """ WITH v3_order_details AS ( - SELECT o.order_date, - oi.product_id, + SELECT oi.product_id, oi.quantity * oi.unit_price AS line_total FROM default.v3.orders o JOIN default.v3.order_items oi ON o.order_id = oi.order_id @@ -176,9 +174,7 @@ async def test_two_filters_on_same_parent_cte_compose_as_and( """ WITH v3_order_details AS ( - SELECT o.order_date, - o.status, - oi.product_id, + SELECT oi.product_id, oi.quantity * oi.unit_price AS line_total FROM default.v3.orders o JOIN default.v3.order_items oi ON o.order_id = oi.order_id @@ -227,9 +223,7 @@ async def test_or_predicate_both_refs_in_same_cte_pushed_down( """ WITH v3_order_details AS ( - SELECT o.order_date, - o.status, - oi.product_id, + SELECT oi.product_id, oi.quantity * oi.unit_price AS line_total FROM default.v3.orders o JOIN default.v3.order_items oi ON o.order_id = oi.order_id @@ -389,8 +383,7 @@ async def test_in_list_filter_pushed_to_parent_cte( """ WITH v3_order_details AS ( - SELECT o.order_date, - oi.product_id, + SELECT oi.product_id, oi.quantity * oi.unit_price AS line_total FROM default.v3.orders o JOIN default.v3.order_items oi ON o.order_id = oi.order_id @@ -549,8 +542,7 @@ async def test_between_filter_pushed_to_parent_cte( """ WITH v3_order_details AS ( - SELECT o.order_date, - oi.product_id, + SELECT oi.product_id, oi.quantity * oi.unit_price AS line_total FROM default.v3.orders o JOIN default.v3.order_items oi ON o.order_id = oi.order_id @@ -754,9 +746,7 @@ async def test_two_roles_of_same_dim_each_filter_pushed_independently( """ WITH v3_order_details AS ( - SELECT o.order_date, - o.from_location_id, - oi.product_id, + SELECT oi.product_id, oi.quantity * oi.unit_price AS line_total FROM default.v3.orders o JOIN default.v3.order_items oi ON o.order_id = oi.order_id @@ -1091,8 +1081,7 @@ async def test_filter_pushed_into_inner_cross_join_when_column_not_in_output( WHERE window_id IN ('7day') ), v3_entity_window_config AS ( - SELECT e.entity_id, - MAX(e.base_value + w.window_size) AS max_bound + SELECT e.entity_id FROM default.v3.entity_facts AS e CROSS JOIN ( SELECT window_id, window_size @@ -1102,7 +1091,7 @@ async def test_filter_pushed_into_inner_cross_join_when_column_not_in_output( GROUP BY e.entity_id ), v3_entity_report AS ( - SELECT c.entity_id, w.window_id + SELECT c.entity_id FROM v3_entity_window_config AS c CROSS JOIN v3_time_window_dim AS w WHERE w.window_id IN ('7day') @@ -1659,7 +1648,7 @@ async def test_snapshot_filter_not_applied_to_transitive_fact( sql, """ WITH v3_xform_alloc AS ( - SELECT a.entity_id, SUM(a.value) AS total_value, a.snap_date + SELECT a.entity_id, a.snap_date FROM default.v3.snap_src AS a WHERE a.snap_date = 20240101 GROUP BY a.entity_id, a.snap_date @@ -1910,7 +1899,7 @@ async def test_dim_label_filter_not_pushed_into_upstream_cte_without_link( sql, """ WITH v3_ml_alloc AS ( - SELECT account_id, is_bot, amount + SELECT is_bot, amount FROM default.v3.ml_alloc_src ), v3_ml_is_bot_dim AS ( @@ -2084,7 +2073,7 @@ async def test_unqualified_shared_dim_filter_does_not_leak_into_role_link( FROM default.v3.shows ), v3_show_activity AS ( - SELECT show_id, region_date, view_secs + SELECT show_id, view_secs FROM default.v3.show_events WHERE region_date BETWEEN 20240101 AND 20240201 ) diff --git a/datajunction-server/tests/construction/build_v3/helpers_test.py b/datajunction-server/tests/construction/build_v3/helpers_test.py index bc9f86bd29..df4b0cb607 100644 --- a/datajunction-server/tests/construction/build_v3/helpers_test.py +++ b/datajunction-server/tests/construction/build_v3/helpers_test.py @@ -3,6 +3,8 @@ from typing import cast from unittest.mock import MagicMock +import duckdb + import pytest from datajunction_server.construction.build_v3.cte import ( @@ -10,6 +12,7 @@ flatten_inner_ctes, get_column_full_name, get_table_references_from_ast, + prune_cte_projections, rewrite_table_references, topological_sort_nodes, ) @@ -36,7 +39,7 @@ from datajunction_server.construction.build_v3.measures import ( _add_table_prefixes_to_filter, _resolve_dim_namespace_refs, - collect_cte_nodes_and_needed_columns, + collect_cte_nodes, ) from datajunction_server.construction.build_v3.types import ( BuildContext, @@ -65,6 +68,10 @@ from datajunction_server.naming import amenable_col_names from datajunction_server.sql.parsing import ast from datajunction_server.sql.parsing.backends.antlr4 import ast, parse +from tests.construction.build_v3 import assert_sql_equal +from tests.construction.build_v3.projection_invariant import ( + unprojected_references, +) class TestDimensionRefParsing: @@ -915,45 +922,31 @@ def test_unknown_column_defaults_to_main_alias(self): class TestResolveDimNamespaceRefs: """Tests for ``_resolve_dim_namespace_refs`` — rewrites dim-namespaced - column refs to use the dim's joined table alias and returns the columns - that must be preserved in each dim's CTE projection.""" + column refs to use the dim's joined table alias.""" @staticmethod def _expr(sql: str) -> ast.Expression: return cast(ast.Expression, parse(f"SELECT {sql}").select.projection[0]) - def test_rewrites_known_dim_and_collects_col(self): + def test_rewrites_known_dim(self): expr = self._expr("v3.customer.tier") - cols = _resolve_dim_namespace_refs( - [expr], - {"v3.customer": "t2"}, - ) + _resolve_dim_namespace_refs([expr], {"v3.customer": "t2"}) assert str(expr) == "t2.tier" - assert cols == {"v3.customer": {"tier"}} def test_skips_unknown_namespace(self): - """Namespaces that don't match a joined dim are left unchanged and - not recorded.""" + """Namespaces that don't match a joined dim are left unchanged.""" expr = self._expr("v3.fact.amount + v3.customer.threshold") - cols = _resolve_dim_namespace_refs( - [expr], - {"v3.customer": "t2"}, - ) + _resolve_dim_namespace_refs([expr], {"v3.customer": "t2"}) rendered = str(expr) assert "v3.fact.amount" in rendered assert "t2.threshold" in rendered - assert cols == {"v3.customer": {"threshold"}} def test_skips_bare_columns(self): """Bare columns have no namespace, so they're not touched here — the main-alias rewrite runs separately.""" expr = self._expr("SUM(amount)") - cols = _resolve_dim_namespace_refs( - [expr], - {"v3.customer": "t2"}, - ) + _resolve_dim_namespace_refs([expr], {"v3.customer": "t2"}) assert str(expr) == "SUM(amount)" - assert cols == {} def test_walks_nested_expression(self): """The walk reaches into function args / CASE branches.""" @@ -961,15 +954,11 @@ def test_walks_nested_expression(self): "SUM(CASE WHEN amount >= v3.customer.threshold " "THEN v3.customer.tier ELSE 'x' END)", ) - cols = _resolve_dim_namespace_refs( - [expr], - {"v3.customer": "t2"}, - ) + _resolve_dim_namespace_refs([expr], {"v3.customer": "t2"}) rendered = str(expr) assert "v3.customer." not in rendered assert "t2.threshold" in rendered assert "t2.tier" in rendered - assert cols == {"v3.customer": {"threshold", "tier"}} class TestBuildComponentExpression: @@ -1675,6 +1664,19 @@ def test_filter_none_column_name(self): projection = result.select.projection assert len(projection) == 2 + def test_distinct_keeps_every_column(self): + """ + The projection is the dedup key, so a direct caller gets it whole. + + ``prune_cte_projections`` settles this earlier for a CTE, so this is + the only guard a caller reaching straight for the helper has. + """ + sql = "SELECT DISTINCT col_a, col_b, col_c FROM t" + + result = filter_cte_projection(parse(sql), {"col_a"}) + + assert str(result) == str(parse(sql)) + class TestFilterCteProjectionPositionalGroupBy: """Tests for filter_cte_projection with positional GROUP BY handling.""" @@ -2348,174 +2350,86 @@ def test_parent_node_preserved_in_grain_group(self): assert result[0].parent_node.name == "v3.my_parent" -class TestCollectCteNodesAndNeededColumns: +class TestCollectCteNodes: """ - Tests for collect_cte_nodes_and_needed_columns. + Tests for ``collect_cte_nodes``, which decides which nodes get a CTE. - Focuses on the case where a dimension node's SQL references another - dimension node via a table alias, and columns used through that alias - must be included in the referenced node's CTE projection. + What each of those CTEs projects is settled later, on the assembled query, + by ``prune_cte_projections``. """ - def test_dim_referencing_dim_via_alias(self): + def test_parent_and_every_join_hop_get_ctes(self): """ - Columns used by one dimension's SQL from another dimension (via a table - alias) must appear in the referenced dimension's needed_columns set. - - Setup: - test.fact --(id)--> test.dim_a (requested: x) - test.fact --(id)--> test.dim_b (requested: p) - - test.dim_a's SQL: - SELECT id, x, b_alias.p, b_alias.q - FROM test.source_a - CROSS JOIN test.dim_b AS b_alias - - Expected needed columns: - test.dim_a: {x, id} -- x requested, id from join key - test.dim_b: {p, q, id} -- p requested, id from join key, - q pulled in because dim_a selects b_alias.q + The parent node and each dimension on a resolved join path need a CTE, + including a hop that is only an intermediate on the way to the target. """ - # --- nodes --- fact_node = MagicMock() fact_node.name = "test.fact" fact_node.type = NodeType.TRANSFORM - fact_node.current.query = "SELECT id, val FROM test.source" - fact_node.current.dimension_links = [] dim_a_node = MagicMock() dim_a_node.name = "test.dim_a" dim_a_node.type = NodeType.DIMENSION - dim_a_node.current.query = ( - "SELECT id, x, b_alias.p, b_alias.q " - "FROM test.source_a " - "CROSS JOIN test.dim_b AS b_alias" - ) dim_b_node = MagicMock() dim_b_node.name = "test.dim_b" dim_b_node.type = NodeType.DIMENSION - dim_b_node.current.query = "SELECT id, p, q, r FROM test.source_b" - - # --- dimension links --- - link_fact_to_dim_a = MagicMock() - link_fact_to_dim_a.dimension = dim_a_node - link_fact_to_dim_a.join_sql = "test.fact.id = test.dim_a.id" - link_fact_to_dim_a.node_revision.name = "test.fact" - link_fact_to_dim_b = MagicMock() - link_fact_to_dim_b.dimension = dim_b_node - link_fact_to_dim_b.join_sql = "test.fact.id = test.dim_b.id" - link_fact_to_dim_b.node_revision.name = "test.fact" + first_hop = MagicMock() + first_hop.dimension = dim_a_node + second_hop = MagicMock() + second_hop.dimension = dim_b_node - # --- context --- ctx = MagicMock() - ctx.temporal_partition_columns = {} ctx.nodes = {"test.dim_a": dim_a_node, "test.dim_b": dim_b_node} - ctx.get_parsed_query.side_effect = lambda node: parse(node.current.query) - # --- resolved dimensions --- resolved_dimensions = [ - ResolvedDimension( - original_ref="test.dim_a.x", - node_name="test.dim_a", - column_name="x", - role=None, - join_path=JoinPath( - links=[link_fact_to_dim_a], - target_dimension=dim_a_node, - ), - is_local=False, - ), ResolvedDimension( original_ref="test.dim_b.p", node_name="test.dim_b", column_name="p", role=None, join_path=JoinPath( - links=[link_fact_to_dim_b], + links=[first_hop, second_hop], target_dimension=dim_b_node, ), is_local=False, ), ] - nodes_for_ctes, needed_columns_by_node = collect_cte_nodes_and_needed_columns( + assert collect_cte_nodes( ctx=ctx, parent_node=fact_node, resolved_dimensions=resolved_dimensions, - grain_col_specs=[], - metric_expressions=[], - ) - - assert fact_node in nodes_for_ctes - assert dim_a_node in nodes_for_ctes - assert dim_b_node in nodes_for_ctes + ) == [fact_node, dim_a_node, dim_b_node] - assert needed_columns_by_node["test.dim_a"] == {"x", "id"} - # q must be present even though it was never explicitly requested — - # it's referenced in dim_a's SQL as b_alias.q - assert needed_columns_by_node["test.dim_b"] == {"p", "q", "id"} - - def test_parent_directly_references_dim_no_alias(self): - """ - Case 1: parent_node's SQL directly selects a column from a dimension - node without an alias (test.dim_a.extra_col). That column must appear - in needed_columns_by_node for that dimension even if it was not - explicitly requested as a dimension attribute. - """ + def test_local_dimension_adds_no_join_cte(self): + """A dimension on the fact itself has no join path to walk.""" fact_node = MagicMock() fact_node.name = "test.fact" fact_node.type = NodeType.TRANSFORM - # Parent selects test.dim_a.extra_col directly — nobody requests it - fact_node.current.query = ( - "SELECT id, val, test.dim_a.extra_col " - "FROM test.source " - "JOIN test.dim_a ON test.source.id = test.dim_a.id" - ) - fact_node.current.dimension_links = [] - - dim_a_node = MagicMock() - dim_a_node.name = "test.dim_a" - dim_a_node.type = NodeType.DIMENSION - dim_a_node.current.query = "SELECT id, x, extra_col FROM test.src_a" - - link_fact_to_dim_a = MagicMock() - link_fact_to_dim_a.dimension = dim_a_node - link_fact_to_dim_a.join_sql = "test.fact.id = test.dim_a.id" - link_fact_to_dim_a.node_revision.name = "test.fact" ctx = MagicMock() - ctx.temporal_partition_columns = {} - ctx.nodes = {"test.dim_a": dim_a_node} - ctx.get_parsed_query.side_effect = lambda node: parse(node.current.query) + ctx.nodes = {} resolved_dimensions = [ ResolvedDimension( - original_ref="test.dim_a.x", - node_name="test.dim_a", - column_name="x", + original_ref="test.fact.region", + node_name="test.fact", + column_name="region", role=None, - join_path=JoinPath( - links=[link_fact_to_dim_a], - target_dimension=dim_a_node, - ), - is_local=False, + join_path=None, + is_local=True, ), ] - _, needed_columns_by_node = collect_cte_nodes_and_needed_columns( + assert collect_cte_nodes( ctx=ctx, parent_node=fact_node, resolved_dimensions=resolved_dimensions, - grain_col_specs=[], - metric_expressions=[], - ) + ) == [fact_node] - # x requested, id from join key, extra_col from parent's SQL body - assert needed_columns_by_node["test.dim_a"] == {"x", "id", "extra_col"} - - def test_source_node_excluded_from_nodes_for_ctes(self): + def test_source_node_excluded(self): """ SOURCE-type nodes must not appear in nodes_for_ctes — they map to physical tables and don't need CTEs. @@ -2523,86 +2437,464 @@ def test_source_node_excluded_from_nodes_for_ctes(self): source_node = MagicMock() source_node.name = "test.src" source_node.type = NodeType.SOURCE - source_node.current.dimension_links = [] ctx = MagicMock() - ctx.temporal_partition_columns = {} ctx.nodes = {} - ctx.get_parsed_query.side_effect = lambda node: parse(node.current.query) - nodes_for_ctes, needed_columns_by_node = collect_cte_nodes_and_needed_columns( - ctx=ctx, - parent_node=source_node, - resolved_dimensions=[], - grain_col_specs=[], - metric_expressions=[], + assert ( + collect_cte_nodes( + ctx=ctx, + parent_node=source_node, + resolved_dimensions=[], + ) + == [] ) - assert source_node not in nodes_for_ctes - assert needed_columns_by_node == {} - def test_metric_expressions_contribute_to_parent_needed_cols(self): +class TestPruneCteProjections: + """ + Tests for ``prune_cte_projections`` on an already-assembled query. + """ + + @staticmethod + def _pruned(sql: str) -> str: + query = parse(sql) + prune_cte_projections(query) + return str(query) + + def test_qualified_reference_keeps_only_what_is_read(self): + pruned = self._pruned( + "WITH a AS (SELECT id, keep, drop_me FROM t) SELECT x.keep FROM a AS x", + ) + assert "keep" in pruned + assert "drop_me" not in pruned + + def test_cte_name_qualifier_resolves_like_an_alias(self): + pruned = self._pruned( + "WITH a AS (SELECT id, keep, drop_me FROM t) SELECT a.keep FROM a", + ) + assert "keep" in pruned + assert "drop_me" not in pruned + + def test_bare_reference_is_kept_in_every_cte_in_scope(self): """ - Columns referenced inside metric expressions must be included in - parent_node's needed columns so filter_cte_projection keeps them. + Nothing says which side supplies a bare column, so both keep it. """ - fact_node = MagicMock() - fact_node.name = "test.fact" - fact_node.type = NodeType.TRANSFORM - fact_node.current.query = "SELECT id, revenue, cost FROM test.src" - fact_node.current.dimension_links = [] - - ctx = MagicMock() - ctx.temporal_partition_columns = {} - ctx.nodes = {} - ctx.get_parsed_query.side_effect = lambda node: parse(node.current.query) + pruned = self._pruned( + "WITH a AS (SELECT id, shared, drop_me FROM t1), " + "b AS (SELECT id, shared, also_drop FROM t2) " + "SELECT shared FROM a JOIN b ON a.id = b.id", + ) + assert pruned.count("shared") == 3 + assert "drop_me" not in pruned + assert "also_drop" not in pruned - metric_expr = parse("SELECT revenue - cost").select.projection[0] + def test_struct_path_keeps_the_first_segment(self): + """ + For ``x.line_item.target_sets`` the producer must project ``line_item``, + the segment right after the qualifier — not the leaf. + """ + pruned = self._pruned( + "WITH a AS (SELECT line_item, drop_me FROM t) " + "SELECT x.line_item.target_sets FROM a AS x", + ) + assert "line_item" in pruned + assert "drop_me" not in pruned - _, needed_columns_by_node = collect_cte_nodes_and_needed_columns( - ctx=ctx, - parent_node=fact_node, - resolved_dimensions=[], - grain_col_specs=[], - metric_expressions=[("profit", metric_expr)], + def test_union_arms_are_pruned_together(self): + """ + A set-operation CTE keeps the same positions in every arm, so the arms + stay union-compatible. + """ + pruned = self._pruned( + "WITH a AS (" + "SELECT id, keep, drop_me FROM t1 " + "UNION ALL " + "SELECT id, keep, drop_me FROM t2" + ") SELECT x.keep FROM a AS x", + ) + assert pruned.count("keep") == 3 + assert "drop_me" not in pruned + + @pytest.mark.parametrize("kind", ["UNION", "INTERSECT", "EXCEPT"]) + def test_dedup_set_operation_arms_are_left_whole(self, kind: str): + """ + For every set op but UNION ALL the projected row is the dedup key, so + dropping a column changes which rows survive. + """ + sql = ( + f"WITH a AS (SELECT id, keep, drop_me FROM t1 {kind} " + f"SELECT id, keep, drop_me FROM t2) SELECT x.keep FROM a AS x" ) + assert self._pruned(sql) == str(parse(sql)) - assert "revenue" in needed_columns_by_node["test.fact"] - assert "cost" in needed_columns_by_node["test.fact"] + def test_distinct_projection_is_left_whole(self): + """ + The projection is the dedup key, so no column is free to go. + """ + sql = ( + "WITH a AS (SELECT DISTINCT id, keep, drop_me FROM t) " + "SELECT x.keep FROM a AS x" + ) + assert self._pruned(sql) == str(parse(sql)) - def test_local_dimension_contributes_to_parent_needed_cols(self): + def test_source_starred_under_distinct_is_left_whole(self): """ - Local dimensions (columns directly on the fact table, no join required) - must be added to parent_node's needed columns, not to a joined dim node. + Narrowing what the star exposes would narrow the dedup key with it. """ - fact_node = MagicMock() - fact_node.name = "test.fact" - fact_node.type = NodeType.TRANSFORM - fact_node.current.query = "SELECT id, val, region FROM test.src" - fact_node.current.dimension_links = [] + sql = ( + "WITH s AS (SELECT id, keep, other FROM t), " + "d AS (SELECT DISTINCT * FROM s) SELECT d.keep FROM d" + ) + assert self._pruned(sql) == str(parse(sql)) - ctx = MagicMock() - ctx.temporal_partition_columns = {} - ctx.nodes = {} - ctx.get_parsed_query.side_effect = lambda node: parse(node.current.query) + def test_projection_name_matches_case_insensitively(self): + """ + Every target dialect reads ``x.region`` off a ``Region`` projection. + """ + assert self._pruned( + "WITH a AS (SELECT Region, drop_me, keep FROM t) " + "SELECT x.region, x.keep FROM a AS x", + ) == str( + parse( + "WITH a AS (SELECT Region, keep FROM t) " + "SELECT x.region, x.keep FROM a AS x", + ), + ) - resolved_dimensions = [ - ResolvedDimension( - original_ref="test.fact.region", - node_name="test.fact", - column_name="region", - role=None, - join_path=None, - is_local=True, + def test_group_by_alias_matches_case_insensitively(self): + """ + A GROUP BY naming ``Total`` holds the ``total`` projection open. + """ + assert self._pruned( + "WITH a AS (SELECT keep, drop_me, SUM(n) AS total FROM t GROUP BY Total) " + "SELECT x.keep FROM a AS x", + ) == str( + parse( + "WITH a AS (SELECT keep, SUM(n) AS total FROM t GROUP BY Total) " + "SELECT x.keep FROM a AS x", ), + ) + + def test_star_passes_its_own_demand_through(self): + """ + A star re-exposes whatever its source has, so a CTE that stars a source + asks of it exactly what its own readers ask. + """ + pruned = self._pruned( + "WITH a AS (SELECT id, keep, drop_me FROM t), " + "b AS (SELECT * FROM a) " + "SELECT y.keep FROM b AS y", + ) + assert "keep" in pruned + assert "drop_me" not in pruned + + def test_star_with_unknown_demand_keeps_everything(self): + """ + The outer select has no readers to narrow it, so a star there leaves its + source whole. + """ + pruned = self._pruned( + "WITH a AS (SELECT id, keep, untouched FROM t) SELECT * FROM a", + ) + assert "untouched" in pruned + + def test_group_by_position_is_renumbered(self): + pruned = self._pruned( + "WITH a AS (SELECT drop_me, keep, COUNT(1) AS n FROM t GROUP BY 2) " + "SELECT x.keep, x.n FROM a AS x", + ) + # ``drop_me`` sits at position 1 and no GROUP BY entry points at it. + assert "drop_me" not in pruned + assert "GROUP BY 1" in pruned + + def test_query_without_ctes_is_untouched(self): + assert self._pruned("SELECT a FROM t") == str(parse("SELECT a FROM t")) + + @staticmethod + def _compact(sql: str) -> str: + """The pruned SQL with runs of whitespace collapsed to one space.""" + query = parse(sql) + prune_cte_projections(query) + return " ".join(str(query).split()) + + def test_a_shadowed_alias_does_not_prune_the_outer_source(self): + assert self._compact( + "WITH a AS (SELECT id, keep FROM t1), b AS (SELECT marker FROM t2) " + "SELECT x.keep, id FROM a AS x " + "WHERE EXISTS (SELECT 1 FROM b AS x WHERE x.marker = 1)", + ) == ( + "WITH a AS ( SELECT id, keep FROM t1 ), " + "b AS ( SELECT marker FROM t2 ) " + "SELECT x.keep, id FROM a AS x " + "WHERE EXISTS (SELECT 1 FROM b AS x WHERE x.marker = 1)" + ) + + def test_using_columns_are_kept_on_both_sides(self): + assert self._compact( + "WITH a AS (SELECT id, keep, drop_me FROM t1), " + "b AS (SELECT id, marker FROM t2) " + "SELECT a.keep FROM a JOIN b USING (id)", + ) == ( + "WITH a AS ( SELECT id, keep FROM t1 ), " + "b AS ( SELECT id FROM t2 ) " + "SELECT a.keep FROM a JOIN b USING (id)" + ) + + def test_a_shadowed_alias_resolves_per_select(self): + assert self._compact( + "WITH a AS (SELECT id, keep FROM t1), " + "b AS (SELECT keep, marker FROM t2) " + "SELECT x.keep, id FROM a AS x " + "WHERE EXISTS (SELECT 1 FROM b AS x WHERE x.marker = 1)", + ) == ( + "WITH a AS ( SELECT id, keep FROM t1 ), " + "b AS ( SELECT marker FROM t2 ) " + "SELECT x.keep, id FROM a AS x " + "WHERE EXISTS (SELECT 1 FROM b AS x WHERE x.marker = 1)" + ) + + def test_a_subquery_reads_an_outer_alias(self): + assert self._compact( + "WITH a AS (SELECT id, keep, drop_me FROM t1), " + "b AS (SELECT id, ref, marker FROM t2) " + "SELECT x.keep FROM a AS x " + "WHERE EXISTS (SELECT 1 FROM b WHERE b.ref = x.id)", + ) == ( + "WITH a AS ( SELECT id, keep FROM t1 ), " + "b AS ( SELECT ref FROM t2 ) " + "SELECT x.keep FROM a AS x " + "WHERE EXISTS (SELECT 1 FROM b WHERE b.ref = x.id)" + ) + + def test_a_natural_join_keeps_both_sides_whole(self): + assert self._compact( + "WITH a AS (SELECT id, keep FROM t1), " + "b AS (SELECT id, marker FROM t2) " + "SELECT a.keep FROM a NATURAL JOIN b", + ) == ( + "WITH a AS ( SELECT id, keep FROM t1 ), " + "b AS ( SELECT id, marker FROM t2 ) " + "SELECT a.keep FROM a NATURAL JOIN b" + ) + + +class TestProjectionInvariantOracle: + """ + Tests for the oracle the pruning tests are gated on. + + It is only worth trusting if it still fails on the shapes that motivated + this work, so those are pinned here directly. + """ + + def test_consistent_query_reports_nothing(self): + assert ( + unprojected_references( + "WITH a AS (SELECT id, keep FROM t) SELECT x.keep FROM a AS x", + ) + == set() + ) + + def test_reports_a_bare_reference_the_producer_dropped(self): + """The original bug: a CTE reads a column its source stopped projecting.""" + assert unprojected_references( + "WITH a AS (SELECT id FROM t), " + "b AS (SELECT CASE WHEN code_a = 'x' THEN 1 END AS flag FROM a) " + "SELECT y.flag FROM b AS y", + ) == {"a.code_a"} + + def test_reports_a_qualified_reference_the_producer_dropped(self): + assert unprojected_references( + "WITH a AS (SELECT id FROM t) SELECT x.gone FROM a AS x", + ) == {"a.gone"} + + def test_reports_per_union_arm(self): + """An arm is read on its own, so its own source is the one charged.""" + assert unprojected_references( + "WITH a AS (SELECT id FROM t1), b AS (SELECT id FROM t2), " + "c AS (SELECT id FROM a UNION ALL SELECT missing FROM b) " + "SELECT z.id FROM c AS z", + ) == {"b.missing"} + + def test_excuses_a_filter_on_the_arm_s_own_output_alias(self): + """ + A filter pushed into a CTE can land on a name the arm introduces with + ``AS``. The source never had a column by that name, so charging it there + would be a false alarm. + """ + assert ( + unprojected_references( + "WITH a AS (SELECT order_date FROM t), " + "b AS (SELECT order_date AS date_id FROM a WHERE date_id = 1) " + "SELECT y.date_id FROM b AS y", + ) + == set() + ) + + def test_still_charges_an_alias_name_used_in_the_projection(self): + """ + The excuse is only for filter clauses. A bare name in the projection is + read from the source, whatever else the arm happens to alias. + """ + assert unprojected_references( + "WITH a AS (SELECT id FROM t), " + "b AS (SELECT channel, id AS channel_id FROM a) " + "SELECT y.channel FROM b AS y", + ) == {"a.channel"} + + +class TestPruneCteProjectionsExecuted: + """ + Pruning faults that neither a parse check nor a string comparison notices. + + Cutting too deep here does not raise. The query still runs and gives back a + different number, or the same numbers in a different order, which is how it + reaches a Spark job before anyone looks. So these run the SQL both ways and + compare what comes back. + """ + + SETUP = ( + "CREATE TABLE t (id INTEGER, grp VARCHAR, keep INTEGER, drop_me INTEGER)", + "INSERT INTO t VALUES " + "(1,'a',10,100),(2,'a',10,101),(3,'b',20,200),(4,'b',20,200)", + ) + + def _execute(self, sql: str): + connection = duckdb.connect(":memory:") + try: + for statement in self.SETUP: + connection.execute(statement) + return connection.execute(sql).fetchall() + finally: + connection.close() + + def _prune(self, sql: str) -> tuple[str, ast.Query]: + query = parse(sql) + prune_cte_projections(query) + return str(query), query + + def _same_results(self, sql: str) -> str: + """Assert pruning did not change the answer; return the pruned SQL.""" + pruned, _ = self._prune(sql) + assert self._execute(pruned) == self._execute(sql), pruned + return pruned + + def test_distinct_producer_keeps_every_column(self): + """ + Under DISTINCT the projection is the dedup key. Narrowing it folds + together rows that were distinct, and the total silently drops. + """ + sql = ( + "WITH a AS (SELECT DISTINCT grp, keep, drop_me FROM t) " + "SELECT SUM(x.keep) AS s FROM a AS x" + ) + assert self._execute(sql) == [(40,)] + pruned, query = self._prune(sql) + assert self._execute(pruned) == [(40,)], pruned + assert [str(entry) for entry in query.ctes[0].select.projection] == [ + "grp", + "keep", + "drop_me", ] - _, needed_columns_by_node = collect_cte_nodes_and_needed_columns( - ctx=ctx, - parent_node=fact_node, - resolved_dimensions=resolved_dimensions, - grain_col_specs=[], - metric_expressions=[], + def test_having_on_the_arm_s_own_alias_survives(self): + """HAVING can name an output alias, so that alias has to stay.""" + sql = ( + "WITH a AS (" + "SELECT grp, SUM(keep) AS s FROM t GROUP BY grp HAVING s > 25" + ") SELECT x.grp FROM a AS x" ) + pruned = self._same_results(sql) + assert self._execute(pruned) == [("b",)] + assert "s" in pruned - assert "region" in needed_columns_by_node["test.fact"] + def test_order_by_on_the_arm_s_own_alias_survives(self): + """ + ORDER BY can name an output alias too. With LIMIT above it the ordering + decides which row survives, so losing the alias is not cosmetic. + """ + sql = ( + "WITH a AS (" + "SELECT id, id * 10 AS ranked FROM t ORDER BY ranked DESC LIMIT 1" + ") SELECT x.id FROM a AS x" + ) + pruned = self._same_results(sql) + assert self._execute(pruned) == [(4,)] + + def test_order_by_position_is_renumbered(self): + """ + A position left alone after the projection shrinks points at whatever + slid into that slot, or off the end of it. + """ + sql = ( + "WITH a AS (SELECT drop_me, id FROM t ORDER BY 2 DESC LIMIT 1) " + "SELECT x.id FROM a AS x" + ) + pruned, query = self._prune(sql) + assert self._execute(pruned) == self._execute(sql) == [(4,)], pruned + arm = query.ctes[0].select + assert [str(entry) for entry in arm.projection] == ["id"] + assert [str(item.expr) for item in arm.organization.order] == ["1"] + + def test_union_producer_keeps_every_column(self): + """ + UNION dedups on the whole row, so a column the reader never names is + still part of what makes two rows different. Pruning it merges them. + """ + sql = ( + "WITH a AS (" + "SELECT grp, keep, drop_me FROM t WHERE id <= 2 " + "UNION " + "SELECT grp, keep, drop_me FROM t WHERE id >= 3" + ") SELECT SUM(x.keep) AS s FROM a AS x" + ) + assert self._execute(sql) == [(40,)] + pruned, _ = self._prune(sql) + assert self._execute(pruned) == [(40,)], pruned + assert_sql_equal(pruned, sql) + + def test_except_producer_keeps_every_column(self): + """ + EXCEPT matches on the whole row too, so a narrowed projection removes + rows the full comparison would have kept. + """ + sql = ( + "WITH a AS (" + "SELECT grp, keep, drop_me FROM t " + "EXCEPT " + "SELECT grp, keep, drop_me FROM t WHERE id = 2" + ") SELECT SUM(x.keep) AS s FROM a AS x" + ) + assert self._execute(sql) == [(30,)] + pruned, _ = self._prune(sql) + assert self._execute(pruned) == [(30,)], pruned + assert_sql_equal(pruned, sql) + + def test_source_starred_under_distinct_keeps_every_column(self): + """ + The star is what the DISTINCT dedups on, so narrowing the source it + draws from narrows the dedup key just as surely. + """ + sql = ( + "WITH base AS (SELECT grp, keep, drop_me FROM t), " + "d AS (SELECT DISTINCT * FROM base) " + "SELECT SUM(x.keep) AS s FROM d AS x" + ) + assert self._execute(sql) == [(40,)] + pruned, _ = self._prune(sql) + assert self._execute(pruned) == [(40,)], pruned + assert_sql_equal(pruned, sql) + + def test_a_column_only_the_producer_s_where_reads_is_still_dropped(self): + """ + WHERE resolves against the FROM, not against the projection, so a column + only the filter uses is free to go. Pinned because protecting it would + undo the pruning this pass exists for. + """ + sql = ( + "WITH a AS (SELECT id, keep FROM t WHERE drop_me > 100) " + "SELECT SUM(x.keep) AS s FROM a AS x" + ) + pruned, query = self._prune(sql) + assert self._execute(pruned) == self._execute(sql) + assert [str(entry) for entry in query.ctes[0].select.projection] == ["keep"] diff --git a/datajunction-server/tests/construction/build_v3/measures_sql_test.py b/datajunction-server/tests/construction/build_v3/measures_sql_test.py index 5a9e61ec4f..005e4957c8 100644 --- a/datajunction-server/tests/construction/build_v3/measures_sql_test.py +++ b/datajunction-server/tests/construction/build_v3/measures_sql_test.py @@ -121,7 +121,7 @@ async def test_no_metrics_raises_error(self, client_with_build_v3): @pytest.mark.asyncio async def test_metrics_v3_no_metrics_raises_422(self, client_with_build_v3): - """GET /sql/metrics/v3/ with no metrics must return 422, not a 500 IndexError.""" + """GET /sql/metrics/v3/ requires at least one metric.""" response = await client_with_build_v3.get( "/sql/metrics/v3/", params={ @@ -3679,16 +3679,7 @@ async def test_no_upstream_pushdown_falls_back_to_parent_cte( WITH v3_order_details AS ( SELECT o.order_id, - oi.line_number, - o.customer_id, - o.order_date, - o.from_location_id, - o.to_location_id, - o.status, - oi.product_id, - oi.quantity, - oi.unit_price, - oi.quantity * oi.unit_price AS line_total + o.order_date FROM default.v3.orders o JOIN default.v3.order_items oi ON o.order_id = oi.order_id ), v3_orders_by_date AS ( @@ -6202,8 +6193,7 @@ async def test_local_filter_on_non_preserved_side_wraps_inside_parent_cte( WITH v3_dates_with_orders AS ( SELECT d.date_id, - o.order_id, - o.status + o.order_id FROM (SELECT * FROM default.v3.orders o WHERE o.status = 'completed') o RIGHT OUTER JOIN default.v3.dates d ON o.order_date = d.date_id ) @@ -6320,16 +6310,9 @@ async def test_filter_on_dim_linked_only_on_upstream( v3_order_details AS ( SELECT o.order_id, - oi.line_number, - o.customer_id, - o.order_date, - o.from_location_id, - o.to_location_id, - o.status, oi.product_id, oi.quantity, - oi.unit_price, - oi.quantity * oi.unit_price AS line_total + oi.unit_price FROM default.v3.orders o JOIN default.v3.order_items oi ON o.order_id = oi.order_id WHERE oi.product_id = 7 @@ -6337,7 +6320,6 @@ async def test_filter_on_dim_linked_only_on_upstream( v3_order_details_wrapper_filter_only AS ( SELECT order_id, - product_id, quantity * unit_price AS line_total FROM v3_order_details WHERE product_id = 7 @@ -6579,7 +6561,7 @@ async def test_filter_only_via_parent_column_annotation( sql, """ WITH v3_status_events AS ( - SELECT log_id, raw_status AS event_status + SELECT log_id FROM default.v3.status_log WHERE raw_status = 'OPEN' ) @@ -6858,17 +6840,11 @@ async def test_layer_2_filter_only_interacts_with_wrapper_cte_absorption( ), v3_order_details AS ( SELECT - o.order_id, - oi.line_number, o.customer_id, o.order_date, - o.from_location_id, - o.to_location_id, - o.status, oi.product_id, oi.quantity, - oi.unit_price, - oi.quantity * oi.unit_price AS line_total + oi.unit_price FROM default.v3.orders o JOIN default.v3.order_items oi ON o.order_id = oi.order_id WHERE oi.product_id = 7 @@ -6877,7 +6853,6 @@ async def test_layer_2_filter_only_interacts_with_wrapper_cte_absorption( SELECT customer_id, order_date, - product_id, quantity * unit_price AS line_total FROM v3_order_details WHERE product_id = 7 @@ -7956,7 +7931,7 @@ async def test_transform_link_target(self, client_with_build_v3): sql, """ WITH v3_events_with_date AS ( - SELECT account_id, event_type, audit_date + SELECT account_id, event_type FROM default.v3.audit_log_txlink_unlinked AS s WHERE audit_date >= 20260101 ), @@ -8055,7 +8030,7 @@ async def test_transform_link_plus_source_link_double_filter( sql, """ WITH v3_events_double_link AS ( - SELECT account_id, event_type, audit_date + SELECT account_id, event_type FROM default.v3.audit_log_txlinkdouble AS s WHERE audit_date >= 20260101 AND s.audit_date >= 20260101 @@ -9074,7 +9049,7 @@ async def test_multiple_filters_all_land_when_pushdown_partial( """ WITH v3_events_by_window_multi AS ( - SELECT e.account_id, e.event_date, e.value, w.window + SELECT e.account_id, e.value, w.window FROM default.v3.events_multi_filter AS e CROSS JOIN default.v3.obs_windows AS w WHERE w.window IN ('1-35') AND e.event_date >= 20260101 @@ -9156,11 +9131,11 @@ async def test_filter_lands_when_parent_cte_is_setop( """ WITH v3_events_union AS ( - SELECT account_id, event_date, value, 'a' AS branch + SELECT value, 'a' AS branch FROM default.v3.events_setop WHERE event_date >= 20260101 UNION ALL - SELECT account_id, event_date, value, 'b' AS branch + SELECT value, 'b' AS branch FROM default.v3.events_setop ) SELECT t1.branch, SUM(t1.value) value_sum_HASH @@ -9429,7 +9404,7 @@ async def test_filter_on_left_joined_side_inside_source_body_skips_pushdown( sql, """ WITH v3_events_with_side AS ( - SELECT e.account_id, s.measure_date + SELECT e.account_id FROM default.v3.events_left_join_primary AS e LEFT JOIN ( SELECT * @@ -9547,7 +9522,7 @@ async def test_filter_pushes_into_nested_subquery_self_reference( sql, """ WITH v3_events_with_nested_self AS ( - SELECT a.account_id, a.event_date + SELECT a.account_id FROM default.v3.events_nested_self_ref AS a LEFT JOIN ( SELECT rev.account_id, rev.lifecycle_id @@ -9746,7 +9721,7 @@ async def test_filter_pushes_into_dim_cte_via_local_link( WHERE column_b = 20260101 ), v3_fact_transform_dual AS ( - SELECT account_id, column_a, value + SELECT account_id, value FROM default.v3.fact_dual WHERE column_a = 20260101 ) @@ -10469,7 +10444,7 @@ async def test_alias_substitution_skips_subquery_alias_collision( WHERE window_label = 'demo' ), v3_alias_collide_xform AS ( - SELECT a.account_id, a.window_label + SELECT a.account_id FROM ( SELECT a.account_id, a.event_date, a.value, w.window_label FROM default.v3.events_alias_collide AS a diff --git a/datajunction-server/tests/construction/build_v3/metrics_sql_test.py b/datajunction-server/tests/construction/build_v3/metrics_sql_test.py index 1ba43384a7..1624493e01 100644 --- a/datajunction-server/tests/construction/build_v3/metrics_sql_test.py +++ b/datajunction-server/tests/construction/build_v3/metrics_sql_test.py @@ -4,6 +4,275 @@ class TestMetricsSQLBasic: + @pytest.mark.asyncio + async def test_dimension_only_query_requires_dimensions(self, client_with_build_v3): + response = await client_with_build_v3.get("/sql/dimensions/v3/") + + assert response.status_code == 422 + assert "at least one dimension" in response.json()["message"].lower() + + @pytest.mark.asyncio + async def test_dimension_only_query(self, client_with_build_v3): + """Attributes from one dimension produce their distinct combinations.""" + response = await client_with_build_v3.get( + "/sql/dimensions/v3/", + params={ + "dimensions": ["v3.customer.name", "v3.customer.email"], + "filters": ["v3.customer.name != 'Unknown'"], + "orderby": ["v3.customer.name"], + "limit": 10, + }, + ) + + assert response.status_code == 200, response.json() + result = response.json() + assert_sql_equal( + result["sql"], + """ + WITH v3_customer AS ( + SELECT customer_id, name, email, registration_date, location_id + FROM default.v3.customers + ) + SELECT DISTINCT name, email + FROM v3_customer + WHERE name != 'Unknown' + ORDER BY name + LIMIT 10 + """, + ) + assert result["columns"] == [ + { + "name": "name", + "type": "string", + "semantic_entity": "v3.customer.name", + "semantic_type": "dimension", + }, + { + "name": "email", + "type": "string", + "semantic_entity": "v3.customer.email", + "semantic_type": "dimension", + }, + ] + + @pytest.mark.asyncio + async def test_cube_scoped_dimension_query(self, client_with_build_v3): + """Cube metrics scope values; cube and request filters combine.""" + create = await client_with_build_v3.post( + "/nodes/cube/", + json={ + "name": "v3.filtered_revenue_cube", + "metrics": ["v3.total_revenue"], + "dimensions": [ + "v3.product.category", + "v3.product.subcategory", + ], + "filters": ["v3.product.category = 'Electronics'"], + "mode": "published", + "description": "Filtered cube for dimension value SQL", + }, + ) + assert create.status_code == 201, create.json() + + response = await client_with_build_v3.get( + "/sql/dimensions/v3/", + params={ + "dimensions": ["v3.product.category"], + "filters": ["v3.product.subcategory = 'Smartphones'"], + "cube": "v3.filtered_revenue_cube", + }, + ) + assert response.status_code == 200, response.json() + assert_sql_equal( + response.json()["sql"], + """ + SELECT DISTINCT dimension_values.category + FROM ( + WITH v3_order_details AS ( + SELECT oi.product_id, oi.quantity * oi.unit_price AS line_total + FROM default.v3.orders o + JOIN default.v3.order_items oi ON o.order_id = oi.order_id + ), + v3_product AS ( + SELECT product_id, category, subcategory + FROM default.v3.products + WHERE category = 'Electronics' AND subcategory = 'Smartphones' + ), + order_details_0 AS ( + SELECT t2.category, SUM(t1.line_total) line_total_sum_e1f61696 + FROM v3_order_details t1 + LEFT OUTER JOIN v3_product t2 ON t1.product_id = t2.product_id + WHERE t2.category = 'Electronics' + AND t2.subcategory = 'Smartphones' + GROUP BY t2.category + ) + SELECT order_details_0.category AS category, + SUM(order_details_0.line_total_sum_e1f61696) AS total_revenue + FROM order_details_0 + WHERE order_details_0.category = 'Electronics' + GROUP BY order_details_0.category + ) AS dimension_values + """, + ) + assert [column["semantic_entity"] for column in response.json()["columns"]] == [ + "v3.product.category", + ] + + availability = await client_with_build_v3.post( + "/data/v3.filtered_revenue_cube/availability/", + json={ + "catalog": "default", + "schema_": "v3", + "table": "filtered_revenue_cube", + "valid_through_ts": 1010129120, + }, + ) + assert availability.status_code in (200, 201), availability.json() + + materialized = await client_with_build_v3.get( + "/sql/dimensions/v3/", + params={ + "dimensions": ["v3.product.category"], + "cube": "v3.filtered_revenue_cube", + "dialect": "druid", + }, + ) + assert materialized.status_code == 200, materialized.json() + assert "filtered_revenue_cube" in materialized.json()["sql"] + assert "default.v3.products" not in materialized.json()["sql"] + + response = await client_with_build_v3.get( + "/sql/dimensions/v3/", + params={ + "dimensions": ["v3.customer.name[customer]"], + "cube": "v3.filtered_revenue_cube", + }, + ) + assert response.status_code == 200, response.json() + assert "v3_customer" in response.json()["sql"] + assert "category = 'Electronics'" in response.json()["sql"] + assert "filtered_revenue_cube" not in response.json()["sql"] + + @pytest.mark.asyncio + async def test_dimension_only_query_rejects_multiple_nodes( + self, + client_with_build_v3, + ): + response = await client_with_build_v3.get( + "/sql/dimensions/v3/", + params={ + "dimensions": ["v3.customer.name", "v3.product.category"], + }, + ) + + assert response.status_code == 422, response.json() + assert "exactly one node" in response.json()["message"] + + @pytest.mark.asyncio + async def test_dimension_only_query_ignores_unknown_cube( + self, + client_with_build_v3, + ): + response = await client_with_build_v3.get( + "/sql/dimensions/v3/", + params={ + "dimensions": ["v3.customer.name"], + "cube": "v3.missing_cube", + }, + ) + + assert response.status_code == 200, response.json() + assert "DISTINCT" in response.json()["sql"] + + @pytest.mark.asyncio + async def test_dimension_only_query_from_transform(self, client_with_build_v3): + response = await client_with_build_v3.get( + "/sql/dimensions/v3/", + params={"dimensions": ["v3.order_details.status"]}, + ) + + assert response.status_code == 200, response.json() + assert_sql_equal( + response.json()["sql"], + """ + WITH v3_order_details AS ( + SELECT o.order_id, oi.line_number, o.customer_id, o.order_date, + o.from_location_id, o.to_location_id, o.status, oi.product_id, + oi.quantity, oi.unit_price, + oi.quantity * oi.unit_price AS line_total + FROM default.v3.orders o + JOIN default.v3.order_items oi ON o.order_id = oi.order_id + ) + SELECT DISTINCT status + FROM v3_order_details + """, + ) + + @pytest.mark.asyncio + async def test_dimension_only_query_rejects_missing_node_with_explicit_dialect( + self, + client_with_build_v3, + ): + response = await client_with_build_v3.get( + "/sql/dimensions/v3/", + params={ + "dimensions": ["missing.dimension.attribute"], + "dialect": "trino", + }, + ) + + assert response.status_code == 422, response.json() + assert "does not exist" in response.json()["message"] + + @pytest.mark.asyncio + async def test_dimension_only_query_rejects_missing_attribute( + self, + client_with_build_v3, + ): + response = await client_with_build_v3.get( + "/sql/dimensions/v3/", + params={ + "dimensions": ["v3.customer.missing"], + "dialect": "trino", + }, + ) + + assert response.status_code == 422, response.json() + assert "does not contain columns" in response.json()["message"] + + @pytest.mark.asyncio + async def test_dimension_only_query_rejects_metric_node( + self, + client_with_build_v3, + ): + response = await client_with_build_v3.get( + "/sql/dimensions/v3/", + params={ + "dimensions": ["v3.total_revenue.value"], + "dialect": "trino", + }, + ) + + assert response.status_code == 422, response.json() + assert "cannot select attributes" in response.json()["message"] + + @pytest.mark.asyncio + async def test_dimension_only_query_aliases_roles_and_accepts_parameters( + self, + client_with_build_v3, + ): + response = await client_with_build_v3.get( + "/sql/dimensions/v3/", + params={ + "dimensions": ["v3.customer.name[buyer]"], + "query_params": '{"unused": "value"}', + }, + ) + + assert response.status_code == 200, response.json() + assert response.json()["columns"][0]["name"] == "name_buyer" + assert "name AS name_buyer" in response.json()["sql"] + @pytest.mark.asyncio async def test_basic_metrics_sql(self, client_with_build_v3): """Test that metrics SQL endpoint returns valid SQL.""" @@ -5818,8 +6087,7 @@ async def test_skip_join_filter_on_dimension_pk_as_fact_fk( result["sql"], """ WITH v3_order_details AS ( - SELECT o.customer_id, o.status, - oi.quantity * oi.unit_price AS line_total + SELECT o.status, oi.quantity * oi.unit_price AS line_total FROM default.v3.orders o JOIN default.v3.order_items oi ON o.order_id = oi.order_id WHERE o.customer_id = 42 @@ -6511,7 +6779,7 @@ async def test_reference_dimension_used_only_as_filter(self, client_with_build_v """ WITH v3_page_views_enriched AS ( - SELECT view_id, page_type + SELECT view_id FROM default.v3.page_views WHERE page_type = 'checkout' ), @@ -6567,7 +6835,7 @@ async def test_metric_on_dimension_node_groups_and_filters_by_its_columns( """ WITH v3_product AS ( - SELECT category, subcategory, price + SELECT category, price FROM default.v3.products WHERE subcategory = 'phones' ), diff --git a/datajunction-server/tests/construction/build_v3/preagg_substitution_test.py b/datajunction-server/tests/construction/build_v3/preagg_substitution_test.py index e2ec14858b..0edd7753eb 100644 --- a/datajunction-server/tests/construction/build_v3/preagg_substitution_test.py +++ b/datajunction-server/tests/construction/build_v3/preagg_substitution_test.py @@ -1014,7 +1014,6 @@ async def test_external_preagg_filter_on_uncovered_column( """ WITH v3_order_details AS ( SELECT o.status, - oi.product_id, oi.quantity * oi.unit_price AS line_total FROM default.v3.orders o JOIN default.v3.order_items oi ON o.order_id = oi.order_id diff --git a/datajunction-server/tests/construction/build_v3/projection_invariant.py b/datajunction-server/tests/construction/build_v3/projection_invariant.py new file mode 100644 index 0000000000..1f9b5172a4 --- /dev/null +++ b/datajunction-server/tests/construction/build_v3/projection_invariant.py @@ -0,0 +1,108 @@ +""" +The invariant every generated query must hold: whatever a CTE reads from +another CTE, that other CTE projects. + +``prune_cte_projections`` trims each CTE to what its readers ask for, so this is +the oracle for whether a trim went too far. It needs nothing but the SQL text. +""" + +from collections.abc import Iterator +from typing import cast + +from datajunction_server.sql.parsing import ast +from datajunction_server.sql.parsing.backends.antlr4 import parse + + +def _projected_names(cte: ast.Query) -> set[str]: + """Names a CTE exposes, taking the first branch of any set operation.""" + names = set() + for expression in cte.select.projection: + names.add(cast(ast.Aliasable, expression).alias_or_name.name) + return names + + +def _branches(select: ast.SelectExpression) -> list[ast.SelectExpression]: + """Each arm of a set operation, or the select itself when there is none.""" + branches = [] + branch: ast.SelectExpression | None = select + while branch is not None: + branches.append(branch) + branch = branch.set_op.right if branch.set_op else None + return branches + + +def _branch_columns( + branch: ast.SelectExpression, +) -> Iterator[tuple[ast.Column, bool]]: + """ + Columns of one arm, not those of the arms unioned after it. + + Each column is flagged with whether it sits in a clause that may name the + select's own output aliases rather than a column of its source. + """ + keyed = [ + *((part, False) for part in branch.projection), + *((part, False) for part in branch.group_by), + *([(branch.from_, False)] if branch.from_ else []), + *([(branch.where, True)] if branch.where else []), + *([(branch.having, True)] if branch.having else []), + ] + for part, may_use_output_alias in keyed: + for column in part.find_all(ast.Column): + yield column, may_use_output_alias + + +def _output_aliases(branch: ast.SelectExpression) -> set[str]: + """ + Names this arm introduces with an explicit ``AS``. + + A filter DJ pushes into a CTE can land on such a name — the alias is the + only handle the column has once the underlying expression is not projected + under its own name. Charging it to the source would be wrong: the source + never had a column by that name. + """ + names = set() + for expression in branch.projection: + alias = getattr(expression, "alias", None) + if alias is not None: + names.add(alias.name) + return names + + +def unprojected_references(sql: str) -> set[str]: + """ + Find every column a CTE reads from another CTE that is not projected there. + + Each arm of a set operation is read on its own, so a bare column in an arm + drawing on a single CTE is charged to that CTE. Returns + ``.`` for each reference the producer does not + project, so an empty set means the query is internally consistent. + """ + query = parse(sql) + ctes = {cte.alias_or_name.name: cte for cte in query.ctes} + + missing = set() + for scope in [*query.ctes, query]: + for branch in _branches(scope.select): + tables = list(branch.from_.find_all(ast.Table)) if branch.from_ else [] + sources = { + (table.alias.name if table.alias else table.name.name): ctes[ + table.name.name + ] + for table in tables + if table.name.name in ctes + } + lone_source = ( + next(iter(sources.values())) if len(tables) == 1 and sources else None + ) + aliases = _output_aliases(branch) + for column, may_use_output_alias in _branch_columns(branch): + qualifier, _, name = column.identifier().rpartition(".") + producer = sources.get(qualifier) if qualifier else lone_source + if producer is None: + continue + if not qualifier and may_use_output_alias and name in aliases: + continue + if name not in _projected_names(producer): + missing.add(f"{producer.alias_or_name.name}.{name}") + return missing diff --git a/datajunction-server/tests/construction/build_v3/set_operations_test.py b/datajunction-server/tests/construction/build_v3/set_operations_test.py index 5097e34808..15163db51a 100644 --- a/datajunction-server/tests/construction/build_v3/set_operations_test.py +++ b/datajunction-server/tests/construction/build_v3/set_operations_test.py @@ -80,11 +80,11 @@ async def test_union_all_transform_metric_generates_sql( """ WITH v3_orders_unified AS ( - SELECT order_id, customer_id, order_date, status + SELECT order_id, status FROM default.v3.orders WHERE status = 'completed' UNION ALL - SELECT order_id, customer_id, order_date, status + SELECT order_id, status FROM default.v3.orders WHERE status = 'shipped' ), @@ -129,11 +129,11 @@ async def test_union_transform_filter_pushed_into_primary_arm_only( """ WITH v3_orders_unified AS ( - SELECT order_id, customer_id, order_date, status + SELECT order_id, status FROM default.v3.orders WHERE status = 'completed' AND status = 'completed' UNION ALL - SELECT order_id, customer_id, order_date, status + SELECT order_id, status FROM default.v3.orders WHERE status = 'shipped' ), diff --git a/datajunction-server/tests/construction/build_v3/transform_query_shapes_test.py b/datajunction-server/tests/construction/build_v3/transform_query_shapes_test.py index ca03bea5a2..9a6b4c3c6d 100644 --- a/datajunction-server/tests/construction/build_v3/transform_query_shapes_test.py +++ b/datajunction-server/tests/construction/build_v3/transform_query_shapes_test.py @@ -15,10 +15,10 @@ from tests.construction.build_v3 import assert_sql_equal -@pytest_asyncio.fixture -async def client_with_edge_shapes(client_with_build_v3: AsyncClient): +@pytest_asyncio.fixture(scope="module") +async def client_with_edge_shapes(module__client_with_build_v3: AsyncClient): """Adds transforms with unusual query shapes + metrics on top of them.""" - r1 = await client_with_build_v3.post( + r1 = await module__client_with_build_v3.post( "/nodes/transform/", json={ "name": "v3.orders_via_derived_table", @@ -37,7 +37,7 @@ async def client_with_edge_shapes(client_with_build_v3: AsyncClient): ) assert r1.status_code == 201, r1.json() - r2 = await client_with_build_v3.post( + r2 = await module__client_with_build_v3.post( "/nodes/transform/", json={ "name": "v3.orders_ranked", @@ -59,7 +59,7 @@ async def client_with_edge_shapes(client_with_build_v3: AsyncClient): ) assert r2.status_code == 201, r2.json() - r3 = await client_with_build_v3.post( + r3 = await module__client_with_build_v3.post( "/nodes/transform/", json={ "name": "v3.orders_self_joined", @@ -89,12 +89,12 @@ async def client_with_edge_shapes(client_with_build_v3: AsyncClient): ("v3.ranked_count", "SELECT COUNT(*) FROM v3.orders_ranked"), ("v3.self_join_count", "SELECT COUNT(*) FROM v3.orders_self_joined"), ]: - r = await client_with_build_v3.post( + r = await module__client_with_build_v3.post( "/nodes/metric/", json={"name": name, "query": query, "mode": "published"}, ) assert r.status_code == 201, r.json() - return client_with_build_v3 + return module__client_with_build_v3 class TestTransformQueryShapes: diff --git a/datajunction-server/tests/database/test_custom_metadata_schema_model.py b/datajunction-server/tests/database/test_custom_metadata_schema_model.py index 73b8c75114..1ff7971407 100644 --- a/datajunction-server/tests/database/test_custom_metadata_schema_model.py +++ b/datajunction-server/tests/database/test_custom_metadata_schema_model.py @@ -31,20 +31,18 @@ async def test_insert_and_read_schema(session): @pytest.mark.asyncio async def test_new_columns_round_trip(session): - """owner, updated_by_id, reserved round-trip through the ORM.""" + """updated_by_id and reserved round-trip through the ORM.""" row = CustomMetadataSchema( key="contact", node_type=None, namespace=None, json_schema={"type": "string"}, - owner="team-platform", updated_by_id=None, reserved=True, ) session.add(row) await session.commit() fetched = await session.get(CustomMetadataSchema, row.id) - assert fetched.owner == "team-platform" assert fetched.updated_by_id is None assert fetched.reserved is True diff --git a/datajunction-server/tests/dj_mcp/test_transport.py b/datajunction-server/tests/dj_mcp/test_transport.py index f0005eadb4..da13c17608 100644 --- a/datajunction-server/tests/dj_mcp/test_transport.py +++ b/datajunction-server/tests/dj_mcp/test_transport.py @@ -2,6 +2,8 @@ Tests for the MCP HTTP transport — ``mount_mcp`` and its lifespan integration. """ +import asyncio + import httpx import pytest import pytest_asyncio @@ -86,6 +88,127 @@ def recorder(scope): assert seen_scope["type"] == "http" +@pytest.mark.asyncio +async def test_shutdown_drains_in_flight(monkeypatch) -> None: + """Lifespan shutdown waits for every in-flight MCP request to respond.""" + from mcp.server.streamable_http_manager import StreamableHTTPSessionManager + from starlette.responses import PlainTextResponse + + events: list[str] = [] + entered: list[str] = [] + + async def slow_handler(_self, scope, receive, send): + wait = scope["query_string"].decode().removeprefix("wait=") + entered.append(wait) + await asyncio.sleep(float(wait)) + await PlainTextResponse("ok")(scope, receive, send) + events.append(wait) + + monkeypatch.setattr(StreamableHTTPSessionManager, "handle_request", slow_handler) + + app = FastAPI() + transport.mount_mcp(app) + + async with httpx.AsyncClient( + transport=ASGITransport(app=app), + base_url="http://test", + follow_redirects=True, + ) as client: + lifespan = app.router.lifespan_context(app) + await lifespan.__aenter__() + requests = [ + asyncio.create_task(client.post(f"/mcp/?wait={wait}")) + for wait in ("0.05", "0.2") + ] + while len(entered) < 2: + await asyncio.sleep(0.01) + await lifespan.__aexit__(None, None, None) + events.append("lifespan done") + responses = await asyncio.gather(*requests) + + assert events == ["0.05", "0.2", "lifespan done"] + assert [(r.status_code, r.text) for r in responses] == [(200, "ok"), (200, "ok")] + + +@pytest.mark.asyncio +async def test_drain_gives_up_after_timeout(monkeypatch) -> None: + """Shutdown stops waiting once the drain timeout passes.""" + from mcp.server.streamable_http_manager import StreamableHTTPSessionManager + from starlette.responses import PlainTextResponse + + events: list[str] = [] + started = asyncio.Event() + + async def slow_handler(_self, scope, receive, send): + started.set() + await asyncio.sleep(0.3) + await PlainTextResponse("ok")(scope, receive, send) + events.append("request done") + + monkeypatch.setattr(StreamableHTTPSessionManager, "handle_request", slow_handler) + + app = FastAPI() + transport.mount_mcp(app, drain_timeout=0.01) + + async with httpx.AsyncClient( + transport=ASGITransport(app=app), + base_url="http://test", + follow_redirects=True, + ) as client: + lifespan = app.router.lifespan_context(app) + await lifespan.__aenter__() + request = asyncio.create_task(client.post("/mcp/")) + await started.wait() + await lifespan.__aexit__(None, None, None) + events.append("lifespan done") + await request + + assert events == ["lifespan done", "request done"] + + +@pytest.mark.asyncio +async def test_drain_rejects_new_requests(monkeypatch) -> None: + """While draining, a new MCP request gets 503 instead of a dropped stream.""" + from mcp.server.streamable_http_manager import StreamableHTTPSessionManager + from starlette.responses import PlainTextResponse + + started = asyncio.Event() + release = asyncio.Event() + + async def blocking_handler(_self, scope, receive, send): + started.set() + await release.wait() + await PlainTextResponse("ok")(scope, receive, send) + + monkeypatch.setattr( + StreamableHTTPSessionManager, + "handle_request", + blocking_handler, + ) + + app = FastAPI() + transport.mount_mcp(app, drain_timeout=0.01) + + async with httpx.AsyncClient( + transport=ASGITransport(app=app), + base_url="http://test", + follow_redirects=True, + ) as client: + lifespan = app.router.lifespan_context(app) + await lifespan.__aenter__() + first = asyncio.create_task(client.post("/mcp/")) + await started.wait() + await lifespan.__aexit__(None, None, None) + late = await client.post("/mcp/") + release.set() + first_response = await first + + assert late.status_code == 503 + assert late.text == "Server shutting down" + assert first_response.status_code == 200 + assert first_response.text == "ok" + + @pytest.mark.asyncio async def test_mount_mcp_request_context_exits_on_handler_error(monkeypatch) -> None: """If MCP request handling raises, the request_context is still exited.""" diff --git a/datajunction-server/tests/helpers/populate_preaggs_template.py b/datajunction-server/tests/helpers/populate_preaggs_template.py new file mode 100644 index 0000000000..f9c7c8c8cf --- /dev/null +++ b/datajunction-server/tests/helpers/populate_preaggs_template.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python +""" +Create the pre-aggregations that ``tests/api/preaggregations_test.py`` needs. + +The target database is already a clone of the main template, so the BUILD_V3 +nodes these preaggs sit on top of are present. This only runs the ten +``/preaggs/plan`` calls and prints the resulting ids, so the caller can turn the +database into a template that every test clones instead of re-planning. + +Run as a subprocess to avoid event loop conflicts with pytest-asyncio, the same +way ``populate_template.py`` is. + +Usage: python populate_preaggs_template.py +""" + +import asyncio +import json +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from helpers.template_app import ( + configure_database_env, + mark_template_populated, + template_app_client, +) + +db_url = sys.argv[1] +reader_db_url = configure_database_env(db_url) + +# Same order as the fixture's preagg1..preagg10, so the printed ids line up +# positionally with the names the tests use. +PREAGG_SPECS: list[dict] = [ + { + "metrics": ["v3.total_revenue", "v3.total_quantity"], + "dimensions": ["v3.order_details.status"], + "strategy": "full", + "schedule": "0 0 * * *", + }, + { + "metrics": ["v3.total_revenue", "v3.avg_unit_price"], + "dimensions": ["v3.order_details.status", "v3.product.category"], + "strategy": "full", + "schedule": "0 * * * *", + }, + { + "metrics": ["v3.max_unit_price"], + "dimensions": ["v3.order_details.status"], + "strategy": "full", + }, + { + "metrics": ["v3.total_revenue"], + "dimensions": ["v3.product.category"], + }, + { + "metrics": ["v3.order_count"], + "dimensions": ["v3.order_details.status"], + "strategy": "full", + "schedule": "0 0 * * *", + }, + { + "metrics": ["v3.min_unit_price"], + "dimensions": ["v3.order_details.status"], + "strategy": "full", + "schedule": "0 0 * * *", + }, + { + "metrics": ["v3.total_revenue"], + "dimensions": ["v3.customer.customer_id"], + }, + { + "metrics": ["v3.page_view_count"], + "dimensions": ["v3.product.category"], + "strategy": "full", + "schedule": "0 0 * * *", + }, + { + "metrics": ["v3.session_count"], + "dimensions": ["v3.product.category"], + "strategy": "full", + "schedule": "0 0 * * *", + }, + { + "metrics": ["v3.visitor_count"], + "dimensions": ["v3.product.category"], + }, +] + + +async def main() -> None: + async with template_app_client(db_url, reader_db_url) as (session, client): + preagg_ids: list[int] = [] + for index, spec in enumerate(PREAGG_SPECS, start=1): + response = await client.post("/preaggs/plan", json=spec) + if response.status_code != 201: + raise RuntimeError( + f"preagg {index} failed ({response.status_code}): {response.text}", + ) + preagg_ids.append(response.json()["preaggs"][0]["id"]) + await session.commit() + + mark_template_populated(db_url) + print("PREAGG_IDS " + json.dumps(preagg_ids)) + + +if __name__ == "__main__": + asyncio.run(main()) diff --git a/datajunction-server/tests/helpers/populate_template.py b/datajunction-server/tests/helpers/populate_template.py index 9338e1e732..be7c09ba29 100644 --- a/datajunction-server/tests/helpers/populate_template.py +++ b/datajunction-server/tests/helpers/populate_template.py @@ -3,7 +3,11 @@ Script to populate the template database with all examples. Run as a subprocess to avoid event loop conflicts with pytest-asyncio. -Usage: python populate_template.py +Usage: python populate_template.py [EXAMPLE_NAME ...] + +With no example names every example is loaded, which is what the shared +all-examples template wants. Naming a subset builds a smaller template for a +module that only needs part of the fixture data. """ # ruff: noqa: E402 - env vars must be set before importing datajunction_server modules @@ -11,93 +15,32 @@ import asyncio import os import sys -from datetime import timedelta from http.client import HTTPException -import httpx -from cachelib.simple import SimpleCache from httpx import AsyncClient -from sqlalchemy import text -from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine -from sqlalchemy.pool import StaticPool - -# Get database URL from command line -template_db_url = sys.argv[1] -reader_db_url = template_db_url.replace("dj:dj@", "readonly_user:readonly@") - -# Set environment variables BEFORE importing any datajunction_server modules -# This ensures the Settings class picks up these values -os.environ["DJ_DATABASE__URI"] = template_db_url -os.environ["WRITER_DB__URI"] = template_db_url -os.environ["READER_DB__URI"] = reader_db_url +from sqlalchemy.ext.asyncio import AsyncSession # Add tests directory to path for examples import sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +import examples as examples_module from examples import COLUMN_MAPPINGS, EXAMPLES, SERVICE_SETUP +from helpers.template_app import ( + configure_database_env, + create_schema, + mark_template_populated, + template_app_client, +) -# Import config first and clear cache to ensure our env vars are used -from datajunction_server.config import DatabaseConfig, Settings -from datajunction_server.utils import get_settings - -# Clear the lru_cache on get_settings to force it to re-read -get_settings.cache_clear() +# Get database URL from command line +template_db_url = sys.argv[1] +examples_to_load = sys.argv[2:] or None +reader_db_url = configure_database_env(template_db_url) -# Now import the rest of the modules - they should use our settings +# Imported after configure_database_env so they read the settings above. from datajunction_server.api.attributes import default_attribute_types -from datajunction_server.api.main import app -from datajunction_server.database.base import Base -from datajunction_server.database.column import Column -from datajunction_server.database.engine import Engine from datajunction_server.database.user import User -from datajunction_server.internal.access.authentication.tokens import ( - create_token, -) -from datajunction_server.internal.access.authorization import ( - PassthroughAuthorizationService, - get_authorization_service, -) from datajunction_server.internal.seed import seed_default_catalogs -from datajunction_server.models.dialect import register_dialect_plugin -from datajunction_server.models.query import QueryCreate, QueryWithResults from datajunction_server.models.user import OAuthProvider -from datajunction_server.service_clients import QueryServiceClient -from datajunction_server.transpilation import SQLTranspilationPlugin -from datajunction_server.typing import QueryState -from datajunction_server.utils import ( - get_query_service_client, - get_session, -) - -# Verify our settings are correct -actual_settings = get_settings() -print(f"Using writer_db: {actual_settings.writer_db.uri}") -print( - f"Using reader_db: {actual_settings.reader_db.uri if actual_settings.reader_db else 'None'}", -) - -# Import seed module to patch its cached settings -from datajunction_server.internal import seed as seed_module - -# Create template settings (matching what get_settings() should return) -template_settings = Settings( - writer_db=DatabaseConfig(uri=template_db_url), - reader_db=DatabaseConfig(uri=reader_db_url), - repository="/path/to/repository", - results_backend=SimpleCache(default_timeout=0), - celery_broker=None, - redis_cache=None, - query_service=None, - secret="a-fake-secretkey", - transpilation_plugins=["default"], -) - -# Patch the cached settings in seed module -seed_module.settings = template_settings - -# Register dialect plugins -register_dialect_plugin("spark", SQLTranspilationPlugin) -register_dialect_plugin("trino", SQLTranspilationPlugin) -register_dialect_plugin("druid", SQLTranspilationPlugin) # Helper functions (copied from conftest.py) @@ -129,7 +72,13 @@ async def load_examples_in_client( # Load only the selected examples if any are specified if examples_to_load is not None: for example_name in examples_to_load: - for endpoint, json in EXAMPLES[example_name]: + # Most names are EXAMPLES keys, but some fixture sets are only + # module-level constants in tests/examples.py. + example = EXAMPLES.get(example_name) or getattr( + examples_module, + example_name, + ) + for endpoint, json in example: await post_and_raise_if_error( client=client, endpoint=endpoint, @@ -172,126 +121,24 @@ async def create_default_user(session: AsyncSession) -> User: async def main(): print(f"Populating template database: {template_db_url}") - engine = create_async_engine( - url=template_db_url, - poolclass=StaticPool, - ) - - # Create all tables - async with engine.begin() as conn: - await conn.execute(text("CREATE EXTENSION IF NOT EXISTS pg_trgm;")) - await conn.run_sync(Base.metadata.create_all) + await create_schema(template_db_url) print("Tables created") - async_session_factory = async_sessionmaker( - bind=engine, - autocommit=False, - expire_on_commit=False, - ) - - async with async_session_factory() as session: - # Seed default data + async with template_app_client( + template_db_url, + reader_db_url, + column_mappings=COLUMN_MAPPINGS, + ) as (session, test_client): await default_attribute_types(session) await seed_default_catalogs(session) await create_default_user(session) print("Default data seeded") - # Create mock query service client - qs_client = QueryServiceClient(uri="query_service:8001") - - def mock_get_columns_for_table( - catalog: str, - schema: str, - table: str, - engine: Engine | None = None, - request_headers: dict[str, str] | None = None, - ) -> list[Column]: - return COLUMN_MAPPINGS.get(f"{catalog}.{schema}.{table}", []) - - def mock_submit_query( - query_create: QueryCreate, - request_headers: dict[str, str] | None = None, - ) -> QueryWithResults: - return QueryWithResults( - id="bd98d6be-e2d2-413e-94c7-96d9411ddee2", - submitted_query=query_create.submitted_query, - state=QueryState.FINISHED, - results=[ - {"columns": [], "rows": [], "sql": query_create.submitted_query}, - ], - errors=[], - ) - - async def mock_get_columns_for_table_async( - catalog: str, - schema: str, - table: str, - request_headers: dict[str, str] | None = None, - engine: Engine | None = None, - ) -> list[Column]: - return mock_get_columns_for_table( - catalog, - schema, - table, - engine, - request_headers, - ) - - async def mock_submit_query_async( - query_create: QueryCreate, - request_headers: dict[str, str] | None = None, - ) -> QueryWithResults: - return mock_submit_query(query_create, request_headers) - - qs_client.get_columns_for_table = mock_get_columns_for_table # type: ignore - qs_client.submit_query = mock_submit_query # type: ignore - qs_client.get_columns_for_table = mock_get_columns_for_table_async # type: ignore - qs_client.submit_query = mock_submit_query_async # type: ignore - - # Override dependencies - def get_session_override() -> AsyncSession: - return session - - def get_settings_override() -> Settings: - return template_settings - - def get_passthrough_auth_service(): - """Override to approve all requests in tests.""" - return PassthroughAuthorizationService() - - def get_query_service_client_override(request=None): - return qs_client - - app.dependency_overrides[get_session] = get_session_override - app.dependency_overrides[get_settings] = get_settings_override - app.dependency_overrides[get_authorization_service] = ( - get_passthrough_auth_service - ) - app.dependency_overrides[get_query_service_client] = ( - get_query_service_client_override - ) - - # Create JWT token - jwt_token = create_token( - {"username": "dj"}, - secret="a-fake-secretkey", - iss="http://localhost:8000/", - expires_delta=timedelta(hours=24), - ) - - # Load ALL examples - print("Loading examples via HTTP client...") - async with httpx.AsyncClient( - transport=httpx.ASGITransport(app=app), - base_url="http://test", - ) as test_client: - test_client.headers.update({"Authorization": f"Bearer {jwt_token}"}) - await load_examples_in_client(test_client, None) # None = load ALL examples + print(f"Loading examples via HTTP client: {examples_to_load or 'ALL'}") + await load_examples_in_client(test_client, examples_to_load) print("Examples loaded") - app.dependency_overrides.clear() - - await engine.dispose() + mark_template_populated(template_db_url) print("Template database populated successfully!") diff --git a/datajunction-server/tests/helpers/template_app.py b/datajunction-server/tests/helpers/template_app.py new file mode 100644 index 0000000000..0cba983025 --- /dev/null +++ b/datajunction-server/tests/helpers/template_app.py @@ -0,0 +1,255 @@ +""" +Shared bootstrap for the scripts that populate template databases. + +Both ``populate_template.py`` and ``populate_preaggs_template.py`` need the same +thing: an authenticated HTTP client talking to the DJ app, with the app pointed +at one specific database. Getting there involves setting environment variables +before ``datajunction_server`` is imported at all, building a ``Settings``, +registering dialect plugins, stubbing the query service and installing four +dependency overrides -- none of which is interesting to either caller. + +Nothing here imports ``datajunction_server`` at module scope: the caller must be +able to ``import`` this module, call :func:`configure_database_env`, and only +then have any DJ module read its settings. +""" + +from __future__ import annotations + +import os +from collections.abc import AsyncIterator +from contextlib import asynccontextmanager +from datetime import timedelta +from typing import Any +from urllib.parse import urlparse + +import httpx +from psycopg import connect + +TEST_SECRET = "a-fake-secretkey" +TEST_ISSUER = "http://localhost:8000/" + + +def configure_database_env(db_url: str) -> str: + """ + Point the DJ settings at ``db_url``, and return the reader URL. + + Must be called before any ``datajunction_server`` module is imported, since + ``Settings`` reads these at import time. + """ + reader_db_url = db_url.replace("dj:dj@", "readonly_user:readonly@") + os.environ["DJ_DATABASE__URI"] = db_url + os.environ["WRITER_DB__URI"] = db_url + os.environ["READER_DB__URI"] = reader_db_url + return reader_db_url + + +def mark_template_populated(template_db_url: str) -> None: + """ + Record that the template at `template_db_url` finished building, via a + marker row on the base `dj` database, not the template itself. + + `clone_database_from_template` calls `pg_terminate_backend` against + connections to the template before cloning it, so a reader connected + straight to the template risks getting killed as collateral damage. + """ + template_name = urlparse(template_db_url).path.lstrip("/") + url = urlparse(template_db_url) + with connect( + host=url.hostname, + port=url.port, + dbname="dj", + user=url.username, + password=url.password, + autocommit=True, + ) as conn: + conn.execute( + "CREATE TABLE IF NOT EXISTS test_template_status (" + "template_name text PRIMARY KEY, populated_at timestamptz NOT NULL DEFAULT now())", + ) + conn.execute( + "INSERT INTO test_template_status (template_name) VALUES (%s) " + "ON CONFLICT (template_name) DO UPDATE SET populated_at = now()", + (template_name,), + ) + + +def build_settings(db_url: str, reader_db_url: str) -> Any: + """Build the Settings the app should use, and patch the seed module's copy.""" + from cachelib.simple import SimpleCache + + from datajunction_server.config import DatabaseConfig, Settings + from datajunction_server.internal import seed as seed_module + from datajunction_server.models.dialect import register_dialect_plugin + from datajunction_server.transpilation import SQLTranspilationPlugin + from datajunction_server.utils import get_settings + + get_settings.cache_clear() + + settings = Settings( + writer_db=DatabaseConfig(uri=db_url), + reader_db=DatabaseConfig(uri=reader_db_url), + repository="/path/to/repository", + results_backend=SimpleCache(default_timeout=0), + celery_broker=None, + redis_cache=None, + query_service=None, + secret=TEST_SECRET, + transpilation_plugins=["default"], + ) + # The seed module caches settings at import time. + seed_module.settings = settings + + for dialect in ("spark", "trino", "druid"): + register_dialect_plugin(dialect, SQLTranspilationPlugin) + + return settings + + +def build_mock_query_service_client(column_mappings: dict | None = None) -> Any: + """ + A query service client that answers from ``column_mappings`` and reports every + submitted query as finished, so template population never needs a live one. + """ + from datajunction_server.database.column import Column + from datajunction_server.database.engine import Engine + from datajunction_server.models.query import QueryCreate, QueryWithResults + from datajunction_server.service_clients import QueryServiceClient + from datajunction_server.typing import QueryState + + mappings = column_mappings or {} + client = QueryServiceClient(uri="query_service:8001") + + def get_columns_for_table( + catalog: str, + schema: str, + table: str, + engine: Engine | None = None, + request_headers: dict[str, str] | None = None, + ) -> list[Column]: + return mappings.get(f"{catalog}.{schema}.{table}", []) + + def submit_query( + query_create: QueryCreate, + request_headers: dict[str, str] | None = None, + ) -> QueryWithResults: + return QueryWithResults( + id="bd98d6be-e2d2-413e-94c7-96d9411ddee2", + submitted_query=query_create.submitted_query, + state=QueryState.FINISHED, + results=[ + {"columns": [], "rows": [], "sql": query_create.submitted_query}, + ], + errors=[], + ) + + async def get_columns_for_table_async( + catalog: str, + schema: str, + table: str, + request_headers: dict[str, str] | None = None, + engine: Engine | None = None, + ) -> list[Column]: + return get_columns_for_table(catalog, schema, table, engine, request_headers) + + async def submit_query_async( + query_create: QueryCreate, + request_headers: dict[str, str] | None = None, + ) -> QueryWithResults: + return submit_query(query_create, request_headers) + + client.get_columns_for_table = get_columns_for_table_async # type: ignore[method-assign] + client.submit_query = submit_query_async # type: ignore[method-assign] + return client + + +async def create_schema(db_url: str) -> None: + """ + Create every table on a fresh database. + + Importing the app is what registers the models on ``Base.metadata``. Without + it ``create_all`` silently creates only whichever subset happens to have + been imported, so tables belonging to newer features go missing and the + failure surfaces much later as ``relation "..." does not exist``. + """ + from sqlalchemy import text + from sqlalchemy.ext.asyncio import create_async_engine + from sqlalchemy.pool import StaticPool + + import datajunction_server.api.main # noqa: F401 registers every model + from datajunction_server.database.base import Base + + engine = create_async_engine(url=db_url, poolclass=StaticPool) + try: + async with engine.begin() as conn: + await conn.execute(text("CREATE EXTENSION IF NOT EXISTS pg_trgm;")) + await conn.run_sync(Base.metadata.create_all) + finally: + await engine.dispose() + + +@asynccontextmanager +async def template_app_client( + db_url: str, + reader_db_url: str, + column_mappings: dict | None = None, +) -> AsyncIterator[tuple[Any, httpx.AsyncClient]]: + """ + Yield ``(session, client)`` for a DJ app bound to ``db_url``. + + The session is the one the app resolves through ``get_session``, so a caller + can read back whatever its HTTP calls wrote. Dependency overrides and the + engine are torn down on exit; committing is left to the caller. + """ + from sqlalchemy.ext.asyncio import async_sessionmaker, create_async_engine + from sqlalchemy.pool import StaticPool + + from datajunction_server.api.main import app + from datajunction_server.internal.access.authentication.tokens import create_token + from datajunction_server.internal.access.authorization import ( + PassthroughAuthorizationService, + get_authorization_service, + ) + from datajunction_server.utils import ( + get_query_service_client, + get_session, + get_settings, + ) + + settings = build_settings(db_url, reader_db_url) + query_service_client = build_mock_query_service_client(column_mappings) + + engine = create_async_engine(url=db_url, poolclass=StaticPool) + session_factory = async_sessionmaker( + bind=engine, + autocommit=False, + expire_on_commit=False, + ) + + try: + async with session_factory() as session: + app.dependency_overrides[get_session] = lambda: session + app.dependency_overrides[get_settings] = lambda: settings + app.dependency_overrides[get_authorization_service] = lambda: ( + PassthroughAuthorizationService() + ) + app.dependency_overrides[get_query_service_client] = lambda request=None: ( + query_service_client + ) + + jwt_token = create_token( + {"username": "dj"}, + secret=TEST_SECRET, + iss=TEST_ISSUER, + expires_delta=timedelta(hours=24), + ) + + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), + base_url="http://test", + ) as client: + client.headers.update({"Authorization": f"Bearer {jwt_token}"}) + yield session, client + + app.dependency_overrides.clear() + finally: + await engine.dispose() diff --git a/datajunction-server/tests/internal/authorization_test.py b/datajunction-server/tests/internal/authorization_test.py index 79f0b1d3c7..1a254fa683 100644 --- a/datajunction-server/tests/internal/authorization_test.py +++ b/datajunction-server/tests/internal/authorization_test.py @@ -11,6 +11,7 @@ from datajunction_server.config import Settings from datajunction_server.database.group_member import GroupMember +from datajunction_server.database.namespace import NodeNamespace from datajunction_server.database.rbac import Role, RoleAssignment, RoleScope from datajunction_server.database.user import PrincipalKind, User from datajunction_server.errors import DJAuthorizationException @@ -24,6 +25,9 @@ RBACAuthorizationService, get_authorization_service, ) +from datajunction_server.internal.access.authorization.service import ( + governed_boundary_rules, +) from datajunction_server.internal.access.group_membership import ( GroupMembershipService, ) @@ -1457,6 +1461,289 @@ def test_settings_parse_json_list_from_environment(self, monkeypatch): ] +class TestGovernedBoundaryScopes: + """Tests for restrictive mutation policy derived from namespace rows.""" + + SERVICE_SETTINGS = ( + "datajunction_server.internal.access.authorization.service.settings" + ) + BOUNDARY = "team.metrics" + + @staticmethod + def request( + action: ResourceAction, + resource_type: ResourceType, + name: str, + ) -> ResourceRequest: + return ResourceRequest( + verb=action, + access_object=Resource(name=name, resource_type=resource_type), + ) + + @classmethod + def context( + cls, + *, + boundaries: tuple[str, ...] | None = None, + explicit_scopes=None, + default_scopes=None, + is_admin: bool = False, + ) -> AuthContext: + assignments = [_assignment(explicit_scopes)] if explicit_scopes else [] + return AuthContext( + user_id=1, + username="boundary-user", + oauth_provider="basic", + role_assignments=assignments, + is_admin=is_admin, + default_scopes=default_scopes or [], + governed_boundaries=boundaries + if boundaries is not None + else (cls.BOUNDARY,), + ) + + def configure(self, mocker, rules=()): + service_settings = mocker.patch(self.SERVICE_SETTINGS) + service_settings.restrictive_scopes = rules + service_settings.default_access_policy = "permissive" + + def test_rules_match_boundary_scope_contract(self): + assert {str(rule) for rule in governed_boundary_rules([self.BOUNDARY])} == { + "write:namespace:team.metrics", + "write:namespace:team.metrics.*", + "write:node:team.metrics.*", + "delete:namespace:team.metrics", + "delete:namespace:team.metrics.*", + "delete:node:team.metrics.*", + "manage:namespace:team.metrics", + "manage:namespace:team.metrics.*", + "manage:node:team.metrics.*", + } + + @pytest.mark.parametrize( + "action", + [ResourceAction.WRITE, ResourceAction.DELETE, ResourceAction.MANAGE], + ) + @pytest.mark.parametrize( + "resource_type,name", + [ + (ResourceType.NAMESPACE, "team.metrics"), + (ResourceType.NAMESPACE, "team.metrics.daily"), + (ResourceType.NODE, "team.metrics.daily_revenue"), + ], + ) + def test_mutations_require_explicit_grants( + self, + mocker, + action, + resource_type, + name, + ): + self.configure(mocker) + + decision = RBACAuthorizationService().authorize( + self.context(), + [self.request(action, resource_type, name)], + )[0] + + assert decision.approved is False + assert decision.reason and decision.reason.startswith("restrictive_scope:") + + @pytest.mark.parametrize("action", [ResourceAction.READ, ResourceAction.EXECUTE]) + def test_read_and_execute_keep_permissive_policy(self, mocker, action): + self.configure(mocker) + + decision = RBACAuthorizationService().authorize( + self.context(), + [self.request(action, ResourceType.NODE, "team.metrics.revenue")], + )[0] + + assert decision.approved is True + assert decision.reason == "default_access_policy_permissive" + + @pytest.mark.parametrize( + "action", + [ResourceAction.WRITE, ResourceAction.DELETE, ResourceAction.MANAGE], + ) + def test_ungoverned_sibling_keeps_permissive_policy(self, mocker, action): + self.configure(mocker) + + decision = RBACAuthorizationService().authorize( + self.context(), + [self.request(action, ResourceType.NODE, "team.other.revenue")], + )[0] + + assert decision.approved is True + assert decision.reason == "default_access_policy_permissive" + + def test_direct_grant_allows_governed_mutation(self, mocker): + self.configure(mocker) + explicit_scope = _scope( + ResourceAction.WRITE, + ResourceType.NODE, + "team.metrics.*", + ) + + decision = RBACAuthorizationService().authorize( + self.context(explicit_scopes=[explicit_scope]), + [ + self.request( + ResourceAction.WRITE, + ResourceType.NODE, + "team.metrics.revenue", + ), + ], + )[0] + + assert decision.approved is True + assert decision.reason == "explicit_grant" + + def test_default_role_cannot_bypass_governed_boundary(self, mocker): + self.configure(mocker) + default_scope = _scope(ResourceAction.WRITE, ResourceType.NODE, "*") + + decision = RBACAuthorizationService().authorize( + self.context(default_scopes=[default_scope]), + [ + self.request( + ResourceAction.WRITE, + ResourceType.NODE, + "team.metrics.revenue", + ), + ], + )[0] + + assert decision.approved is False + assert decision.reason and decision.reason.startswith("restrictive_scope:") + + def test_admin_keeps_audited_boundary_bypass(self, mocker, caplog): + self.configure(mocker) + + with caplog.at_level(logging.WARNING, logger="datajunction.audit.rbac"): + decision = RBACAuthorizationService().authorize( + self.context(is_admin=True), + [ + self.request( + ResourceAction.MANAGE, + ResourceType.NAMESPACE, + self.BOUNDARY, + ), + ], + )[0] + + assert decision.approved is True + assert decision.reason == "admin_bypass" + assert "event=rbac_admin_bypass" in caplog.messages[0] + + def test_configured_and_governed_rules_combine(self, mocker): + self.configure(mocker, rules=["delete:node:legacy.secure.*"]) + requests = [ + self.request( + ResourceAction.WRITE, + ResourceType.NODE, + "team.metrics.revenue", + ), + self.request( + ResourceAction.DELETE, + ResourceType.NODE, + "legacy.secure.revenue", + ), + self.request(ResourceAction.WRITE, ResourceType.NODE, "open.revenue"), + ] + + decisions = RBACAuthorizationService().authorize(self.context(), requests) + + assert [decision.approved for decision in decisions] == [False, False, True] + assert decisions[1].reason == "restrictive_scope:delete:node:legacy.secure.*" + + def test_multiple_boundaries_are_independent(self, mocker): + self.configure(mocker) + context = self.context(boundaries=("finance.cubes", "team.metrics")) + requests = [ + self.request( + ResourceAction.WRITE, + ResourceType.NODE, + "team.metrics.revenue", + ), + self.request( + ResourceAction.WRITE, + ResourceType.NODE, + "finance.cubes.revenue", + ), + self.request( + ResourceAction.WRITE, + ResourceType.NODE, + "finance.other.revenue", + ), + ] + + decisions = RBACAuthorizationService().authorize(context, requests) + + assert [decision.approved for decision in decisions] == [False, False, True] + + +@pytest.mark.asyncio +async def test_governed_boundary_allows_group_grant( + session: AsyncSession, + default_user: User, + mocker, +): + boundary = NodeNamespace( + namespace="group_governed", + is_governed_boundary=True, + ) + group = User( + username="governed-writers", + kind=PrincipalKind.GROUP, + oauth_provider="basic", + ) + role = Role(name="governed-group-role", created_by_id=default_user.id) + session.add_all([boundary, group, role]) + await session.flush() + session.add_all( + [ + GroupMember(group_id=group.id, member_id=default_user.id), + RoleScope( + role_id=role.id, + action=ResourceAction.WRITE, + scope_type=ResourceType.NODE, + scope_value="group_governed.*", + ), + RoleAssignment( + principal_id=group.id, + role_id=role.id, + granted_by_id=default_user.id, + ), + ], + ) + await session.commit() + + context_settings = mocker.patch( + "datajunction_server.internal.access.authorization.context.settings", + ) + context_settings.default_access_role = None + service_settings = mocker.patch( + "datajunction_server.internal.access.authorization.service.settings", + ) + service_settings.restrictive_scopes = [] + service_settings.default_access_policy = "permissive" + + user = await get_user(username=default_user.username, session=session) + context = await AuthContext.from_user(session, user) + request = ResourceRequest( + verb=ResourceAction.WRITE, + access_object=Resource( + name="group_governed.revenue", + resource_type=ResourceType.NODE, + ), + ) + + decision = RBACAuthorizationService().authorize(context, [request])[0] + + assert decision.approved is True + assert decision.reason == "explicit_grant" + + @pytest.mark.asyncio class TestGroupBasedPermissions: """Tests for group-based role assignments.""" @@ -2141,6 +2428,40 @@ async def test_execute_implies_read( class TestAuthContext: """Tests for AuthContext and effective assignments.""" + async def test_from_user_loads_only_retained_governed_boundaries( + self, + default_user: User, + session: AsyncSession, + ): + session.add_all( + [ + NodeNamespace( + namespace="context_governed", + is_governed_boundary=True, + ), + NodeNamespace(namespace="context_governed.child"), + NodeNamespace( + namespace="context_missing.parent.boundary", + is_governed_boundary=True, + ), + NodeNamespace( + namespace="context_deactivated", + is_governed_boundary=True, + deactivated_at=datetime.now(UTC), + ), + NodeNamespace(namespace="context_ungoverned"), + ], + ) + await session.commit() + + context = await AuthContext.from_user(session, default_user) + + assert context.governed_boundaries == ( + "context_deactivated", + "context_governed", + "context_missing.parent.boundary", + ) + async def test_auth_context_from_user_direct_assignments_only( self, default_user: User, diff --git a/datajunction-server/tests/internal/deployment/fingerprints_test.py b/datajunction-server/tests/internal/deployment/fingerprints_test.py new file mode 100644 index 0000000000..314c8491cb --- /dev/null +++ b/datajunction-server/tests/internal/deployment/fingerprints_test.py @@ -0,0 +1,559 @@ +from typing import get_args +from unittest.mock import MagicMock + +import pytest + +from datajunction_server.internal.deployment.fingerprints import ( + SEMANTIC_PARENT_RESOLVERS, + SemanticFingerprintGraph, + _candidate_parts, + _parent_candidates, + _resolved_proposed_specs, + build_deployment_fingerprints, +) +from datajunction_server.models.deployment import ( + ColumnSpec, + CubeSpec, + DimensionJoinLinkSpec, + DimensionSpec, + MetricSpec, + NodeSpec, + NodeUnion, + SourceSpec, + TransformSpec, +) +from datajunction_server.models.semantic_fingerprint import ( + UNKNOWN_SEMANTIC_FINGERPRINT, + SemanticFingerprint, +) +from datajunction_server.semantic_fingerprints.engine import local_node_fingerprint + + +def source_spec( + name: str, + *, + namespace: str = "ns", + table: str | None = None, + columns: list[ColumnSpec] | None = None, + dimension_links: list[DimensionJoinLinkSpec] | None = None, +) -> SourceSpec: + return SourceSpec( + namespace=namespace, + name=name, + catalog="catalog", + schema_="schema", + table=table or name, + columns=columns, + dimension_links=dimension_links or [], + ) + + +def transform_spec( + name: str, + query: str, + *, + namespace: str = "ns", +) -> TransformSpec: + return TransformSpec(namespace=namespace, name=name, query=query) + + +def spec_map(*specs: NodeSpec) -> dict[str, NodeSpec]: + return {spec.rendered_name: spec for spec in specs} + + +def linked_dimension( + name: str, + *targets: str, + query: str = "SELECT 1", +) -> DimensionSpec: + return DimensionSpec( + namespace="cycle", + name=name, + query=query, + dimension_links=[ + DimensionJoinLinkSpec( + dimension_node=f"cycle.{target}", + join_type="cross", + ) + for target in targets + ], + ) + + +def test_parent_candidates_follow_oss_semantic_relationships(): + transform = TransformSpec( + namespace="ns", + name="transform", + query="SELECT * FROM ${prefix}source", + dimension_links=[ + DimensionJoinLinkSpec( + dimension_node="${prefix}dimension", + join_on="${prefix}transform.id = ${prefix}dimension.id", + ), + ], + ) + metric = MetricSpec( + namespace="ns", + name="metric", + query="SELECT SUM(value) FROM ${prefix}transform", + required_dimensions=["${prefix}dimension.id"], + ) + cube = CubeSpec( + namespace="ns", + name="cube", + metrics=["${prefix}metric"], + dimensions=["${prefix}dimension.id"], + filters=["${prefix}filter_dimension.id = 1"], + ) + + assert _parent_candidates(transform) == {"ns.source", "ns.dimension"} + assert _parent_candidates(metric) == {"ns.transform", "ns.dimension"} + assert _parent_candidates(cube) == { + "ns.metric", + "ns.dimension", + "ns.filter_dimension", + } + node_union = get_args(NodeUnion)[0] + assert set(SEMANTIC_PARENT_RESOLVERS) == set(get_args(node_union)) + + cache = {} + first = _candidate_parts(transform, cache) + assert _candidate_parts(transform, cache) is first + assert len(cache) == 1 + with pytest.raises(TypeError, match="No semantic parent resolver for NodeSpec"): + _candidate_parts(NodeSpec(name="unknown", node_type="source"), {}) + + +def test_unparseable_cube_filter_remains_in_local_fingerprint(): + broken = CubeSpec( + namespace="ns", + name="cube", + filters=["not valid sql !!!"], + ) + without_filter = broken.model_copy(update={"filters": []}) + + assert _parent_candidates(broken) == set() + assert local_node_fingerprint(broken) != local_node_fingerprint(without_filter) + + +def test_merkle_fingerprints_propagate_changes_through_all_descendants(): + source = source_spec( + "source", + table="table", + columns=[ColumnSpec(name="id", type="bigint")], + ) + transform = transform_spec( + "transform", + "SELECT id FROM ${prefix}source", + ) + metric = MetricSpec( + namespace="ns", + name="metric", + query="SELECT COUNT(*) FROM ${prefix}transform", + ) + specs = spec_map(source, transform, metric) + original = SemanticFingerprintGraph(specs).fingerprints() + + changed_source = source.model_copy(update={"table": "changed"}) + changed_specs = {**specs, changed_source.rendered_name: changed_source} + changed = SemanticFingerprintGraph(changed_specs).fingerprints() + + assert all(original[name] != changed[name] for name in specs) + + +def test_graph_fingerprint_uses_its_parent_snapshot(): + orders_v1 = source_spec("orders", table="orders_v1") + orders_v2 = orders_v1.model_copy(update={"table": "orders_v2"}) + revenue = transform_spec( + "revenue", + "SELECT * FROM ${prefix}orders", + ) + + current = SemanticFingerprintGraph(spec_map(orders_v1, revenue)) + proposed = SemanticFingerprintGraph(spec_map(orders_v2, revenue)) + + current_fingerprint = current.fingerprint(revenue.rendered_name) + assert current.fingerprint(revenue.rendered_name) is current_fingerprint + assert current_fingerprint != proposed.fingerprint( + revenue.rendered_name, + ) + assert ( + SemanticFingerprintGraph(spec_map(revenue)).fingerprint(revenue.rendered_name) + == UNKNOWN_SEMANTIC_FINGERPRINT + ) + + +def test_merkle_fingerprints_support_deep_dependency_chains(): + specs = {} + for index in range(1_100): + links = ( + [ + DimensionJoinLinkSpec( + dimension_node=f"deep.node_{index - 1}", + join_type="cross", + ), + ] + if index + else [] + ) + spec = source_spec( + f"node_{index}", + namespace="deep", + table=f"table_{index}", + dimension_links=links, + ) + specs[spec.rendered_name] = spec + + target = "deep.node_1099" + assert ( + SemanticFingerprintGraph(specs).fingerprint(target) + != UNKNOWN_SEMANTIC_FINGERPRINT + ) + + +@pytest.mark.parametrize( + ("name", "query"), + [ + ("unresolved", "SELECT * FROM missing.parent"), + ("invalid", "SELECT ("), + ], +) +def test_unavailable_fingerprint_propagates_to_dependents(name: str, query: str): + unavailable = transform_spec(name, query) + dependent = transform_spec( + "dependent", + f"SELECT * FROM ${{prefix}}{name}", + ) + specs = spec_map(unavailable, dependent) + + assert SemanticFingerprintGraph(specs).fingerprints() == { + unavailable.rendered_name: UNKNOWN_SEMANTIC_FINGERPRINT, + dependent.rendered_name: UNKNOWN_SEMANTIC_FINGERPRINT, + } + + +def test_direct_required_dimension_paths_use_persisted_identity(): + parent = source_spec("parent") + authored = MetricSpec( + namespace="ns", + name="metric", + query="SELECT COUNT(*) FROM ${prefix}parent", + required_dimensions=["${prefix}parent.id"], + ) + persisted = authored.model_copy(update={"required_dimensions": ["id"]}) + authored_specs = spec_map(parent, authored) + persisted_specs = spec_map(parent, persisted) + + assert SemanticFingerprintGraph(authored_specs).fingerprint( + authored.rendered_name, + ) == SemanticFingerprintGraph(persisted_specs).fingerprint( + persisted.rendered_name, + ) + + dimension = DimensionSpec(namespace="ns", name="dimension", query="SELECT 1") + base_metric = MetricSpec( + namespace="ns", + name="base_metric", + query="SELECT COUNT(*) FROM ${prefix}parent", + ) + authored_derived = MetricSpec( + namespace="ns", + name="derived_metric", + query="SELECT ${prefix}base_metric * 2", + required_dimensions=[ + "${prefix}base_metric.id", + "${prefix}dimension.id", + ], + ) + persisted_derived = authored_derived.model_copy( + update={"required_dimensions": ["id", "ns.dimension.id"]}, + ) + authored_specs = spec_map(parent, dimension, base_metric, authored_derived) + persisted_specs = spec_map(parent, dimension, base_metric, persisted_derived) + assert SemanticFingerprintGraph(authored_specs).fingerprint( + authored_derived.rendered_name, + ) == SemanticFingerprintGraph(persisted_specs).fingerprint( + persisted_derived.rendered_name, + ) + + +def test_nested_required_dimensions_use_existing_node_boundary(): + parent = source_spec("parent") + authored = MetricSpec( + namespace="ns", + name="metric", + query="SELECT COUNT(*) FROM ${prefix}parent", + required_dimensions=["${prefix}parent.address.city"], + ) + persisted = authored.model_copy( + update={"required_dimensions": ["address.city"]}, + ) + + authored_fingerprint = SemanticFingerprintGraph( + spec_map(parent, authored), + ).fingerprint(authored.rendered_name) + persisted_fingerprint = SemanticFingerprintGraph( + spec_map(parent, persisted), + ).fingerprint(persisted.rendered_name) + + assert authored_fingerprint != UNKNOWN_SEMANTIC_FINGERPRINT + assert authored_fingerprint == persisted_fingerprint + + +def test_nested_cube_dimension_paths_use_existing_node_boundary(): + dimension = DimensionSpec(namespace="ns", name="dimension", query="SELECT 1") + metric = MetricSpec(namespace="ns", name="metric", query="SELECT COUNT(*)") + cube = CubeSpec( + namespace="ns", + name="cube", + metrics=["${prefix}metric"], + dimensions=["${prefix}dimension.address.city"], + filters=["${prefix}dimension.address.city = 'LA'"], + ) + original_specs = spec_map(dimension, metric, cube) + original = SemanticFingerprintGraph(original_specs).fingerprint( + cube.rendered_name, + ) + changed_dimension = dimension.model_copy(update={"query": "SELECT 2"}) + changed_specs = { + **original_specs, + changed_dimension.rendered_name: changed_dimension, + } + + assert original != UNKNOWN_SEMANTIC_FINGERPRINT + assert original != SemanticFingerprintGraph(changed_specs).fingerprint( + cube.rendered_name, + ) + + missing = cube.model_copy( + update={ + "name": "missing_cube", + "dimensions": [], + "filters": ["missing.dimension.address.city = 'LA'"], + }, + ) + assert ( + SemanticFingerprintGraph(spec_map(metric, missing)).fingerprint( + missing.rendered_name, + ) + == UNKNOWN_SEMANTIC_FINGERPRINT + ) + + +def test_cycle_hashing_is_stable_and_propagates_member_changes(): + first = linked_dimension("first", "second") + second = linked_dimension("second", "third") + third = linked_dimension("third", "first") + downstream = transform_spec( + "downstream", + "SELECT * FROM ${prefix}first", + namespace="cycle", + ) + specs = spec_map(first, second, third, downstream) + original = SemanticFingerprintGraph(specs).fingerprints() + reversed_input = SemanticFingerprintGraph( + dict(reversed(list(specs.items()))), + ).fingerprints() + assert original == reversed_input + assert all( + fingerprint != UNKNOWN_SEMANTIC_FINGERPRINT for fingerprint in original.values() + ) + + changed_second = second.model_copy(update={"query": "SELECT 2"}) + changed_specs = { + **specs, + changed_second.rendered_name: changed_second, + } + changed = SemanticFingerprintGraph(changed_specs).fingerprints() + assert all(original[name] != changed[name] for name in specs) + + broken_third = linked_dimension("third") + broken_specs = {**specs, broken_third.rendered_name: broken_third} + broken = SemanticFingerprintGraph(broken_specs).fingerprints() + assert all(original[name] != broken[name] for name in specs) + + +def test_self_link_and_external_parent_cycles_have_stable_hashes(): + self_link = linked_dimension("self", "self") + assert ( + SemanticFingerprintGraph( + {self_link.rendered_name: self_link}, + ).fingerprint(self_link.rendered_name) + != UNKNOWN_SEMANTIC_FINGERPRINT + ) + + parent = source_spec( + "parent", + namespace="cycle", + table="table", + ) + first = linked_dimension( + "first", + "second", + query="SELECT * FROM ${prefix}parent", + ) + second = linked_dimension("second", "first") + specs = spec_map(parent, first, second) + original = SemanticFingerprintGraph(specs).fingerprints() + changed_parent = parent.model_copy(update={"table": "changed"}) + changed_specs = {**specs, changed_parent.rendered_name: changed_parent} + changed = SemanticFingerprintGraph(changed_specs).fingerprints() + assert all(original[name] != changed[name] for name in specs) + + +@pytest.mark.parametrize("query", ["SELECT (", "SELECT * FROM missing.parent"]) +def test_deleted_legacy_node_returns_unknown_without_failure(query: str): + legacy = transform_spec("legacy", query) + graph = SemanticFingerprintGraph( + {legacy.rendered_name: legacy}, + ignored_parse_errors={legacy.rendered_name}, + ) + assert graph.fingerprint(legacy.rendered_name) == UNKNOWN_SEMANTIC_FINGERPRINT + + +def test_fingerprint_build_failures_are_unavailable(): + invalid_metric = MetricSpec( + namespace="ns", + name="invalid_metric", + query="SELECT (", + required_dimensions=["ns.dimension.id"], + ) + invalid_source = source_spec( + "invalid_source", + columns=[ColumnSpec(name="id", type="bigint")], + ) + invalid_source.columns[0].type = object() # type: ignore[assignment] + specs = spec_map(invalid_metric, invalid_source) + + graph = SemanticFingerprintGraph(specs) + assert graph.fingerprints([]) == {} + assert graph.fingerprints() == { + invalid_metric.rendered_name: UNKNOWN_SEMANTIC_FINGERPRINT, + invalid_source.rendered_name: UNKNOWN_SEMANTIC_FINGERPRINT, + } + assert SemanticFingerprintGraph( + specs, + ignored_parse_errors=set(specs), + ).fingerprints() == { + invalid_metric.rendered_name: UNKNOWN_SEMANTIC_FINGERPRINT, + invalid_source.rendered_name: UNKNOWN_SEMANTIC_FINGERPRINT, + } + + +def test_proposed_sources_reuse_resolved_columns_and_remove_deletes(): + source = source_spec( + "source", + table="table", + columns=[ColumnSpec(name="id", type="bigint")], + ) + deleted = source_spec( + "deleted", + table="deleted", + ) + proposed = source.model_copy(update={"columns": None}) + resolved = _resolved_proposed_specs( + { + source.rendered_name: source, + deleted.rendered_name: deleted, + }, + [proposed], + {deleted.rendered_name}, + ) + + assert local_node_fingerprint( + resolved[source.rendered_name], + ) == local_node_fingerprint(source) + assert deleted.rendered_name not in resolved + + +@pytest.mark.asyncio +async def test_build_deployment_fingerprints_without_external_parents(): + source = source_spec("source", table="table") + transform = transform_spec( + "transform", + "SELECT * FROM ${prefix}source", + ) + current, proposed = await build_deployment_fingerprints( + MagicMock(), + {}, + [source, transform], + [], + ) + + assert current == {} + graph = SemanticFingerprintGraph(spec_map(source, transform)) + source_fingerprint = proposed[source.rendered_name] + assert isinstance(source_fingerprint, SemanticFingerprint) + assert source_fingerprint == graph.fingerprint(source.rendered_name) + assert proposed[transform.rendered_name] == graph.fingerprint( + transform.rendered_name, + ) + + +@pytest.mark.asyncio +async def test_build_deployment_fingerprints_loads_external_ancestors(session): + transform = transform_spec( + "external_child", + "SELECT * FROM default.hard_hat", + ) + _, proposed = await build_deployment_fingerprints( + session, + {}, + [transform], + [], + ) + + assert proposed[transform.rendered_name] != UNKNOWN_SEMANTIC_FINGERPRINT + assert proposed[transform.rendered_name] != local_node_fingerprint(transform) + + unavailable = [ + transform_spec("missing_child", "SELECT * FROM missing.parent"), + transform_spec("invalid_child", "SELECT ("), + ] + _, proposed = await build_deployment_fingerprints( + session, + {}, + unavailable, + [], + ) + assert proposed == { + spec.rendered_name: UNKNOWN_SEMANTIC_FINGERPRINT for spec in unavailable + } + + metric = MetricSpec( + namespace="ns", + name="external_metric", + query="SELECT COUNT(*)", + ) + cube = CubeSpec( + namespace="ns", + name="external_cube", + metrics=["${prefix}external_metric"], + filters=["default.hard_hat.address.city = 'LA'"], + ) + _, proposed = await build_deployment_fingerprints( + session, + {}, + [metric, cube], + [], + ) + assert proposed[cube.rendered_name] != UNKNOWN_SEMANTIC_FINGERPRINT + + legacy = transform_spec("legacy", "SELECT (") + current, _ = await build_deployment_fingerprints( + session, + {legacy.rendered_name: legacy}, + [], + [legacy], + ) + assert current == {legacy.rendered_name: UNKNOWN_SEMANTIC_FINGERPRINT} + + current, proposed = await build_deployment_fingerprints( + session, + {}, + [], + [], + additional_target_names=["default.hard_hat"], + ) + assert current["default.hard_hat"] != UNKNOWN_SEMANTIC_FINGERPRINT + assert proposed["default.hard_hat"] != UNKNOWN_SEMANTIC_FINGERPRINT diff --git a/datajunction-server/tests/internal/deployment/orchestration_test.py b/datajunction-server/tests/internal/deployment/orchestration_test.py index a163ba5ab0..aa167a92a9 100644 --- a/datajunction-server/tests/internal/deployment/orchestration_test.py +++ b/datajunction-server/tests/internal/deployment/orchestration_test.py @@ -24,6 +24,9 @@ from datajunction_server.database.tag import Tag from datajunction_server.database.user import OAuthProvider, User from datajunction_server.errors import DJError, DJInvalidDeploymentConfig, ErrorCode +from datajunction_server.internal.deployment.fingerprints import ( + SemanticFingerprintGraph, +) from datajunction_server.internal.deployment.orchestrator import ( DeploymentOrchestrator, DeploymentPlan, @@ -66,6 +69,7 @@ from datajunction_server.models.node import MetricUnit, NodeStatus from datajunction_server.models.node_type import NodeType from datajunction_server.models.partition import Granularity +from datajunction_server.models.semantic_fingerprint import SemanticFingerprint from datajunction_server.sql.parsing.types import StringType @@ -837,6 +841,76 @@ def test_filter_nodes_to_deploy_without_force( assert len(to_skip) == len(sample_deployment_spec.nodes) assert to_delete == [] + def test_filter_nodes_records_revalidation_only( + self, + orchestrator, + sample_deployment_spec, + ): + """An unchanged node that is stuck INVALID is queued for revalidation.""" + existing_specs = { + node.rendered_name: node for node in sample_deployment_spec.nodes + } + stuck = sample_deployment_spec.nodes[1] + orchestrator.registry.add_nodes( + { + stuck.rendered_name: SimpleNamespace( + current=SimpleNamespace( + status=NodeStatus.INVALID, + parents=[], + ), + ), + }, + ) + to_deploy, to_skip, _ = orchestrator.filter_nodes_to_deploy(existing_specs) + assert to_deploy == [stuck] + assert orchestrator._revalidation_only == {stuck.rendered_name} + assert [spec.rendered_name for spec in to_skip] == [ + spec.rendered_name + for spec in sample_deployment_spec.nodes + if spec is not stuck + ] + + def test_filter_nodes_uses_normalized_query_and_source_columns(self): + incoming = [ + TransformSpec(name="transform", query=" SELECT id\nFROM source "), + SourceSpec( + name="source", + catalog="catalog", + schema_="schema", + table="table", + columns=None, + ), + ] + orchestrator = DeploymentOrchestrator( + deployment_spec=DeploymentSpec(namespace="test", nodes=incoming), + deployment_id="normalized-filter", + session=MagicMock(), + context=MagicMock(), + ) + existing = { + "test.transform": TransformSpec( + name="transform", + namespace="test", + query="SELECT id FROM source", + ), + "test.source": SourceSpec( + name="source", + namespace="test", + catalog="catalog", + schema_="schema", + table="table", + columns=[ColumnSpec(name="id", type="bigint")], + ), + } + + to_deploy, to_skip, _ = orchestrator.filter_nodes_to_deploy(existing) + assert to_deploy == [] + assert to_skip == incoming + + incoming[1].columns = [ColumnSpec(name="id", type="string")] + to_deploy, _, _ = orchestrator.filter_nodes_to_deploy(existing) + assert to_deploy == [incoming[1]] + def test_filter_nodes_to_deploy_with_force( self, session, @@ -897,6 +971,7 @@ async def test_execute_full_deployment_success(self, orchestrator): mock_plan.to_deploy = [] mock_plan.to_delete = [] mock_plan.to_delete_namespaces = [] + mock_plan.existing_specs = {} mock_create_plan.return_value = (mock_plan, []) # Execute @@ -924,6 +999,8 @@ async def test_execute_empty_deployment(self, orchestrator): mock_plan.to_deploy = [] mock_plan.to_delete = [] mock_plan.to_delete_namespaces = [] + mock_plan.existing_specs = {} + mock_plan.deletable_specs = [] mock_create_plan.return_value = (mock_plan, []) mock_handle_no_changes.return_value = [] @@ -1157,6 +1234,13 @@ async def test_create_cubes_from_validation_invalid_cubes(self, orchestrator): orchestrator._generate_changelog = AsyncMock( return_value=([], [], ChangeTier.NONE), ) + cube = invalid_results[0].spec + cube_fingerprint = SemanticFingerprintGraph( + {cube.rendered_name: cube}, + ).fingerprint(cube.rendered_name) + orchestrator._proposed_semantic_fingerprints = { + invalid_results[0].spec.rendered_name: cube_fingerprint, + } with patch( "datajunction_server.internal.deployment.orchestrator.get_node_namespace", @@ -1174,6 +1258,8 @@ async def test_create_cubes_from_validation_invalid_cubes(self, orchestrator): assert len(revisions) == 1 assert len(results) == 1 assert results[0].status == "invalid" + assert results[0].change_tier == "major" + assert results[0].semantic_fingerprint == cube_fingerprint @pytest.mark.asyncio async def test_cube_column_partition_applied_from_spec( @@ -2630,6 +2716,73 @@ async def test_deploy_reference_link_on_invalid_node( assert "INVALID" in result.message +@pytest.mark.asyncio +async def test_deploy_join_link_without_join_on( + session, + mock_deployment_context, +): + """A join link with no join_on and a stray node_column is rejected, not written.""" + node = await Node.get_by_name(session, "default.repair_orders") + dimension_node = await Node.get_by_name(session, "default.hard_hat") + node.current.status = NodeStatus.INVALID + + link_spec = DimensionJoinLinkSpec( + dimension_node="default.hard_hat", + node_column="hard_hat_id", + ) + link_spec.namespace = "default" + source_spec = SourceSpec( + name="repair_orders", + namespace="default", + catalog="default", + schema="roads", + table="repair_orders", + columns=[], + ) + + orch = DeploymentOrchestrator( + deployment_spec=DeploymentSpec(namespace="default", nodes=[]), + deployment_id="join-link-test", + session=session, + context=mock_deployment_context, + dry_run=True, + ) + orch.registry.nodes[node.name] = node + orch.registry.nodes[dimension_node.name] = dimension_node + + result = await orch._process_node_dimension_link( + node_spec=source_spec, + link_spec=link_spec, + ) + # join_sql is NOT NULL, so a written link would fail here. + await session.flush() + + assert result.status == DeploymentResult.Status.FAILED + assert result.operation == DeploymentResult.Operation.CREATE + assert result.name == "default.repair_orders -> default.hard_hat" + assert result.message == ( + "Dimension link from default.repair_orders to default.hard_hat sets " + "node_column, which only applies to reference links. Express the join " + "in join_on instead.\n" + "Dimension link from default.repair_orders to default.hard_hat has no " + "join_on clause. Set join_on to the equality between this node's " + "foreign key column(s) and the dimension's primary key." + ) + + +def test_cross_join_link_needs_no_join_on(): + """A CROSS join has no ON clause, so it is stored as-is.""" + link_spec = DimensionJoinLinkSpec( + dimension_node="default.hard_hat", + join_type=JoinType.CROSS, + ) + problems = DeploymentOrchestrator._join_link_problems( + link_spec, + "default.repair_orders", + ) + assert problems == [] + + @pytest.mark.asyncio async def test_delete_nodes_bulk_deletes_existing_node( session, @@ -2647,9 +2800,13 @@ async def test_delete_nodes_bulk_deletes_existing_node( context=mock_deployment_context, dry_run=False, ) - # "default.hard_hat" is present in the pre-loaded roads example DB. - spec = Mock() - spec.rendered_name = "default.hard_hat" + # "default.hard_hat" is present in the pre-loaded roads example DB. Its + # unparseable legacy query must not prevent deletion. + spec = TransformSpec( + name="hard_hat", + namespace="default", + query="SELECT (", + ) # No external references block the delete. with patch.object(orch, "_validate_node_deletion", AsyncMock(return_value={})): results = await orch._delete_nodes([spec]) @@ -2658,6 +2815,8 @@ async def test_delete_nodes_bulk_deletes_existing_node( assert results[0].status == DeploymentResult.Status.SUCCESS assert results[0].operation == DeploymentResult.Operation.DELETE assert results[0].name == "default.hard_hat" + assert results[0].change_tier == "major" + assert results[0].semantic_fingerprint is None # The node row is gone. gone = ( @@ -2668,6 +2827,32 @@ async def test_delete_nodes_bulk_deletes_existing_node( assert gone is None +@pytest.mark.asyncio +@pytest.mark.parametrize(("dry_run", "expects_lock"), [(True, False), (False, True)]) +async def test_validate_node_deletion_only_locks_wet_runs( + current_user, + mock_deployment_context, + dry_run, + expects_lock, +): + session = AsyncMock() + session.execute.return_value = [] + orch = DeploymentOrchestrator( + deployment_spec=DeploymentSpec(namespace="default", nodes=[]), + deployment_id="delete-lock-test", + session=session, + context=mock_deployment_context, + dry_run=dry_run, + ) + + await orch._validate_node_deletion( + [TransformSpec(name="target", query="SELECT 1")], + ) + + stmt = session.execute.await_args.args[0] + assert (stmt._for_update_arg is not None) is expects_lock + + @pytest.mark.asyncio async def test_delete_nodes_reports_referenced_and_missing( session, @@ -2686,6 +2871,10 @@ async def test_delete_nodes_reports_referenced_and_missing( referenced.rendered_name = "default.referenced" absent = Mock() absent.rendered_name = "default.does_not_exist" + orch._current_semantic_fingerprints = { + "default.referenced": SemanticFingerprint(digest="b" * 64), + "default.does_not_exist": SemanticFingerprint(digest="c" * 64), + } with patch.object( orch, @@ -2699,6 +2888,8 @@ async def test_delete_nodes_reports_referenced_and_missing( assert "referenced by" in by_name["default.referenced"].message assert by_name["default.does_not_exist"].status == DeploymentResult.Status.FAILED assert "not found" in by_name["default.does_not_exist"].message + assert all(result.change_tier == "major" for result in results) + assert by_name["default.referenced"].semantic_fingerprint.digest == "b" * 64 class TestGenerateChangelog: @@ -2720,7 +2911,7 @@ async def test_no_changed_fields_with_dimension_links( transform_spec = TransformSpec( name="test_node", namespace="default", - query="SELECT id FROM default.source_table", + query=" SELECT id\nFROM default.source_table ", dimension_links=[dim_link], ) # existing_spec is identical — diff() will return [] @@ -2763,6 +2954,80 @@ async def test_no_changed_fields_with_dimension_links( assert changelog == ["└─ Updated dimension_links"] assert change_tier == ChangeTier.NONE + @pytest.mark.asyncio + async def test_source_column_type_change_is_major(self): + existing_spec = SourceSpec( + name="source", + namespace="test", + catalog="catalog", + schema_="schema", + table="table", + columns=[ColumnSpec(name="id", type="bigint")], + ) + proposed = existing_spec.model_copy(deep=True) + proposed.columns[0].type = "string" + existing = MagicMock() + existing.current.columns = [] + existing.to_spec = AsyncMock(return_value=existing_spec) + orchestrator = DeploymentOrchestrator( + deployment_spec=DeploymentSpec(namespace="test", nodes=[]), + deployment_id="source-changelog", + session=MagicMock(), + context=MagicMock(), + ) + orchestrator.registry.nodes["test.source"] = existing + result = NodeValidationResult( + spec=proposed, + status=NodeStatus.VALID, + inferred_columns=proposed.columns, + errors=[], + dependencies=[], + ) + + changelog, changed_fields, tier = await orchestrator._generate_changelog( + result, + ) + + assert changed_fields == ["columns"] + assert tier == ChangeTier.MAJOR + assert changelog[-1] == "└─ Updated columns" + + @pytest.mark.asyncio + async def test_source_without_declared_columns_uses_inferred_columns(self): + source_spec = SourceSpec( + name="source", + namespace="test", + catalog="catalog", + schema_="schema", + table="table", + columns=[], + ) + existing = MagicMock() + existing.current.columns = [] + existing.to_spec = AsyncMock(return_value=source_spec) + orchestrator = DeploymentOrchestrator( + deployment_spec=DeploymentSpec(namespace="test", nodes=[]), + deployment_id="source-inferred-columns", + session=MagicMock(), + context=MagicMock(), + ) + orchestrator.registry.nodes["test.source"] = existing + result = NodeValidationResult( + spec=source_spec, + status=NodeStatus.VALID, + inferred_columns=[ColumnSpec(name="id", type="bigint")], + errors=[], + dependencies=[], + ) + + changelog, changed_fields, tier = await orchestrator._generate_changelog( + result, + ) + + assert changed_fields == [] + assert tier == ChangeTier.NONE + assert changelog == [] + @pytest.mark.asyncio async def test_cube_column_change_uses_role_qualified_identity( self, diff --git a/datajunction-server/tests/internal/deployment/test_dimension_reachability.py b/datajunction-server/tests/internal/deployment/test_dimension_reachability.py index 3522c708ad..49067f3213 100644 --- a/datajunction-server/tests/internal/deployment/test_dimension_reachability.py +++ b/datajunction-server/tests/internal/deployment/test_dimension_reachability.py @@ -10,6 +10,9 @@ DimensionReachability, find_reference_dimensions_batch, ) +from datajunction_server.internal.deployment.utils import ( + extract_dimension_refs_from_filters, +) class TestDimensionReachabilityInMemory: @@ -271,57 +274,37 @@ async def test_build_with_local_names_no_targets(self): class TestExtractDimensionRefsFromFilters: - """Tests for _extract_dimension_refs_from_filters.""" + """Tests for extract_dimension_refs_from_filters.""" def test_single_filter(self): - from datajunction_server.internal.deployment.orchestrator import ( - _extract_dimension_refs_from_filters, - ) - - result = _extract_dimension_refs_from_filters( + result = extract_dimension_refs_from_filters( ["ns.hard_hat.state = 'CA'"], ) assert result == [("ns.hard_hat", "state")] def test_multiple_filters(self): - from datajunction_server.internal.deployment.orchestrator import ( - _extract_dimension_refs_from_filters, - ) - - result = _extract_dimension_refs_from_filters( + result = extract_dimension_refs_from_filters( ["ns.hard_hat.state = 'CA'", "ns.date_dim.year > 2020"], ) assert sorted(result) == [("ns.date_dim", "year"), ("ns.hard_hat", "state")] def test_empty_filters(self): - from datajunction_server.internal.deployment.orchestrator import ( - _extract_dimension_refs_from_filters, - ) - - assert _extract_dimension_refs_from_filters([]) == [] + assert extract_dimension_refs_from_filters([]) == [] def test_unparseable_filter(self): - from datajunction_server.internal.deployment.orchestrator import ( - _extract_dimension_refs_from_filters, - ) - - result = _extract_dimension_refs_from_filters(["not valid sql !!!"]) + result = extract_dimension_refs_from_filters(["not valid sql !!!"]) assert result == [] def test_filter_with_no_namespace(self): - from datajunction_server.internal.deployment.orchestrator import ( - _extract_dimension_refs_from_filters, - ) + result = extract_dimension_refs_from_filters(["x > 5"]) + assert result == [] - result = _extract_dimension_refs_from_filters(["x > 5"]) + def test_filter_with_single_namespace_segment(self): + result = extract_dimension_refs_from_filters(["hard_hat.state = 'CA'"]) assert result == [] def test_filter_with_multiple_refs_in_one_expression(self): - from datajunction_server.internal.deployment.orchestrator import ( - _extract_dimension_refs_from_filters, - ) - - result = _extract_dimension_refs_from_filters( + result = extract_dimension_refs_from_filters( ["ns.dim_a.col1 > 5 AND ns.dim_b.col2 = 'x'"], ) assert sorted(result) == [("ns.dim_a", "col1"), ("ns.dim_b", "col2")] diff --git a/datajunction-server/tests/internal/deployment/test_type_inference.py b/datajunction-server/tests/internal/deployment/test_type_inference.py index 72d050b30d..6676d207dd 100644 --- a/datajunction-server/tests/internal/deployment/test_type_inference.py +++ b/datajunction-server/tests/internal/deployment/test_type_inference.py @@ -2491,6 +2491,18 @@ def test_multi_segment_dim_attribute_ref_does_not_trigger_namespace_error(self): class TestCoverageGaps: """Tests targeting specific uncovered branches in type_inference.py.""" + def test_unresolved_references_are_deduplicated(self): + result = validate_node_query( + "SELECT missing + missing AS doubled FROM default.orders", + _col_map(ORDERS_COLS), + ) + + assert any( + "Unresolved column reference(s): missing." in error + for error in result.errors + ) + assert not any("missing, missing" in error for error in result.errors) + def test_inline_table_parens_with_alias_list_resolves_columns(self): """(VALUES (1), (2)) AS t(x) — parenthesized-VALUES + alias list currently parses with both node.alias and node.name None, so the diff --git a/datajunction-server/tests/internal/deployment/validation_test.py b/datajunction-server/tests/internal/deployment/validation_test.py index c252035a57..ebc6cda33f 100644 --- a/datajunction-server/tests/internal/deployment/validation_test.py +++ b/datajunction-server/tests/internal/deployment/validation_test.py @@ -253,6 +253,37 @@ async def test_validate_query_node_successful_path_for_comparison( or "No columns could be inferred" in error.message ) + @pytest.mark.asyncio + async def test_validate_query_node_flags_unmatched_declared_column( + self, + validation_context: ValidationContext, + ): + """A declared column that doesn't match any query output column is invalid. + + Otherwise its metadata (display_name/description) is silently dropped + and the node churns a version bump on every redeploy. + """ + spec = TransformSpec( + name="transform", + query="SELECT id, name FROM test.parent", + description="A test transform", + mode="published", + columns=[ + ColumnSpec(name="id"), + ColumnSpec(name="full_name", display_name="Full Name"), + ], + ) + validator = NodeSpecBulkValidator(validation_context) + result = validator.validate_query_node(spec) + + assert result.status == NodeStatus.INVALID + error_codes = [e.code for e in result.errors] + assert ErrorCode.INVALID_COLUMN in error_codes + message = next( + e.message for e in result.errors if e.code == ErrorCode.INVALID_COLUMN + ) + assert "full_name" in message + @pytest.mark.asyncio async def test_validate_query_node_flags_hardcoded_namespace( self, diff --git a/datajunction-server/tests/internal/deployment_test.py b/datajunction-server/tests/internal/deployment_test.py index 2d8eae590b..5c5af1b2a0 100644 --- a/datajunction-server/tests/internal/deployment_test.py +++ b/datajunction-server/tests/internal/deployment_test.py @@ -40,6 +40,7 @@ NodeType, ) from datajunction_server.models.node_type import NodeType +from datajunction_server.sql.parsing.backends.exceptions import DJParseException from datajunction_server.sql.parsing.types import IntegerType, StringType @@ -96,6 +97,13 @@ def test_extract_node_graph(basic_nodes): } +def test_extract_node_graph_rejects_unparseable_queries(): + invalid = TransformSpec(name="lunch.mystery_meat", query="SELECT (") + + with pytest.raises(DJParseException): + extract_node_graph([invalid]) + + def test_graph_complex(): # Base source nodes clicks = SourceSpec( @@ -508,6 +516,7 @@ async def test_delete_nodes_success( status=DeploymentResult.Status.SUCCESS, operation=DeploymentResult.Operation.DELETE, message="Node catalog.dim.categories has been removed.", + change_tier="major", ), ] assert await Node.get_by_name(session, categories.name) is None @@ -534,6 +543,7 @@ async def test_delete_nodes_missing( status=DeploymentResult.Status.FAILED, operation=DeploymentResult.Operation.DELETE, message="Node catalog.dim.categoriesbogus not found.", + change_tier="major", ), ] diff --git a/datajunction-server/tests/internal/impact_test.py b/datajunction-server/tests/internal/impact_test.py index b9921701e1..3d7f8045c5 100644 --- a/datajunction-server/tests/internal/impact_test.py +++ b/datajunction-server/tests/internal/impact_test.py @@ -12,9 +12,19 @@ from datajunction_server.database.column import Column as DBColumn from datajunction_server.database.namespace import NodeNamespace -from datajunction_server.database.node import Node, NodeRelationship, NodeRevision +from datajunction_server.database.node import ( + BoundDimensionsRelationship, + Node, + NodeRelationship, + NodeRevision, +) from datajunction_server.database.user import User -from datajunction_server.internal.impact import _merge_impacts, propagate_impact +from datajunction_server.internal.impact import ( + _build_propagation_context, + _merge_impacts, + _propagate_via_parent_graph, + propagate_impact, +) from datajunction_server.models.impact import DownstreamImpact, ImpactType from datajunction_server.models.node import NodeStatus, NodeType from datajunction_server.models.user import OAuthProvider @@ -141,6 +151,63 @@ async def test_propagate_impact_valid_parent_may_affect(session, current_user: U assert child_rev.status == NodeStatus.VALID +@pytest.mark.asyncio +async def test_required_dimension_on_older_revision_is_discovered( + session, + current_user: User, +): + session.add(NodeNamespace(namespace="ns")) + dimension, current_dimension_rev = _make_node( + "ns.dimension", + NodeType.DIMENSION, + NodeStatus.VALID, + current_user.id, + version="v2.0", + columns=[("id", IntegerType())], + ) + older_dimension_rev = NodeRevision( + name=dimension.name, + type=NodeType.DIMENSION, + node=dimension, + version="v1.0", + status=NodeStatus.VALID, + query="SELECT 1 AS id", + created_by_id=current_user.id, + columns=[DBColumn(name="id", type=IntegerType())], + ) + metric, metric_rev = _make_node( + "ns.metric", + NodeType.METRIC, + NodeStatus.VALID, + current_user.id, + ) + await _persist( + session, + dimension, + older_dimension_rev, + current_dimension_rev, + metric, + metric_rev, + ) + await _persist( + session, + BoundDimensionsRelationship( + metric_id=metric_rev.id, + bound_dimension_id=older_dimension_rev.columns[0].id, + ), + ) + + ctx = await _build_propagation_context( + session, + "ns", + {dimension.name}, + frozenset(), + ) + impacts = await _propagate_via_parent_graph(session, ctx) + + assert [impact.name for impact in impacts] == [metric.name] + + @pytest.mark.asyncio async def test_propagate_impact_invalid_parent_will_invalidate( session, diff --git a/datajunction-server/tests/internal/materializations_test.py b/datajunction-server/tests/internal/materializations_test.py index a0ab11e38e..45d21aabf7 100644 --- a/datajunction-server/tests/internal/materializations_test.py +++ b/datajunction-server/tests/internal/materializations_test.py @@ -283,6 +283,7 @@ def test_stop_materialization_workflows_reports_every_failure(): def _swap( rebuilt_names: list[str], backfill: CoverageBackfill | None = None, + is_branch_deploy: bool = False, ) -> CubeMaterializationSwap: """A swap with one superseded materialization and the given rebuilt names.""" return CubeMaterializationSwap( @@ -298,6 +299,7 @@ def _swap( ), ], backfill=backfill, + is_branch_deploy=is_branch_deploy, ) @@ -334,6 +336,7 @@ async def test_apply_cube_swap_reports_a_scheduled_push(): materialization_names=["druid_cube_v3"], query_service_client=query_service_client, request_headers={"cookie": "a-cookie"}, + is_branch_deploy=False, ), ] assert query_service_client.deactivate_workflows.call_args_list == [ @@ -344,6 +347,37 @@ async def test_apply_cube_swap_reports_a_scheduled_push(): ] +@pytest.mark.asyncio +async def test_apply_cube_swap_threads_branch_deploy_through_to_scheduling(): + """ + A swap computed on a branch-preview deploy tells the query service so, rather + than leaving it to assume main. + """ + query_service_client = mock.MagicMock() + session = mock.MagicMock() + + with mock.patch( + "datajunction_server.internal.materializations.schedule_materialization_jobs", + new=mock.AsyncMock(), + ) as schedule: + await apply_cube_materialization_swap( + session, + _swap(["druid_cube_v3"], is_branch_deploy=True), + query_service_client, + ) + + assert schedule.call_args_list == [ + mock.call( + session, + node_revision_id=42, + materialization_names=["druid_cube_v3"], + query_service_client=query_service_client, + request_headers=None, + is_branch_deploy=True, + ), + ] + + @pytest.mark.asyncio async def test_apply_cube_swap_reports_a_rejected_push(): """ diff --git a/datajunction-server/tests/internal/nodes/propagate_to_cubes_test.py b/datajunction-server/tests/internal/nodes/propagate_to_cubes_test.py new file mode 100644 index 0000000000..6783203b8a --- /dev/null +++ b/datajunction-server/tests/internal/nodes/propagate_to_cubes_test.py @@ -0,0 +1,446 @@ +""" +Propagation of an upstream change into downstream cubes. + +A cube used to be excluded from downstream propagation entirely, so an upstream +edit left it serving a materialized table built against a definition that no +longer existed. The only way to recompile it was a no-op edit to the cube itself. +These tests pin that an upstream change now bumps the cube, at the upstream's own +tier, whether or not the cube's shape moved. +""" + +from unittest.mock import patch + +import pytest +import pytest_asyncio +from httpx import AsyncClient +from sqlalchemy import select, text +from sqlalchemy.ext.asyncio import AsyncSession + +import datajunction_server.internal.nodes as nodes_module +from datajunction_server.database.node import Node +from datajunction_server.database.user import User +from datajunction_server.internal.nodes import _propagate_update_downstream +from datajunction_server.models.deployment import ChangeTier, version_change_tier + +# The upstream transform, reduced to the three columns these tests care about so a +# change to it is easy to read. `price` and `where` are the two things they vary: +# `price` moves a column type the cube can see, `where` moves only which rows the +# cube counts. +FACT_QUERY = """SELECT + repair_orders.repair_order_id, + repair_orders.hard_hat_id, + {price} AS price +FROM + default.repair_orders repair_orders +JOIN + default.repair_order_details repair_order_details +ON repair_orders.repair_order_id = repair_order_details.repair_order_id +{where}""" + +FACT = "default.repair_orders_fact" +CUBE = "default.repairs_by_state" +UNRELATED_CUBE = "default.employment_by_state" + + +async def _patch_fact(client: AsyncClient, price: str, where: str = "") -> str: + """Rewrite the upstream transform's query and return its new version.""" + response = await client.patch( + f"/nodes/{FACT}", + json={"query": FACT_QUERY.format(price=price, where=where)}, + ) + assert response.status_code == 200, response.text + return response.json()["version"] + + +async def _version(client: AsyncClient, name: str) -> str: + response = await client.get(f"/nodes/{name}") + assert response.status_code == 200, response.text + return response.json()["version"] + + +async def _columns(client: AsyncClient, name: str) -> list[tuple[str, str]]: + response = await client.get(f"/nodes/{name}") + assert response.status_code == 200, response.text + return [(col["name"], col["type"]) for col in response.json()["columns"]] + + +async def _state(client: AsyncClient, name: str) -> tuple[str, str]: + """The node's version and status, which move together on a breaking change.""" + response = await client.get(f"/nodes/{name}") + assert response.status_code == 200, response.text + body = response.json() + return body["version"], body["status"] + + +@pytest_asyncio.fixture +async def client_with_cube_downstream(client_with_roads: AsyncClient) -> AsyncClient: + """ + Roads, narrowed to a transform -> metric -> cube chain the tests can move. + + A second cube sits on a metric and a dimension that are not downstream of the + transform at all, so "everything bumped" and "the right thing bumped" are + distinguishable. + """ + await _patch_fact(client_with_roads, price="repair_order_details.price") + response = await client_with_roads.post( + "/nodes/metric/", + json={ + "name": "default.total_price", + "description": "Total price", + "mode": "published", + "query": f"SELECT sum(price) FROM {FACT}", + }, + ) + assert response.status_code == 201, response.text + response = await client_with_roads.post( + "/nodes/cube/", + json={ + "name": CUBE, + "metrics": ["default.total_price"], + "dimensions": ["default.hard_hat.state"], + "description": "Repairs by state", + "mode": "published", + }, + ) + assert response.status_code == 201, response.text + response = await client_with_roads.post( + "/nodes/cube/", + json={ + "name": UNRELATED_CUBE, + "metrics": ["default.avg_length_of_employment"], + "dimensions": ["default.hard_hat.state"], + "description": "Employment by state", + "mode": "published", + }, + ) + assert response.status_code == 201, response.text + return client_with_roads + + +@pytest.mark.asyncio +async def test_upstream_column_change_bumps_and_recompiles_cube( + client_with_cube_downstream: AsyncClient, +): + """ + The reported bug: an upstream change that materially changes the cube left the + cube's version alone, so it kept serving a table built against the old upstream. + """ + client = client_with_cube_downstream + assert await _version(client, CUBE) == "v1.0" + assert await _columns(client, CUBE) == [ + ("default.total_price", "double"), + ("default.hard_hat.state", "string"), + ] + + assert ( + await _patch_fact(client, price="CAST(repair_order_details.price AS int)") + == "v3.0" + ) + + assert await _version(client, CUBE) == "v2.0" + # The cube's columns were re-resolved against the new upstream revision, not + # copied forward: sum(int) is a bigint where sum(double) was a double. + assert await _columns(client, CUBE) == [ + ("default.total_price", "bigint"), + ("default.hard_hat.state", "string"), + ] + + +@pytest.mark.asyncio +async def test_upstream_filter_change_bumps_cube_with_unchanged_shape( + client_with_cube_downstream: AsyncClient, +): + """ + An upstream filter change moves no metric, no dimension, no column type and no + metric component identity -- `is_non_trivial_cube_change` returns False for it -- + yet every row in the cube's materialized table was computed under the old + filter. The bump must not be gated on the cube's own shape. + """ + client = client_with_cube_downstream + columns_before = await _columns(client, CUBE) + + assert ( + await _patch_fact( + client, + price="repair_order_details.price", + where="WHERE repair_order_details.discount > 0.0", + ) + == "v3.0" + ) + + assert await _version(client, CUBE) == "v2.0" + assert await _columns(client, CUBE) == columns_before + assert columns_before == [ + ("default.total_price", "double"), + ("default.hard_hat.state", "string"), + ] + + +@pytest.mark.asyncio +async def test_upstream_minor_change_bumps_cube_minor( + client_with_cube_downstream: AsyncClient, +): + """A cube inherits the upstream's tier, so a minor upstream edit is minor here.""" + client = client_with_cube_downstream + + response = await client.patch( + f"/nodes/{FACT}", + json={"description": "Fact transform, redescribed"}, + ) + assert response.status_code == 200, response.text + assert response.json()["version"] == "v2.1" + + assert await _version(client, CUBE) == "v1.1" + + +@pytest.mark.asyncio +async def test_cube_not_downstream_is_untouched( + client_with_cube_downstream: AsyncClient, +): + """Propagation reaches the cubes below the changed node and no others.""" + client = client_with_cube_downstream + + await _patch_fact(client, price="CAST(repair_order_details.price AS int)") + + assert await _version(client, CUBE) == "v2.0" + assert await _version(client, UNRELATED_CUBE) == "v1.0" + + +@pytest.mark.asyncio +async def test_cube_failure_does_not_abort_remaining_downstreams( + client_with_roads: AsyncClient, +): + """ + Bumping a cube can trigger a materialization rebuild, so one cube that cannot be + rebuilt must not cost the remaining downstreams their propagation. + """ + await _patch_fact(client_with_roads, price="repair_order_details.price") + response = await client_with_roads.post( + "/nodes/metric/", + json={ + "name": "default.total_price", + "description": "Total price", + "mode": "published", + "query": f"SELECT sum(price) FROM {FACT}", + }, + ) + assert response.status_code == 201, response.text + for name in ("default.cube_one", "default.cube_two"): + response = await client_with_roads.post( + "/nodes/cube/", + json={ + "name": name, + "metrics": ["default.total_price"], + "dimensions": ["default.hard_hat.state"], + "description": "A cube", + "mode": "published", + }, + ) + assert response.status_code == 201, response.text + + real_save = nodes_module.save_new_cube_revision + calls: list[str] = [] + + async def fail_first(session, node_revision, *args, **kwargs): + calls.append(node_revision.name) + if len(calls) == 1: + raise RuntimeError("cannot rebuild this cube") + return await real_save(session, node_revision, *args, **kwargs) + + with patch.object(nodes_module, "save_new_cube_revision", fail_first): + await _patch_fact( + client_with_roads, + price="CAST(repair_order_details.price AS int)", + ) + + assert len(calls) == 2 + versions = sorted( + [ + await _version(client_with_roads, "default.cube_one"), + await _version(client_with_roads, "default.cube_two"), + ], + ) + # The cube whose rebuild raised keeps its version; the one after it in the + # propagation order is still bumped. + assert versions == ["v1.0", "v2.0"] + + +@pytest.mark.asyncio +async def test_a_cube_failing_after_one_succeeded_keeps_the_success( + client_with_roads: AsyncClient, +): + """ + The rollback that recovers from a failed rebuild must not undo an earlier one. + + Failing the *first* cube says nothing about this: there is no committed work for + the rollback to reach. `save_new_cube_revision` commits per cube, so the bump + before the failure is already durable -- this pins that, because the recovery + path rolls the session back and a single transaction spanning the walk would + silently lose the earlier cube's revision. + """ + await _patch_fact(client_with_roads, price="repair_order_details.price") + response = await client_with_roads.post( + "/nodes/metric/", + json={ + "name": "default.total_price", + "description": "Total price", + "mode": "published", + "query": f"SELECT sum(price) FROM {FACT}", + }, + ) + assert response.status_code == 201, response.text + for name in ("default.cube_one", "default.cube_two"): + response = await client_with_roads.post( + "/nodes/cube/", + json={ + "name": name, + "metrics": ["default.total_price"], + "dimensions": ["default.hard_hat.state"], + "description": "A cube", + "mode": "published", + }, + ) + assert response.status_code == 201, response.text + + real_save = nodes_module.save_new_cube_revision + calls: list[str] = [] + + async def fail_second(session, node_revision, *args, **kwargs): + calls.append(node_revision.name) + if len(calls) == 2: + raise RuntimeError("cannot rebuild this cube") + return await real_save(session, node_revision, *args, **kwargs) + + with patch.object(nodes_module, "save_new_cube_revision", fail_second): + await _patch_fact( + client_with_roads, + price="CAST(repair_order_details.price AS int)", + ) + + assert len(calls) == 2 + versions = sorted( + [ + await _version(client_with_roads, "default.cube_one"), + await _version(client_with_roads, "default.cube_two"), + ], + ) + # The first cube's bump survived the rollback that the second one triggered. + assert versions == ["v1.0", "v2.0"] + + +@pytest.mark.asyncio +async def test_no_tier_change_leaves_cube_alone( + client_with_cube_downstream: AsyncClient, + session: AsyncSession, +): + """ + A propagation carrying no tier invents no cube revision. `bump_version` would + hand back the version the cube already has, and a second revision at the same + version is not a thing a cube can have. + """ + client = client_with_cube_downstream + user = ( + await session.execute(select(User).where(User.username == "dj")) + ).scalar_one() + fact = await Node.get_by_name(session, FACT, raise_if_not_exists=True) + assert fact is not None + + async def discard_history(event, session): # noqa: ARG001 + return None + + await _propagate_update_downstream( + session=session, + node=fact, + current_user=user, + save_history=discard_history, + change_tier=ChangeTier.NONE, + ) + + assert await _version(client, CUBE) == "v1.0" + + +@pytest.mark.asyncio +async def test_column_order_backfill_upstream_does_not_rebuild_the_cube( + client_with_cube_downstream: AsyncClient, + session: AsyncSession, +): + """ + The one upstream change that moves nothing at all. + + A stored column with no `order` is DJ's own bookkeeping. Revalidating the + upstream fills it in against the current revision, so the upstream does not turn + over -- and a node that does not turn over has nothing to propagate, which is + what leaves the cube alone. Before the backfill was reclassified this earned the + upstream a major bump, and every consumer of a bump would have followed. + + Pinned on the whole downstream chain rather than the cube alone, since the + guarantee is "no revision, so nothing moved", not something specific to cubes. + """ + client = client_with_cube_downstream + await session.execute( + text( + """ + UPDATE "column" SET "order" = NULL + WHERE node_revision_id IN ( + SELECT id FROM noderevision WHERE name = :name + ) + """, + ), + {"name": FACT}, + ) + await session.commit() + session.expire_all() + + response = await client.post(f"/nodes/{FACT}/validate/") + assert response.status_code == 200, response.text + assert response.json()["status"] == "valid" + + assert [ + (name, await _version(client, name)) + for name in (FACT, "default.total_price", CUBE, UNRELATED_CUBE) + ] == [ + (FACT, "v2.0"), + ("default.total_price", "v1.0"), + (CUBE, "v1.0"), + (UNRELATED_CUBE, "v1.0"), + ] + + +def test_version_change_tier(): + """`bump_version` read backwards, including the no-change case.""" + assert version_change_tier("v1.0", "v2.0") == ChangeTier.MAJOR + assert version_change_tier("v1.3", "v2.0") == ChangeTier.MAJOR + assert version_change_tier("v1.0", "v1.1") == ChangeTier.MINOR + assert version_change_tier("v1.0", "v1.0") == ChangeTier.NONE + + +@pytest.mark.asyncio +async def test_a_removed_column_a_metric_uses_invalidates_it_and_the_cube( + client_with_cube_downstream: AsyncClient, +): + """ + Dropping a column the metric aggregates carries all the way to the cube. + + The transform stays valid -- its own query is fine -- while the metric can no + longer infer a type for `sum(price)` and the cube built on that metric follows + it down. Each bumps, so nothing is left quietly serving a definition that no + longer resolves. Pinned because the failure is only visible downstream: the + edit that causes it looks entirely successful at the node being edited. + """ + client = client_with_cube_downstream + response = await client.patch( + f"/nodes/{FACT}", + json={ + "query": ( + "SELECT repair_orders.repair_order_id, repair_orders.hard_hat_id " + "FROM default.repair_orders repair_orders" + ), + }, + ) + assert response.status_code == 200, response.text + + assert await _state(client, FACT) == ("v3.0", "valid") + assert await _state(client, "default.total_price") == ("v2.0", "invalid") + assert await _state(client, CUBE) == ("v2.0", "invalid") + # The cube that shares only a dimension with the edited transform is untouched. + assert await _state(client, UNRELATED_CUBE) == ("v1.0", "valid") diff --git a/datajunction-server/tests/internal/nodes/revalidate_column_tier_test.py b/datajunction-server/tests/internal/nodes/revalidate_column_tier_test.py new file mode 100644 index 0000000000..699c80d6d3 --- /dev/null +++ b/datajunction-server/tests/internal/nodes/revalidate_column_tier_test.py @@ -0,0 +1,289 @@ +""" +The version tier `revalidate_node` gives a node whose columns moved. + +Any column difference used to earn a major bump. That is wrong for the additive +cases -- nothing written before a column existed can be referencing it -- and it +matters more now that downstream cubes inherit their upstream's tier, since a +major bump rebuilds a cube's materialized table. A type change stays major: +everything reading the column is reading a different type than it was written +against, and so is a removal, which breaks anything that referenced it. + +A missing column `order` is not a change at all: it is DJ's own bookkeeping, +backfilled in place with no new revision. + +The column differences are induced by editing the stored columns directly, the +same trick the existing revalidate tests use, because they are differences +between what is *stored* on the revision and what the validator recomputes -- +which no API payload can produce on its own. +""" + +import pytest +from httpx import AsyncClient +from sqlalchemy import text +from sqlalchemy.ext.asyncio import AsyncSession + +NODE = "default.tier_node" +COLUMNS = [("repair_order_id", "int"), ("price", "float")] + + +async def _create_node(client: AsyncClient) -> None: + response = await client.post( + "/nodes/transform/", + json={ + "name": NODE, + "description": "A transform whose columns the tests move", + "mode": "published", + "query": "SELECT repair_order_id, price FROM default.repair_order_details", + }, + ) + assert response.status_code == 201, response.text + assert response.json()["version"] == "v1.0" + assert [(col["name"], col["type"]) for col in response.json()["columns"]] == COLUMNS + + +async def _revalidate(client: AsyncClient) -> None: + response = await client.post(f"/nodes/{NODE}/validate/") + assert response.status_code == 200, response.text + assert response.json()["status"] == "valid" + + +async def _node(client: AsyncClient) -> tuple[str, list[tuple[str, str]]]: + response = await client.get(f"/nodes/{NODE}") + assert response.status_code == 200, response.text + return ( + response.json()["version"], + [(col["name"], col["type"]) for col in response.json()["columns"]], + ) + + +async def _stored_order(session: AsyncSession) -> list[tuple[str, int | None]]: + """(name, order) for the current revision's stored columns, by name.""" + session.expire_all() + rows = await session.execute( + text( + """ + SELECT c.name, c."order" + FROM "column" c + JOIN noderevision nr ON nr.id = c.node_revision_id + JOIN node n ON n.id = nr.node_id AND n.current_version = nr.version + WHERE nr.name = :name + ORDER BY c.name + """, + ), + {"name": NODE}, + ) + return [(name, order) for name, order in rows.all()] + + +async def _revalidate_events(client: AsyncClient) -> list[dict]: + response = await client.get(f"/history?node={NODE}") + assert response.status_code == 200, response.text + return [ + entry["details"] + for entry in response.json() + if entry["activity_type"] == "update" + ] + + +async def _edit_stored_columns( + session: AsyncSession, + statement: str, + column: str = "price", +) -> None: + await session.execute( + text( + f""" + {statement} + WHERE name = :column AND node_revision_id IN ( + SELECT id FROM noderevision WHERE name = :name + ) + """, + ), + {"name": NODE, "column": column}, + ) + await session.commit() + session.expire_all() + + +async def _clear_all_orders(session: AsyncSession) -> None: + """What a revision written before the `order` field existed looks like.""" + await session.execute( + text( + """ + UPDATE "column" SET "order" = NULL + WHERE node_revision_id IN ( + SELECT id FROM noderevision WHERE name = :name + ) + """, + ), + {"name": NODE}, + ) + await session.commit() + session.expire_all() + + +@pytest.mark.asyncio +async def test_added_column_is_a_major_bump( + client_with_roads: AsyncClient, + session: AsyncSession, +): + """ + A column the revision does not have yet still came from a query the node did + not have before, and the cube rebuild below skips only NONE, so demoting the + addition would not spare any downstream work. It earns v2.0. + """ + await _create_node(client_with_roads) + await _edit_stored_columns(session, 'DELETE FROM "column"') + + await _revalidate(client_with_roads) + + assert await _node(client_with_roads) == ("v2.0", COLUMNS) + + +@pytest.mark.asyncio +async def test_column_type_change_is_a_major_bump( + client_with_roads: AsyncClient, + session: AsyncSession, +): + """A column whose type moved breaks everything reading it, so it earns v2.0.""" + await _create_node(client_with_roads) + await _edit_stored_columns(session, "UPDATE \"column\" SET type = 'int'") + + await _revalidate(client_with_roads) + + assert await _node(client_with_roads) == ("v2.0", COLUMNS) + + +@pytest.mark.asyncio +async def test_removed_column_is_a_major_bump( + client_with_roads: AsyncClient, + session: AsyncSession, +): + """ + A column the query no longer produces is breaking for anything that referenced + it, and must not survive onto the new revision advertising a value the node + cannot supply. + """ + await _create_node(client_with_roads) + # A column stored on the revision that the query does not produce -- what a + # node whose query stopped selecting a field looks like from the validator's + # side. Cloned from an existing row so every non-null column is populated. + await session.execute( + text( + """ + INSERT INTO "column" (name, type, node_revision_id, "order") + SELECT 'ghost', c.type, c.node_revision_id, 99 + FROM "column" c + JOIN noderevision nr ON nr.id = c.node_revision_id + WHERE nr.name = :name AND c.name = 'price' + """, + ), + {"name": NODE}, + ) + await session.commit() + session.expire_all() + + await _revalidate(client_with_roads) + + assert await _node(client_with_roads) == ("v2.0", COLUMNS) + + +@pytest.mark.asyncio +async def test_backfilled_column_order_creates_no_revision( + client_with_roads: AsyncClient, + session: AsyncSession, +): + """ + A missing `order` is DJ filling in metadata it should already have stored. No + query changed and no name, type or value moved, so there is no new revision -- + and so nothing downstream is rebuilt for it. The value is written to the + current revision in place. + """ + await _create_node(client_with_roads) + await _clear_all_orders(session) + assert await _stored_order(session) == [("price", None), ("repair_order_id", None)] + + await _revalidate(client_with_roads) + + assert await _node(client_with_roads) == ("v1.0", COLUMNS) + assert await _stored_order(session) == [("price", 1), ("repair_order_id", 0)] + + +@pytest.mark.asyncio +async def test_backfilled_column_order_still_writes_a_history_event( + client_with_roads: AsyncClient, + session: AsyncSession, +): + """ + Losing the version bump must not mean losing the record that DJ touched the + row: a column that changes position needs an explanation somewhere. + """ + await _create_node(client_with_roads) + await _clear_all_orders(session) + + await _revalidate(client_with_roads) + + assert await _revalidate_events(client_with_roads) == [ + { + "version": "v1.0", + "reason": "column order backfill", + "order_fixed": ["repair_order_id", "price"], + }, + ] + + +@pytest.mark.asyncio +async def test_backfill_rides_along_on_a_revision_earned_elsewhere( + client_with_roads: AsyncClient, + session: AsyncSession, +): + """ + A legacy revision with no `order` that also earns a bump for a real reason. + + The backfill no longer creates a revision of its own, but a revision created + for a type change deep-copies the unordered columns along with everything + else. Filling the index in on the way through is what keeps the new revision + from inheriting the gap and carrying it forward forever -- the copy is the + only chance to fix it, since the next revalidate will find nothing changed. + """ + await _create_node(client_with_roads) + await _edit_stored_columns(session, "UPDATE \"column\" SET type = 'int'") + await _clear_all_orders(session) + assert await _stored_order(session) == [("price", None), ("repair_order_id", None)] + + await _revalidate(client_with_roads) + + # The type change earns the bump; the backfill contributes nothing to the tier. + assert await _node(client_with_roads) == ("v2.0", COLUMNS) + # The new revision's copies come out ordered, not NULL. + assert await _stored_order(session) == [("price", 1), ("repair_order_id", 0)] + # One event, explaining both what earned the bump and what rode along on it. + assert await _revalidate_events(client_with_roads) == [ + { + "version": "v2.0", + "reason": "revalidate", + "type_changes": [{"column": "price", "from": "int", "to": "float"}], + "order_fixed": ["repair_order_id", "price"], + }, + ] + + +@pytest.mark.asyncio +async def test_backfill_leaves_orders_that_are_already_set( + client_with_roads: AsyncClient, + session: AsyncSession, +): + """ + Only the missing values are filled. A partially-ordered revision is the case + where the backfill genuinely moves a column: `price` with no order sorts behind + every ordered column, and filling it in puts it back at the position the query + always projected it in. + """ + await _create_node(client_with_roads) + await _edit_stored_columns(session, 'UPDATE "column" SET "order" = NULL') + assert await _stored_order(session) == [("price", None), ("repair_order_id", 0)] + + await _revalidate(client_with_roads) + + assert await _node(client_with_roads) == ("v1.0", COLUMNS) + assert await _stored_order(session) == [("price", 1), ("repair_order_id", 0)] diff --git a/datajunction-server/tests/internal/test_custom_metadata_deploy.py b/datajunction-server/tests/internal/test_custom_metadata_deploy.py new file mode 100644 index 0000000000..78d79e15f4 --- /dev/null +++ b/datajunction-server/tests/internal/test_custom_metadata_deploy.py @@ -0,0 +1,619 @@ +"""Tests for upsert_schema_specs — namespace-scoped schema registration via deploy.""" + +import datetime + +import pytest +from sqlalchemy import select, text +from unittest.mock import MagicMock + +from datajunction_server.database.custom_metadata_schema import CustomMetadataSchema +from datajunction_server.errors import ( + DJAlreadyExistsException, + DJInvalidInputException, +) +from datajunction_server.internal.custom_metadata import upsert_schema_specs +from datajunction_server.internal.deployment.orchestrator import DeploymentOrchestrator +from datajunction_server.internal.deployment.utils import DeploymentContext +from datajunction_server.models.deployment import ( + CustomMetadataSchemaSpec, + DeploymentSpec, +) +from datajunction_server.models.node_type import NodeType + + +@pytest.mark.asyncio +async def test_deploy_registers_namespace_scoped_schema(session, current_user): + """Insert path: a new schema spec creates a namespace-scoped row.""" + await upsert_schema_specs( + session, + namespace="finance", + specs=[ + CustomMetadataSchemaSpec(key="grain", json_schema={"type": "string"}), + ], + current_user_id=current_user.id, + ) + row = ( + await session.execute( + select(CustomMetadataSchema).where(CustomMetadataSchema.key == "grain"), + ) + ).scalar_one() + assert row.namespace == "finance" + assert row.value_kind == "string" + + +@pytest.mark.asyncio +async def test_deploy_updates_existing_schema(session, current_user): + """Update path: a pre-existing row is updated in-place, not duplicated.""" + # Pre-seed a row for the same key/namespace/node_type=None + session.add( + CustomMetadataSchema( + key="grain", + namespace="finance", + json_schema={"type": "string"}, + value_kind="string", + filterable=True, + ), + ) + await session.commit() + + # Call upsert again with a changed schema (now integer) + await upsert_schema_specs( + session, + namespace="finance", + specs=[ + CustomMetadataSchemaSpec( + key="grain", + json_schema={"type": "integer"}, + filterable=False, + description="updated", + ), + ], + current_user_id=current_user.id, + ) + + rows = ( + ( + await session.execute( + select(CustomMetadataSchema).where( + CustomMetadataSchema.key == "grain", + CustomMetadataSchema.namespace == "finance", + ), + ) + ) + .scalars() + .all() + ) + # Should still be exactly one row (updated, not duplicated) + assert len(rows) == 1 + assert rows[0].value_kind == "integer" + assert rows[0].filterable is False + assert rows[0].description == "updated" + + +@pytest.mark.asyncio +async def test_deploy_node_type_scoped_schema(session, current_user): + """Node-type-scoped spec creates a row with node_type set.""" + await upsert_schema_specs( + session, + namespace="eng", + specs=[ + CustomMetadataSchemaSpec( + key="owner_team", + node_type=NodeType.METRIC, + json_schema={"type": "string"}, + ), + ], + current_user_id=current_user.id, + ) + row = ( + await session.execute( + select(CustomMetadataSchema).where( + CustomMetadataSchema.key == "owner_team", + CustomMetadataSchema.namespace == "eng", + ), + ) + ).scalar_one() + assert row.node_type == NodeType.METRIC.value + assert row.value_kind == "string" + + +@pytest.mark.asyncio +async def test_deploy_empty_specs_on_an_empty_namespace(session, current_user): + """An empty spec list against a namespace with no rows registers nothing. + + `[]` means "this manifest manages schemas and declares none", so it retires + whatever the namespace had -- which here is nothing. + """ + await upsert_schema_specs( + session, + namespace="empty_ns", + specs=[], + current_user_id=current_user.id, + ) + rows = ( + ( + await session.execute( + select(CustomMetadataSchema).where( + CustomMetadataSchema.namespace == "empty_ns", + ), + ) + ) + .scalars() + .all() + ) + assert rows == [] + + +@pytest.mark.asyncio +async def test_upsert_invalid_json_schema_raises(session, current_user): + """upsert_schema_specs raises DJInvalidInputException for an invalid JSON Schema.""" + with pytest.raises(DJInvalidInputException) as exc_info: + await upsert_schema_specs( + session, + namespace="validation_test", + specs=[ + CustomMetadataSchemaSpec( + key="bad_schema_key", + json_schema={"type": "not-a-type"}, + ), + ], + current_user_id=current_user.id, + ) + assert "bad_schema_key" in exc_info.value.message + assert "Invalid JSON Schema" in exc_info.value.message + + +@pytest.mark.asyncio +async def test_upsert_filterable_numeric_builds_index(session, current_user): + """upsert_schema_specs with a filterable numeric spec creates the expression index.""" + unique_key = "deploy_numeric_score_idx_test" + await upsert_schema_specs( + session, + namespace="index_test_ns", + specs=[ + CustomMetadataSchemaSpec( + key=unique_key, + json_schema={"type": "number"}, + filterable=True, + ), + ], + current_user_id=current_user.id, + ) + # Verify the index was created in pg_indexes + result = await session.execute( + text( + "SELECT indexname FROM pg_indexes WHERE indexname = :idx", + ), + {"idx": f"ix_cm_{unique_key}"}, + ) + row = result.fetchone() + assert row is not None, f"Expected index ix_cm_{unique_key} to exist in pg_indexes" + + +@pytest.mark.asyncio +async def test_orchestrator_setup_calls_upsert_schema_specs(session, current_user): + """The deploy wiring: _setup_deployment_resources calls upsert_schema_specs + when custom_metadata_schemas is non-empty, registering the row in the DB.""" + spec = DeploymentSpec( + namespace="wiring_test", + nodes=[], + custom_metadata_schemas=[ + CustomMetadataSchemaSpec(key="region", json_schema={"type": "string"}), + ], + ) + mock_context = MagicMock(spec=DeploymentContext) + mock_context.current_user = current_user + orchestrator = DeploymentOrchestrator( + deployment_spec=spec, + deployment_id="wiring-test-id", + session=session, + context=mock_context, + ) + await orchestrator._setup_deployment_resources() + + row = ( + await session.execute( + select(CustomMetadataSchema).where( + CustomMetadataSchema.key == "region", + CustomMetadataSchema.namespace == "wiring_test", + ), + ) + ).scalar_one() + assert row.value_kind == "string" + + +@pytest.mark.asyncio +async def test_a_retired_key_can_be_registered_again(session, current_user): + """ + Soft-deleting a key must not make its scope permanently unusable. + + The unique index spans deactivated rows while every read hides them, so an + insert beside a tombstone violates the constraint. The upsert revives the + row instead, which also keeps its id and created_at: a key that comes back + is the same registration, not a new one. + """ + spec = CustomMetadataSchemaSpec(key="lifecycle", json_schema={"type": "string"}) + await upsert_schema_specs( + session, + namespace="revive_ns", + specs=[spec], + current_user_id=current_user.id, + ) + original = ( + await session.execute( + select(CustomMetadataSchema).where( + CustomMetadataSchema.key == "lifecycle", + CustomMetadataSchema.namespace == "revive_ns", + ), + ) + ).scalar_one() + original_id, original_created = original.id, original.created_at + + original.deactivated_at = datetime.datetime.now(datetime.UTC) + await session.commit() + + await upsert_schema_specs( + session, + namespace="revive_ns", + specs=[spec], + current_user_id=current_user.id, + ) + revived = ( + ( + await session.execute( + select(CustomMetadataSchema).where( + CustomMetadataSchema.key == "lifecycle", + CustomMetadataSchema.namespace == "revive_ns", + ), + ) + ) + .scalars() + .all() + ) + assert len(revived) == 1 + assert revived[0].id == original_id + assert revived[0].created_at == original_created + assert revived[0].deactivated_at is None + + +@pytest.mark.asyncio +async def test_a_key_the_manifest_drops_is_retired(session, current_user): + """Reconciliation: the manifest is the whole truth for its own namespace.""" + await upsert_schema_specs( + session, + namespace="reconcile_ns", + specs=[ + CustomMetadataSchemaSpec(key="keep", json_schema={"type": "string"}), + CustomMetadataSchemaSpec(key="drop", json_schema={"type": "string"}), + ], + current_user_id=current_user.id, + ) + await upsert_schema_specs( + session, + namespace="reconcile_ns", + specs=[ + CustomMetadataSchemaSpec(key="keep", json_schema={"type": "string"}), + ], + current_user_id=current_user.id, + ) + live = sorted( + row.key + for row in ( + await session.execute( + select(CustomMetadataSchema).where( + CustomMetadataSchema.namespace == "reconcile_ns", + CustomMetadataSchema.deactivated_at.is_(None), + ), + ) + ) + .scalars() + .all() + ) + assert live == ["keep"] + + +@pytest.mark.asyncio +async def test_reconciliation_leaves_global_rows_alone(session, current_user): + """No namespace owns a global key, so no deployment may retire one.""" + session.add( + CustomMetadataSchema( + key="global_key", + namespace=None, + json_schema={"type": "string"}, + value_kind="string", + ), + ) + await session.commit() + + await upsert_schema_specs( + session, + namespace="some_ns", + specs=[], + current_user_id=current_user.id, + ) + row = ( + await session.execute( + select(CustomMetadataSchema).where( + CustomMetadataSchema.key == "global_key", + ), + ) + ).scalar_one() + assert row.deactivated_at is None + + +@pytest.mark.asyncio +async def test_a_deploy_cannot_shadow_a_reserved_global_key(session, current_user): + """ + The API refuses this; the deploy path used to be the way around it. + + Both writers hit the same table, so a check only one of them makes is not a + check at all. + """ + session.add( + CustomMetadataSchema( + key="lifecycle", + namespace=None, + reserved=True, + json_schema={"type": "string", "enum": ["beta"]}, + value_kind="string", + ), + ) + await session.commit() + + with pytest.raises(DJAlreadyExistsException) as exc_info: + await upsert_schema_specs( + session, + namespace="sneaky_ns", + specs=[ + CustomMetadataSchemaSpec( + key="lifecycle", + json_schema={"type": "string"}, + ), + ], + current_user_id=current_user.id, + ) + assert "reserved globally" in exc_info.value.message + + +@pytest.mark.asyncio +async def test_a_deploy_records_who_registered_the_schema(session, current_user): + """Provenance, which the deploy path was not setting at all.""" + await upsert_schema_specs( + session, + namespace="prov_ns", + specs=[ + CustomMetadataSchemaSpec( + key="grain", + json_schema={"type": "string"}, + ), + ], + current_user_id=current_user.id, + ) + row = ( + await session.execute( + select(CustomMetadataSchema).where( + CustomMetadataSchema.namespace == "prov_ns", + ), + ) + ).scalar_one() + assert row.created_by_id == current_user.id + assert row.updated_by_id == current_user.id + assert row.reserved is False + + +@pytest.mark.asyncio +async def test_upsert_leaves_the_transaction_to_its_caller(session, current_user): + """ + Registration must be rollback-able, which is what makes a dry run safe. + + `POST /deployments/impact` runs the whole orchestrator inside a SAVEPOINT + purely to report what a deployment would do, then unwinds it. A commit in + here releases that SAVEPOINT, so analysing a spec would permanently + register its schemas. + """ + await upsert_schema_specs( + session, + namespace="rollback_ns", + specs=[ + CustomMetadataSchemaSpec(key="grain", json_schema={"type": "string"}), + ], + current_user_id=current_user.id, + ) + await session.rollback() + + rows = ( + ( + await session.execute( + select(CustomMetadataSchema).where( + CustomMetadataSchema.namespace == "rollback_ns", + ), + ) + ) + .scalars() + .all() + ) + assert rows == [] + + +@pytest.mark.asyncio +async def test_a_dry_run_builds_no_index(session, current_user): + """ + Index DDL is the one thing a rolled-back SAVEPOINT would do for nothing, + and impact analysis needs no index to report impact. + """ + unique_key = "dry_run_no_index_probe" + await upsert_schema_specs( + session, + namespace="dry_run_ns", + specs=[ + CustomMetadataSchemaSpec( + key=unique_key, + json_schema={"type": "number"}, + filterable=True, + ), + ], + current_user_id=current_user.id, + build_indexes=False, + ) + found = ( + await session.execute( + text("SELECT indexname FROM pg_indexes WHERE indexname = :idx"), + {"idx": f"ix_cm_{unique_key}"}, + ) + ).fetchone() + assert found is None + + +@pytest.mark.asyncio +async def test_an_omitted_section_leaves_schemas_alone(session, current_user): + """ + None and [] are different manifests. + + Every deployment that predates this field sends no section at all. If that + read as "declares none", the next deploy of any existing repo would retire + every schema it had. + """ + session.add( + CustomMetadataSchema( + key="pre_existing", + namespace="untouched_ns", + json_schema={"type": "string"}, + value_kind="string", + ), + ) + await session.commit() + + spec = DeploymentSpec(namespace="untouched_ns", nodes=[]) + assert spec.custom_metadata_schemas is None + + mock_context = MagicMock(spec=DeploymentContext) + mock_context.current_user = current_user + orchestrator = DeploymentOrchestrator( + deployment_spec=spec, + deployment_id="omitted-section-id", + session=session, + context=mock_context, + ) + await orchestrator._setup_deployment_resources() + + row = ( + await session.execute( + select(CustomMetadataSchema).where( + CustomMetadataSchema.key == "pre_existing", + ), + ) + ).scalar_one() + assert row.deactivated_at is None + + +@pytest.mark.asyncio +async def test_a_schema_can_be_scoped_to_a_sub_namespace(session, current_user): + """ + Rolling a vocabulary out to part of a repo's graph before all of it. + + The USG rollout gates conformed dimensions first, which needs a schema scoped + narrower than the namespace being deployed. + """ + await upsert_schema_specs( + session, + namespace="shared", + specs=[ + CustomMetadataSchemaSpec( + key="system", + namespace="shared.conformed", + json_schema={"type": "object"}, + ), + ], + current_user_id=current_user.id, + ) + row = ( + await session.execute( + select(CustomMetadataSchema).where(CustomMetadataSchema.key == "system"), + ) + ).scalar_one() + assert row.namespace == "shared.conformed" + + +@pytest.mark.asyncio +async def test_a_sub_namespace_declaration_spares_its_siblings(session, current_user): + """ + Reconciliation covers what the specs name, not the whole subtree. + + `shared.finance` is deployed by whoever owns it. A deployment of `shared` that + declares a schema for `shared.conformed` says nothing about it, so retiring it + would be one repo reaching into another's rows. + """ + session.add( + CustomMetadataSchema( + key="system", + namespace="shared.finance", + json_schema={"type": "object"}, + value_kind="object", + ), + ) + await session.commit() + + await upsert_schema_specs( + session, + namespace="shared", + specs=[ + CustomMetadataSchemaSpec( + key="system", + namespace="shared.conformed", + json_schema={"type": "object"}, + ), + ], + current_user_id=current_user.id, + ) + live = sorted( + row.namespace + for row in ( + await session.execute( + select(CustomMetadataSchema).where( + CustomMetadataSchema.key == "system", + CustomMetadataSchema.deactivated_at.is_(None), + ), + ) + ) + .scalars() + .all() + ) + assert live == ["shared.conformed", "shared.finance"] + + +@pytest.mark.asyncio +async def test_an_empty_list_still_retires_the_deploying_namespace( + session, + current_user, +): + """ + The deploying namespace stays in scope even when no spec names it, so `[]` + keeps meaning "manages schemas, declares none". + """ + await upsert_schema_specs( + session, + namespace="retire_ns", + specs=[ + CustomMetadataSchemaSpec(key="system", json_schema={"type": "object"}), + ], + current_user_id=current_user.id, + ) + await upsert_schema_specs( + session, + namespace="retire_ns", + specs=[], + current_user_id=current_user.id, + ) + live = ( + ( + await session.execute( + select(CustomMetadataSchema).where( + CustomMetadataSchema.namespace == "retire_ns", + CustomMetadataSchema.deactivated_at.is_(None), + ), + ) + ) + .scalars() + .all() + ) + assert live == [] diff --git a/datajunction-server/tests/migrations_test.py b/datajunction-server/tests/migrations_test.py index 868bc08c50..b6a0704fb7 100644 --- a/datajunction-server/tests/migrations_test.py +++ b/datajunction-server/tests/migrations_test.py @@ -47,10 +47,17 @@ def test_migrations_are_current(connection): context.configure(connection=connection) context.run_migrations() + def include_object(object_, name, type_, reflected, compare_to): + # test_template_status is test-only infrastructure created via raw + # SQL by tests/helpers/template_app.py, not part of the app schema. + if type_ == "table" and name == "test_template_status": + return False + return True + # Don't use compare_type due to false positives. migrations_state = MigrationContext.configure( connection, - opts={"compare_type": False}, + opts={"compare_type": False, "include_object": include_object}, ) diff = compare_metadata(migrations_state, target_metadata) assert diff == [], "The alembic migrations do not match the models." diff --git a/datajunction-server/tests/models/deployment_test.py b/datajunction-server/tests/models/deployment_test.py index 43f5f28517..b21af11c64 100644 --- a/datajunction-server/tests/models/deployment_test.py +++ b/datajunction-server/tests/models/deployment_test.py @@ -1,5 +1,9 @@ import json +import os +import subprocess +import sys from datetime import date +from decimal import Decimal import pytest from pydantic import ValidationError @@ -12,6 +16,7 @@ ChangeTier, ColumnSpec, CubeSpec, + CustomMetadataSchemaSpec, DeploymentSpec, DimensionJoinLinkSpec, DimensionReferenceLinkSpec, @@ -38,6 +43,18 @@ MaterializationStrategy, ) from datajunction_server.models.node import MetricUnit, NodeMode, NodeType +from datajunction_server.models.semantic_fingerprint import SemanticFingerprint +from datajunction_server.semantic_fingerprints.engine import ( + compose_node_fingerprint, + local_node_fingerprint, +) +from datajunction_server.semantic_fingerprints.normalization import ( + canonical_json, + normalize_field, + normalize_sequence, + normalize_value, +) +from datajunction_server.semantic_fingerprints.v1 import semantic_fields def test_source_spec(): @@ -263,6 +280,7 @@ def test_deployment_spec(): "tags": [], "hierarchies": [], "preaggregations": [], + "custom_metadata_schemas": None, "source": None, "auto_register_sources": True, "force": False, @@ -486,6 +504,45 @@ def test_dimension_join_link_spec_with_join_cardinality(): assert hash(default_link) != hash(fanout_link) +def test_diff_does_not_flag_description_mentioning_prefix(): + """ + `NodeSpec.diff()` renders `${prefix}` on both sides before comparing. + + An exported spec's `description`/`custom_metadata` keep any `${prefix}` + substring verbatim -- those fields are stored exactly as authored, never + rendered. Rendering only the incoming spec (not the exported one) would + replace `${prefix}` on one side but not the other, so a description that + happens to mention a sibling node via `${prefix}` would never compare + equal to itself. + """ + namespace = "test" + description = "See also ${prefix}other_node for context." + custom_metadata = {"see_also": "${prefix}other_node"} + + exported = DimensionSpec( + name=f"{namespace}.some_node", + namespace=namespace, + query="SELECT 1 AS id", + primary_key=["id"], + description=description, + custom_metadata=custom_metadata, + ) + declared = DimensionSpec( + name="some_node", + namespace=namespace, + query="SELECT 1 AS id", + primary_key=["id"], + description=description, + custom_metadata=custom_metadata, + owners=["someone_else"], + ) + + changed = exported.diff(declared) + assert "description" not in changed + assert "custom_metadata" not in changed + assert changed == ["owners"] + + def test_source_spec_with_dimension_link_default_value(): """Test SourceSpec with dimension_links including default_value.""" source_spec = SourceSpec( @@ -642,13 +699,35 @@ def test_deployment_results_property_getter(): "status": "success", "operation": "create", "message": "Created", + "change_tier": "major", + "semantic_fingerprint": { + "digest": "a" * 64, + }, + }, + { + "name": "legacy_node", + "deploy_type": "node", + "status": "invalid", + "operation": "update", + "semantic_fingerprint": "unknown", + }, + { + "name": "test_node -> test_dimension", + "deploy_type": "link", + "status": "success", + "operation": "create", }, ], ) results = deployment.deployment_results - assert len(results) == 1 + assert len(results) == 3 assert results[0].name == "test_node" assert results[0].status == DeploymentResult.Status.SUCCESS + assert results[0].change_tier == "major" + assert results[0].semantic_fingerprint == SemanticFingerprint(digest="a" * 64) + assert results[1].semantic_fingerprint == "unknown" + assert results[2].change_tier is None + assert results[2].semantic_fingerprint is None def test_deployment_spec_preserves_explicit_preagg_namespace(): @@ -883,6 +962,12 @@ def test_every_spec_field_has_an_explicit_change_tier(): if spec_class.unclassified_fields() } assert unclassified == {} + unclassified_order = { + spec_class.__name__: spec_class.unclassified_list_order_fields() + for spec_class in all_node_spec_classes() + if spec_class.unclassified_list_order_fields() + } + assert unclassified_order == {} def test_change_tier_lookup_walks_the_mro(): @@ -904,7 +989,7 @@ def test_change_tier_lookup_walks_the_mro(): assert TransformSpec.field_change_tier("primary_key") == ChangeTier.MAJOR assert TransformSpec.field_change_tier("tags") == ChangeTier.MINOR assert TransformSpec.order_sensitive_fields() == [] - assert CubeSpec.order_sensitive_fields() == ["metrics", "dimensions", "filters"] + assert CubeSpec.order_sensitive_fields() == ["metrics", "dimensions"] def test_fold_change_tiers(): @@ -993,7 +1078,7 @@ def test_cube_spec_order_diff(): assert one.order_diff( a_cube(dimensions=["ns.d.two", "ns.d.one"], filters=["x = 1", "y = 2"]), ) == ["dimensions"] - assert one.order_diff(a_cube(filters=["y = 2", "x = 1"])) == ["filters"] + assert one.order_diff(a_cube(filters=["y = 2", "x = 1"])) == [] # A set change is not a reorder — diff() reports that one instead. assert one.order_diff(a_cube(metrics=["ns.a"], filters=["x = 1", "y = 2"])) == [] assert one.diff(a_cube(metrics=["ns.a"], filters=["x = 1", "y = 2"])) == ["metrics"] @@ -1574,3 +1659,629 @@ def test_cube_spec_materialization_diff_ignores_declaration_order(): ) == [] ) + + +def test_a_schema_namespace_defaults_to_the_deployment(): + """Omitting it keeps the existing behaviour: the deploying namespace.""" + spec = DeploymentSpec( + namespace="shared", + nodes=[], + custom_metadata_schemas=[ + CustomMetadataSchemaSpec(key="system", json_schema={"type": "object"}), + ], + ) + assert spec.custom_metadata_schemas[0].namespace == "shared" + + +def test_a_schema_may_be_scoped_to_a_sub_namespace(): + """Narrower than the deployment is how a vocabulary rolls out in stages.""" + spec = DeploymentSpec( + namespace="shared", + nodes=[], + custom_metadata_schemas=[ + CustomMetadataSchemaSpec( + key="system", + namespace="shared.conformed", + json_schema={"type": "object"}, + ), + ], + ) + assert spec.custom_metadata_schemas[0].namespace == "shared.conformed" + + +@pytest.mark.parametrize( + "outside", + ["arc", "shared_other", "other.shared", "sharedx"], +) +def test_a_schema_namespace_outside_the_deployment_is_rejected(outside): + """ + A manifest may scope a schema narrower than itself, never wider or sideways -- + otherwise one repo governs another repo's nodes. `shared_other` and `sharedx` + are the prefix trap: they start with the namespace but are not beneath it. + """ + with pytest.raises(DJInvalidDeploymentConfig) as exc_info: + DeploymentSpec( + namespace="shared", + nodes=[], + custom_metadata_schemas=[ + CustomMetadataSchemaSpec( + key="system", + namespace=outside, + json_schema={"type": "object"}, + ), + ], + ) + assert "not 'shared' or beneath it" in str(exc_info.value) + + +def semantic_specs() -> dict[str, NodeSpec]: + """Representative inputs for each concrete node type.""" + return { + "source": SourceSpec( + namespace="analytics", + name="orders", + catalog="warehouse", + schema="sales", + table="orders", + columns=[ColumnSpec(name="order_id", type="bigint")], + primary_key=["order_id"], + ), + "transform": TransformSpec( + namespace="analytics", + name="clean_orders", + query=( + "SELECT order_id AS id, amount FROM ${prefix}orders WHERE amount > 0" + ), + ), + "dimension": DimensionSpec( + namespace="analytics", + name="order", + query="SELECT order_id, status FROM ${prefix}orders", + ), + "metric": MetricSpec( + namespace="analytics", + name="total_amount", + query="SELECT SUM(amount) AS value FROM ${prefix}orders", + required_dimensions=["${prefix}order.status"], + ), + "cube": CubeSpec( + namespace="analytics", + name="order_cube", + metrics=["${prefix}total_amount", "${prefix}order_count"], + dimensions=["${prefix}order.status", "${prefix}order.order_id"], + filters=["${prefix}order.status != 'cancelled'", "amount > 0"], + columns=[ + ColumnSpec( + name="${prefix}order.status", + partition=PartitionSpec(type=PartitionType.CATEGORICAL), + ), + ], + ), + } + + +def fingerprint(spec: NodeSpec) -> SemanticFingerprint: + return local_node_fingerprint(spec) + + +GOLDEN_FINGERPRINTS = { + "source": "71dcbc388988c2bdd850670427710384687b58565ee38ca392dc220adfed868d", + "transform": "978e692880c7bcfb1bd78ece85895a1ec1558e85377b064a8dac3f3719cff2a5", + "dimension": "f7b3c87a61fdadf9997432fd9334befdf43f2488874ef555e3f7d4c4ba86e3e1", + "metric": "a3fde7af5dbd00d194805af33fc213bca52244c7cdedf1f7363ec52d2f6d4116", + "cube": "9b0a56d974d1e3769bc2db94e2cfbae7a6a4839f664eebd2a4387ef112ceea81", +} + + +@pytest.mark.parametrize("node_type", GOLDEN_FINGERPRINTS) +def test_semantic_fingerprint_golden_digests(node_type): + spec = semantic_specs()[node_type] + result = fingerprint(spec) + assert result == SemanticFingerprint(digest=GOLDEN_FINGERPRINTS[node_type]) + assert result == fingerprint(spec) + assert result.version == 1 + + +def test_semantic_fingerprint_is_independent_of_python_hash_seed(): + script = """ +from datajunction_server.api.main import app +from datajunction_server.models.deployment import ColumnSpec, SourceSpec +from datajunction_server.semantic_fingerprints.engine import local_node_fingerprint +spec = SourceSpec(name="s", catalog="c", schema_="s", table="t", + columns=[ColumnSpec(name="id", attributes=["z", "primary_key", "a"])]) +print(local_node_fingerprint(spec).digest) +""" + + def digest_for(seed): + return subprocess.check_output( + [sys.executable, "-c", script], + env={**os.environ, "PYTHONHASHSEED": seed}, + text=True, + ).splitlines()[-1] + + assert digest_for("1") == digest_for("42") + + +def test_semantic_fingerprint_normalizes_empty_and_resolved_source_columns(): + common = {"name": "source", "catalog": "c", "schema_": "s", "table": "t"} + unspecified = SourceSpec(**common, columns=None) + empty = SourceSpec(**common, columns=[]) + columns = [ColumnSpec(name="id", type="bigint")] + resolved = SourceSpec(**common, columns=columns) + duplicated = SourceSpec(**common, columns=[*columns, columns[0].model_copy()]) + + assert fingerprint(unspecified) == fingerprint(empty) + assert local_node_fingerprint( + unspecified, + resolved_columns=columns, + ) == fingerprint(resolved) + assert fingerprint(resolved) == fingerprint(duplicated) + assert fingerprint( + CubeSpec(name="cube", metrics=[], dimensions=[], filters=None), + ) == fingerprint(CubeSpec(name="cube", metrics=[], dimensions=[], filters=[])) + + +def test_semantic_fingerprint_normalized_values_are_stable(): + first = {"outer": {"a": 1, "b": 2}, "value": 3} + second = {"value": 3, "outer": {"b": 2, "a": 1}} + assert canonical_json(normalize_value(first)) == canonical_json( + normalize_value(second), + ) + with pytest.raises(TypeError, match="string keys"): + normalize_value({1: "value"}) + assert normalize_sequence( + [1, 2], + preserve_order=True, + ) != normalize_sequence( + [2, 1], + preserve_order=True, + ) + with pytest.raises(TypeError, match="Unsupported"): + normalize_value({"bad": object()}) + with pytest.raises(ValueError, match="must be finite"): + normalize_value({"bad": float("nan")}) + with pytest.raises(ValueError, match="must be finite"): + normalize_value(Decimal("NaN")) + assert normalize_value(True) is True + assert normalize_value(1.5) == 1.5 + assert normalize_value(Decimal("1.0")) == 1 + assert normalize_value(Decimal("1.50")) == {"decimal": "1.5"} + + +def test_semantic_fingerprint_normalizes_equivalent_numbers(): + assert normalize_value({"value": 1}) == normalize_value({"value": 1.0}) + assert normalize_value({"value": -0.0}) == normalize_value({"value": 0}) + + sql_integer = TransformSpec(name="sql_number", query="SELECT 1") + sql_integral_float = TransformSpec(name="sql_number", query="SELECT 1.0") + assert sql_integer.semantic_diff(sql_integral_float) == ([], []) + assert fingerprint(sql_integer) == fingerprint(sql_integral_float) + + +@pytest.mark.parametrize( + "spec_type", + [SourceSpec, TransformSpec, DimensionSpec, MetricSpec, CubeSpec], +) +def test_semantic_fingerprint_v1_projection_is_explicit(spec_type): + current_major_fields = { + field + for field, field_info in spec_type.model_fields.items() + if field not in {"name", "namespace", "node_type"} + and field_info.exclude is not True + and spec_type.field_change_tier(field) == ChangeTier.MAJOR + } + assert set(semantic_fields(spec_type)) == current_major_fields + + +def test_semantic_fingerprint_v1_rejects_unregistered_spec_type(): + with pytest.raises(TypeError, match="No semantic fingerprint fields for NodeSpec"): + semantic_fields(NodeSpec) + + +def test_semantic_fingerprint_renders_prefixes_and_normalizes_sql(): + from datajunction_server.models.dialect import Dialect + from datajunction_server.sql.parsing.ast import render_for_dialect + + parameterized = TransformSpec( + namespace="analytics", + name="orders", + query="SELECT\n id AS order_id\nFROM ${prefix}raw_orders", + ) + rendered = TransformSpec( + namespace="analytics", + name="orders", + query="SELECT id AS order_id FROM analytics.raw_orders", + ) + assert parameterized.query_ast.compare(rendered.query_ast) + assert fingerprint(parameterized) == fingerprint(rendered) + dialect_query = TransformSpec( + name="dialect", + query="SELECT COLLECT_LIST(value) AS values FROM source", + ) + dialect_fingerprint = fingerprint(dialect_query) + with render_for_dialect(Dialect.TRINO): + assert fingerprint(parameterized) == fingerprint(rendered) + assert fingerprint(dialect_query) == dialect_fingerprint + assert fingerprint( + TransformSpec( + name="typed", + query="SELECT CAST(value AS DECIMAL(10, 2)) FROM source", + ), + ).digest + assert fingerprint(TransformSpec(name="blank", query="")).digest + explicit = TransformSpec(name="orders", query="SELECT id AS order_id FROM raw") + implicit = TransformSpec(name="orders", query="SELECT id order_id FROM raw") + assert not explicit.query_ast.compare(implicit.query_ast) + assert fingerprint(explicit) != fingerprint(implicit) + + +def test_semantic_diff_and_fingerprint_share_change_rules(): + original = TransformSpec(name="node", query="SELECT id AS value FROM source") + formatted = TransformSpec( + name="node", + query=" SELECT id AS value\nFROM source ", + ) + changed, reordered = original.semantic_diff(formatted) + assert (changed, reordered) == ([], []) + assert TransformSpec.change_tier(changed, reordered) == ChangeTier.NONE + assert fingerprint(original) == fingerprint(formatted) + + source = SourceSpec( + name="source", + catalog="c", + schema_="s", + table="t", + columns=[ColumnSpec(name="id", type="bigint")], + ) + source_changed = source.model_copy(deep=True) + source_changed.columns[0].type = "string" + changed, reordered = source.semantic_diff(source_changed) + assert (changed, reordered) == (["columns"], []) + assert SourceSpec.change_tier(changed, reordered) == ChangeTier.MAJOR + assert fingerprint(source) != fingerprint(source_changed) + assert source.semantic_diff(original) == (["node_type"], []) + + cube = CubeSpec(name="cube", metrics=["a", "b"], dimensions=[]) + reordered_cube = cube.model_copy(update={"metrics": ["b", "a", "a"]}) + changed, reordered = cube.semantic_diff(reordered_cube) + assert (changed, reordered) == ([], ["metrics"]) + assert CubeSpec.change_tier(changed, reordered) == ChangeTier.MINOR + assert fingerprint(cube) == fingerprint(reordered_cube) + + legacy_metric = MetricSpec( + name="metric", + query="SELECT 1", + unit="dollar", + ) + structured_metric = MetricSpec( + name="metric", + query="SELECT 1", + direction="neutral", + unit={"kind": "currency", "code": "USD"}, + ) + assert legacy_metric.semantic_diff(structured_metric) == ([], []) + changed, reordered = MetricSpec( + name="metric", + query="SELECT 1", + ).semantic_diff(legacy_metric) + assert (changed, reordered) == (["unit_enum"], []) + assert MetricSpec.change_tier(changed, reordered) == ChangeTier.MINOR + + +def test_semantic_diff_canonicalizes_required_dimension_identity(): + bare = MetricSpec( + namespace="analytics", + name="order_count", + query="SELECT COUNT(*) FROM analytics.orders", + required_dimensions=["order_id"], + ) + qualified = bare.model_copy( + update={"required_dimensions": ["analytics.orders.order_id"]}, + ) + + assert bare.canonical_required_dimensions == qualified.canonical_required_dimensions + assert bare.diff(bare) == [] + assert bare.diff(qualified) == [] + assert bare.semantic_diff(qualified) == ([], []) + assert fingerprint(bare) == fingerprint(qualified) + + +def test_semantic_diff_canonicalizes_required_dimension_with_multiple_parents(): + bare = MetricSpec( + namespace="analytics", + name="order_count", + query=( + "SELECT COUNT(*) FROM analytics.orders " + "JOIN analytics.customers " + "ON analytics.orders.customer_id = analytics.customers.customer_id" + ), + required_dimensions=["order_id"], + ) + qualified = bare.model_copy( + update={"required_dimensions": ["analytics.orders.order_id"]}, + ) + + assert bare.canonical_required_dimensions == qualified.canonical_required_dimensions + assert bare.semantic_diff(qualified) == ([], []) + assert fingerprint(bare) == fingerprint(qualified) + + +def test_semantic_diff_compares_unparseable_queries_as_raw_sql(): + original = TransformSpec(name="node", query="SELECT (") + same = TransformSpec(name="node", query="SELECT (") + changed = TransformSpec(name="node", query="SELECT )") + + assert original.semantic_diff(same) == ([], []) + assert original.semantic_diff(changed) == (["query"], []) + + metric = MetricSpec( + name="metric", + query="SELECT (", + required_dimensions=["id"], + ) + same_metric = metric.model_copy(deep=True) + changed_metric = metric.model_copy(update={"required_dimensions": ["other_id"]}) + assert metric.semantic_diff(same_metric) == ([], []) + assert metric.semantic_diff(changed_metric) == (["required_dimensions"], []) + + +@pytest.mark.parametrize( + ("field", "value"), + [ + ("owners", ["other"]), + ("display_name", "Orders"), + ("description", "Updated description"), + ("tags", ["certified"]), + ("mode", NodeMode.DRAFT), + ("custom_metadata", {"team": "analytics"}), + ], +) +def test_minor_base_node_fields_preserve_semantic_fingerprint(field, value): + original = semantic_specs()["source"] + changed = original.model_copy(update={field: value}) + assert type(original).field_change_tier(field) == ChangeTier.MINOR + assert fingerprint(original) == fingerprint(changed) + + +def test_metric_presentation_fields_preserve_semantic_fingerprint(): + baseline = MetricSpec(name="metric", query="SELECT 1") + presentations = [ + MetricSpec( + name="metric", + query="SELECT 1", + direction="higher_is_better", + unit="dollar", + significant_digits=3, + min_decimal_exponent=-2, + max_decimal_exponent=4, + ), + MetricSpec( + name="metric", + query="SELECT 1", + unit={"kind": "currency", "code": "USD"}, + ), + ] + fields = ( + set(MetricSpec.model_fields) + - set(NodeSpec.model_fields) + - { + "query", + "columns", + "required_dimensions", + } + ) + assert all( + MetricSpec.field_change_tier(field) == ChangeTier.MINOR for field in fields + ) + assert all(fingerprint(spec) == fingerprint(baseline) for spec in presentations) + + +@pytest.mark.parametrize( + ("node_type", "field", "value"), + [ + ("source", "catalog", "other"), + ("source", "schema_", "other"), + ("source", "table", "other"), + ("source", "primary_key", ["amount"]), + ("transform", "query", "SELECT amount FROM analytics.orders"), + ("dimension", "query", "SELECT order_id FROM analytics.orders"), + ("metric", "query", "SELECT COUNT(*) AS value FROM analytics.orders"), + ("metric", "required_dimensions", ["analytics.order.order_id"]), + ("cube", "metrics", ["analytics.order_count"]), + ("cube", "dimensions", ["analytics.order.order_id"]), + ("cube", "filters", ["amount >= 0"]), + ], +) +def test_major_node_fields_change_semantic_fingerprint(node_type, field, value): + original = semantic_specs()[node_type] + changed = original.model_copy(update={field: value}) + assert type(original).field_change_tier(field) == ChangeTier.MAJOR + assert fingerprint(original) != fingerprint(changed) + + +def test_semantic_fingerprint_column_rules_match_equality(): + source = SourceSpec( + name="source", + catalog="c", + schema_="s", + table="t", + columns=[ + ColumnSpec(name="id", type="bigint", attributes=["primary_key", "id"]), + ColumnSpec(name="value", type="string"), + ], + ) + source_reordered = source.model_copy(deep=True) + source_reordered.columns = list(reversed(source_reordered.columns or [])) + source_reordered.columns[1].attributes = ["id", "primary_key"] + source_type_changed = source.model_copy(deep=True) + source_type_changed.columns[0].type = "integer" + assert eq_columns(source.columns, source_reordered.columns) + assert fingerprint(source) == fingerprint(source_reordered) + assert not eq_columns(source.columns, source_type_changed.columns) + assert fingerprint(source) != fingerprint(source_type_changed) + + for spec_class in (TransformSpec, DimensionSpec): + original = spec_class( + name="derived", + query="SELECT id FROM source", + columns=[ColumnSpec(name="id", type="bigint")], + ) + inferred_type_changed = original.model_copy(deep=True) + inferred_type_changed.columns[0].type = "string" + metadata_changed = original.model_copy(deep=True) + metadata_changed.columns[0].attributes = ["identifier"] + assert eq_columns(original.columns, inferred_type_changed.columns, False) + assert fingerprint(original) == fingerprint(inferred_type_changed) + assert not eq_columns(original.columns, metadata_changed.columns, False) + assert fingerprint(original) != fingerprint(metadata_changed) + + +def test_semantic_fingerprint_dimension_link_rules_match_equality(): + from datajunction_server.models.dimensionlink import SparkJoinStrategy + + links = [ + DimensionReferenceLinkSpec( + node_column="customer_id", + dimension="${prefix}customer.id", + role="customer", + ), + DimensionJoinLinkSpec( + dimension_node="${prefix}date", + join_on="${prefix}orders.date_id = ${prefix}date.id", + role="date", + ), + ] + original = TransformSpec( + namespace="analytics", + name="orders", + query="SELECT 1", + dimension_links=links, + ) + reordered = original.model_copy(update={"dimension_links": list(reversed(links))}) + changed = original.model_copy(deep=True) + changed.dimension_links[0].role = "buyer" + hint_changed = original.model_copy( + update={ + "dimension_links": [ + links[0], + links[1].model_copy( + update={"spark_hints": SparkJoinStrategy.BROADCAST}, + ), + ], + }, + ) + assert original == reordered + assert fingerprint(original) == fingerprint(reordered) + assert original == hint_changed + assert fingerprint(original) == fingerprint(hint_changed) + assert original != changed + assert fingerprint(original) != fingerprint(changed) + + +def test_semantic_fingerprint_normalizes_primary_keys_and_cube_ordering(): + source = semantic_specs()["source"] + assert fingerprint(source) == fingerprint( + source.model_copy( + update={"primary_key": ["order_id", "order_id"]}, + ), + ) + + cube = semantic_specs()["cube"] + reordered = cube.model_copy(deep=True) + reordered.metrics.reverse() + reordered.dimensions.reverse() + reordered.filters = list(reversed(reordered.filters or [])) + assert fingerprint(cube) == fingerprint(reordered) + metric = MetricSpec( + name="metric", + query="SELECT 1", + required_dimensions=["one", "two"], + ) + reordered_metric = metric.model_copy( + update={"required_dimensions": ["two", "one", "one"]}, + ) + assert metric == reordered_metric + assert metric.semantic_diff(reordered_metric) == ([], []) + assert fingerprint(metric) == fingerprint(reordered_metric) + assert MetricSpec.field_change_tier("columns") == ChangeTier.NONE + assert fingerprint(cube) == fingerprint( + cube.model_copy( + update={"metrics": [*cube.metrics, "${prefix}order_count"]}, + ), + ) + assert fingerprint(cube) != fingerprint( + cube.model_copy( + update={"metrics": [*cube.metrics, "${prefix}average_amount"]}, + ), + ) + assert fingerprint(cube) != fingerprint( + cube.model_copy( + update={"filters": ["amount > 1"]}, + ), + ) + partition_changed = cube.model_copy(deep=True) + partition_changed.columns[0].partition.type = PartitionType.TEMPORAL + assert fingerprint(cube) != fingerprint(partition_changed) + + +def test_required_dimensions_normalization_falls_back_for_invalid_query(): + metric = MetricSpec( + namespace="analytics", + name="orders", + query="SELECT (", + required_dimensions=["${prefix}orders.order_id"], + ) + + assert normalize_field(metric, "required_dimensions") == [ + "analytics.orders.order_id", + ] + + +@pytest.mark.parametrize( + "digest", + ["a" * 63, "a" * 65, "A" * 64, "g" * 64], +) +def test_semantic_fingerprint_digest_validation(digest): + with pytest.raises(ValidationError): + SemanticFingerprint(digest=digest) + + +def test_semantic_fingerprint_rejects_unknown_version(): + with pytest.raises(ValidationError): + SemanticFingerprint(version=2, digest="a" * 64) + with pytest.raises(ValueError, match="Unsupported semantic fingerprint version: 2"): + local_node_fingerprint(semantic_specs()["source"], version=2) + + +def test_semantic_fingerprint_combines_sorted_parent_hashes(): + node = TransformSpec(name="node", query="SELECT id FROM parent") + first = SourceSpec(name="first", catalog="c", schema_="s", table="first") + second = SourceSpec(name="second", catalog="c", schema_="s", table="second") + first_hash = fingerprint(first) + second_hash = fingerprint(second) + + expected = compose_node_fingerprint( + node, + parent_fingerprints=[first_hash, second_hash], + ) + assert expected == compose_node_fingerprint( + node, + parent_fingerprints=[second_hash, first_hash, first_hash], + ) + assert expected != compose_node_fingerprint( + node, + parent_fingerprints=[ + first_hash, + fingerprint( + SourceSpec( + name="second", + catalog="c", + schema_="s", + table="changed", + ), + ), + ], + ) + mismatched = SemanticFingerprint.model_construct(version=2, digest="b" * 64) + with pytest.raises(ValueError, match="Parent fingerprint version"): + compose_node_fingerprint(node, parent_fingerprints=[mismatched]) diff --git a/datajunction-server/tests/service_clients_test.py b/datajunction-server/tests/service_clients_test.py index fd7f70d123..ad7221e1a6 100644 --- a/datajunction-server/tests/service_clients_test.py +++ b/datajunction-server/tests/service_clients_test.py @@ -371,15 +371,11 @@ async def test_query_service_client_submit_query( ) @pytest.mark.asyncio - async def test_submit_query_ignores_request_headers( + async def test_submit_query_forwards_full_cache_policy( self, mocker: MockerFixture, ) -> None: - """``request_headers`` is intentionally not forwarded to DJQS — it stays - on the API for caller compatibility, but only the static ``accept`` header - actually goes on the wire. This guards against accidentally forwarding - the FastAPI request's ``Accept-Encoding`` (e.g. ``zstd``) and getting - back a body httpx can't auto-decompress.""" + """Only Cache-Control is propagated; request credentials stay local.""" mock_response = MagicMock() mock_response.status_code = 200 mock_response.json.return_value = { @@ -417,12 +413,16 @@ async def test_submit_query_ignores_request_headers( request_headers={ "X-DJ-User": "alice", "Accept-Encoding": "zstd", + "Cache-Control": "max-age=86400, stale-while-revalidate", }, ) mock_request.assert_called_with( "POST", "/queries/", - headers={"accept": "application/json"}, + headers={ + "accept": "application/json", + "cache-control": "max-age=86400, stale-while-revalidate", + }, json=ANY, ) diff --git a/datajunction-server/tests/sql/functions_test.py b/datajunction-server/tests/sql/functions_test.py index e20bce09c2..10a1290df2 100644 --- a/datajunction-server/tests/sql/functions_test.py +++ b/datajunction-server/tests/sql/functions_test.py @@ -3,6 +3,7 @@ """ import pytest +import pytest_asyncio from sqlalchemy.ext.asyncio import AsyncSession import datajunction_server.sql.functions as F @@ -37,6 +38,18 @@ ) +@pytest_asyncio.fixture(scope="module") +async def session(module__session: AsyncSession) -> AsyncSession: + """ + Override the function-scoped ``session`` fixture with a module-scoped one. + + Nothing in this module writes to the database -- these tests only compile + expressions and assert inferred types -- so cloning a database per test is + wasted work. + """ + return module__session + + @pytest.mark.asyncio async def test_missing_functions() -> None: """ diff --git a/datajunction-server/tests/sql/parsing/test_ast.py b/datajunction-server/tests/sql/parsing/test_ast.py index 66a4c69e37..c07bbc09ba 100644 --- a/datajunction-server/tests/sql/parsing/test_ast.py +++ b/datajunction-server/tests/sql/parsing/test_ast.py @@ -5,6 +5,7 @@ from typing import cast import pytest +from sqlglot import exp as sqlglot_exp from httpx import AsyncClient from sqlalchemy.ext.asyncio import AsyncSession @@ -1527,3 +1528,87 @@ def test_to_sql_transpiles_functions_for_dialect(monkeypatch): # No dialect => no transpilation, canonical names returned verbatim. assert "COLLECT_LIST" in ast.to_sql(query, None).upper() + + +def test_to_sql_passes_dj_table_schema_to_sqlglot(monkeypatch): + """Compiled DJ table metadata is retained for SQLGlot type annotation.""" + from types import SimpleNamespace + + from datajunction_server.models.dialect import DialectRegistry + from datajunction_server.transpilation import SQLGlotTranspilationPlugin + + monkeypatch.setitem( + DialectRegistry._registry, + "bigquery", + SQLGlotTranspilationPlugin, + ) + query = parse( + "SELECT events.payload['name'] FROM catalog.schema.events AS events", + ) + table = query.select.from_.relations[0].primary + table.set_dj_node( + SimpleNamespace( + columns=[ + SimpleNamespace( + name="payload", + type=types.StructType( + types.NestedField("name", types.StringType()), + ), + ), + ], + ), + ) + + assert "`events`.`payload`.name" in ast.to_sql(query, Dialect.BIGQUERY) + + +@pytest.mark.parametrize( + ("dj_type", "expected"), + [ + (types.NullType(), "NULL"), + (types.FixedType(8), "BINARY(8)"), + (types.DecimalType(12, 3), "DECIMAL(12, 3)"), + (types.BooleanType(), "BOOLEAN"), + (types.TinyIntType(), "TINYINT"), + (types.SmallIntType(), "SMALLINT"), + (types.IntegerType(), "INT"), + (types.LongType(), "BIGINT"), + (types.BigIntType(), "BIGINT"), + (types.FloatType(), "FLOAT"), + (types.DoubleType(), "DOUBLE"), + (types.DateType(), "DATE"), + (types.TimeType(), "TIME"), + (types.TimestampType(), "TIMESTAMP"), + (types.TimestamptzType(), "TIMESTAMPTZ"), + (types.StringType(), "TEXT"), + (types.VarcharType(20), "VARCHAR(20)"), + (types.UUIDType(), "UUID"), + (types.BinaryType(), "BINARY"), + (types.DayTimeIntervalType(), "INTERVAL DAY TO SECOND"), + (types.YearMonthIntervalType(), "INTERVAL YEAR TO MONTH"), + (types.UnknownType(), "UNKNOWN"), + (types.WildcardType(), "UNKNOWN"), + ], +) +def test_sqlglot_type_maps_dj_types(dj_type, expected): + assert ast._sqlglot_type(dj_type).sql() == expected + + +def test_sqlglot_type_maps_nested_types(): + dj_type = types.StructType( + types.NestedField("id", types.IntegerType(), is_optional=False), + types.NestedField( + "attributes", + types.MapType( + types.StringType(), + types.ListType(types.TimestampType()), + ), + ), + ) + + result = ast._sqlglot_type(dj_type) + + assert result.this == sqlglot_exp.DataType.Type.STRUCT + assert result.sql() == ( + "STRUCT>>" + ) diff --git a/datajunction-server/tests/sql/parsing/test_structural.py b/datajunction-server/tests/sql/parsing/test_structural.py new file mode 100644 index 0000000000..dc50964fb7 --- /dev/null +++ b/datajunction-server/tests/sql/parsing/test_structural.py @@ -0,0 +1,61 @@ +from dataclasses import dataclass +from decimal import Decimal + +import pytest + +from datajunction_server.sql.parsing import ast +from datajunction_server.sql.parsing.backends.antlr4 import parse +from datajunction_server.sql.parsing.structural import ( + _serialize_number_v1, + serialize_ast, +) + + +def test_serialize_ast_is_structural_and_numeric(): + integer = serialize_ast(parse("SELECT 1")) + integral_float = serialize_ast(parse("SELECT 1.0")) + typed = serialize_ast( + parse("SELECT CAST(value AS DECIMAL(10, 2)) FROM source"), + ) + decimal = serialize_ast(ast.Number(Decimal("1.50"))) + + assert integer == integral_float + assert integer["type"] == "Query" + assert "datajunction_server." not in str(integer) + assert typed["type"] == "Query" + assert decimal["fields"]["value"] == {"decimal": "1.5"} + + +@pytest.mark.parametrize( + ("value", "expected"), + [ + (1, 1), + (1.5, 1.5), + (Decimal("1.0"), 1), + (Decimal("1.50"), {"decimal": "1.5"}), + ], +) +def test_serialize_number_v1(value, expected): + assert _serialize_number_v1(value) == expected + + +def test_serialize_number_v1_rejects_booleans_and_nonfinite_values(): + with pytest.raises(TypeError, match="Boolean values are not SQL numbers"): + _serialize_number_v1(True) + for value in (float("inf"), Decimal("NaN")): + with pytest.raises(ValueError, match="must be finite"): + _serialize_number_v1(value) + + +def test_serialize_ast_rejects_unclassified_nodes_and_versions(): + @dataclass(eq=False) + class FutureNode(ast.Node): + def __str__(self) -> str: + return "future" + + with pytest.raises(TypeError, match="Unsupported structural SQL node: FutureNode"): + serialize_ast(FutureNode()) + with pytest.raises(TypeError, match="Unsupported structural SQL value: object"): + serialize_ast(ast.Name(name=object())) # type: ignore[arg-type] + with pytest.raises(ValueError, match="Unsupported structural SQL serialization"): + serialize_ast(parse("SELECT 1"), version=2) diff --git a/datajunction-server/tests/test_route_coverage.py b/datajunction-server/tests/test_route_coverage.py index b098453fad..cdd327f52b 100644 --- a/datajunction-server/tests/test_route_coverage.py +++ b/datajunction-server/tests/test_route_coverage.py @@ -93,6 +93,7 @@ def flatten(buckets: dict[str, list[tuple[str, str]]]) -> set[tuple[str, str]]: ("POST", "/engines"), ("POST", "/tags"), ("PATCH", "/tags/{name}"), + ("DELETE", "/tags/{name}"), ("POST", "/attributes"), ("POST", "/measures"), ("PATCH", "/measures/{measure_name}"), diff --git a/datajunction-server/tests/transpilation_test.py b/datajunction-server/tests/transpilation_test.py index c5fe438652..1d95e4993b 100644 --- a/datajunction-server/tests/transpilation_test.py +++ b/datajunction-server/tests/transpilation_test.py @@ -2,6 +2,8 @@ from unittest import mock +import pytest + from datajunction_server.models.dialect import Dialect, DialectRegistry from datajunction_server.models.engine import Dialect from datajunction_server.models.metric import TranslatedSQL @@ -96,6 +98,72 @@ def test_sqlglot_transpile_success(): assert result == "SELECT * FROM bar" +def test_sqlglot_transpile_uses_schema_to_annotate_types(): + """Schema types are attached before dialect-specific SQL generation.""" + plugin = SQLGlotTranspilationPlugin() + result = plugin.transpile_sql( + "SELECT t.payload['name'] FROM catalog.schema.events AS t", + input_dialect=Dialect.SPARK, + output_dialect=Dialect.BIGQUERY, + schema={ + "catalog": { + "schema": { + "events": {"payload": "struct"}, + }, + }, + }, + ) + + assert result == ( + "SELECT\n `t`.`payload`.name AS `name`\n" + "FROM `catalog`.`schema`.`events` AS `t`" + ) + + +# TODO: Remove this skip after upgrading to a SQLGlot release containing the +# map EXPLODE type-annotation fix. +@pytest.mark.skip(reason="Requires the unreleased SQLGlot map EXPLODE fix") +def test_sqlglot_transpile_map_explode_with_schema(): + """A typed Spark map EXPLODE becomes a two-column Trino UNNEST.""" + plugin = SQLGlotTranspilationPlugin() + result = plugin.transpile_sql( + """SELECT + CAST(test_id AS BIGINT) AS test_id, + key, + value +FROM ( + SELECT + test_id, + explode(map_column) AS (key, value) + FROM some_table T +)""", + input_dialect=Dialect.SPARK, + output_dialect=Dialect.TRINO, + schema={ + "some_table": { + "test_id": "INT", + "map_column": "MAP", + }, + }, + ) + + assert ( + result + == '''SELECT + TRY_CAST("_0"."test_id" AS BIGINT) AS "test_id", + "_0"."key" AS "key", + "_0"."value" AS "value" +FROM ( + SELECT + "t"."test_id" AS "test_id", + _u_2."key" AS "key", + _u_2."value" AS "value" + FROM "some_table" AS "t" + CROSS JOIN UNNEST("t"."map_column") AS _u_2("key", "value") +) AS "_0"''' + ) + + def test_default_transpile_success(): with mock.patch( "datajunction_server.transpilation.settings.transpilation_plugins", diff --git a/datajunction-ui/package.json b/datajunction-ui/package.json index 25e8b7e35a..877896311d 100644 --- a/datajunction-ui/package.json +++ b/datajunction-ui/package.json @@ -1,6 +1,6 @@ { "name": "datajunction-ui", - "version": "0.0.223", + "version": "0.0.244", "description": "DataJunction UI", "type": "module", "module": "src/index.tsx", diff --git a/datajunction-ui/src/app/pages/NodePage/NodeMaterializationTab.jsx b/datajunction-ui/src/app/pages/NodePage/NodeMaterializationTab.jsx index 28836d1fa7..7a2b34803c 100644 --- a/datajunction-ui/src/app/pages/NodePage/NodeMaterializationTab.jsx +++ b/datajunction-ui/src/app/pages/NodePage/NodeMaterializationTab.jsx @@ -43,8 +43,10 @@ export default function NodeMaterializationTab({ const materializationsByRevision = useMemo(() => { return filteredMaterializations.reduce((acc, mat) => { - // Extract version from materialization config - const matVersion = mat.config?.cube?.version || node?.version; + // `config.cube.version` is absent for most job types (e.g. Druid cube + // jobs), so it silently grouped every materialization under the current + // node version regardless of which revision it actually belongs to. + const matVersion = mat.node_version || node?.version; if (!acc[matVersion]) { acc[matVersion] = []; @@ -137,7 +139,7 @@ export default function NodeMaterializationTab({ // Determine which versions have only inactive materializations const versionHasOnlyInactive = {}; rawMaterializations.forEach(mat => { - const matVersion = mat.config?.cube?.version || node.version; + const matVersion = mat.node_version || node.version; if (!versionHasOnlyInactive[matVersion]) { versionHasOnlyInactive[matVersion] = { hasActive: false, @@ -205,7 +207,7 @@ export default function NodeMaterializationTab({ // Check if latest version has any materializations (including inactive ones) const hasLatestVersionMaterialization = rawMaterializations.some(mat => { - const matVersion = mat.config?.cube?.version || node?.version; + const matVersion = mat.node_version || node?.version; return matVersion === node?.version; }); diff --git a/datajunction-ui/src/app/pages/NodePage/__tests__/NodeMaterializationTab.test.jsx b/datajunction-ui/src/app/pages/NodePage/__tests__/NodeMaterializationTab.test.jsx index 6d53a627df..1de4f28a3c 100644 --- a/datajunction-ui/src/app/pages/NodePage/__tests__/NodeMaterializationTab.test.jsx +++ b/datajunction-ui/src/app/pages/NodePage/__tests__/NodeMaterializationTab.test.jsx @@ -187,4 +187,41 @@ describe('', () => { expect(link).toHaveAttribute('href', `https://www.foobar.com/dashboard`); }); }); + + it('groups a materialization by its own node_version, not the current node version', async () => { + // DruidCubeMaterializationJob configs (and most other job types) have no + // `config.cube`, so grouping must key off `node_version` from the API. + mockDjClient.materializations.mockReturnValue([ + { + name: 'druid_cube_v3', + config: {}, + schedule: '@daily', + job: 'DruidCubeMaterializationJob', + backfills: [], + strategy: 'incremental_time', + output_tables: ['table1'], + urls: ['https://example.com/'], + deactivated_at: null, + node_version: 'v1.0', + }, + ]); + mockDjClient.availabilityStates.mockReturnValue([]); + mockDjClient.materializationInfo.mockReturnValue({ + job_types: [], + strategies: [], + }); + + render( + , + ); + await waitFor(() => { + // Belongs to v1.0, not the node's current version (v2.0). + expect(screen.getByText('v1.0')).toBeInTheDocument(); + expect(screen.queryByText('v2.0 (latest)')).not.toBeInTheDocument(); + expect(screen.getByText('Druid Cube')).toBeInTheDocument(); + }); + }); }); diff --git a/datajunction-ui/src/app/pages/NodePage/__tests__/NodePage.test.jsx b/datajunction-ui/src/app/pages/NodePage/__tests__/NodePage.test.jsx index 17c80b76ad..7fe47c5ffc 100644 --- a/datajunction-ui/src/app/pages/NodePage/__tests__/NodePage.test.jsx +++ b/datajunction-ui/src/app/pages/NodePage/__tests__/NodePage.test.jsx @@ -7,7 +7,7 @@ import DJClientContext from '../../../providers/djclient'; // React.lazy + Suspense boundary which adds an async resolution step that // can race testing-library's waitFor on slower CI runners. import { NodePage } from '../index'; -import { MemoryRouter, Route, Routes } from 'react-router-dom'; +import { MemoryRouter, Route, Routes, useParams } from 'react-router-dom'; import userEvent from '@testing-library/user-event'; // Mock cronstrue for NodePreAggregationsTab @@ -97,6 +97,11 @@ describe('', () => { upstreamsGQL: vi.fn().mockResolvedValue([]), downstreamsGQL: vi.fn().mockResolvedValue([]), findCubesWithMetrics: vi.fn().mockResolvedValue([]), + // The server answers DELETE /nodes/{name}/ with 200 + a message body. + deactivate: vi.fn().mockResolvedValue({ + status: 200, + json: { message: 'Node `default.num_repair_orders` deleted.' }, + }), }, }; }; @@ -445,6 +450,127 @@ describe('', () => { expect( screen.queryByRole('button', { name: 'Edit' }), ).not.toBeInTheDocument(); + // ...nor is Delete. + expect(screen.queryByRole('button', { name: /Delete/ })).toBeNull(); + }, 60000); + + // Renders the node page under a router that also serves the namespace + // route, so a successful delete can be asserted by the redirect landing. + const renderNodePageWithNamespaceRoute = djClient => { + const element = ( + + + + ); + const NamespaceLanding = () => { + const { namespace } = useParams(); + return
{namespace}
; + }; + return render( + + + + } /> + + , + ); + }; + + it('deletes the node and redirects to its namespace after confirmation', async () => { + const djClient = mockDJClient(); + djClient.DataJunctionAPI.node.mockReturnValue(mocks.mockMetricNode); + djClient.DataJunctionAPI.getMetric.mockResolvedValue( + mocks.mockMetricNodeJson, + ); + const confirm = vi.spyOn(window, 'confirm').mockReturnValue(true); + renderNodePageWithNamespaceRoute(djClient); + + const deleteButton = await screen.findByRole('button', { + name: /Delete/, + }); + fireEvent.click(deleteButton); + + await waitFor(() => { + expect(djClient.DataJunctionAPI.deactivate).toHaveBeenCalledWith( + 'default.num_repair_orders', + ); + }); + // The redirect landed on the parent namespace, not the node page. + expect(await screen.findByTestId('namespace-landing')).toHaveTextContent( + 'default', + ); + confirm.mockRestore(); + }, 60000); + + it('does not delete the node when the confirmation is dismissed', async () => { + const djClient = mockDJClient(); + djClient.DataJunctionAPI.node.mockReturnValue(mocks.mockMetricNode); + djClient.DataJunctionAPI.getMetric.mockResolvedValue( + mocks.mockMetricNodeJson, + ); + const confirm = vi.spyOn(window, 'confirm').mockReturnValue(false); + renderNodePageWithNamespaceRoute(djClient); + + fireEvent.click(await screen.findByRole('button', { name: /Delete/ })); + + expect(djClient.DataJunctionAPI.deactivate).not.toHaveBeenCalled(); + expect(screen.queryByTestId('namespace-landing')).toBeNull(); + confirm.mockRestore(); + }, 60000); + + it('alerts and stays on the page when the delete is rejected', async () => { + const djClient = mockDJClient(); + djClient.DataJunctionAPI.node.mockReturnValue(mocks.mockMetricNode); + djClient.DataJunctionAPI.getMetric.mockResolvedValue( + mocks.mockMetricNodeJson, + ); + djClient.DataJunctionAPI.deactivate.mockResolvedValue({ + status: 409, + json: { message: 'Node has downstream dependencies' }, + }); + const confirm = vi.spyOn(window, 'confirm').mockReturnValue(true); + const alert = vi.spyOn(window, 'alert').mockImplementation(() => {}); + renderNodePageWithNamespaceRoute(djClient); + + fireEvent.click(await screen.findByRole('button', { name: /Delete/ })); + + await waitFor(() => { + expect(alert).toHaveBeenCalledWith( + 'Unable to delete node default.num_repair_orders: Node has downstream dependencies', + ); + }); + expect(screen.queryByTestId('namespace-landing')).toBeNull(); + confirm.mockRestore(); + alert.mockRestore(); + }, 60000); + + it('alerts and re-enables the button when the delete request throws', async () => { + const djClient = mockDJClient(); + djClient.DataJunctionAPI.node.mockReturnValue(mocks.mockMetricNode); + djClient.DataJunctionAPI.getMetric.mockResolvedValue( + mocks.mockMetricNodeJson, + ); + djClient.DataJunctionAPI.deactivate.mockRejectedValue( + new Error('Failed to fetch'), + ); + const confirm = vi.spyOn(window, 'confirm').mockReturnValue(true); + const alert = vi.spyOn(window, 'alert').mockImplementation(() => {}); + renderNodePageWithNamespaceRoute(djClient); + + const deleteButton = await screen.findByRole('button', { name: /Delete/ }); + fireEvent.click(deleteButton); + + await waitFor(() => { + expect(alert).toHaveBeenCalledWith( + 'Unable to delete node default.num_repair_orders: Failed to fetch', + ); + }); + // The in-flight guard is released, so a retry is possible. + await waitFor(() => { + expect(screen.getByRole('button', { name: /Delete/ })).not.toBeDisabled(); + }); + confirm.mockRestore(); + alert.mockRestore(); }, 60000); it('renders the NodeInfo tab correctly for cube nodes', async () => { diff --git a/datajunction-ui/src/app/pages/NodePage/index.jsx b/datajunction-ui/src/app/pages/NodePage/index.jsx index 92a7de5bbe..90a802e52d 100644 --- a/datajunction-ui/src/app/pages/NodePage/index.jsx +++ b/datajunction-ui/src/app/pages/NodePage/index.jsx @@ -17,6 +17,7 @@ import WatchButton from './WatchNodeButton'; import NodesWithDimension from './NodesWithDimension'; import NodeColumnLineage from './NodeLineageTab'; import EditIcon from '../../icons/EditIcon'; +import DeleteIcon from '../../icons/DeleteIcon'; import ChartIcon from '../../icons/ChartIcon'; import AlertIcon from '../../icons/AlertIcon'; import LoadingIcon from '../../icons/LoadingIcon'; @@ -37,6 +38,7 @@ export function NodePage() { // undefined = not yet known; NamespaceHeader reports the read-only verdict // (git_only, flat/root shape, or git-deployed) once its config + sources load. const [isReadOnly, setIsReadOnly] = useState(undefined); + const [isDeleting, setIsDeleting] = useState(false); const onClickTab = id => () => { // Preview tab redirects to Query Planner instead of showing content @@ -193,6 +195,36 @@ export function NodePage() { whiteSpace: 'nowrap', }; + const deleteButtonStyle = { + ...buttonStyle, + color: '#b91c1c', + borderColor: '#fecaca', + cursor: isDeleting ? 'not-allowed' : 'pointer', + opacity: isDeleting ? 0.6 : 1, + }; + + const onDelete = async () => { + if (!window.confirm(`Deleting node ${node?.name}. Are you sure?`)) { + return; + } + setIsDeleting(true); + try { + const { status, json } = await djClient.deactivate(node?.name); + if (status === 200 || status === 201 || status === 204) { + // Nodes always live under a namespace, but fall back to the root + // listing rather than routing to a nameless /namespaces/ URL. + const parentNamespace = node?.name?.split('.').slice(0, -1).join('.'); + navigate(parentNamespace ? `/namespaces/${parentNamespace}` : '/'); + } else { + window.alert(`Unable to delete node ${node?.name}: ${json?.message}`); + } + } catch (error) { + window.alert(`Unable to delete node ${node?.name}: ${error.message}`); + } finally { + setIsDeleting(false); + } + }; + const NodeButtons = () => { return (
@@ -211,6 +243,17 @@ export function NodePage() { {node?.type === 'cube' && ( )} + + {isReadOnly === false && ( + + )}
); }; diff --git a/datajunction-ui/src/app/services/DJService.js b/datajunction-ui/src/app/services/DJService.js index 82b4f6c956..cec6006531 100644 --- a/datajunction-ui/src/app/services/DJService.js +++ b/datajunction-ui/src/app/services/DJService.js @@ -1565,7 +1565,12 @@ export const DataJunctionAPI = { return results; }, - nodeData: async function (nodeName, selection = null) { + nodeData: async function ( + nodeName, + selection = null, + maxAge = 86400, + staleWhileRevalidate = false, + ) { if (selection === null) { selection = { dimensions: [], @@ -1581,11 +1586,16 @@ export const DataJunctionAPI = { } params.append('limit', '1000'); params.append('async_', 'true'); + const cacheControl = staleWhileRevalidate + ? `max-age=${maxAge}, stale-while-revalidate` + : `max-age=${maxAge}`; return await ( await fetch(`${DJ_URL}/data/${nodeName}?${params}`, { credentials: 'include', - headers: { 'Cache-Control': 'max-age=86400' }, + headers: { + 'Cache-Control': cacheControl, + }, }) ).json(); }, @@ -1943,7 +1953,9 @@ export const DataJunctionAPI = { }, credentials: 'include', }); - return { status: response.status, json: await response.json() }; + // A 204 carries no body, and a gateway error page carries no JSON. + const json = await response.json().catch(() => ({})); + return { status: response.status, json }; }, addNamespace: async function (namespace) { const response = await fetch(`${DJ_URL}/namespaces/${namespace}`, { diff --git a/datajunction-ui/src/app/services/__tests__/DJService.test.jsx b/datajunction-ui/src/app/services/__tests__/DJService.test.jsx index be0ca63120..0335179ca6 100644 --- a/datajunction-ui/src/app/services/__tests__/DJService.test.jsx +++ b/datajunction-ui/src/app/services/__tests__/DJService.test.jsx @@ -733,6 +733,38 @@ describe('DataJunctionAPI', () => { ); }); + it('uses a caller-provided nodeData max age', () => { + fetch.mockResponseOnce(JSON.stringify({})); + + DataJunctionAPI.nodeData('transform1', null, 604800); + + expect(fetch).toHaveBeenCalledWith( + `${DJ_URL}/data/transform1?limit=1000&async_=true`, + { + credentials: 'include', + headers: { + 'Cache-Control': 'max-age=604800', + }, + }, + ); + }); + + it('uses caller-provided nodeData stale-while-revalidate', () => { + fetch.mockResponseOnce(JSON.stringify({})); + + DataJunctionAPI.nodeData('transform1', null, 604800, true); + + expect(fetch).toHaveBeenCalledWith( + `${DJ_URL}/data/transform1?limit=1000&async_=true`, + { + credentials: 'include', + headers: { + 'Cache-Control': 'max-age=604800, stale-while-revalidate', + }, + }, + ); + }); + it('calls dag correctly and processes response', async () => { const mockResponse = [ { @@ -897,6 +929,14 @@ describe('DataJunctionAPI', () => { }); }); + // A gateway can answer with an HTML error page. Parsing that as JSON throws, + // so without the guard the call rejects and the caller sees a silent no-op. + it('tolerates a non-JSON error body', async () => { + fetch.mockResponseOnce('502 Bad Gateway', { status: 502 }); + const result = await DataJunctionAPI.deactivate('default.transform1'); + expect(result).toEqual({ status: 502, json: {} }); + }); + it('calls attributes correctly', async () => { fetch.mockResponseOnce(JSON.stringify(mocks.attributes)); await DataJunctionAPI.attributes(); diff --git a/docs/content/0.1.0/docs/data-modeling/custom-metadata.md b/docs/content/0.1.0/docs/data-modeling/custom-metadata.md new file mode 100644 index 0000000000..0486add7f7 --- /dev/null +++ b/docs/content/0.1.0/docs/data-modeling/custom-metadata.md @@ -0,0 +1,134 @@ +--- +weight: 9 +title: "Custom Metadata" +--- + +You can optionally set a `custom_metadata` field on any node. It is a free-form object, so you can put whatever you want in it. DJ stores it, returns it, and by default does not interpret it at all. + +```yaml +name: default.repair_orders +node_type: source +custom_metadata: + team: logistics + cost_center: "4400" +``` + +## Registering a schema + +You can register a JSON Schema for a key. Once a schema exists, DJ validates every write against it and rejects values that do not match. You declare schemas in the deployment manifest: + +```yaml +namespace: analytics.sales + +custom_metadata_schemas: + - key: sla + description: Freshness commitment for scheduled outputs. + json_schema: + type: object + properties: + max_staleness_hours: + type: integer + minimum: 1 + pager_rotation: + type: string + required: [max_staleness_hours] + + - key: review_status + node_type: metric + description: Where a metric definition sits in review. + json_schema: + type: string + enum: [draft, in_review, approved] +``` + +With that deployed, a node in `analytics.sales` setting `custom_metadata.review_status: approved` is accepted, and one setting `Approved` is rejected with the allowed values in the error. + +You can also register a schema through the API, which is useful for one-offs and for namespaces that are not managed by a repo: + +```sh +curl -X POST $DJ_SERVER/metadata-schemas/ \ + -H 'Content-Type: application/json' \ + -d '{ + "key": "review_status", + "namespace": "analytics.sales", + "node_type": "metric", + "json_schema": {"type": "string", "enum": ["draft", "in_review", "approved"]} + }' +``` + +If a repo manages the namespace, this is refused. A deployment reconciles that namespace to exactly what its manifest declares, so anything you register here would be undone on the next push. The error tells you to declare it in the repo instead. + +## What validation does and does not do + +Validation is **lax about keys it does not know**. A key with no registered schema passes untouched. That means registering your first schema breaks nothing that already exists, and the feature is inert until you opt into it. + +Validation is **strict about keys it does know**. A registered key is checked on every write and a failure rejects the deploy. There is no warning mode, so a schema is a hard gate from the moment it registers. + +Keep that distinction in mind when you decide what to put in a schema. Constraining the *shape* of a value, meaning its type and its allowed values, is safe, because a wrong value is always a mistake. Requiring a key to be *present* is a different thing: adding `required` to a schema will fail every existing node that lacks it, on its next deploy. If you want to phase presence in, leave it out of the schema and check for it another way until the values are backfilled. + +To find nodes that would fail a schema you are about to tighten, ask for its violations: + +```sh +curl $DJ_SERVER/metadata-schemas/12/violations +``` + +That returns a count and a sample of offending nodes without changing anything. + +## Scoping + +You can apply a schema to everything, or narrow it to a namespace, a node type, or both: + +| Registered with | Applies to | +|---|---| +| neither | every node, which makes it a global schema | +| `namespace` | that namespace and everything beneath it | +| `node_type` | nodes of that type, in any namespace | +| both | that node type, in that namespace and below | + +When more than one schema exists for the same key, the most specific one wins. Namespace counts for more than node type, so a schema registered on `analytics.sales` beats one registered for all metrics. + +Two more rules apply to global schemas. Registering one takes an administrator, since a schema with no namespace governs every node on the server. An administrator can also mark a global schema **reserved**, which stops any namespace from shadowing it. A reserved key always resolves to the global schema no matter what else is registered, so a platform team can guarantee that a key means one thing everywhere. + +A manifest can only scope a schema to its own namespace or one beneath it. Declaring `analytics.sales.customer` from a deployment of `analytics.sales` is fine, and it lets you roll a set of values out to part of a graph before it applies to all of it. Anything outside is rejected, so one repo cannot register schemas that govern another repo's nodes. + +## Reconciliation + +A deployment manages the complete set of schemas for its scope, the same way it manages nodes. Declared keys are created, updated, or revived if they were previously retired, and keys in scope that the manifest no longer names are retired. + +Because of that, an absent section and an empty one mean different things: + +- **Omitting `custom_metadata_schemas`** leaves existing schemas alone. This is what most manifests do. +- **Declaring it as an empty list** says this manifest manages schemas and declares none, which retires them. + +## Filtering + +Registered or not, you can search for nodes by what is in their metadata. Dots address nested values, and a backslash escapes a dot that is part of a key name: + +```graphql +{ + findNodesPaginated( + customMetadataFilters: [ + {key: "sla.max_staleness_hours", op: LTE, value: 24} + {key: "review_status", op: EQ, value: "approved"} + ] + limit: 100 + ) { + edges { node { name } } + totalCount + } +} +``` + +Available operators are `EQ`, `NE`, `EXISTS`, `GT`, `GTE`, `LT`, `LTE`, and `CONTAINS` for arrays and objects. Multiple filters are combined with AND. + +An index serves equality filters, so they stay fast on a large graph. Numeric comparisons are indexed too, but only for keys whose registered schema declares a numeric type, since DJ builds that index when you register the schema. You can still compare a key with no schema, or a non-numeric one, but the query will scan. + +## Choosing between a schema and a tag + +Both let you attach a controlled vocabulary to nodes, and the difference is what it costs to add a value. + +A **registered schema** suits a closed set that changes rarely, where adding a value should go through review. A lifecycle state, a review status, or a service tier all fit. The set lives in one place and DJ enforces it. + +A **tag** suits an open set that grows continuously, where anyone should be able to add a value without editing a schema. A domain, a project, or a team label all fit. + +If you find yourself editing a schema every week to add another allowed value, it probably wanted to be a tag.