[DJ Semantic Fingerprint 4] Bulk endpoint for viewing fingerprints + testing fixtures

datajunction-clients/javascript/package.json

@@ -1,6 +1,6 @@
 {
   "name": "datajunction",
-  "version": "0.0.241",
+  "version": "0.0.244",
   "description": "A Javascript client for interacting with a DataJunction server",
   "module": "src/index.js",
   "scripts": {

datajunction-clients/python/datajunction/__about__.py

@@ -2,4 +2,4 @@
 Version for Hatch
 """
 
-__version__ = "0.0.241"
+__version__ = "0.0.244"

datajunction-clients/python/datajunction/models.py

@@ -406,6 +406,9 @@ class DeploymentResult:
     deploy_type: str = ""
     change_tier: Literal["none", "minor", "major"] | None = None
     semantic_fingerprint: SemanticFingerprintValue | None = None
+    # True when the node was re-deployed only to retry a pre-existing failure.
+    # None on responses from servers that predate the field.
+    revalidation_only: bool | None = None
 
     @classmethod
     def from_dict(cls, d: dict) -> DeploymentResult:
@@ -419,6 +422,7 @@ def from_dict(cls, d: dict) -> DeploymentResult:
             deploy_type=d.get("deploy_type", ""),
             change_tier=d.get("change_tier"),
             semantic_fingerprint=_parse_semantic_fingerprint(fingerprint),
+            revalidation_only=d.get("revalidation_only"),
         )
 
 

datajunction-clients/python/tests/test_deploy.py

@@ -1418,6 +1418,7 @@ def test_get_impact_calls_api(self, tmp_path, monkeypatch):
                         "version": 1,
                         "digest": "a" * 64,
                     },
+                    "revalidation_only": True,
                 },
             ],
             "downstream_impacts": [
@@ -1448,6 +1449,7 @@ def test_get_impact_calls_api(self, tmp_path, monkeypatch):
         parsed = DeploymentInfo.from_dict(result)
         assert parsed.results[0].deploy_type == "node"
         assert parsed.results[0].change_tier == "none"
+        assert parsed.results[0].revalidation_only is True
         assert parsed.results[0].semantic_fingerprint == SemanticFingerprint(
             digest="a" * 64,
         )

datajunction-query/djqs/__about__.py

@@ -2,4 +2,4 @@
 Version for Hatch
 """
 
-__version__ = "0.0.241"
+__version__ = "0.0.244"

datajunction-reflection/datajunction_reflection/__about__.py

@@ -2,4 +2,4 @@
 Version for Hatch
 """
 
-__version__ = "0.0.241"
+__version__ = "0.0.244"

datajunction-server/datajunction_server/__about__.py

@@ -2,4 +2,4 @@
 Version for Hatch
 """
 
-__version__ = "0.0.241"
+__version__ = "0.0.244"

datajunction-server/datajunction_server/api/namespaces.py

@@ -25,6 +25,9 @@
     AccessDenialMode,
     get_access_checker,
 )
+from datajunction_server.internal.deployment.fingerprints import (
+    build_current_fingerprint_graph,
+)
 from datajunction_server.internal.git.github_service import (
     GitHubService,
     GitHubServiceError,
@@ -64,6 +67,8 @@
     NamespaceSourcesResponse,
 )
 from datajunction_server.models.namespace import (
+    NamespaceFingerprintNode,
+    NamespaceFingerprintResponse,
     NamespaceProvisionRequest,
     NamespaceProvisionResponse,
     NamespaceWriteStatus,
@@ -73,6 +78,10 @@
     NodeMinimumDetail,
 )
 from datajunction_server.models.node_type import NodeType
+from datajunction_server.models.semantic_fingerprint import (
+    LATEST_SEMANTIC_FINGERPRINT_VERSION,
+    SUPPORTED_SEMANTIC_FINGERPRINT_VERSIONS,
+)
 from datajunction_server.service_clients import QueryServiceClient
 from datajunction_server.utils import (
     get_current_user,
@@ -289,6 +298,57 @@ async def list_nodes_in_namespace(
     return [node for node in nodes if node.name in approved_names]
 
 
+@router.get(
+    "/namespaces/{namespace}/semantic-fingerprints",
+    response_model=NamespaceFingerprintResponse,
+)
+async def get_namespace_semantic_fingerprints(
+    namespace: str,
+    version: int = Query(default=LATEST_SEMANTIC_FINGERPRINT_VERSION),
+    *,
+    session: AsyncSession = Depends(get_session),
+    access_checker: AccessChecker = Depends(get_access_checker),
+) -> NamespaceFingerprintResponse:
+    """Return current semantic fingerprints for a namespace and its descendants."""
+    if version not in SUPPORTED_SEMANTIC_FINGERPRINT_VERSIONS:
+        raise DJInvalidInputException(
+            f"Unsupported semantic fingerprint version: {version}",
+        )
+
+    access_checker.add_namespace(namespace, ResourceAction.READ)
+    await access_checker.check(on_denied=AccessDenialMode.RAISE)
+
+    nodes = await NodeNamespace.list_all_nodes(
+        session,
+        namespace,
+        options=Node.export_load_options(),
+    )
+    access_checker.add_nodes(nodes, ResourceAction.READ)
+    await access_checker.check(on_denied=AccessDenialMode.RAISE)
+
+    ordered_nodes = sorted(nodes, key=lambda node: node.name)
+    specs = [await node.to_spec(session) for node in ordered_nodes]
+    graph = await build_current_fingerprint_graph(
+        session,
+        specs,
+        version=version,
+    )
+    fingerprints = graph.fingerprints(spec.rendered_name for spec in specs)
+    return NamespaceFingerprintResponse(
+        namespace=namespace,
+        version=version,
+        nodes=[
+            NamespaceFingerprintNode(
+                name=node.name,
+                node_type=node.type,
+                owners=sorted(owner.username for owner in node.owners),
+                semantic_fingerprint=fingerprints[node.name],
+            )
+            for node in ordered_nodes
+        ],
+    )
+
+
 @router.delete("/namespaces/{namespace}/", status_code=HTTPStatus.OK)
 async def deactivate_a_namespace(
     namespace: str,

datajunction-server/datajunction_server/database/namespace.py

@@ -259,7 +259,7 @@ async def list_all_nodes(
             select(Node)
             .where(
                 or_(
-                    Node.namespace.like(f"{namespace}.%"),
+                    Node.namespace.startswith(f"{namespace}.", autoescape=True),
                     Node.namespace == namespace,
                 ),
             )

datajunction-server/datajunction_server/internal/deployment/fingerprints.py

@@ -563,3 +563,25 @@ async def build_deployment_fingerprints(
         submitted_names | additional_target_names,
     )
     return current, proposed_hashes
+
+
+async def build_current_fingerprint_graph(
+    session: AsyncSession,
+    specs: Iterable[NodeSpec],
+    *,
+    version: int = LATEST_SEMANTIC_FINGERPRINT_VERSION,
+) -> SemanticFingerprintGraph:
+    """Build the current semantic graph for a node set."""
+    specs_by_name = {spec.rendered_name: spec for spec in specs}
+    parent_cache: ParentCandidateCache = {}
+    external = await _load_external_specs(
+        session,
+        specs_by_name.values(),
+        ignored_parse_errors=set(),
+        parent_cache=parent_cache,
+    )
+    return SemanticFingerprintGraph(
+        {**external, **specs_by_name},
+        parent_cache=parent_cache,
+        version=version,
+    )

datajunction-server/datajunction_server/internal/deployment/orchestrator.py

@@ -386,6 +386,8 @@ def __init__(
         self._cubes_bumped_by_upstream: dict[str, list[str]] = {}
         # Node name -> the tier its change earned, for those cubes to inherit.
         self._change_tiers: dict[str, ChangeTier] = {}
+        # Unchanged nodes re-deployed only to retry a pre-existing failure.
+        self._revalidation_only: set[str] = set()
         self._current_semantic_fingerprints: FingerprintMap = {}
         self._proposed_semantic_fingerprints: FingerprintMap = {}
 
@@ -3999,6 +4001,7 @@ async def _create_cubes_from_validation(
                 semantic_fingerprint=self._proposed_semantic_fingerprints.get(
                     cube_spec.rendered_name,
                 ),
+                revalidation_only=cube_spec.rendered_name in self._revalidation_only,
             )
 
             deployment_results.append(deployment_result)
@@ -4415,6 +4418,9 @@ def filter_nodes_to_deploy(
         version. So `force` and the INVALID re-deploy below can re-process a node
         without that implying anything about what changed.
 
+        Those re-deployed only for that retry are recorded in `_revalidation_only`
+        and marked on their `DeploymentResult`.
+
         A cube whose own spec is unchanged is still processed when something
         upstream of it is changing, matching what `_propagate_update_downstream`
         does for a `PATCH`: the cube names the same metrics and dimensions, but
@@ -4424,6 +4430,7 @@ def filter_nodes_to_deploy(
         to_create: list[NodeSpec] = []
         to_update: list[NodeSpec] = []
         to_skip: list[NodeSpec] = []
+        revalidation_only: set[str] = set()
         force = self.deployment_spec.force
         for node_spec in self.deployment_spec.nodes:
             existing_spec = existing_nodes_map.get(node_spec.rendered_name)
@@ -4461,9 +4468,11 @@ def filter_nodes_to_deploy(
                     and existing_node.current.status == NodeStatus.INVALID
                 ):
                     to_update.append(node_spec)
+                    revalidation_only.add(node_spec.rendered_name)
                 else:
                     to_skip.append(node_spec)
 
+        self._revalidation_only = revalidation_only
         changed_names = {spec.rendered_name for spec in to_create + to_update}
         self._cubes_bumped_by_upstream = self._cubes_below_changed_nodes(
             to_skip,
@@ -5089,6 +5098,7 @@ async def _process_valid_node_deploy(
             semantic_fingerprint=self._proposed_semantic_fingerprints.get(
                 result.spec.rendered_name,
             ),
+            revalidation_only=result.spec.rendered_name in self._revalidation_only,
         )
         return deployment_result, new_node, new_revision
 

datajunction-server/datajunction_server/models/deployment.py

@@ -1783,6 +1783,9 @@ class Type(str, Enum):
     changed_fields: list[str] = Field(default_factory=list)
     change_tier: ChangeTierName | None = None
     semantic_fingerprint: SemanticFingerprintValue | None = None
+    # True when the node was re-deployed only to retry a pre-existing failure.
+    # Failure reasons are not compared. Nullable for older persisted rows.
+    revalidation_only: bool | None = None
 
 
 class DeploymentInfo(BaseModel):

datajunction-server/datajunction_server/models/namespace.py

@@ -2,6 +2,8 @@
 
 from pydantic import BaseModel, Field
 
+from datajunction_server.models.node_type import NodeType
+from datajunction_server.models.semantic_fingerprint import SemanticFingerprintValue
 from datajunction_server.typing import StrEnum
 
 
@@ -73,3 +75,20 @@ class NamespaceProvisionResponse(BaseModel):
     namespace: str
     owner_role: str
     deployer_role: str | None
+
+
+class NamespaceFingerprintNode(BaseModel):
+    """A node and its current semantic fingerprint."""
+
+    name: str
+    node_type: NodeType
+    owners: list[str] = Field(default_factory=list)
+    semantic_fingerprint: SemanticFingerprintValue
+
+
+class NamespaceFingerprintResponse(BaseModel):
+    """Current semantic fingerprints for a namespace tree."""
+
+    namespace: str
+    version: int
+    nodes: list[NamespaceFingerprintNode] = Field(default_factory=list)