Record bundle_id and fit provenance in every model bundle

packages/kumo-core/src/kumo_core/artifacts.py

@@ -13,7 +13,14 @@
 import tempfile
 from dataclasses import dataclass, field
 from pathlib import Path
-from typing import Any, ClassVar, Mapping, Protocol, runtime_checkable
+from typing import (
+    Any,
+    ClassVar,
+    Iterable,
+    Mapping,
+    Protocol,
+    runtime_checkable,
+)
 
 from .io.filesystem import atomic_write_bytes, fsync_directory, local_path
 from .io.json import load_json, save_json
@@ -248,14 +255,17 @@ def _sha256(path: Path) -> str:
         return digest.hexdigest()
 
 
+#: Earlier name of the bundle manifest; also kept out of the bundle id.
+_OLD_PROVENANCE_FILE = 'provenance.json'
+
+
 def verify_model_bundle(bundle: Path) -> str:
     """Verify fetched artifacts before parsing or staging a model bundle.
 
-    Preserves the local SDK import verifier's error and content-identity
-    contract. Paths must stay inside the bundle after symlink resolution;
-    declared sizes and SHA-256 digests must match. Older manifests may omit
-    sizes or leave training-spec.json unlisted; its digest still participates
-    in the returned content identity.
+    Returns the bundle id, a SHA-256 over every verified file except
+    ``provenance.json``. Paths must stay inside the bundle after symlink
+    resolution; declared sizes and SHA-256 digests must match. Older
+    manifests may omit sizes or leave training-spec.json unlisted.
     """
     from .sources import layout
     from .sources.errors import PermanentSourceError
@@ -273,7 +283,7 @@ def verify_model_bundle(bundle: Path) -> str:
 
     artifacts = manifest.get('artifacts')
     problems: list[str] = []
-    verified: list[tuple[str, str, str]] = []
+    verified: list[tuple[str, str]] = []
     seen = set()
     claimed = set()
     if not artifacts:
@@ -325,7 +335,7 @@ def verify_model_bundle(bundle: Path) -> str:
         if VitalArtifactPublicationPolicy._sha256(path) != expected:
             problems.append(f'{relative} does not match its digest')
             continue
-        verified.append((role, safe, expected))
+        verified.append((safe, expected))
     if problems:
         raise ValueError(
             f"The model bundle at {bundle} did not survive the transfer: "
@@ -335,11 +345,47 @@ def verify_model_bundle(bundle: Path) -> str:
     if not spec_path.resolve().is_relative_to(bundle):
         raise ValueError('Training specification is not below the bundle')
     if 'training-spec.json' not in seen and spec_path.is_file():
-        verified.append(('training_spec', 'training-spec.json',
+        verified.append(('training-spec.json',
                          VitalArtifactPublicationPolicy._sha256(spec_path)))
-    payload = json.dumps(sorted(verified), sort_keys=True,
-                         separators=(',', ':'), default=str)
-    return hashlib.sha256(payload.encode('utf-8')).hexdigest()[:16]
+    return _bundle_id(verified)
+
+
+def bundle_id_from_manifest(bundle: Path) -> str:
+    """The bundle id from the digests a freshly published bundle lists."""
+    manifest_path = Path(bundle) / VitalArtifactPublicationPolicy.MANIFEST_NAME
+    manifest = json.loads(manifest_path.read_text(encoding='utf-8'))
+    return _bundle_id((str(entry['relative_path']), str(entry['sha256']))
+                      for entry in manifest['artifacts'])
+
+
+def _bundle_id(pairs: Iterable[tuple[str, str]]) -> str:
+    payload = json.dumps(
+        sorted(pair for pair in pairs
+               if pair[0] not in (PROVENANCE_FILE, _OLD_PROVENANCE_FILE)),
+        separators=(',', ':'))
+    return hashlib.sha256(payload.encode('utf-8')).hexdigest()
+
+
+def add_bundle_file(bundle_dir: Path, role: str, relative_path: str) -> None:
+    """List one file already written in a bundle in its publication manifest."""
+    bundle_dir = Path(bundle_dir)
+    path = bundle_dir / relative_path
+    manifest_path = bundle_dir / VitalArtifactPublicationPolicy.MANIFEST_NAME
+    manifest = json.loads(manifest_path.read_text(encoding='utf-8'))
+    manifest['artifacts'] = [
+        entry for entry in manifest.get('artifacts') or []
+        if entry.get('relative_path') != relative_path
+        and entry.get('role') != role
+    ] + [
+        PublishedArtifact(
+            role=role,
+            relative_path=relative_path,
+            size_bytes=path.stat().st_size,
+            sha256=VitalArtifactPublicationPolicy._sha256(path),
+        ).to_dict()
+    ]
+    payload = json.dumps(manifest, indent=2, sort_keys=True) + '\n'
+    atomic_write_bytes(manifest_path, payload.encode('utf-8'))
 
 
 @runtime_checkable

packages/kumo-core/tests/test_artifacts.py

@@ -15,7 +15,10 @@
     FilesystemVitalArtifactPublisher,
     VitalArtifactPublicationPolicy,
     VitalArtifactSources,
+    add_bundle_file,
+    bundle_id_from_manifest,
     resolve_vital_artifact_sources,
+    verify_model_bundle,
 )
 from kumo_core.config import NativeRuntimeConfig
 from kumo_core.manifest import RunManifest, RunPaths
@@ -237,3 +240,27 @@ def best_ckpt(self):
     assert sources.selected_checkpoint == checkpoints[selected_mode]
     assert sources.graph_manifest == bundle / "graph.json"
     assert json.loads(sources.metrics.read_text()) == {"test_accuracy": 0.8}
+
+
+def test_bundle_id_ignores_provenance_only(tmp_path: Path) -> None:
+    sources, _ = _vital_sources(tmp_path)
+    bundle = tmp_path / "bundle"
+    VitalArtifactPublicationPolicy().publish(sources, bundle)
+    bundle_id = verify_model_bundle(bundle)
+
+    for name, payload in (("_provenance.json", '{"a": 1}'),
+                          ("_provenance.json", '{"a": 2}'),
+                          ("provenance.json", '{"a": 3}')):
+        (bundle / name).write_text(payload)
+        add_bundle_file(bundle, "provenance", name)
+        assert verify_model_bundle(bundle) == bundle_id
+        assert bundle_id_from_manifest(bundle) == bundle_id
+    roles = [
+        entry["role"] for entry in json.loads(
+            (bundle / "publication-manifest.json").read_text())["artifacts"]
+    ]
+    assert roles.count("provenance") == 1
+
+    (bundle / "metrics.json").write_text('{"auc": 0}')
+    add_bundle_file(bundle, "metrics", "metrics.json")
+    assert verify_model_bundle(bundle) != bundle_id

packages/kumo-engine/docs/mlops.md

@@ -35,3 +35,14 @@ A generated or caller-supplied training table carries `_provenance.json` in its
 `TrainingTable.id` is deterministic: the same snapshot, query, plan, split seed and engine build give the same id. A caller-supplied source without a version gets a random id. An export to files (`TrainingTable.export`) writes the manifest next to the Parquet parts. `TrainingTable.load_from_connector(connector, path=..., pq=..., snapshot=...)` materializes an export again.
 
 With MLOps on, each table is stored once under `<root>/training_tables/<id>/`.
+
+### Models
+
+Every fitted bundle carries `_provenance.json` (`ModelManifest`):
+- its `id`, the bundle id (a content hash that leaves the manifest file out);
+- when it was fitted and with which fit mode;
+- its parents: the training table's manifest, embedded with that table's snapshot, and the warm-start model's manifest, if any.
+
+An embedded model keeps only the ids of its own model parents, so a long chain of warm-start retrains does not grow each manifest. `fit` returns a new `TrainingResult` per call (`.id`, `.manifest`); `Trainer.result` and `Trainer.manifest` describe the latest fit or the loaded bundle.
+
+With MLOps on, each bundle is stored under `<root>/models/<bundle id>/`. A complete copy that is already there is kept; a partial one is replaced.

packages/kumo-engine/kumo/engine/__init__.py

@@ -25,7 +25,11 @@
 from kumoapi.plugin import ModelRegistryLocation
 from kumoapi.table import Column
 
-from kumo.engine._manifest import SnapshotManifest, TrainingTableManifest
+from kumo.engine._manifest import (
+    ModelManifest,
+    SnapshotManifest,
+    TrainingTableManifest,
+)
 from kumo.engine._mlops import MLOps, mlops
 from kumo.engine._version import __version__
 from kumo.engine.config import configure
@@ -89,6 +93,7 @@
     "mlops",
     "SnapshotManifest",
     "TrainingTableManifest",
+    "ModelManifest",
     "PredictiveQuery",
     "TrainingTable",
     "PredictionTable",

packages/kumo-engine/kumo/engine/_manifest.py

@@ -224,6 +224,32 @@ class TrainingTableManifest(Manifest):
     split_seed: Optional[int] = None
 
 
+@dataclass(frozen=True, kw_only=True)
+class ModelParents(Record):
+    training_table: Optional[TrainingTableManifest] = None
+    warm_start: Optional['ModelManifest'] = None
+    derived_from: Optional['ModelManifest'] = omit_none()
+
+
+@dataclass(frozen=True, kw_only=True)
+class FitTiming(Record):
+    started_at: str
+    finished_at: str
+    seconds: float
+
+
+@dataclass(frozen=True, kw_only=True)
+class ModelManifest(Manifest):
+    """How a model bundle was fitted: from which training table (embedded,
+    with its snapshot) and warm-start model, when, and with which fit
+    mode. Its ``id`` is the bundle id."""
+
+    kind: str = 'model'
+    parents: Optional[ModelParents] = None
+    fit: Optional[FitTiming] = None
+    fit_mode: Optional[str] = None
+
+
 def write_manifest(path: Path, manifest: Manifest) -> None:
     payload = json.dumps(manifest.to_dict(), indent=2, sort_keys=True) + '\n'
     atomic_write_bytes(path, payload.encode('utf-8'))

packages/kumo-engine/kumo/engine/trainer/_provenance.py

@@ -0,0 +1,87 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026, NVIDIA CORPORATION & AFFILIATES.
+# All rights reserved.
+# SPDX-License-Identifier: Apache-2.0
+"""A model bundle's ``_provenance.json``; excluded from the bundle id."""
+
+from dataclasses import replace
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+from kumo.engine._manifest import (
+    FitTiming,
+    ModelManifest,
+    ModelParents,
+    embedded,
+    read_manifest,
+    stamp,
+    stub,
+    write_manifest,
+)
+from kumo.engine.pquery.training_table import TrainingTable
+from kumo.engine.trainer._spec import PROVENANCE_ROLE, listed_file
+
+
+def write_provenance(
+    bundle_dir: Path,
+    *,
+    bundle_id: str,
+    train_table: TrainingTable,
+    warm_start: Optional[ModelManifest],
+    started_at: datetime,
+    fit_mode: Optional[str],
+) -> None:
+    from kumo_core.artifacts import PROVENANCE_FILE, add_bundle_file
+
+    finished_at = datetime.now(timezone.utc)
+    table = train_table.manifest
+    write_manifest(
+        bundle_dir / PROVENANCE_FILE,
+        ModelManifest(
+            id=bundle_id,
+            parents=ModelParents(
+                training_table=embedded(table) if table is not None else
+                stub('training_table', train_table.id or train_table.tt_id),
+                warm_start=None
+                if warm_start is None else embedded_model(warm_start)),
+            fit=FitTiming(
+                started_at=started_at.isoformat(timespec='seconds'),
+                finished_at=finished_at.isoformat(timespec='seconds'),
+                seconds=round((finished_at - started_at).total_seconds(), 3),
+            ),
+            fit_mode=fit_mode,
+            **stamp(),
+        ))
+    add_bundle_file(bundle_dir, PROVENANCE_ROLE, PROVENANCE_FILE)
+
+
+def embedded_model(manifest: ModelManifest) -> ModelManifest:
+    r"""``manifest`` as a child model embeds it: its own model parents cut
+    to stubs, so a chain of retrains does not grow each manifest."""
+    copy = embedded(manifest)
+    parents = copy.parents
+    if parents is None:
+        return copy
+
+    def cut(model: Optional[ModelManifest]) -> Optional[ModelManifest]:
+        return None if model is None else ModelManifest(id=model.id,
+                                                        reconstructed=True)
+
+    return replace(copy, parents=replace(
+        parents, warm_start=cut(parents.warm_start),
+        derived_from=cut(parents.derived_from)))
+
+
+def parent_model(bundle_dir: Path, bundle_id: Optional[str]
+                 ) -> Optional[ModelManifest]:
+    r"""The manifest of the bundle at ``bundle_dir``, or a stub."""
+    manifest = read_provenance(bundle_dir)
+    if manifest is not None:
+        return manifest
+    return None if bundle_id is None else ModelManifest(id=bundle_id,
+                                                        reconstructed=True)
+
+
+def read_provenance(bundle_dir: Path) -> Optional[ModelManifest]:
+    path = listed_file(bundle_dir, PROVENANCE_ROLE)
+    return None if path is None else read_manifest(path, ModelManifest)