Let any sink hold files: FileStoreSink protocol and a chunked filesystem

packages/kumo-api/kumoapi/json_serde.py

@@ -9,7 +9,7 @@
 from decimal import Decimal
 from enum import Enum
 from pathlib import PurePath
-from typing import Any, Dict, Type, TypeVar, Union
+from typing import Any, Dict, Type, TypeVar, Union, cast
 from uuid import UUID
 
 from pydantic import BaseModel, SecretStr
@@ -139,5 +139,5 @@ def from_json(obj: Any, cls: Union[Type[T], object]) -> T:
     if isinstance(obj, str):
         obj = json.loads(obj)
     from pydantic import TypeAdapter
-    adapter = TypeAdapter(cls)
+    adapter = TypeAdapter(cast(Any, cls))
     return adapter.validate_python(obj)

packages/kumo-api/kumoapi/plugin.py

@@ -29,6 +29,7 @@
     Mapping,
     Optional,
     Protocol,
+    Tuple,
     TypeVar,
     runtime_checkable,
 )
@@ -43,6 +44,7 @@
 __all__ = [
     'ArtifactLocation',
     'ArtifactSink',
+    'FileStoreSink',
     'IngestResult',
     'ModelFetchingSink',
     'ModelListingSink',
@@ -604,6 +606,22 @@ def write_files(
         ...
 
 
+@runtime_checkable
+class FileStoreSink(Protocol):
+    r"""A sink that also holds files: snapshot exports, model bundles and
+    export manifests go through its filesystem."""
+    def holds_files(self, connector: SourceConnector, path: str) -> bool:
+        r"""Whether ``path`` is a file location of this sink rather than a
+        table name; decided from the path alone, without I/O."""
+        ...
+
+    def filesystem(self, connector: SourceConnector,
+                   path: str) -> Tuple[Any, str]:
+        r"""A ``pyarrow.fs.FileSystem`` and ``path`` resolved on it, for a
+        path :meth:`holds_files` accepts."""
+        ...
+
+
 @runtime_checkable
 class ModelPublishingSink(ArtifactSink, Protocol):
     r"""A sink that can also receive a trained model bundle.

packages/kumo-core/src/kumo_core/artifacts.py

@@ -19,6 +19,10 @@
 from .io.json import load_json, save_json
 from .manifest import RunManifest, RunPaths
 
+#: Every engine artifact's manifest file; underscore-prefixed so Parquet
+#: readers skip it.
+PROVENANCE_FILE = '_provenance.json'
+
 
 @runtime_checkable
 class ArtifactStore(Protocol):