Skip to content

Commit 6b4d331

Browse files
committed
feat: import legacy processing records as legacy activities
Assisted-by: Claude Opus 5.5
1 parent 4736e21 commit 6b4d331

7 files changed

Lines changed: 773 additions & 34 deletions

File tree

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ Status: experimental, under development. Nothing is published yet.
1010
- Shared replay preparation (eligibility codes), the `exact` comparison rule and verification reports.
1111
- HDF5 provenance block (`/DataLab_Provenance`) read and written with h5py alone; links and virtual datasets are refused.
1212
- Integrity primitives: RFC 8785 canonical JSON, SHA-256 digests, the `datalab-signal-v1` and `datalab-image-v1` state fingerprints (data, coordinates or calibration, units). A state also records the digest of its ROI definition, and an analysis result the digest of its stored value: a saved object whose data, calibration, units or ROI differ from its state is `altered`, and a result replaced by a later analysis is never attributed to the earlier one. Fingerprints are streamed into SHA-256 without copying the arrays.
13+
- Legacy workspaces: processing records stored by DataLab in object metadata (`__processing_parameters`, `__analysis_parameters`) of objects unknown to the ledger become `legacy_metadata` activities. They keep the record as stored and invent nothing: inputs are bound to the recorded objects (state unknown), the call is opaque without parameters, environment, edition and times are null, and import issues (missing ancestry, unavailable inputs, unsupported operations, version warnings...) are listed. They are never replayed.
1314
- Capsules (`.dlcapsule`): a ZIP archive holding `workspace.h5` and an RO-Crate 1.3 manifest (`ro-crate-metadata.json`) that conforms to Process Run Crate 0.6 and to the experimental [DataLab capsule profile 0.1](https://datalab-platform.com/capsule/0.1) (terms: `https://datalab-platform.com/capsule/terms#`). The reader refuses unsafe archives (path traversal, duplicates, links, encryption, size and compression-ratio limits) and never accesses the network. It holds the whole archive in memory: it checks the archive size before reading it, then the size of each entry. Each edition passes its own `SizePolicy`; the default one (4 GiB workspace) is an accepted upper bound, not a tested capacity.
1415
- Pure Python for CPython 3.9+ and Pyodide (Python 3.12). The core only needs the standard library and `jsonschema`; HDF5 helpers use NumPy and h5py.
1516
- The package never imports DataLab, Sigima, SigimaX or Qt, and never executes code named in a file.

‎src/datalab_capsule/calls.py‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,8 @@
1010
Binding kinds:
1111
1212
- ``{"state_id": ...}``: a concrete state (used by activities);
13-
- ``{"object_uuids": [...]}``: an object selection (reserved);
13+
- ``{"object_uuids": [...]}``: objects whose state at execution time is unknown
14+
(only in ``legacy_metadata`` activities, see :mod:`datalab_capsule.legacy`);
1415
- ``{"step": ..., "role": ...}``: the output of an upstream step (reserved).
1516
"""
1617

@@ -35,6 +36,7 @@
3536
OPERATION_ID_PATTERN = re.compile(r"^[a-z0-9]+(?:[._-][a-z0-9]+)*$")
3637
BINDING_KINDS = ("state_id", "object_uuids", "step")
3738
_IMPLEMENTED_BINDINGS = ("state_id",)
39+
_UUID = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$")
3840

3941

4042
class CallError(ProvenanceError, ValueError):
@@ -83,22 +85,37 @@ def is_opaque(call: dict[str, Any]) -> bool:
8385
return call.get("operation") is None
8486

8587

86-
def _check_binding(binding: Any, where: str) -> None:
88+
def _check_binding(binding: Any, where: str, allow_objects: bool) -> None:
8789
if not isinstance(binding, dict) or len(binding) == 0:
8890
raise CallError(f"{where}: binding must be a non-empty object")
8991
kinds = [kind for kind in BINDING_KINDS if kind in binding]
9092
if len(kinds) != 1:
9193
raise CallError(f"{where}: binding must hold exactly one of {BINDING_KINDS}")
9294
kind = kinds[0]
95+
if kind == "object_uuids" and allow_objects:
96+
uuids = binding["object_uuids"]
97+
if (
98+
set(binding) != {"object_uuids"}
99+
or not isinstance(uuids, list)
100+
or not uuids
101+
or not all(isinstance(u, str) and _UUID.match(u) for u in uuids)
102+
):
103+
raise CallError(f"{where}: object binding must list object UUIDs")
104+
return
93105
if kind not in _IMPLEMENTED_BINDINGS:
94106
raise ReservedBindingError(f"{where}: binding kind {kind!r} is reserved")
95107
if set(binding) != {"state_id"} or not isinstance(binding["state_id"], str):
96108
raise CallError(f"{where}: state binding must be {{'state_id': <str>}}")
97109

98110

99-
def validate_call(call: Any) -> None:
111+
def validate_call(call: Any, *, allow_object_bindings: bool = False) -> None:
100112
"""Validate the structure of an operation call.
101113
114+
Args:
115+
call: Call to validate.
116+
allow_object_bindings: Accept ``object_uuids`` bindings (legacy
117+
activities only).
118+
102119
Raises:
103120
ReservedBindingError: If a reserved binding kind is used.
104121
CallError: If the call is malformed.
@@ -131,4 +148,4 @@ def validate_call(call: Any) -> None:
131148
raise CallError(f"{where}: an input holds exactly role and binding")
132149
if not isinstance(item["role"], str) or not item["role"]:
133150
raise CallError(f"{where}: role must be a non-empty string")
134-
_check_binding(item["binding"], where)
151+
_check_binding(item["binding"], where, allow_object_bindings)

‎src/datalab_capsule/ledger.py‎

Lines changed: 84 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -40,8 +40,9 @@
4040

4141
LEDGER_SCHEMA = "datalab-provenance-ledger"
4242
LEDGER_SCHEMA_VERSION = 1
43-
#: Origins written today; ``recipe``, ``legacy_metadata`` and ``opaque`` are
44-
#: reserved and accepted when reading.
43+
#: Origins written by :meth:`Ledger.record_activity`; ``legacy_metadata`` is
44+
#: written by :meth:`Ledger.record_legacy_activity`; ``recipe`` and ``opaque``
45+
#: are reserved and accepted when reading.
4546
ORIGINS = ("ordinary", "recompute_in_place", "history_replay")
4647

4748

@@ -106,17 +107,26 @@ def validate_ledger(data: Any) -> None:
106107
if act_id in activity_ids:
107108
problems.append(f"{where}: duplicate activity_id")
108109
activity_ids.add(act_id)
110+
legacy = activity["origin"] == "legacy_metadata"
109111
try:
110-
validate_call(activity["call"])
112+
validate_call(activity["call"], allow_object_bindings=legacy)
111113
except CallError as exc:
112114
problems.append(f"{where}: {exc}")
113115
continue
114116
for item in activity["call"]["inputs"]:
115-
if item["binding"]["state_id"] not in states:
117+
state_id = item["binding"].get("state_id")
118+
if state_id is not None and state_id not in states:
116119
problems.append(f"{where}: unknown input state")
117-
if is_opaque(activity["call"]) and activity["implementation"] is None:
120+
if legacy and not is_opaque(activity["call"]):
121+
problems.append(f"{where}: a legacy activity is never qualified")
122+
if (
123+
is_opaque(activity["call"])
124+
and activity["implementation"] is None
125+
and not legacy
126+
):
118127
problems.append(f"{where}: an opaque call needs an implementation")
119-
if activity["environment_id"] not in environments:
128+
env_id = activity["environment_id"]
129+
if env_id is not None and env_id not in environments:
120130
problems.append(f"{where}: unknown environment")
121131
for output in activity["outputs"]:
122132
state_id = output.get("state_id")
@@ -385,3 +395,71 @@ def record_activity(
385395
self._activity_index[activity_id] = len(self._data["activities"])
386396
self._data["activities"].append(activity)
387397
return activity
398+
399+
def record_legacy_activity(
400+
self,
401+
*,
402+
inputs: Iterable[tuple[str, str]],
403+
outputs: Iterable[tuple[str, str, dict[str, Any]]],
404+
legacy: dict[str, Any],
405+
limits: Iterable[str] = (),
406+
) -> dict[str, Any]:
407+
"""Append an activity rebuilt from processing metadata of an old workspace.
408+
409+
What the metadata does not say stays unknown: inputs are bound to objects
410+
(their state at execution time is unknown), the call is opaque with no
411+
parameters, and environment, edition, implementation and times are null.
412+
Such an activity is never replayed.
413+
414+
Args:
415+
inputs: Ordered ``(role, object_uuid)`` pairs.
416+
outputs: Ordered ``(role, object_uuid, facts)`` triples, observed when
417+
the workspace was opened.
418+
legacy: ``{"key", "object_uuid", "record", "saved_by_version",
419+
"issues"}`` (see :mod:`datalab_capsule.legacy`).
420+
limits: Reasons why the activity is not replayable.
421+
422+
Returns:
423+
The appended activity.
424+
425+
Raises:
426+
LedgerError: If the activity is inconsistent.
427+
"""
428+
call = {
429+
"operation": None,
430+
"parameters": None,
431+
"inputs": [
432+
{"role": role, "binding": {"object_uuids": [obj_uuid]}}
433+
for role, obj_uuid in inputs
434+
],
435+
}
436+
try:
437+
validate_call(call, allow_object_bindings=True)
438+
except CallError as exc:
439+
raise LedgerError(str(exc)) from exc
440+
activity_id = str(uuid.uuid4())
441+
activity = {
442+
"activity_id": activity_id,
443+
"command_id": None,
444+
"call": call,
445+
"implementation": None,
446+
"outputs": [
447+
{
448+
"role": role,
449+
"state_id": self._new_state(obj_uuid, facts, activity_id),
450+
}
451+
for role, obj_uuid, facts in outputs
452+
],
453+
"context": {},
454+
"limits": list(limits),
455+
"environment_id": None,
456+
"edition": None,
457+
"origin": "legacy_metadata",
458+
"status": "completed",
459+
"started_at": None,
460+
"finished_at": None,
461+
"legacy": copy.deepcopy(legacy),
462+
}
463+
self._activity_index[activity_id] = len(self._data["activities"])
464+
self._data["activities"].append(activity)
465+
return activity

0 commit comments

Comments
 (0)