From 0d948450d2dcc5b1c668a6841dce57b3bd63a520 Mon Sep 17 00:00:00 2001 From: Pierre Raybaut <1311787+PierreRaybaut@users.noreply.github.com> Date: Sun, 25 Oct 2026 08:42:07 +0100 Subject: [PATCH] refactor: use the NPZ serialization of Sigima instead of a copy serialization_npz.py was a copy of the NPZ serialization of the DataLab Web API, which diverged from it: non-uniform image coordinates had to be added twice, and object metadata holding NumPy arrays nested in lists (e.g. the regions of interest of an image) raised a TypeError when an object was sent to DataLab. Sigima 1.4 provides the implementation shared with DataLab (sigima.io.serialize_object_to_npz and deserialize_object_from_npz): datalab_kernel.serialization_npz now re-exports it, and sigima>=1.4.0 is required. A test checks that an image with regions of interest round-trips. Closes #5 Assisted-by: Claude Opus 5.5 --- README.md | 2 +- datalab_kernel/serialization_npz.py | 310 +------------------- datalab_kernel/tests/test_webapi_backend.py | 18 +- doc/release_notes/release_0.02.md | 1 + pyproject.toml | 2 +- requirements.txt | 2 +- 6 files changed, 29 insertions(+), 306 deletions(-) diff --git a/README.md b/README.md index bebd6a0..a3f5d2c 100644 --- a/README.md +++ b/README.md @@ -188,7 +188,7 @@ The kernel requires: - Python 3.10 or later - `xeus-python>=0.17.0` - The xeus-based Python kernel - `xeus-python-shell>=0.6.0` - Python shell utilities for xeus-python -- `sigima>=1.0` - Scientific signal and image processing +- `sigima>=1.4.0` - Scientific signal and image processing - `numpy>=1.22`, `h5py>=3.0`, `matplotlib>=3.5` Optional dependencies (via extras): diff --git a/datalab_kernel/serialization_npz.py b/datalab_kernel/serialization_npz.py index 582c101..3b79e8f 100644 --- a/datalab_kernel/serialization_npz.py +++ b/datalab_kernel/serialization_npz.py @@ -5,311 +5,17 @@ NPZ Serialization ================= -Serialization of SignalObj and ImageObj to/from NumPy NPZ format. +Serialization of SignalObj and ImageObj to/from NumPy NPZ format, used by the +DataLab Web API for efficient binary data transfer. -This module provides the client-side implementation of the NPZ format -used by the DataLab Web API for efficient binary data transfer. - -Format Specification (V1) -------------------------- - -The NPZ archive contains: - -For SignalObj: - - ``x.npy``: X coordinates (1D float64 array) - - ``y.npy``: Y data (1D float64 array) - - ``dx.npy``: X uncertainties (optional) - - ``dy.npy``: Y uncertainties (optional) - - ``metadata.json``: JSON-encoded metadata - -For ImageObj: - - ``data.npy``: Image data (2D array, preserves dtype) - - ``xcoords.npy``, ``ycoords.npy``: Pixel center coordinates along X and Y - (1D float arrays, only for non-uniform coordinates) - - ``metadata.json``: JSON-encoded metadata - -Metadata JSON includes: - - ``type``: "signal" or "image" - - ``title``, ``xlabel``, ``ylabel``, etc. - - ``x0``, ``y0``, ``dx``, ``dy`` (for images) - - ``is_uniform_coords`` (for images): False when the coordinates are given by - ``xcoords.npy`` and ``ycoords.npy``. Archives without this key (written by - earlier versions) have uniform coordinates. +The format is implemented by Sigima (:mod:`sigima.io.npz`, where it is +described), so that DataLab and DataLab-Kernel share one implementation: +:func:`serialize_object_to_npz` and :func:`deserialize_object_from_npz` are those +of :mod:`sigima.io`. """ from __future__ import annotations -import io -import json -import zipfile -from typing import TYPE_CHECKING - -import numpy as np -from sigima import ImageObj, SignalObj - -if TYPE_CHECKING: - DataObject = SignalObj | ImageObj - - -def _serialize_obj_metadata(obj_metadata: dict) -> dict: - """Convert object metadata to JSON-serializable format. - - Handles numpy arrays (from GeometryResult.coords, TableResult, etc.) - by converting them to nested lists. - - Args: - obj_metadata: The object's metadata dictionary. - - Returns: - JSON-serializable dictionary. - """ - result = {} - for key, value in obj_metadata.items(): - if isinstance(value, dict): - # Recursively handle nested dicts (like GeometryResult.to_dict()) - result[key] = _serialize_obj_metadata(value) - elif isinstance(value, np.ndarray): - result[key] = value.tolist() - else: - result[key] = value - return result - - -def _deserialize_obj_metadata(serialized: dict) -> dict: - """Convert deserialized metadata back to original types. - - Converts nested lists back to numpy arrays where appropriate - (for coords fields in geometry results). - - Args: - serialized: The JSON-deserialized metadata. - - Returns: - Metadata with numpy arrays restored. - """ - result = {} - for key, value in serialized.items(): - if isinstance(value, dict): - # Recursively handle nested dicts - result[key] = _deserialize_obj_metadata(value) - elif isinstance(value, list) and key == "coords": - # Convert coords back to numpy array - result[key] = np.array(value) - else: - result[key] = value - return result - - -def serialize_object_to_npz(obj: DataObject, *, compress: bool = True) -> bytes: - """Serialize a SignalObj or ImageObj to NPZ format. - - Args: - obj: The object to serialize. - compress: If True (default), use ZIP deflate compression. - Set to False for faster serialization at the cost of larger size. - For incompressible data (random images), False can be 10x faster. - - Returns: - Bytes containing the NPZ archive. - - Raises: - TypeError: If object type is not supported. - """ - buffer = io.BytesIO() - obj_type = type(obj).__name__ - - if obj_type == "SignalObj": - _serialize_signal(obj, buffer, compress=compress) - elif obj_type == "ImageObj": - _serialize_image(obj, buffer, compress=compress) - else: - raise TypeError(f"Unsupported object type: {obj_type}") - - return buffer.getvalue() - - -def _serialize_signal(obj, buffer: io.BytesIO, *, compress: bool = True) -> None: - """Serialize a SignalObj to NPZ format.""" - metadata = { - "type": "signal", - "title": obj.title, - "xlabel": obj.xlabel, - "ylabel": obj.ylabel, - "xunit": obj.xunit, - "yunit": obj.yunit, - } - # Include object metadata (contains Geometry_*, Table_* results) - obj_metadata = obj.metadata - if obj_metadata: - metadata["obj_metadata"] = _serialize_obj_metadata(obj_metadata) - - compression = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED - with zipfile.ZipFile(buffer, "w", compression) as zf: - _write_array_to_zip(zf, "x.npy", obj.x) - _write_array_to_zip(zf, "y.npy", obj.y) - - if obj.dx is not None: - _write_array_to_zip(zf, "dx.npy", obj.dx) - if obj.dy is not None: - _write_array_to_zip(zf, "dy.npy", obj.dy) - - zf.writestr("metadata.json", json.dumps(metadata, ensure_ascii=False)) - - -def _serialize_image(obj, buffer: io.BytesIO, *, compress: bool = True) -> None: - """Serialize an ImageObj to NPZ format.""" - metadata = { - "type": "image", - "title": obj.title, - "xlabel": obj.xlabel, - "ylabel": obj.ylabel, - "zlabel": obj.zlabel, - "xunit": obj.xunit, - "yunit": obj.yunit, - "zunit": obj.zunit, - "x0": obj.x0, - "y0": obj.y0, - "dx": obj.dx, - "dy": obj.dy, - "is_uniform_coords": bool(obj.is_uniform_coords), - } - # Include object metadata (contains Geometry_*, Table_* results) - obj_metadata = obj.metadata - if obj_metadata: - metadata["obj_metadata"] = _serialize_obj_metadata(obj_metadata) - - compression = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED - with zipfile.ZipFile(buffer, "w", compression) as zf: - _write_array_to_zip(zf, "data.npy", obj.data) - if not obj.is_uniform_coords: - _write_array_to_zip(zf, "xcoords.npy", obj.xcoords) - _write_array_to_zip(zf, "ycoords.npy", obj.ycoords) - zf.writestr("metadata.json", json.dumps(metadata, ensure_ascii=False)) - - -def _write_array_to_zip(zf: zipfile.ZipFile, name: str, arr: np.ndarray) -> None: - """Write a numpy array to a zip file.""" - arr_buffer = io.BytesIO() - np.save(arr_buffer, arr, allow_pickle=False) - zf.writestr(name, arr_buffer.getvalue()) - - -def deserialize_object_from_npz(data: bytes) -> DataObject: - """Deserialize a SignalObj or ImageObj from NPZ format. - - Args: - data: Bytes containing the NPZ archive. - - Returns: - SignalObj or ImageObj. - - Raises: - ValueError: If the archive format is invalid. - """ - buffer = io.BytesIO(data) - - with zipfile.ZipFile(buffer, "r") as zf: - if "metadata.json" not in zf.namelist(): - raise ValueError("Invalid NPZ format: missing metadata.json") - - metadata = json.loads(zf.read("metadata.json")) - obj_type = metadata.get("type") - - if obj_type == "signal": - return _deserialize_signal(zf, metadata) - if obj_type == "image": - return _deserialize_image(zf, metadata) - - raise ValueError(f"Unknown object type in NPZ: {obj_type}") - - -def _read_array_from_zip(zf: zipfile.ZipFile, name: str) -> np.ndarray | None: - """Read a numpy array from a zip file.""" - if name not in zf.namelist(): - return None - arr_buffer = io.BytesIO(zf.read(name)) - return np.load(arr_buffer, allow_pickle=False) - - -def _deserialize_signal(zf: zipfile.ZipFile, metadata: dict) -> DataObject: - """Deserialize a SignalObj from NPZ archive.""" - x = _read_array_from_zip(zf, "x.npy") - y = _read_array_from_zip(zf, "y.npy") - dx = _read_array_from_zip(zf, "dx.npy") - dy = _read_array_from_zip(zf, "dy.npy") - - if x is None or y is None: - raise ValueError("Invalid signal NPZ: missing x.npy or y.npy") - - obj = SignalObj() - obj.set_xydata(x, y, dx=dx, dy=dy) - - if metadata.get("title"): - obj.title = metadata["title"] - if metadata.get("xlabel"): - obj.xlabel = metadata["xlabel"] - if metadata.get("ylabel"): - obj.ylabel = metadata["ylabel"] - if metadata.get("xunit"): - obj.xunit = metadata["xunit"] - if metadata.get("yunit"): - obj.yunit = metadata["yunit"] - - # Restore object metadata (Geometry_*, Table_* results) - if metadata.get("obj_metadata"): - obj.metadata = _deserialize_obj_metadata(metadata["obj_metadata"]) - - return obj - - -def _deserialize_image(zf: zipfile.ZipFile, metadata: dict) -> DataObject: - """Deserialize an ImageObj from NPZ archive.""" - data = _read_array_from_zip(zf, "data.npy") - if data is None: - raise ValueError("Invalid image NPZ: missing data.npy") - - obj = ImageObj() - obj.data = data - - if metadata.get("title"): - obj.title = metadata["title"] - if metadata.get("xlabel"): - obj.xlabel = metadata["xlabel"] - if metadata.get("ylabel"): - obj.ylabel = metadata["ylabel"] - if metadata.get("zlabel"): - obj.zlabel = metadata["zlabel"] - if metadata.get("xunit"): - obj.xunit = metadata["xunit"] - if metadata.get("yunit"): - obj.yunit = metadata["yunit"] - if metadata.get("zunit"): - obj.zunit = metadata["zunit"] - - # Set coordinate info - if metadata.get("is_uniform_coords", True): - obj.x0 = metadata.get("x0", 0.0) - obj.y0 = metadata.get("y0", 0.0) - obj.dx = metadata.get("dx", 1.0) - obj.dy = metadata.get("dy", 1.0) - else: - xcoords = _read_array_from_zip(zf, "xcoords.npy") - ycoords = _read_array_from_zip(zf, "ycoords.npy") - if xcoords is None or ycoords is None: - raise ValueError( - "Invalid image NPZ: missing xcoords.npy or ycoords.npy " - "for non-uniform coordinates" - ) - nrows, ncols = data.shape - if xcoords.shape != (ncols,) or ycoords.shape != (nrows,): - raise ValueError( - f"Invalid image NPZ: coordinates of shape {xcoords.shape} (X) and " - f"{ycoords.shape} (Y) do not match image shape {data.shape}" - ) - obj.set_coords(xcoords, ycoords) - - # Restore object metadata (Geometry_*, Table_* results) - if metadata.get("obj_metadata"): - obj.metadata = _deserialize_obj_metadata(metadata["obj_metadata"]) +from sigima.io import deserialize_object_from_npz, serialize_object_to_npz - return obj +__all__ = ["deserialize_object_from_npz", "serialize_object_to_npz"] diff --git a/datalab_kernel/tests/test_webapi_backend.py b/datalab_kernel/tests/test_webapi_backend.py index 5f7e642..9b9464d 100644 --- a/datalab_kernel/tests/test_webapi_backend.py +++ b/datalab_kernel/tests/test_webapi_backend.py @@ -25,7 +25,11 @@ deserialize_object_from_npz, serialize_object_to_npz, ) -from datalab_kernel.tests.data import make_test_image, make_test_image_nonuniform +from datalab_kernel.tests.data import ( + make_test_image, + make_test_image_nonuniform, + make_test_image_rich, +) class TestNPZSerialization: @@ -154,6 +158,18 @@ def test_image_preserves_dtype(self): f"Expected {dtype}, got {result.data.dtype}" ) + def test_image_rich_metadata_round_trip(self): + """Metadata holding NumPy arrays nested in lists (ROIs) are transferred.""" + obj = make_test_image_rich() + result = deserialize_object_from_npz(serialize_object_to_npz(obj)) + np.testing.assert_array_equal(result.data, obj.data) + assert set(result.metadata) == set(obj.metadata) + assert len(result.roi.single_rois) == len(obj.roi.single_rois) + rois = zip(result.roi.single_rois, obj.roi.single_rois, strict=True) + for roi, expected in rois: + np.testing.assert_array_equal(roi.coords, expected.coords) + assert roi.title == expected.title + @pytest.mark.parametrize("compress", [False, True]) def test_image_nonuniform_round_trip(self, compress): """Non-uniform coordinates are kept, in the same layout as DataLab.""" diff --git a/doc/release_notes/release_0.02.md b/doc/release_notes/release_0.02.md index 7ccc8fd..d15108e 100644 --- a/doc/release_notes/release_0.02.md +++ b/doc/release_notes/release_0.02.md @@ -5,6 +5,7 @@ * Python 3.10 or later is now required. Python 3.9 users must keep an earlier compatible release. * Kernel language information now reports the Python version actually running the kernel, rather than always announcing Python 3.9. * Fixed images with non-uniform coordinates losing their coordinates when transferred to or from DataLab, or when the workspace was saved to and loaded from an HDF5 file: they came back with uniform coordinates equal to the pixel indices, so that their physical positions were wrong. The pixel center coordinates are now kept, in the same NPZ format as DataLab (1.4.0 or later); data and files written by earlier versions are still read with uniform coordinates. +* Sigima 1.4.0 or later is now required: objects are transferred to and from DataLab with the NPZ serialization of Sigima, shared with DataLab, instead of a copy of it. Objects whose metadata hold arrays in lists (for example images with regions of interest) can now be transferred: they raised an error. ## DataLab-Kernel Version 0.2.11 (2026-02-15) ## diff --git a/pyproject.toml b/pyproject.toml index 58cebda..30e62f4 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -33,7 +33,7 @@ dependencies = [ "httpx>=0.25.0", "h5py>=3.0", "matplotlib>=3.5", - "sigima>=1.0", + "sigima>=1.4.0", ] dynamic = ["version"] diff --git a/requirements.txt b/requirements.txt index 9aaaf9d..6cf55f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ pydata-sphinx-theme>=0.14 pytest-cov>=4.0 pytest>=7.0 ruff>=0.1.0 -sigima>=1.0 +sigima>=1.4.0 sphinx-copybutton>=0.5 sphinx-design>=0.5 sphinx>=7.0