diff --git a/datalab/tests/webapi_extra_test.py b/datalab/tests/webapi_extra_test.py index 42553a183..b8e7093ab 100644 --- a/datalab/tests/webapi_extra_test.py +++ b/datalab/tests/webapi_extra_test.py @@ -40,8 +40,6 @@ set_server_url, ) from datalab.webapi.serialization import ( - _deserialize_obj_metadata, - _make_json_serializable, _to_python_scalar, deserialize_object_from_npz, object_to_metadata, @@ -148,7 +146,7 @@ def test_workspace_adapter_resolves_uuid_and_rejects_ambiguous_titles() -> None: # ============================================================================= -# Serialization helpers — _to_python_scalar / _make_json_serializable +# Serialization helpers — _to_python_scalar # ============================================================================= @@ -180,71 +178,6 @@ def test_passthrough(self) -> None: assert _to_python_scalar(3.14) == pytest.approx(3.14) -class TestMakeJsonSerializable: - """Tests for the ``_make_json_serializable`` recursive helper.""" - - def test_nested_dict_with_numpy(self) -> None: - """Numpy values nested in dicts are recursively converted.""" - data = { - "outer": { - "ints": np.int32(2), - "floats": np.float64(2.5), - "bools": np.bool_(False), - "arr": np.array([1, 2, 3]), - } - } - result = _make_json_serializable(data) - assert result == { - "outer": { - "ints": 2, - "floats": 2.5, - "bools": False, - "arr": [1, 2, 3], - } - } - # Round-trip through JSON to confirm true serialisability: - json.dumps(result) - - def test_tuple_becomes_list(self) -> None: - """Tuples are converted to lists for JSON compatibility.""" - result = _make_json_serializable((1, np.float64(2.0), "x")) - assert result == [1, 2.0, "x"] - - def test_list_with_arrays(self) -> None: - """Lists of numpy arrays/scalars are recursively converted.""" - result = _make_json_serializable([np.array([1, 2]), np.int8(5)]) - assert result == [[1, 2], 5] - - def test_passthrough_basic_types(self) -> None: - """Basic JSON-compatible values are returned unchanged.""" - assert _make_json_serializable(42) == 42 - assert _make_json_serializable("abc") == "abc" - assert _make_json_serializable(None) is None - - -class TestDeserializeObjMetadata: - """Tests for ``_deserialize_obj_metadata`` (coords → ndarray restoration).""" - - def test_coords_restored_to_array(self) -> None: - """Geometry ``coords`` lists are restored to numpy arrays.""" - serialized = {"Geometry_circle": {"coords": [[1.0, 2.0], [3.0, 4.0]]}} - result = _deserialize_obj_metadata(serialized) - coords = result["Geometry_circle"]["coords"] - assert isinstance(coords, np.ndarray) - np.testing.assert_array_equal(coords, [[1.0, 2.0], [3.0, 4.0]]) - - def test_non_coords_list_kept_as_list(self) -> None: - """Non-``coords`` list entries remain Python lists.""" - result = _deserialize_obj_metadata({"labels": ["a", "b"]}) - assert result == {"labels": ["a", "b"]} - assert isinstance(result["labels"], list) - - def test_scalar_passthrough(self) -> None: - """Scalar metadata values are returned unchanged.""" - result = _deserialize_obj_metadata({"title": "T", "count": 7}) - assert result == {"title": "T", "count": 7} - - # ============================================================================= # Serialization end-to-end edge cases # ============================================================================= diff --git a/datalab/webapi/serialization.py b/datalab/webapi/serialization.py index b593f21dd..f50039ea2 100644 --- a/datalab/webapi/serialization.py +++ b/datalab/webapi/serialization.py @@ -8,50 +8,36 @@ Binary serialization of DataLab objects using NumPy's NPZ format. This module handles the data plane of the Web API, efficiently transferring -large numerical arrays between DataLab and clients. - -Format Specification (V1) -------------------------- - -The NPZ archive contains: - -For SignalObj: - - ``x.npy``: X coordinates (1D float64 array) - - ``y.npy``: Y data (1D float64 array) - - ``dx.npy``: X uncertainties (optional) - - ``dy.npy``: Y uncertainties (optional) - - ``metadata.json``: JSON-encoded metadata - -For ImageObj: - - ``data.npy``: Image data (2D array, preserves dtype) - - ``xcoords.npy``, ``ycoords.npy``: Pixel center coordinates along X and Y - (1D float arrays, only for non-uniform coordinates) - - ``metadata.json``: JSON-encoded metadata - -Metadata JSON includes: - - ``type``: "signal" or "image" - - ``title``, ``xlabel``, ``ylabel``, etc. - - ``x0``, ``y0``, ``dx``, ``dy`` (for images) - - ``is_uniform_coords`` (for images): False when the coordinates are given by - ``xcoords.npy`` and ``ycoords.npy``. Archives without this key (written by - earlier versions) have uniform coordinates. +large numerical arrays between DataLab and clients. The NPZ format (signals and +images with their labels, units, uncertainties, uniform or non-uniform +coordinates and metadata) is implemented by Sigima +(:mod:`sigima.io.npz`, where it is described), so that DataLab and its clients +(e.g. DataLab-Kernel) share one implementation: :func:`serialize_object_to_npz` +and :func:`deserialize_object_from_npz` are those of :mod:`sigima.io`. The +metadata returned by the API for an object (:func:`object_to_metadata`) are +specific to DataLab. """ from __future__ import annotations -import io -import json -import zipfile from typing import TYPE_CHECKING, Union import numpy as np -from sigima.objects import ImageObj, SignalObj +from sigima.io import deserialize_object_from_npz, serialize_object_to_npz from datalab.objectmodel import get_uuid if TYPE_CHECKING: + from sigima.objects import ImageObj, SignalObj + DataObject = Union[SignalObj, ImageObj] +__all__ = [ + "deserialize_object_from_npz", + "object_to_metadata", + "serialize_object_to_npz", +] + def _to_python_scalar(value): """Convert numpy scalar types to Python native types for JSON serialization. @@ -71,302 +57,6 @@ def _to_python_scalar(value): return value -def _make_json_serializable(value): # pylint: disable=too-many-return-statements - """Recursively convert a value to be JSON serializable. - - Handles numpy arrays, numpy scalars, nested dicts, and lists. - - Args: - value: Any value that may contain numpy types. - - Returns: - JSON-serializable value. - """ - if isinstance(value, dict): - return {k: _make_json_serializable(v) for k, v in value.items()} - if isinstance(value, list): - return [_make_json_serializable(item) for item in value] - if isinstance(value, tuple): - return [_make_json_serializable(item) for item in value] - if isinstance(value, np.ndarray): - return value.tolist() - if isinstance(value, np.integer): - return int(value) - if isinstance(value, np.floating): - return float(value) - if isinstance(value, np.bool_): - return bool(value) - return value - - -def _serialize_obj_metadata(obj_metadata: dict) -> dict: - """Convert object metadata to JSON-serializable format. - - Handles numpy arrays (from GeometryResult.coords, TableResult, etc.) - by converting them to nested lists, and numpy scalar types (int64, float64) - to Python native types. - - Args: - obj_metadata: The object's metadata dictionary. - - Returns: - JSON-serializable dictionary. - """ - return _make_json_serializable(obj_metadata) - - -def _deserialize_obj_metadata(serialized: dict) -> dict: - """Convert deserialized metadata back to original types. - - Converts nested lists back to numpy arrays where appropriate - (for coords fields in geometry results). - - Args: - serialized: The JSON-deserialized metadata. - - Returns: - Metadata with numpy arrays restored. - """ - result = {} - for key, value in serialized.items(): - if isinstance(value, dict): - # Recursively handle nested dicts - result[key] = _deserialize_obj_metadata(value) - elif isinstance(value, list) and key == "coords": - # Convert coords back to numpy array - result[key] = np.array(value) - else: - result[key] = value - return result - - -def serialize_object_to_npz(obj: DataObject, *, compress: bool = True) -> bytes: - """Serialize a SignalObj or ImageObj to NPZ format. - - Args: - obj: The object to serialize. - compress: If True (default), use ZIP deflate compression. - Set to False for faster serialization at the cost of larger size. - For incompressible data (random images), False can be 10-30x faster. - - Returns: - Bytes containing the NPZ archive. - - Raises: - TypeError: If object type is not supported. - """ - buffer = io.BytesIO() - - # Detect object type - obj_type = type(obj).__name__ - - if obj_type == "SignalObj": - _serialize_signal(obj, buffer, compress=compress) - elif obj_type == "ImageObj": - _serialize_image(obj, buffer, compress=compress) - else: - raise TypeError(f"Unsupported object type: {obj_type}") - - return buffer.getvalue() - - -def _serialize_signal(obj, buffer: io.BytesIO, *, compress: bool = True) -> None: - """Serialize a SignalObj to NPZ format.""" - # Build metadata dict - metadata = { - "type": "signal", - "title": obj.title, - "xlabel": obj.xlabel, - "ylabel": obj.ylabel, - "xunit": obj.xunit, - "yunit": obj.yunit, - } - # Include object metadata (contains Geometry_*, Table_* results) - obj_metadata = obj.metadata - if obj_metadata: - metadata["obj_metadata"] = _serialize_obj_metadata(obj_metadata) - - # Create zip archive - compression = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED - with zipfile.ZipFile(buffer, "w", compression) as zf: - # Write arrays - _write_array_to_zip(zf, "x.npy", obj.x) - _write_array_to_zip(zf, "y.npy", obj.y) - - if obj.dx is not None: - _write_array_to_zip(zf, "dx.npy", obj.dx) - if obj.dy is not None: - _write_array_to_zip(zf, "dy.npy", obj.dy) - - # Write metadata - zf.writestr("metadata.json", json.dumps(metadata, ensure_ascii=False)) - - -def _serialize_image(obj, buffer: io.BytesIO, *, compress: bool = True) -> None: - """Serialize an ImageObj to NPZ format.""" - # Build metadata dict - use _to_python_scalar for numeric values that may be - # numpy scalars after HDF5 deserialization - metadata = { - "type": "image", - "title": obj.title, - "xlabel": obj.xlabel, - "ylabel": obj.ylabel, - "zlabel": obj.zlabel, - "xunit": obj.xunit, - "yunit": obj.yunit, - "zunit": obj.zunit, - "x0": _to_python_scalar(obj.x0), - "y0": _to_python_scalar(obj.y0), - "dx": _to_python_scalar(obj.dx), - "dy": _to_python_scalar(obj.dy), - "is_uniform_coords": bool(obj.is_uniform_coords), - } - # Include object metadata (contains Geometry_*, Table_* results) - obj_metadata = obj.metadata - if obj_metadata: - metadata["obj_metadata"] = _serialize_obj_metadata(obj_metadata) - - # Create zip archive - compression = zipfile.ZIP_DEFLATED if compress else zipfile.ZIP_STORED - with zipfile.ZipFile(buffer, "w", compression) as zf: - _write_array_to_zip(zf, "data.npy", obj.data) - if not obj.is_uniform_coords: - _write_array_to_zip(zf, "xcoords.npy", obj.xcoords) - _write_array_to_zip(zf, "ycoords.npy", obj.ycoords) - zf.writestr("metadata.json", json.dumps(metadata, ensure_ascii=False)) - - -def _write_array_to_zip(zf: zipfile.ZipFile, name: str, arr: np.ndarray) -> None: - """Write a numpy array to a zip file.""" - arr_buffer = io.BytesIO() - np.save(arr_buffer, arr, allow_pickle=False) - zf.writestr(name, arr_buffer.getvalue()) - - -def deserialize_object_from_npz(data: bytes) -> DataObject: - """Deserialize a SignalObj or ImageObj from NPZ format. - - Args: - data: Bytes containing the NPZ archive. - - Returns: - SignalObj or ImageObj. - - Raises: - ValueError: If the archive format is invalid. - """ - buffer = io.BytesIO(data) - - with zipfile.ZipFile(buffer, "r") as zf: - # Read metadata - if "metadata.json" not in zf.namelist(): - raise ValueError("Invalid NPZ format: missing metadata.json") - - metadata = json.loads(zf.read("metadata.json")) - obj_type = metadata.get("type") - - if obj_type == "signal": - return _deserialize_signal(zf, metadata) - if obj_type == "image": - return _deserialize_image(zf, metadata) - - raise ValueError(f"Unknown object type in NPZ: {obj_type}") - - -def _read_array_from_zip(zf: zipfile.ZipFile, name: str) -> np.ndarray | None: - """Read a numpy array from a zip file.""" - if name not in zf.namelist(): - return None - arr_buffer = io.BytesIO(zf.read(name)) - return np.load(arr_buffer, allow_pickle=False) - - -def _deserialize_signal(zf: zipfile.ZipFile, metadata: dict) -> DataObject: - """Deserialize a SignalObj from NPZ archive.""" - x = _read_array_from_zip(zf, "x.npy") - y = _read_array_from_zip(zf, "y.npy") - dx = _read_array_from_zip(zf, "dx.npy") - dy = _read_array_from_zip(zf, "dy.npy") - - if x is None or y is None: - raise ValueError("Invalid signal NPZ: missing x.npy or y.npy") - - obj = SignalObj() - obj.set_xydata(x, y, dx=dx, dy=dy) - - # Set metadata - if metadata.get("title"): - obj.title = metadata["title"] - if metadata.get("xlabel"): - obj.xlabel = metadata["xlabel"] - if metadata.get("ylabel"): - obj.ylabel = metadata["ylabel"] - if metadata.get("xunit"): - obj.xunit = metadata["xunit"] - if metadata.get("yunit"): - obj.yunit = metadata["yunit"] - - # Restore object metadata (Geometry_*, Table_* results) - if metadata.get("obj_metadata"): - obj.metadata = _deserialize_obj_metadata(metadata["obj_metadata"]) - - return obj - - -def _deserialize_image(zf: zipfile.ZipFile, metadata: dict) -> DataObject: - """Deserialize an ImageObj from NPZ archive.""" - data = _read_array_from_zip(zf, "data.npy") - if data is None: - raise ValueError("Invalid image NPZ: missing data.npy") - - obj = ImageObj() - obj.data = data - - # Set metadata - if metadata.get("title"): - obj.title = metadata["title"] - if metadata.get("xlabel"): - obj.xlabel = metadata["xlabel"] - if metadata.get("ylabel"): - obj.ylabel = metadata["ylabel"] - if metadata.get("zlabel"): - obj.zlabel = metadata["zlabel"] - if metadata.get("xunit"): - obj.xunit = metadata["xunit"] - if metadata.get("yunit"): - obj.yunit = metadata["yunit"] - if metadata.get("zunit"): - obj.zunit = metadata["zunit"] - - # Set coordinate info - if metadata.get("is_uniform_coords", True): - obj.x0 = metadata.get("x0", 0.0) - obj.y0 = metadata.get("y0", 0.0) - obj.dx = metadata.get("dx", 1.0) - obj.dy = metadata.get("dy", 1.0) - else: - xcoords = _read_array_from_zip(zf, "xcoords.npy") - ycoords = _read_array_from_zip(zf, "ycoords.npy") - if xcoords is None or ycoords is None: - raise ValueError( - "Invalid image NPZ: missing xcoords.npy or ycoords.npy " - "for non-uniform coordinates" - ) - nrows, ncols = data.shape - if xcoords.shape != (ncols,) or ycoords.shape != (nrows,): - raise ValueError( - f"Invalid image NPZ: coordinates of shape {xcoords.shape} (X) and " - f"{ycoords.shape} (Y) do not match image shape {data.shape}" - ) - obj.set_coords(xcoords, ycoords) - - # Restore object metadata (Geometry_*, Table_* results) - if metadata.get("obj_metadata"): - obj.metadata = _deserialize_obj_metadata(metadata["obj_metadata"]) - - return obj - - def object_to_metadata(obj: DataObject, name: str) -> dict: """Extract metadata from an object for API responses. diff --git a/doc/features/advanced/webapi.rst b/doc/features/advanced/webapi.rst index 7560da38a..a03c4750f 100644 --- a/doc/features/advanced/webapi.rst +++ b/doc/features/advanced/webapi.rst @@ -220,7 +220,10 @@ Computation (Auth Required) Data Format ----------- -Objects are transferred using NumPy's NPZ format for efficiency. +Objects are transferred using NumPy's NPZ format for efficiency. The format is +implemented by Sigima (``sigima.io.serialize_object_to_npz`` and +``sigima.io.deserialize_object_from_npz``), so that Python clients can read and +write these archives with the same code as DataLab. SignalObj NPZ Structure ^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/doc/locale/fr/LC_MESSAGES/features/advanced/webapi.po b/doc/locale/fr/LC_MESSAGES/features/advanced/webapi.po index 99c5a26ee..1e1955f4e 100644 --- a/doc/locale/fr/LC_MESSAGES/features/advanced/webapi.po +++ b/doc/locale/fr/LC_MESSAGES/features/advanced/webapi.po @@ -276,8 +276,8 @@ msgstr "Exécuter une fonction de calcul DataLab sur la sélection courante" msgid "Data Format" msgstr "Format des données" -msgid "Objects are transferred using NumPy's NPZ format for efficiency." -msgstr "Les objets sont transférés au format NPZ de NumPy pour plus d'efficacité." +msgid "Objects are transferred using NumPy's NPZ format for efficiency. The format is implemented by Sigima (``sigima.io.serialize_object_to_npz`` and ``sigima.io.deserialize_object_from_npz``), so that Python clients can read and write these archives with the same code as DataLab." +msgstr "Les objets sont transférés au format NPZ de NumPy pour plus d'efficacité. Le format est implémenté par Sigima (``sigima.io.serialize_object_to_npz`` et ``sigima.io.deserialize_object_from_npz``), de sorte que les clients Python peuvent lire et écrire ces archives avec le même code que DataLab." msgid "SignalObj NPZ Structure" msgstr "Structure NPZ de SignalObj"