diff --git a/doc/locale/fr/LC_MESSAGES/release_notes/release_1.04.po b/doc/locale/fr/LC_MESSAGES/release_notes/release_1.04.po new file mode 100644 index 00000000..60c06eb2 --- /dev/null +++ b/doc/locale/fr/LC_MESSAGES/release_notes/release_1.04.po @@ -0,0 +1,44 @@ +# French translations for Sigima. +# Copyright (C) 2026 ORGANIZATION +# This file is distributed under the same license as the Sigima project. +# FIRST AUTHOR , 2026. +# +#, fuzzy +msgid "" +msgstr "" +"Project-Id-Version: Sigima VERSION\n" +"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" +"POT-Creation-Date: 2026-10-22 21:34+0200\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" +"Language: fr\n" +"Language-Team: fr \n" +"Plural-Forms: nplurals=2; plural=(n > 1);\n" +"MIME-Version: 1.0\n" +"Content-Type: text/plain; charset=utf-8\n" +"Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" + +msgid "Version 1.4" +msgstr "Version 1.4" + +msgid "Sigima Version 1.4.0" +msgstr "Sigima Version 1.4.0" + +msgid "✨ New features since version 1.3.0" +msgstr "✨ Nouveautés depuis la version 1.3.0" + +msgid "**Origin of experimental data**: Objects imported from experimental files or captured from instruments can record where their data came from in the new `experimental_source` metadata entry: source kind, file path or instrument address, scan or dataset, selected part, timestamps, applied conversions and other details. The entry is a versioned dictionary made of JSON types only, so it remains readable without the reader or connector that created it, and it is restored with the same Python types by Sigima's native signal and image files (and by DataLab workspaces): a restored entry can be serialized to JSON or stored again. Values are limited to what HDF5 files restore exactly: `None` and keys that are not plain names are rejected, and NumPy scalars are converted to Python values. It is created and accessed with `make_experimental_source`, `set_experimental_source` and `get_experimental_source` (`sigima.objects`). The existing `source` metadata entry is unchanged." +msgstr "**Origine des données expérimentales** : les objets importés depuis des fichiers expérimentaux ou acquis par des instruments peuvent enregistrer l'origine de leurs données dans la nouvelle entrée de métadonnées `experimental_source` : type de source, chemin du fichier ou adresse de l'instrument, scan ou jeu de données, partie sélectionnée, horodatages, conversions appliquées et autres détails. Cette entrée est un dictionnaire versionné composé uniquement de types JSON ; elle reste donc lisible sans le lecteur ou connecteur qui l'a créée, et elle est restaurée avec les mêmes types Python par les formats natifs de signaux et d'images de Sigima (et par les espaces de travail DataLab) : une entrée restaurée peut être sérialisée en JSON ou stockée à nouveau. Les valeurs sont limitées à ce que les fichiers HDF5 restituent à l'identique : `None` et les clés qui ne sont pas de simples noms sont rejetés, et les scalaires NumPy sont convertis en valeurs Python. Elle est créée et consultée avec `make_experimental_source`, `set_experimental_source` et `get_experimental_source` (`sigima.objects`). L'entrée de métadonnées existante `source` reste inchangée." + +msgid "**Group UUID from the remote client**: `add_group` of the remote client (`sigima.client`) now returns the UUID of the created group, so that objects can be added to it with `group_id=...` (DataLab ≥ 1.4.0; older versions return `None`)." +msgstr "**UUID de groupe depuis le client distant** : `add_group` du client distant (`sigima.client`) renvoie désormais l'UUID du groupe créé, afin d'y ajouter des objets avec `group_id=...` (DataLab ≥ 1.4.0 ; les versions antérieures renvoient `None`)." + +msgid "The [user guide](../user_guide/overview.rst) shows how readers and connectors attach experimental-source metadata, save and restore it, and send an acquisition to a DataLab group. Source recording is opt-in and does not make the experiment reproducible." +msgstr "Le [guide utilisateur](../user_guide/overview.rst) montre comment les lecteurs et connecteurs attachent les métadonnées d'origine expérimentale, les enregistrent et les restaurent, et envoient une acquisition dans un groupe DataLab. L'enregistrement de l'origine est explicite et ne rend pas l'expérience reproductible." + +msgid "🛠️ Bug Fixes since version 1.3.0" +msgstr "🛠️ Corrections de bugs depuis la version 1.3.0" + +msgid "**Exact integer import**: Integer data that does not fit any integer type supported by the destination object (for example `uint32` counts above 2 147 483 647 in an image, or large integers in a signal) is now converted to a floating-point type that represents every value exactly. Such data was previously converted to `float32` when the source used 4-byte integers, which rounded counts above 16 777 216. A warning is now emitted when no supported floating-point type is exact (integers beyond 2⁵³)." +msgstr "**Import exact des entiers** : les données entières qui ne tiennent dans aucun type entier pris en charge par l'objet de destination (par exemple des comptages `uint32` supérieurs à 2 147 483 647 dans une image, ou de grands entiers dans un signal) sont désormais converties vers un type flottant représentant chaque valeur exactement. Ces données étaient auparavant converties en `float32` lorsque la source utilisait des entiers sur 4 octets, ce qui arrondissait les comptages au-delà de 16 777 216. Un avertissement est désormais émis lorsqu'aucun type flottant pris en charge n'est exact (entiers au-delà de 2⁵³)." diff --git a/doc/locale/fr/LC_MESSAGES/user_guide/overview.po b/doc/locale/fr/LC_MESSAGES/user_guide/overview.po index ef99b996..fc584600 100644 --- a/doc/locale/fr/LC_MESSAGES/user_guide/overview.po +++ b/doc/locale/fr/LC_MESSAGES/user_guide/overview.po @@ -6,11 +6,18 @@ #, fuzzy msgid "" msgstr "" +"Project-Id-Version: PROJECT VERSION\n" +"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" +"POT-Creation-Date: 2026-10-22 21:34+0200\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" "Language: fr\n" +"Language-Team: fr \n" "Plural-Forms: nplurals=2; plural=(n > 1);\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" msgid "Overview" msgstr "Vue d'ensemble" @@ -123,6 +130,36 @@ msgstr "Considérez une mesure de température au cours du temps. Avec un simple msgid "This signal object can now be processed, analyzed, saved, and loaded while preserving all its context. When you apply operations like filtering, peak detection, or statistics, the units, labels, and error bars remain attached to your data." msgstr "Cet objet signal peut maintenant être traité, analysé, sauvegardé et chargé tout en préservant son contexte. Lorsque vous appliquez des opérations comme le filtrage, la détection de pics ou les statistiques, les unités, étiquettes et barres d'erreur restent attachées à vos données." +msgid "Experimental Data Origin" +msgstr "Origine des données expérimentales" + +msgid "Readers and instrument connectors can attach a structured ``experimental_source`` entry to a signal or image. This is opt-in: creating an object does not identify its experimental origin automatically. The existing ``source`` metadata entry is unchanged." +msgstr "Les lecteurs et connecteurs d'instruments peuvent attacher une entrée structurée ``experimental_source`` à un signal ou une image. Cette démarche est explicite : créer un objet n'identifie pas automatiquement son origine expérimentale. L'entrée de métadonnées existante ``source`` reste inchangée." + +msgid "The required fields identify the source kind (for example ``nexus``, ``spec`` or ``tango``) and its location (file path or instrument address). Optional fields describe the scan or dataset (``entry``), selected columns or indices (``selection``), timestamps, applied conversions and additional details. The versioned dictionary accepts the JSON values that HDF5 files restore exactly: dictionaries with non-empty string keys (without ``/`` and not starting with ``__``), lists, strings, booleans, 64-bit integers and finite numbers. ``None`` (JSON ``null``) is rejected, because HDF5 files would restore it as an empty string: readers use ``\".\"`` for a dimension without axis, as NeXus does. NumPy scalars and tuples are converted to Python values and lists; convert arrays to lists and timestamps to strings. Unknown fields, unsupported versions and non-finite values are rejected." +msgstr "Les champs obligatoires identifient le type de source (par exemple ``nexus``, ``spec`` ou ``tango``) et son emplacement (chemin de fichier ou adresse d'instrument). Les champs facultatifs décrivent le scan ou jeu de données (``entry``), les colonnes ou indices sélectionnés (``selection``), les horodatages, les conversions appliquées et des détails supplémentaires. Le dictionnaire versionné accepte les valeurs JSON que les fichiers HDF5 restituent à l'identique : dictionnaires à clés chaînes non vides (sans ``/`` et ne commençant pas par ``__``), listes, chaînes, booléens, entiers 64 bits et nombres finis. ``None`` (``null`` en JSON) est rejeté, car les fichiers HDF5 le restitueraient sous forme de chaîne vide : les lecteurs utilisent ``\".\"`` pour une dimension sans axe, comme NeXus. Les scalaires NumPy et les tuples sont convertis en valeurs et listes Python ; convertissez les tableaux en listes et les horodatages en chaînes. Les champs inconnus, versions non prises en charge et valeurs non finies sont rejetés." + +msgid "This example records a scan, then saves and restores its origin using the native signal format:" +msgstr "Cet exemple décrit un scan, puis enregistre et restaure son origine avec le format natif des signaux :" + +msgid "The native image format (``.h5ima``) preserves this entry too, with its Python types: a restored source can be serialized to JSON or stored again as is. The setter stores a copy, and the getter returns a copy, so editing either dictionary does not alter the other. The information remains readable without the original reader or instrument connector. It identifies a measurement; it does not make the acquisition reproducible or replace the record of subsequent processing. NeXus import and declarative HDF5 mappings are provided by SigimaX, not by these metadata helpers; naming a source kind does not install its reader or connector." +msgstr "Le format natif des images (``.h5ima``) préserve aussi cette entrée, avec ses types Python : une source restaurée peut être sérialisée en JSON ou stockée à nouveau telle quelle. La fonction d'affectation stocke une copie, et la fonction de lecture renvoie une copie : modifier l'un des dictionnaires ne modifie pas l'autre. Ces informations restent lisibles sans le lecteur ou connecteur d'instrument d'origine. Elles identifient une mesure ; elles ne rendent pas l'acquisition reproductible et ne remplacent pas l'enregistrement des traitements ultérieurs. L'import NeXus et les profils HDF5 déclaratifs sont fournis par SigimaX, pas par ces fonctions de métadonnées ; nommer un type de source n'installe pas son lecteur ou connecteur." + +msgid "Integer Precision on Import" +msgstr "Précision des entiers à l'import" + +msgid "When an integer array does not fit an integer type supported by its destination object, Sigima selects a supported floating-point type with sufficient precision when possible. For example, ``uint32`` image counts up to 4 294 967 295 are preserved exactly in ``float64`` rather than clipped to ``int32`` or rounded to ``float32``. This preserves values, not necessarily the original dtype. If no supported float type can represent the integer range exactly, Sigima emits a warning; ``float64`` guarantees exact integers only up to ``+/-2**53``." +msgstr "Lorsqu'un tableau entier ne tient dans aucun type entier pris en charge par son objet de destination, Sigima sélectionne si possible un type flottant pris en charge suffisamment précis. Par exemple, les comptages d'image ``uint32`` jusqu'à 4 294 967 295 sont préservés exactement en ``float64`` plutôt qu'écrêtés en ``int32`` ou arrondis en ``float32``. Cela préserve les valeurs, pas nécessairement le dtype original. Si aucun type flottant pris en charge ne peut représenter exactement la plage d'entiers, Sigima émet un avertissement ; ``float64`` ne garantit les entiers exacts que jusqu'à ``+/-2**53``." + +msgid "Sending Acquisitions to a DataLab Group" +msgstr "Envoi d'acquisitions dans un groupe DataLab" + +msgid "With a running DataLab 1.4 or later, a remote client can use the UUID returned by ``add_group`` to add measured objects to that group, without looking it up by title:" +msgstr "Avec DataLab 1.4 ou ultérieur en cours d'exécution, un client distant peut utiliser l'UUID renvoyé par ``add_group`` pour ajouter des objets mesurés à ce groupe, sans le rechercher par son titre :" + +msgid "This example uses the signal defined above. Older DataLab servers return ``None`` instead of a group UUID; this grouping workflow requires DataLab 1.4." +msgstr "Cet exemple utilise le signal défini ci-dessus. Les anciens serveurs DataLab renvoient ``None`` au lieu d'un UUID de groupe ; cette procédure de regroupement nécessite DataLab 1.4." + msgid "Specialized Features" msgstr "Fonctionnalités spécialisées" diff --git a/doc/release_notes/release_1.04.md b/doc/release_notes/release_1.04.md new file mode 100644 index 00000000..48576e72 --- /dev/null +++ b/doc/release_notes/release_1.04.md @@ -0,0 +1,14 @@ +# Version 1.4 # + +## Sigima Version 1.4.0 ## + +### ✨ New features since version 1.3.0 ### + +* **Origin of experimental data**: Objects imported from experimental files or captured from instruments can record where their data came from in the new `experimental_source` metadata entry: source kind, file path or instrument address, scan or dataset, selected part, timestamps, applied conversions and other details. The entry is a versioned dictionary made of JSON types only, so it remains readable without the reader or connector that created it, and it is restored with the same Python types by Sigima's native signal and image files (and by DataLab workspaces): a restored entry can be serialized to JSON or stored again. Values are limited to what HDF5 files restore exactly: `None` and keys that are not plain names are rejected, and NumPy scalars are converted to Python values. It is created and accessed with `make_experimental_source`, `set_experimental_source` and `get_experimental_source` (`sigima.objects`). The existing `source` metadata entry is unchanged. +* **Group UUID from the remote client**: `add_group` of the remote client (`sigima.client`) now returns the UUID of the created group, so that objects can be added to it with `group_id=...` (DataLab ≥ 1.4.0; older versions return `None`). + +The [user guide](../user_guide/overview.rst) shows how readers and connectors attach experimental-source metadata, save and restore it, and send an acquisition to a DataLab group. Source recording is opt-in and does not make the experiment reproducible. + +### 🛠️ Bug Fixes since version 1.3.0 ### + +* **Exact integer import**: Integer data that does not fit any integer type supported by the destination object (for example `uint32` counts above 2 147 483 647 in an image, or large integers in a signal) is now converted to a floating-point type that represents every value exactly. Such data was previously converted to `float32` when the source used 4-byte integers, which rounded counts above 16 777 216. A warning is now emitted when no supported floating-point type is exact (integers beyond 2⁵³). diff --git a/doc/user_guide/overview.rst b/doc/user_guide/overview.rst index 9bd5b208..c36577f9 100644 --- a/doc/user_guide/overview.rst +++ b/doc/user_guide/overview.rst @@ -100,6 +100,101 @@ Consider a temperature measurement over time. With a simple NumPy array, you jus This signal object can now be processed, analyzed, saved, and loaded while preserving all its context. When you apply operations like filtering, peak detection, or statistics, the units, labels, and error bars remain attached to your data. +Experimental Data Origin +^^^^^^^^^^^^^^^^^^^^^^^^ + +Readers and instrument connectors can attach a structured ``experimental_source`` +entry to a signal or image. This is opt-in: creating an object does not identify +its experimental origin automatically. The existing ``source`` metadata entry +is unchanged. + +The required fields identify the source kind (for example ``nexus``, ``spec`` +or ``tango``) and its location (file path or instrument address). Optional +fields describe the scan or dataset (``entry``), selected columns or indices +(``selection``), timestamps, applied conversions and additional details. +The versioned dictionary accepts the JSON values that HDF5 files restore +exactly: dictionaries with non-empty string keys (without ``/`` and not +starting with ``__``), lists, strings, booleans, 64-bit integers and finite +numbers. ``None`` (JSON ``null``) is rejected, because HDF5 files would restore +it as an empty string: readers use ``"."`` for a dimension without axis, as +NeXus does. NumPy scalars and tuples are converted to Python values and lists; +convert arrays to lists and timestamps to strings. Unknown fields, unsupported +versions and non-finite values are rejected. + +This example records a scan, then saves and restores its origin using the +native signal format: + +.. code-block:: python + + from pathlib import Path + from tempfile import TemporaryDirectory + + import numpy as np + import sigima.io + from sigima.objects import ( + create_signal, + get_experimental_source, + make_experimental_source, + set_experimental_source, + ) + + signal = create_signal( + "Transmission", np.array([8.0, 8.1, 8.2]), np.array([10.0, 7.0, 4.0]) + ) + signal.xunit = "keV" + source = make_experimental_source( + "spec", + "scan.spec", + entry="12.1", + selection={"x": "Energy", "y": "I0"}, + timestamps={"source": "2026-10-20T08:15:00+02:00"}, + details={"command": "ascan energy 8 8.2 2 0.1"}, + ) + set_experimental_source(signal, source) + with TemporaryDirectory() as directory: + filename = str(Path(directory) / "transmission.h5sig") + sigima.io.write_signal(filename, signal) + restored = sigima.io.read_signal(filename) + assert get_experimental_source(restored) == source + +The native image format (``.h5ima``) preserves this entry too, with its Python +types: a restored source can be serialized to JSON or stored again as is. The +setter stores a copy, and the getter returns a copy, so editing either +dictionary does not alter the other. The information remains readable without the original reader +or instrument connector. It identifies a measurement; it does not make the +acquisition reproducible or replace the record of subsequent processing. +NeXus import and declarative HDF5 mappings are provided by SigimaX, not by these +metadata helpers; naming a source kind does not install its reader or connector. + +Integer Precision on Import +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +When an integer array does not fit an integer type supported by its destination +object, Sigima selects a supported floating-point type with sufficient precision +when possible. For example, ``uint32`` image counts up to 4 294 967 295 are +preserved exactly in ``float64`` rather than clipped to ``int32`` or rounded +to ``float32``. This preserves values, not necessarily the original dtype. +If no supported float type can represent the integer range exactly, Sigima +emits a warning; ``float64`` guarantees exact integers only up to ``+/-2**53``. + +Sending Acquisitions to a DataLab Group +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +With a running DataLab 1.4 or later, a remote client can use the UUID returned by +``add_group`` to add measured objects to that group, without looking it up by +title: + +.. code-block:: python + + from sigima.client import SimpleRemoteProxy + + proxy = SimpleRemoteProxy() + group_id = proxy.add_group("Scan 12.1", panel="signal") + proxy.add_object(signal, group_id=group_id) + +This example uses the signal defined above. Older DataLab servers return +``None`` instead of a group UUID; this grouping workflow requires DataLab 1.4. + Specialized Features ^^^^^^^^^^^^^^^^^^^^ diff --git a/sigima/_metadata.py b/sigima/_metadata.py index 821be8af..7e446c31 100644 --- a/sigima/_metadata.py +++ b/sigima/_metadata.py @@ -4,7 +4,7 @@ from __future__ import annotations -__version__ = "1.3.0" +__version__ = "1.4.0" __docurl__ = "https://sigima.readthedocs.io/" __homeurl__ = "https://github.com/DataLab-Platform/Sigima" __supporturl__ = "https://github.com/DataLab-Platform/sigima/issues/new/choose" diff --git a/sigima/client/base.py b/sigima/client/base.py index 9ce1dc80..f67da991 100644 --- a/sigima/client/base.py +++ b/sigima/client/base.py @@ -312,13 +312,16 @@ def load_from_directory(self, path: str, param: gds.DataSet | None = None) -> No @abc.abstractmethod def add_group( self, title: str, panel: str | None = None, select: bool = False - ) -> None: + ) -> str | None: """Add group to DataLab. Args: title: Group title panel: Panel name (valid values: "signal", "image"). Defaults to None. select: Select the group after creation. Defaults to False. + + Returns: + UUID of the created group (None with DataLab versions before 1.4) """ @abc.abstractmethod @@ -739,15 +742,18 @@ def load_from_files( def add_group( self, title: str, panel: str | None = None, select: bool = False - ) -> None: + ) -> str | None: """Add group to DataLab. Args: title: Group title panel: Panel name (valid values: "signal", "image"). Defaults to None. select: Select the group after creation. Defaults to False. + + Returns: + UUID of the created group (None with DataLab versions before 1.4) """ - self._datalab.add_group(title, panel, select) + return self._datalab.add_group(title, panel, select) def get_sel_object_uuids(self, include_groups: bool = False) -> list[str]: """Return selected objects uuids. diff --git a/sigima/io/common/converters.py b/sigima/io/common/converters.py index cfcca8a6..7f6f9a54 100644 --- a/sigima/io/common/converters.py +++ b/sigima/io/common/converters.py @@ -8,6 +8,7 @@ from __future__ import annotations +import warnings from typing import Any, Sequence import numpy as np @@ -69,8 +70,9 @@ def _convert_int_array( """Convert an integer array to a standard type. Select the smallest supported integer dtype that can represent all values in the - array. If no suitable integer dtype is found, convert the array to a supported - float type. + array. If no suitable integer dtype is found, convert the array to the smallest + supported float type, at least as large as the input, that represents all values + exactly. A warning is emitted when no supported float type is exact. Args: array: Input numpy array of integer type. @@ -92,11 +94,48 @@ def _convert_int_array( new_type = np.dtype(code).newbyteorder("=") break else: - new_type = _convert_float_array(array, supported_data_types).dtype + new_type = _exact_float_dtype(array, amin, amax, supported_data_types) return array.astype(new_type, copy=False) +def _exact_float_dtype( + array: np.ndarray, amin: Any, amax: Any, supported_data_types: tuple[np.dtype] +) -> np.dtype: + """Return the float dtype used to store an integer array. + + Args: + array: Input numpy array of integer type. + amin: Minimum value of the array. + amax: Maximum value of the array. + supported_data_types: Tuple of supported numpy dtypes for destination object. + + Returns: + Smallest supported float dtype, at least as large as the input, whose + significand represents all values exactly (the largest one otherwise). + + Raises: + ValueError: If no supported float dtype exists. + """ + float_codes = dtypes_to_sorted_short_codes(supported_data_types, kind_filter="f") + if not float_codes: + raise ValueError("Unsupported data type") + magnitude = max(abs(int(amin)), abs(int(amax))) + for code in float_codes: + dtype = np.dtype(code) + exact_limit = 2 ** (np.finfo(dtype).nmant + 1) + if dtype.itemsize >= array.dtype.itemsize and magnitude <= exact_limit: + return dtype.newbyteorder("=") + dtype = np.dtype(float_codes[-1]) + warnings.warn( + f"Integer values up to {magnitude} ({array.dtype}) cannot be represented " + f"exactly as {dtype}: they are rounded to the nearest {dtype} value.", + UserWarning, + stacklevel=4, + ) + return dtype.newbyteorder("=") + + def _convert_float_array( array: np.ndarray, supported_data_types: tuple[np.dtype] ) -> np.ndarray: diff --git a/sigima/objects/__init__.py b/sigima/objects/__init__.py index 8c3a55ca..bb894717 100644 --- a/sigima/objects/__init__.py +++ b/sigima/objects/__init__.py @@ -13,6 +13,7 @@ - :mod:`sigima.objects.scalar` - :mod:`sigima.objects.image` - :mod:`sigima.objects.signal` +- :mod:`sigima.objects.source` .. code-block:: python @@ -111,12 +112,27 @@ .. autodataset:: sigima.objects.ROI2DParam .. autoclass:: sigima.objects.ImageROI .. autoclass:: sigima.objects.ImageDatatypes + +Experimental data origin +^^^^^^^^^^^^^^^^^^^^^^^^ + +Readers and instrument connectors can record their objects' experimental +origin in the ``experimental_source`` metadata entry: a versioned dictionary made of +JSON types only, readable without the reader or connector that created it. +This is opt-in and identifies a measurement, not a reproducible acquisition. +See :doc:`../user_guide/overview` for a complete creation and persistence example. + +.. autofunction:: sigima.objects.make_experimental_source +.. autofunction:: sigima.objects.set_experimental_source +.. autofunction:: sigima.objects.get_experimental_source """ __all__ = [ "ANNOTATION_FORMAT", "ANNOTATION_VERSION", "CREATION_PARAMS_VERSION", + "EXPERIMENTAL_SOURCE_KEY", + "EXPERIMENTAL_SOURCE_VERSION", "NO_ROI", "PEAK_PARAMETERIZATION", "AnnotationKind", @@ -224,11 +240,14 @@ "filter_table_by_roi", "flip_annotation_horizontally", "flip_annotation_vertically", + "get_experimental_source", "is_graphical_annotation_dict", "legacy_plotpy_payload_to_annotations", + "make_experimental_source", "migrate_legacy_plotpy_annotations", "rotate_annotation", "scale_annotation", + "set_experimental_source", "transform_annotation", "translate_annotation", "transpose_annotation", @@ -364,3 +383,10 @@ create_signal_roi, validate_peak_creation_params, ) +from sigima.objects.source import ( + EXPERIMENTAL_SOURCE_KEY, + EXPERIMENTAL_SOURCE_VERSION, + get_experimental_source, + make_experimental_source, + set_experimental_source, +) diff --git a/sigima/objects/image/object.py b/sigima/objects/image/object.py index d3887d44..9d0fd7be 100644 --- a/sigima/objects/image/object.py +++ b/sigima/objects/image/object.py @@ -35,6 +35,7 @@ from sigima.config import _ from sigima.objects import base from sigima.objects.image.roi import ImageROI +from sigima.objects.source import normalize_experimental_source_metadata from sigima.tools.datatypes import clip_astype @@ -90,6 +91,15 @@ def __init__(self, title=None, comment=None, icon=""): base.BaseObj.__init__(self) self._dicom_template = None + def deserialize(self, reader) -> None: + """Deserialize object, restoring the JSON types of its experimental source + + Args: + reader: guidata reader (HDF5, JSON or INI) + """ + super().deserialize(reader) + normalize_experimental_source_metadata(self.metadata) + @staticmethod def get_roi_class() -> Type[ImageROI]: """Return ROI class""" diff --git a/sigima/objects/signal/object.py b/sigima/objects/signal/object.py index f87b9714..39307c7a 100644 --- a/sigima/objects/signal/object.py +++ b/sigima/objects/signal/object.py @@ -38,6 +38,7 @@ VALID_TIME_UNITS, ) from sigima.objects.signal.roi import SignalROI +from sigima.objects.source import normalize_experimental_source_metadata from sigima.tools.datatypes import datetime64_to_seconds @@ -134,6 +135,15 @@ def __init__(self, title=None, comment=None, icon=""): gds.DataSet.__init__(self, title, comment, icon) base.BaseObj.__init__(self) + def deserialize(self, reader) -> None: + """Deserialize object, restoring the JSON types of its experimental source + + Args: + reader: guidata reader (HDF5, JSON or INI) + """ + super().deserialize(reader) + normalize_experimental_source_metadata(self.metadata) + @staticmethod def get_roi_class() -> Type[SignalROI]: """Return ROI class""" diff --git a/sigima/objects/source.py b/sigima/objects/source.py new file mode 100644 index 00000000..3bc9552e --- /dev/null +++ b/sigima/objects/source.py @@ -0,0 +1,280 @@ +# Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file. + +""" +Origin of experimental data +=========================== + +Objects imported from experimental files or captured from instruments record their +origin in the ``experimental_source`` metadata entry. It is a versioned dictionary +made of JSON types only, so that it stays readable without the reader or the +instrument connector that created it (saved workspaces, DataLab-Web, provenance). + +Values are restricted to the JSON types that HDF5 files restore exactly: objects +with non-empty string keys, arrays, strings, booleans, 64-bit integers and finite +numbers. ``null`` is not allowed. NumPy scalars and tuples are converted to Python +values and lists. + +It identifies the data origin; it does not make the acquisition reproducible. +The existing ``source`` metadata entry is left untouched. +""" + +from __future__ import annotations + +import copy +import math +from typing import TYPE_CHECKING, Any + +import numpy as np + +if TYPE_CHECKING: + from sigima.objects.image import ImageObj + from sigima.objects.signal import SignalObj + +__all__ = [ + "EXPERIMENTAL_SOURCE_KEY", + "EXPERIMENTAL_SOURCE_VERSION", + "get_experimental_source", + "make_experimental_source", + "normalize_experimental_source_metadata", + "set_experimental_source", +] + +#: Metadata key of the experimental data origin +EXPERIMENTAL_SOURCE_KEY = "experimental_source" + +#: Version of the ``experimental_source`` dictionary layout +EXPERIMENTAL_SOURCE_VERSION = 1 + +_OPTIONAL_FIELDS = ("entry", "selection", "timestamps", "conversions", "details") + +_INT64_MIN, _INT64_MAX = -(2**63), 2**63 - 1 + + +def _check_text(name: str, value: Any) -> None: + """Raise if value is not a non-empty string.""" + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"Experimental source `{name}` must be a non-empty string") + + +def _check_key(key: Any, where: str) -> None: + """Raise if key cannot be stored as a dictionary key in HDF5 files.""" + if not isinstance(key, str): + raise TypeError( + f"Experimental source `{where or 'source'}` keys must be strings: {key!r}" + ) + # HDF5 paths use "/" and "."; guidata reserves "__seq", "__dict" and "*__type__" + if ( + not key + or "/" in key + or key in (".", "..") + or key.startswith("__") + or key.endswith("__type__") + ): + raise ValueError( + f"Experimental source `{where or 'source'}` has an invalid key: {key!r}" + ) + + +def _normalize(value: Any, where: str = "", strict: bool = True) -> Any: + """Return a copy of value made of Python JSON types. + + Args: + value: value to normalize + where: path of the value in the source, for error messages + strict: if True, raise on values that HDF5 files do not restore exactly; + otherwise, convert NumPy values and bytes, and copy other values as is + + Returns: + Normalized copy of value + + Raises: + TypeError: if a value or a key has an unsupported type (strict mode) + ValueError: if a value or a key is invalid (strict mode) + """ + if isinstance(value, dict): + result = {} + for key, item in value.items(): + if strict: + _check_key(key, where) + result[key] = _normalize(item, f"{where}.{key}" if where else key, strict) + return result + if isinstance(value, (list, tuple)) or ( + not strict and isinstance(value, np.ndarray) + ): + return [ + _normalize(item, f"{where}[{index}]", strict) + for index, item in enumerate(value) + ] + return _normalize_scalar(value, where, strict) + + +def _normalize_scalar(value: Any, where: str, strict: bool) -> Any: + """Return a copy of a value that is neither a dictionary nor a sequence, made + of Python JSON types. + + Args: + value: value to normalize + where: path of the value in the source, for error messages + strict: if True, raise on values that HDF5 files do not restore exactly; + otherwise, convert NumPy values and bytes, and copy other values as is + + Returns: + Normalized copy of value + + Raises: + TypeError: if the value has an unsupported type (strict mode) + ValueError: if the value is invalid (strict mode) + """ + if isinstance(value, np.generic): + value = value.item() + if not strict and isinstance(value, bytes): + value = value.decode("utf-8", errors="replace") + if isinstance(value, bool): + return value + if isinstance(value, int): + if strict and not _INT64_MIN <= value <= _INT64_MAX: + raise ValueError( + f"Experimental source `{where}` must be a 64-bit integer: {value}" + ) + return int(value) + if isinstance(value, float): + if strict and not math.isfinite(value): + raise ValueError(f"Experimental source `{where}` must be finite: {value}") + return float(value) + if isinstance(value, str): + return str(value) + if not strict: + return copy.deepcopy(value) + if value is None: + raise ValueError( + f"Experimental source `{where}` must not be None: " + "null values are not preserved by HDF5 files" + ) + raise TypeError( + f"Experimental source `{where}` must contain JSON types, " + f"not {type(value).__name__}" + ) + + +def _validate(source: dict[str, Any]) -> dict[str, Any]: + """Return a normalized copy of source, raise if it is not a valid + ``experimental_source`` dictionary.""" + if not isinstance(source, dict): + raise TypeError("Experimental source must be a dictionary") + source = _normalize(source) + version = source.get("version") + # bool is an int subclass: True must not be taken for version 1 + if ( + isinstance(version, bool) + or not isinstance(version, int) + or version != EXPERIMENTAL_SOURCE_VERSION + ): + raise ValueError(f"Unsupported experimental source version: {version!r}") + _check_text("kind", source.get("kind")) + _check_text("location", source.get("location")) + unknown = set(source) - {"version", "kind", "location", *_OPTIONAL_FIELDS} + if unknown: + raise ValueError(f"Unknown experimental source fields: {sorted(unknown)}") + if "entry" in source: + _check_text("entry", source["entry"]) + for name in ("selection", "timestamps", "details"): + if name in source and not isinstance(source[name], dict): + raise TypeError(f"Experimental source `{name}` must be a dictionary") + if "conversions" in source and not ( + isinstance(source["conversions"], list) + and all(isinstance(item, str) for item in source["conversions"]) + ): + raise TypeError("Experimental source `conversions` must be a list of strings") + return source + + +def make_experimental_source( + kind: str, + location: str, + *, + entry: str | None = None, + selection: dict[str, Any] | None = None, + timestamps: dict[str, Any] | None = None, + conversions: list[str] | None = None, + details: dict[str, Any] | None = None, +) -> dict[str, Any]: + """Return a validated ``experimental_source`` dictionary. + + Values are restricted to the JSON types restored exactly by HDF5 files: + dictionaries with non-empty string keys (without ``/``, not starting with + ``__``), lists, strings, booleans, 64-bit integers and finite numbers. ``None`` + is not allowed. NumPy scalars and tuples are converted to Python values and + lists. + + Args: + kind: source kind, e.g. ``"spec"``, ``"nexus"``, ``"hdf5"`` or ``"tango"`` + location: file path or instrument address + entry: item inside the source, e.g. scan, dataset path or attribute name + selection: selected part of the entry, e.g. columns, indices or slices + timestamps: timestamps as ISO 8601 strings, e.g. source and reception times + conversions: human-readable conversions applied to the source values + details: other JSON-compatible information (units, calibration, quality...) + + Returns: + Dictionary made of JSON types only + + Raises: + TypeError: if a field, a value or a key has an invalid type + ValueError: if a field, a value or a key is invalid + """ + source: dict[str, Any] = { + "version": EXPERIMENTAL_SOURCE_VERSION, + "kind": kind, + "location": location, + } + values = (entry, selection, timestamps, conversions, details) + for name, value in zip(_OPTIONAL_FIELDS, values): + if value is not None: + source[name] = value + return _validate(source) + + +def set_experimental_source(obj: SignalObj | ImageObj, source: dict[str, Any]) -> None: + """Store the experimental data origin in object metadata. + + Args: + obj: signal or image object + source: dictionary returned by :func:`make_experimental_source` or + :func:`get_experimental_source` + + Raises: + TypeError: if a field, a value or a key has an invalid type + ValueError: if a field, a value or a key is invalid + """ + obj.metadata[EXPERIMENTAL_SOURCE_KEY] = _validate(source) + + +def get_experimental_source(obj: SignalObj | ImageObj) -> dict[str, Any] | None: + """Return a copy of the experimental data origin stored in object metadata. + + NumPy values, which HDF5 files return for numbers, are converted to Python + values. + + Args: + obj: signal or image object + + Returns: + ``experimental_source`` dictionary, or None if the object has none + """ + source = obj.metadata.get(EXPERIMENTAL_SOURCE_KEY) + return None if source is None else _normalize(source, strict=False) + + +def normalize_experimental_source_metadata(metadata: dict[str, Any]) -> None: + """Convert the NumPy values of the ``experimental_source`` metadata entry to + Python values, in place. + + HDF5 files return numbers as NumPy scalars: objects call this function once + deserialized, so that their metadata keeps JSON types only. + + Args: + metadata: object metadata + """ + source = metadata.get(EXPERIMENTAL_SOURCE_KEY) + if source is not None: + metadata[EXPERIMENTAL_SOURCE_KEY] = _normalize(source, strict=False) diff --git a/sigima/tests/common/converters_unit_test.py b/sigima/tests/common/converters_unit_test.py index 047fd821..a42211c9 100644 --- a/sigima/tests/common/converters_unit_test.py +++ b/sigima/tests/common/converters_unit_test.py @@ -37,6 +37,30 @@ def test_int_arrays(self) -> None: assert result.dtype == np.int32 np.testing.assert_array_almost_equal(result, arr, decimal=6) + @pytest.mark.parametrize( + "values, dtype, valid_dtypes, expected", + [ + ([0, 2**32 - 1], np.uint32, ImageObj.VALID_DTYPES, np.float64), + ([0, 2**24 + 1], np.int32, SignalObj.VALID_DTYPES, np.float64), + ([0, 2**53], np.uint64, ImageObj.VALID_DTYPES, np.float64), + ([-(2**53), 7], np.int64, SignalObj.VALID_DTYPES, np.float64), + ], + ids=["uint32-max", "int32-above-float32", "uint64-2^53", "int64-neg-2^53"], + ) + def test_int_to_float_is_exact(self, values, dtype, valid_dtypes, expected) -> None: + """Integers outside the supported integer types are converted exactly.""" + arr = np.array(values, dtype=dtype) + result = converters.convert_array_to_valid_dtype(arr, valid_dtypes) + assert result.dtype == expected + assert [int(value) for value in result] == values + + def test_int_to_float_rounding_warns(self) -> None: + """Integers beyond the exact range of float64 are rounded with a warning.""" + arr = np.array([0, 2**53 + 1], dtype=np.uint64) + with pytest.warns(UserWarning, match="cannot be represented exactly"): + result = converters.convert_array_to_valid_dtype(arr, ImageObj.VALID_DTYPES) + assert result.dtype == np.float64 + def test_float_arrays(self) -> None: """Test conversion of float numpy arrays.""" arr = np.array([1.1, 2.2, 3.3], dtype=np.float32) diff --git a/sigima/tests/common/experimental_source_unit_test.py b/sigima/tests/common/experimental_source_unit_test.py new file mode 100644 index 00000000..de83047d --- /dev/null +++ b/sigima/tests/common/experimental_source_unit_test.py @@ -0,0 +1,228 @@ +# Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file. + +""" +Unit tests for the experimental data origin stored in object metadata +""" + +from __future__ import annotations + +import json +import os.path as osp + +import numpy as np +import pytest + +import sigima.io +from sigima.objects import ( + EXPERIMENTAL_SOURCE_KEY, + create_image, + create_signal, + get_experimental_source, + make_experimental_source, + set_experimental_source, +) +from sigima.tests.helpers import WorkdirRestoringTempDir + + +def _spec_source() -> dict: + """Return a typical source of a signal imported from a SPEC scan.""" + return make_experimental_source( + "spec", + "C:/data/sample.spec", + entry="12.1", + selection={"x": "Energy", "y": "I0"}, + timestamps={"source": "2026-10-20T08:15:00+02:00"}, + conversions=["int32 -> float64"], + details={"command": "ascan th 0 1 10 0.1", "motors": {"th": 0.5}}, + ) + + +def test_make_experimental_source() -> None: + """Optional fields are kept only when provided.""" + source = make_experimental_source("tango", "tango://host:10000/lab/cam/1") + assert source == { + "version": 1, + "kind": "tango", + "location": "tango://host:10000/lab/cam/1", + } + assert _spec_source()["selection"] == {"x": "Energy", "y": "I0"} + + +@pytest.mark.parametrize( + "kwargs, error", + [ + ({"kind": "", "location": "f"}, ValueError), + ({"kind": "spec", "location": " "}, ValueError), + ({"kind": "spec", "location": "f", "entry": ""}, ValueError), + ({"kind": "spec", "location": "f", "selection": ["x"]}, TypeError), + ({"kind": "spec", "location": "f", "conversions": "a"}, TypeError), + ({"kind": "spec", "location": "f", "details": {"v": float("nan")}}, ValueError), + ({"kind": "spec", "location": "f", "details": {"v": None}}, ValueError), + ({"kind": "spec", "location": "f", "selection": {"a": [None]}}, ValueError), + ({"kind": "spec", "location": "f", "details": {1: "a"}}, TypeError), + ({"kind": "spec", "location": "f", "details": {"": 1}}, ValueError), + ({"kind": "spec", "location": "f", "details": {"a/b": 1}}, ValueError), + ({"kind": "spec", "location": "f", "details": {"__seq": 1}}, ValueError), + ({"kind": "spec", "location": "f", "details": {"v": 2**63}}, ValueError), + ({"kind": "spec", "location": "f", "details": {"v": 1j}}, TypeError), + ({"kind": "spec", "location": "f", "details": {"v": b"a"}}, TypeError), + ], + ids=[ + "kind", + "location", + "entry", + "selection", + "conversions", + "nan", + "none", + "none-in-list", + "int-key", + "empty-key", + "slash-key", + "reserved-key", + "int64-overflow", + "complex", + "bytes", + ], +) +def test_invalid_experimental_source(kwargs: dict, error: type) -> None: + """Invalid fields and values that HDF5 files do not restore are rejected.""" + with pytest.raises(error): + make_experimental_source(**kwargs) + + +def test_numpy_values_are_converted() -> None: + """NumPy scalars and tuples are stored as Python values and lists.""" + source = make_experimental_source( + "spec", + "f", + selection={"indices": (np.int32(1), np.int64(2))}, + details={ + "gain": np.float32(0.5), + "valid": np.bool_(True), + "name": np.str_("a"), + }, + ) + assert source["selection"] == {"indices": [1, 2]} + assert source["details"] == {"gain": 0.5, "valid": True, "name": "a"} + for value in (*source["selection"]["indices"], *source["details"].values()): + assert not isinstance(value, np.generic) + json.dumps(source) + + +def test_set_rejects_unknown_version_and_fields() -> None: + """Only the current layout can be stored.""" + obj = create_signal("s", np.arange(3.0), np.arange(3.0)) + with pytest.raises(ValueError): + set_experimental_source(obj, {**_spec_source(), "version": 2}) + with pytest.raises(ValueError): + set_experimental_source(obj, {**_spec_source(), "version": True}) + with pytest.raises(ValueError): + set_experimental_source(obj, {**_spec_source(), "extra": 1}) + assert get_experimental_source(obj) is None + + +def test_source_is_copied() -> None: + """Stored and returned sources are independent of the caller's dictionary.""" + source = _spec_source() + obj = create_signal("s", np.arange(3.0), np.arange(3.0)) + set_experimental_source(obj, source) + source["details"]["motors"]["th"] = 9.0 + returned = get_experimental_source(obj) + assert returned["details"]["motors"]["th"] == 0.5 + returned["entry"] = "1.1" + assert obj.metadata[EXPERIMENTAL_SOURCE_KEY]["entry"] == "12.1" + assert get_experimental_source(obj.copy()) == _spec_source() + + +def _assert_json_types(value) -> None: + """Assert that value is made of Python JSON types only.""" + if isinstance(value, dict): + for key, item in value.items(): + # Exact type: str subclasses, such as numpy.str_, are not JSON types + assert type(key) is str # pylint: disable=unidiomatic-typecheck + _assert_json_types(item) + elif isinstance(value, list): + for item in value: + _assert_json_types(item) + else: + assert type(value) in (str, int, float, bool), repr(value) + + +def _round_trip(signal, image) -> tuple: + """Write and read objects with Sigima's native signal and image formats.""" + with WorkdirRestoringTempDir() as tmpdir: + sigima.io.write_signal(osp.join(tmpdir, "s.h5sig"), signal) + sigima.io.write_image(osp.join(tmpdir, "i.h5ima"), image) + return ( + sigima.io.read_signal(osp.join(tmpdir, "s.h5sig")), + sigima.io.read_image(osp.join(tmpdir, "i.h5ima")), + ) + + +def test_source_survives_file_round_trip() -> None: + """The source is restored by Sigima's native signal and image formats, with + its JSON types, so that it can be stored again.""" + signal = create_signal("s", np.arange(3.0), np.arange(3.0)) + image = create_image("i", np.zeros((2, 3))) + for obj in (signal, image): + set_experimental_source(obj, _spec_source()) + for obj in _round_trip(signal, image): + source = get_experimental_source(obj) + assert source == _spec_source() + version = source["version"] + assert isinstance(version, int) and not isinstance(version, bool) + _assert_json_types(source) + _assert_json_types(obj.metadata[EXPERIMENTAL_SOURCE_KEY]) + json.dumps(source) + set_experimental_source(obj, source) + + +@pytest.mark.parametrize( + "value", + [ + [], + {}, + [[0, 1], [2, 3]], + [{"a": 1}, {"b": "x"}], + {"a": {"b": [1.5, -2]}}, + [True, False], + "µm é", + "", + ["a", 1, 2.5, False], + ".", + -(2**63), + 2**63 - 1, + ], + ids=[ + "empty-list", + "empty-dict", + "nested-list", + "list-of-dicts", + "nested-dict", + "bools", + "unicode", + "empty-string", + "mixed-list", + "dot", + "int64-min", + "int64-max", + ], +) +def test_json_values_survive_file_round_trip(value) -> None: + """Every accepted JSON value is restored with its type.""" + source = make_experimental_source( + "hdf5", "f.h5", selection={"axes": [".", "x"]}, details={"v": value} + ) + signal = create_signal("s", np.arange(3.0), np.arange(3.0)) + image = create_image("i", np.zeros((2, 3))) + for obj in (signal, image): + set_experimental_source(obj, source) + for obj in _round_trip(signal, image): + restored = get_experimental_source(obj) + assert restored == source + _assert_json_types(restored) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"])