From 196c74245c9ebc1193f4b31b0e42167f7af01a3a Mon Sep 17 00:00:00 2001 From: Pierre Raybaut <1311787+PierreRaybaut@users.noreply.github.com> Date: Wed, 21 Oct 2026 06:07:49 +0200 Subject: [PATCH 01/20] feat(h5): browse HDF5 files without reading whole datasets Also stops clipping uint32 images and adds NodeFactory.unregister for plugin reloads. Assisted-by: Claude Opus 5.5 --- sigimax/h5/common.py | 29 ++- sigimax/h5/generic.py | 236 ++++++++---------- sigimax/h5/utils.py | 46 ++++ .../tests/hdf5/test_h5_lazy_classification.py | 160 ++++++++++++ 4 files changed, 331 insertions(+), 140 deletions(-) create mode 100644 sigimax/tests/hdf5/test_h5_lazy_classification.py diff --git a/sigimax/h5/common.py b/sigimax/h5/common.py index 1dd76e6..c88743d 100644 --- a/sigimax/h5/common.py +++ b/sigimax/h5/common.py @@ -20,6 +20,9 @@ from sigimax.config import get_conf +#: Attributes with more elements are not copied to object metadata +MAX_ATTRIBUTE_ITEMS = 10000 + def data_to_xy(data: np.ndarray) -> list[np.ndarray]: """Convert 2-D array into a list of 1-D array data (x, y, dx, dy). @@ -62,6 +65,7 @@ def __init__(self, h5file, dname): self.metadata = {} self.__obj = None self.children = [] + # Kept for applications reading it: uint32 data is no longer clipped self.uint32_wng = False @property @@ -133,9 +137,15 @@ def collect_attributes(self): ``REFERENCE_LIST`` arrays produced by ``h4toh5convert``) are exposed by h5py as ``object``-dtype arrays of :class:`h5py.h5r.Reference`, which raise ``TypeError: no default __reduce__`` when DataLab pickles the - object to run a computation in a worker process. + object to run a computation in a worker process. Attributes with more than + :data:`MAX_ATTRIBUTE_ITEMS` elements are skipped without being read. """ - for key, value in self.dset.attrs.items(): + attrs = self.dset.attrs + for key in attrs: + shape = attrs.get_id(key).shape + if shape and int(np.prod(shape, dtype=np.int64)) > MAX_ATTRIBUTE_ITEMS: + continue + value = attrs[key] if isinstance(value, bytes): value = to_string(value) if isinstance(value, np.ndarray): @@ -180,12 +190,7 @@ def set_signal_data(self, obj: SignalObj) -> None: def set_image_data(self, obj: ImageObj) -> None: """Set image data (handles various issues)""" - data = self.data - if data.dtype == np.uint32: - self.uint32_wng = data.max() > np.iinfo(np.int32).max - clipped_data = data.clip(0, np.iinfo(np.int32).max) - data = np.array(clipped_data, dtype=np.int32) - obj.data = convert_array_to_valid_dtype(data, ImageObj.VALID_DTYPES) + obj.data = convert_array_to_valid_dtype(self.data, ImageObj.VALID_DTYPES) class H5Importer: @@ -248,6 +253,14 @@ def register(self, cls, is_generic=False): else: self.__thirdparty_classes.append(cls) + def unregister(self, cls) -> None: + """Unregister node class and its post triggers, if registered. + This lets an application plugin remove its node classes when it is unloaded""" + for classes in (self.__thirdparty_classes, self.__generic_classes): + while cls in classes: + classes.remove(cls) + self.__post_triggers.pop(cls, None) + def get(self, dset): """Return node class that matches h5 dataset""" for name in to_string(dset.name).split("/"): diff --git a/sigimax/h5/generic.py b/sigimax/h5/generic.py index c49d7c5..bf5e9ad 100644 --- a/sigimax/h5/generic.py +++ b/sigimax/h5/generic.py @@ -6,6 +6,8 @@ # pylint: disable=invalid-name # Allows short reference names like x, y, ... +from __future__ import annotations + import h5py import numpy as np from guidata.utils.misc import to_string @@ -13,6 +15,12 @@ from sigimax.h5 import common, utils +#: Maximum number of elements read to show the textual preview of an array +TEXT_PREVIEW_MAX_ITEMS = 100 + +#: Number of compound records read to decide if their string fields matter +COMPOUND_SAMPLE_SIZE = 1000 + # ============================================================================= # Encoding and Data Reading Utilities # ============================================================================= @@ -163,7 +171,7 @@ class BaseGenericNode(common.BaseNode): @classmethod def match(cls, dset): """Return True if h5 dataset matches this node pattern.""" - return not isinstance(dset, h5py.Group) + return isinstance(dset, h5py.Dataset) @property def icon_name(self): @@ -205,13 +213,10 @@ class GenericScalarNode(BaseGenericNode): @classmethod def match(cls, dset): """Match scalar numeric data.""" - if not super().match(dset): - return False - data = safe_read_dataset(dset) return ( - data is not None - and isinstance(data, np.generic) - and utils.is_supported_num_dtype(data) + super().match(dset) + and dset.shape == () + and utils.is_supported_num_dtype(dset) ) @@ -220,18 +225,12 @@ class GenericTextNode(BaseGenericNode): @classmethod def match(cls, dset): - """Match text or string data.""" - if not super().match(dset): - return False - data = safe_read_dataset(dset) - if data is None: - # Try to match based on dtype for unreadable data - try: - dtype = dset.dtype - return dtype.kind in ["S", "a", "U"] or "str" in str(dtype) - except Exception: # pylint: disable=broad-except - return False - return isinstance(data, bytes) or utils.is_supported_str_dtype(data) + """Match scalar text or string data.""" + return ( + super().match(dset) + and dset.shape == () + and utils.is_string_dtype(dset.dtype) + ) @property def dtype_str(self): @@ -273,47 +272,58 @@ class GenericArrayNode(BaseGenericNode): IS_ARRAY = True + def __init__(self, h5file, dname): + super().__init__(h5file, dname) + self.__layout: tuple[tuple[int, ...], np.dtype] | None = None + @classmethod def match(cls, dset): """Match numeric array data, including convertible compound data.""" - if not super().match(dset): - return False - data = safe_read_dataset(dset) - - if data is None: - return False - - # First check direct numeric arrays - if ( - utils.is_supported_num_dtype(data) - and isinstance(data, np.ndarray) - and len(data.shape) in (1, 2) - ): - return True - - # Then check compound data that can be converted to numeric arrays - if ( - isinstance(data, np.ndarray) - and hasattr(data.dtype, "names") - and data.dtype.names is not None - ): - return cls._can_convert_compound_to_numeric(data) - - return False + return super().match(dset) and cls.numeric_layout(dset) is not None @classmethod - def _can_convert_compound_to_numeric(cls, data): - """Check if compound data can be converted to a supported numeric array.""" - try: - numeric_array = cls.extract_numeric_from_compound(data) - return ( - numeric_array is not None - and utils.is_supported_num_dtype(numeric_array) - and isinstance(numeric_array, np.ndarray) - and len(numeric_array.shape) in (1, 2) - ) - except Exception: # pylint: disable=broad-except - return False + def numeric_layout(cls, dset) -> tuple[tuple[int, ...], np.dtype] | None: + """Return shape and dtype of the numeric array read from a dataset. + + Only the dataset metadata is used, except for compound datasets mixing + numeric and string fields: their first records are read to check whether + the string fields carry information that a numeric array would lose. + + Args: + dset: HDF5 dataset + + Returns: + Shape and dtype of the 1D or 2D numeric array, or None if the dataset + does not provide one + """ + dtype, shape = dset.dtype, dset.shape + if not shape: + return None + if dtype.names is None: + if utils.is_supported_num_dtype(dset) and len(shape) in (1, 2): + return shape, dtype + return None + fields = {name: dtype.fields[name][0] for name in dtype.names} + numeric = [ + name for name, fdt in fields.items() if np.issubdtype(fdt, np.number) + ] + if not numeric: + return None + if len(numeric) != len(fields): + sample, _truncated = utils.read_bounded(dset, COMPOUND_SAMPLE_SIZE) + for name in fields: + if name not in numeric and cls._has_meaningful_string_data( + sample[name] + ): + return None + if len(numeric) == 1: + new_dtype = fields[numeric[0]] + else: + new_dtype = np.result_type(*(fields[name] for name in numeric)) + shape = shape + (len(numeric),) + if not new_dtype.name.startswith(("int", "uint", "float", "complex")): + return None + return (shape, new_dtype) if len(shape) in (1, 2) else None @classmethod def extract_numeric_from_compound(cls, data): @@ -400,14 +410,21 @@ def data(self): return raw_data + @property + def layout(self) -> tuple[tuple[int, ...], np.dtype]: + """Shape and dtype of the numeric array associated to node""" + if self.__layout is None: + self.__layout = self.numeric_layout(self.dset) + return self.__layout + def is_supported(self) -> bool: """Return True if node is associated to supported data""" - return self.data.size > 1 + return int(np.prod(self.layout[0])) > 1 @property def __is_signal(self): """Return True if array represents a signal""" - shape = self.data.shape + shape = self.layout[0] return len(shape) == 1 or shape[0] in (1, 2) or shape[1] in (1, 2) @property @@ -420,17 +437,20 @@ def icon_name(self): @property def shape_str(self): """Return string representation of node shape, if any""" - return " x ".join([str(size) for size in self.data.shape]) + return " x ".join([str(size) for size in self.layout[0]]) @property def dtype_str(self): """Return string representation of node data type, if any""" - return str(self.data.dtype) + return str(self.layout[1]) @property def text(self): """Return node textual representation""" - return str(self.data) + data, truncated = utils.read_bounded(self.dset, TEXT_PREVIEW_MAX_ITEMS) + if self.dset.dtype.names is not None: + data = self.extract_numeric_from_compound(data) + return str(data) + (" ..." if truncated else "") def create_native_object(self): """Create native object, if supported""" @@ -457,55 +477,12 @@ class GenericCompoundNode(BaseGenericNode): @classmethod def match(cls, dset): """Match compound/structured data that cannot be converted to numeric arrays.""" - if not super().match(dset): - return False - - data = safe_read_dataset(dset) - if data is None: - # Try to match based on dtype if we can't read the data - try: - return dset.dtype.names is not None - except Exception: # pylint: disable=broad-except - return False - - # Check if it's compound data (structured array) - if not ( - isinstance(data, np.ndarray) - and hasattr(data.dtype, "names") - and data.dtype.names is not None - ): - return False - - # IMPORTANT: Only match if GenericArrayNode cannot handle this data - # Try to convert to a numeric array first - if cls._can_convert_to_numeric_array(data): - return False # Let GenericArrayNode handle it - - return True # We handle compound data that can't be converted - - @classmethod - def _can_convert_to_numeric_array(cls, data): - """Check if compound data can be converted to a numeric array.""" - try: - # Try to extract numeric fields and create a pure numeric array - numeric_array = cls.extract_numeric_array(data) - if numeric_array is None: - return False - - # Check if the resulting array would be supported by GenericArrayNode - return ( - utils.is_supported_num_dtype(numeric_array) - and isinstance(numeric_array, np.ndarray) - and len(numeric_array.shape) in (1, 2) - ) - except Exception: # pylint: disable=broad-except - return False - - @classmethod - def extract_numeric_array(cls, data): - """Try to extract a pure numeric array from compound data.""" - # Use the same logic as GenericArrayNode - return GenericArrayNode.extract_numeric_from_compound(data) + return ( + super().match(dset) + and dset.dtype.names is not None + and bool(dset.shape) + and GenericArrayNode.numeric_layout(dset) is None + ) @property def dtype_str(self): @@ -525,47 +502,42 @@ def dtype_str(self): @property def text(self): """Return formatted compound data.""" - if self.data is None: + try: + data, truncated = utils.read_first_items(self.dset, 3) + except (UnicodeDecodeError, TypeError, ValueError, OSError): return "" try: - return self._format_compound_data() + if data.size == 1 and not truncated: + return self._format_single_record(data) + return self._format_multiple_records(data, truncated) except Exception: # pylint: disable=broad-except - return f"" + return f"" - def _format_compound_data(self): - """Format compound data for display.""" - if not (hasattr(self.data.dtype, "names") and self.data.dtype.names): - return super().text - if self.data.size == 1: - return self._format_single_record() - return self._format_multiple_records() - - def _format_single_record(self): + @staticmethod + def _format_single_record(data): """Format a single compound record.""" parts = [] - for field_name in self.data.dtype.names: - field_value = self.data[field_name].item() + for field_name in data.dtype.names: + field_value = data[field_name].item() if isinstance(field_value, bytes): field_value = safe_decode_bytes(field_value) parts.append(f"{field_name}: {field_value}") return f"({', '.join(parts)})" - def _format_multiple_records(self): - """Format multiple compound records.""" + @staticmethod + def _format_multiple_records(data, truncated): + """Format the first compound records.""" records = [] - for i, record in enumerate(self.data.flat): - if i >= 3: # Show max 3 records - records.append("...") - break - + for record in data.flat: parts = [] - for field_name in self.data.dtype.names: + for field_name in data.dtype.names: field_value = record[field_name] if isinstance(field_value, bytes): field_value = safe_decode_bytes(field_value) parts.append(f"{field_name}: {field_value}") records.append(f"({', '.join(parts)})") - + if truncated: + records.append("...") return f"[{', '.join(records)}]" diff --git a/sigimax/h5/utils.py b/sigimax/h5/utils.py index 0678ded..312fa0b 100644 --- a/sigimax/h5/utils.py +++ b/sigimax/h5/utils.py @@ -6,6 +6,11 @@ # pylint: disable=invalid-name # Allows short reference names like x, y, ... +from __future__ import annotations + +from typing import Any + +import h5py import numpy as np from guidata.utils.misc import to_string @@ -93,3 +98,44 @@ def is_single_str_array(data): def is_supported_str_dtype(data): """Return True if data type is a string type supported by preview""" return data.dtype.name.startswith("string") or is_single_str_array(data) + + +def is_string_dtype(dtype: np.dtype) -> bool: + """Return True if dtype stores fixed-length or variable-length strings""" + return dtype.kind in "SU" or h5py.check_string_dtype(dtype) is not None + + +def get_size(dset: h5py.Dataset) -> int: + """Return the number of elements of a dataset, without reading it""" + return 0 if dset.shape is None else int(np.prod(dset.shape, dtype=np.int64)) + + +def read_bounded(dset: h5py.Dataset, max_items: int) -> tuple[Any, bool]: + """Read the first elements of a dataset, along each axis + + Args: + dset: HDF5 dataset + max_items: maximum number of elements to read + + Returns: + Data that was read and True if it is only part of the dataset + """ + shape = dset.shape + if not shape or get_size(dset) <= max_items: + return dset[()], False + per_axis = max(1, int(max_items ** (1 / len(shape)))) + return dset[tuple(slice(0, min(size, per_axis)) for size in shape)], True + + +def read_first_items(dset: h5py.Dataset, count: int) -> tuple[Any, bool]: + """Read the first elements of a dataset, in C order, from its first row + + Args: + dset: HDF5 dataset with at least one dimension + count: maximum number of elements to read + + Returns: + Data that was read and True if it is only part of the dataset + """ + selection = (0,) * (len(dset.shape) - 1) + (slice(0, count),) + return dset[selection], get_size(dset) > count diff --git a/sigimax/tests/hdf5/test_h5_lazy_classification.py b/sigimax/tests/hdf5/test_h5_lazy_classification.py new file mode 100644 index 0000000..f58f112 --- /dev/null +++ b/sigimax/tests/hdf5/test_h5_lazy_classification.py @@ -0,0 +1,160 @@ +# Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file. + +""" +HDF5 lazy classification unit tests +----------------------------------- + +Browsing an HDF5 file must not read whole datasets: nodes are classified and +described from the dataset shape and type, and textual previews read only the +first elements. Data is read entirely only when an object is created. +""" + +from __future__ import annotations + +import uuid +from collections.abc import Generator + +import h5py +import numpy as np +import pytest + +from sigimax.h5 import H5Importer +from sigimax.h5.common import MAX_ATTRIBUTE_ITEMS, NODE_FACTORY, BaseNode +from sigimax.h5.generic import ( + GenericArrayNode, + GenericCompoundNode, + GenericScalarNode, + GenericTextNode, +) + +pytestmark = pytest.mark.unit + +#: Datasets larger than this must never be read entirely while browsing +FULL_READ_LIMIT = 1000 + + +@pytest.fixture(name="h5fname") +def fixture_h5fname(tmp_path) -> str: + """HDF5 file covering the generic node kinds, with large datasets.""" + fname = str(tmp_path / f"{uuid.uuid4()}.h5") + with h5py.File(fname, "w") as h5file: + h5file["scalar"] = np.int32(5) + h5file["text"] = "unicode text" + h5file["signal"] = np.arange(50_000.0) + h5file["image"] = np.ones((300, 400), dtype=np.uint16) + counts = [[0, 2**32 - 1, 3], [7, 2**31, 4], [1, 2, 5]] + h5file["counts"] = np.array(counts, np.uint32) + h5file["cube"] = np.zeros((10, 30, 40)) + records = np.zeros(5000, dtype=[("x", "f8"), ("y", "f4")]) + h5file["records"] = records + mixed = np.zeros(5000, dtype=[("name", "S8"), ("v", "f8")]) + mixed["name"] = b"label" + h5file["mixed"] = mixed + h5file["signal"].attrs["unit"] = "s" + h5file["signal"].attrs["big"] = np.zeros(MAX_ATTRIBUTE_ITEMS + 1, np.uint8) + return fname + + +@pytest.fixture(name="full_reads") +def fixture_full_reads(monkeypatch) -> Generator[list[str], None, None]: + """Record whole reads of datasets larger than :data:`FULL_READ_LIMIT`.""" + reads: list[str] = [] + original = h5py.Dataset.__getitem__ + + def getitem(self, args, *rest, **kwargs): + selection = args if isinstance(args, tuple) else (args,) + whole = selection in ((), (Ellipsis,)) or all( + isinstance(item, slice) and item == slice(None) for item in selection + ) + if whole and self.size > FULL_READ_LIMIT: + reads.append(self.name) + return original(self, args, *rest, **kwargs) + + monkeypatch.setattr(h5py.Dataset, "__getitem__", getitem) + yield reads + + +def test_browsing_reads_no_large_dataset(h5fname: str, full_reads: list) -> None: + """Classification and descriptions use metadata and bounded reads only.""" + importer = H5Importer(h5fname) + try: + nodes = {node.id: node for node in importer.nodes} + for node in nodes.values(): + _description = ( + node.shape_str, + node.dtype_str, + node.text, + node.icon_name, + node.is_supported(), + node.metadata, + ) + assert full_reads == [] + assert isinstance(nodes["/scalar"], GenericScalarNode) + assert isinstance(nodes["/text"], GenericTextNode) + assert isinstance(nodes["/records"], GenericArrayNode) + assert isinstance(nodes["/mixed"], GenericCompoundNode) + assert "/cube" not in nodes + assert nodes["/signal"].shape_str == "50000" + assert nodes["/signal"].text.endswith(" ...") + assert nodes["/records"].shape_str == "5000 x 2" + assert nodes["/records"].dtype_str == "float64" + assert nodes["/image"].icon_name == "image.svg" + assert nodes["/mixed"].text.endswith("...]") + assert nodes["/signal"].metadata["unit"] == "s" + assert "big" not in nodes["/signal"].metadata + + # Creating the object reads the whole dataset, on demand + signal = nodes["/signal"].get_native_object() + assert full_reads == ["/signal"] + assert signal.y.size == 50_000 + finally: + importer.close() + + +def test_uint32_image_is_exact(h5fname: str) -> None: + """uint32 counts beyond the int32 range are imported without clipping.""" + importer = H5Importer(h5fname) + try: + node = importer.get("/counts") + assert node.icon_name == "image.svg" + image = node.get_native_object() + assert image.data.dtype == np.float64 + assert image.data[0, 1] == 2**32 - 1 + assert image.data[1, 1] == 2**31 + assert not node.uint32_wng + finally: + importer.close() + + +def test_unregister_node_class(h5fname: str) -> None: + """A node class registered by an application can be removed again.""" + + class CubeNode(BaseNode): + """Node matching the 3D dataset only.""" + + @classmethod + def match(cls, dset): + return isinstance(dset, h5py.Dataset) and dset.ndim == 3 + + triggered: list[str] = [] + NODE_FACTORY.register(CubeNode) + NODE_FACTORY.add_post_trigger( + CubeNode, lambda node, _imp: triggered.append(node.id) + ) + try: + importer = H5Importer(h5fname) + assert isinstance(importer.get("/cube"), CubeNode) + importer.close() + finally: + NODE_FACTORY.unregister(CubeNode) + NODE_FACTORY.unregister(CubeNode) + importer = H5Importer(h5fname) + try: + assert "/cube" not in [node.id for node in importer.nodes] + finally: + importer.close() + assert triggered == ["/cube"] + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From fad037f7532ec4bc98f51b8d7612621c5921f522 Mon Sep 17 00:00:00 2001 From: Pierre Raybaut <1311787+PierreRaybaut@users.noreply.github.com> Date: Wed, 21 Oct 2026 06:07:54 +0200 Subject: [PATCH 02/20] docs: release notes for HDF5 browsing and import Assisted-by: Claude Opus 5.5 --- doc/release_notes/release_1.01.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/doc/release_notes/release_1.01.md b/doc/release_notes/release_1.01.md index 58585d0..ea8632f 100644 --- a/doc/release_notes/release_1.01.md +++ b/doc/release_notes/release_1.01.md @@ -32,6 +32,13 @@ * The Planckian fit dialog starts from a curve matching the data peak (its initial amplitude was about 147 times too low on blackbody data in physical units), and its "width factor" is now labelled "temperature factor": for wavelengths in meters, the temperature is `h c σ / (5 k_B x₀)`. * The exponential fit dialog now evaluates the same model as Sigima, and its rate slider covers short time constants whatever the X unit (e.g. 1 ms with X in seconds). +### HDF5 import ### + +* Browsing an HDF5 file no longer reads its datasets entirely: generic nodes are classified and described from the dataset shape and data type, and value previews read only the first elements (`TEXT_PREVIEW_MAX_ITEMS`). Compound datasets mixing numeric and string fields are checked on their first records only. Data is read entirely when the object is created, for preview or import. +* Dataset attributes with more than `MAX_ATTRIBUTE_ITEMS` (10 000) values are no longer copied to object metadata. +* Images with unsigned 32-bit values above 2 147 483 647 are no longer clipped to `int32`: they are converted with Sigima's converter, which keeps every value exactly with Sigima ≥ 1.4.0. `BaseNode.uint32_wng` is kept for compatibility and is always `False`. +* `NodeFactory.unregister` removes a node class and its post triggers, so that an application plugin can remove the node classes it registered when it is unloaded. + ### Requirements ### * Sigima 1.3.0 or later is required for the portable annotation model, PlotPy conversion helpers and curve fitting model formulas. \ No newline at end of file From 3caf121e7d987bbcc48ac0228df17302f4463cff Mon Sep 17 00:00:00 2001 From: Pierre Raybaut <1311787+PierreRaybaut@users.noreply.github.com> Date: Wed, 21 Oct 2026 06:48:14 +0200 Subject: [PATCH 03/20] feat(h5): bounded previews and views of N-dimensional datasets Assisted-by: Claude Opus 5.5 --- sigimax/h5/common.py | 11 ++ sigimax/h5/generic.py | 66 +++++++++++- sigimax/h5/selection.py | 102 ++++++++++++++++++ sigimax/locale/fr/LC_MESSAGES/sigimax.po | 3 + .../tests/hdf5/test_h5_lazy_classification.py | 72 ++++++++++++- sigimax/tests/hdf5/test_h5browser_preview.py | 78 ++++++++++++++ sigimax/widgets/h5browser.py | 74 +++++++++++-- 7 files changed, 391 insertions(+), 15 deletions(-) create mode 100644 sigimax/h5/selection.py create mode 100644 sigimax/tests/hdf5/test_h5browser_preview.py diff --git a/sigimax/h5/common.py b/sigimax/h5/common.py index c88743d..df19a39 100644 --- a/sigimax/h5/common.py +++ b/sigimax/h5/common.py @@ -128,6 +128,17 @@ def get_native_object(self): self.__obj = obj return self.__obj + def create_preview_object(self, max_items: int): # pylint: disable=unused-argument + """Return the object shown as a preview of the node, if supported. + + Subclasses may return a decimated object reading at most ``max_items`` + elements, instead of the native object. + + Args: + max_items: maximum number of data elements to read + """ + return self.get_native_object() + def collect_attributes(self): """Collect attributes from node. diff --git a/sigimax/h5/generic.py b/sigimax/h5/generic.py index bf5e9ad..4a729e3 100644 --- a/sigimax/h5/generic.py +++ b/sigimax/h5/generic.py @@ -8,12 +8,16 @@ from __future__ import annotations +import math + import h5py import numpy as np from guidata.utils.misc import to_string -from sigima.objects import create_image, create_signal +from sigima.io.common.converters import convert_array_to_valid_dtype +from sigima.objects import ImageObj, SignalObj, create_image, create_signal from sigimax.h5 import common, utils +from sigimax.h5.selection import NDSelection, preview_step, read_selection #: Maximum number of elements read to show the textual preview of an array TEXT_PREVIEW_MAX_ITEMS = 100 @@ -468,6 +472,65 @@ def create_native_object(self): obj = None return obj + def create_preview_object(self, max_items: int): + """Return the native object, or a decimated one if the dataset is larger + than ``max_items`` (1D signals and images only)""" + shape = self.layout[0] + if ( + math.prod(shape) <= max_items + or self.dset.dtype.names is not None + or (self.__is_signal and len(shape) == 2) + ): + return self.get_native_object() + step = preview_step(shape, max_items) + data = read_selection(self.dset, NDSelection(tuple(range(len(shape)))), step) + if len(shape) == 1: + x = np.arange(0, shape[0], step, dtype=float) + y = convert_array_to_valid_dtype(data, SignalObj.VALID_DTYPES) + return create_signal(self.object_title, x, y) + obj = create_image( + self.object_title, + convert_array_to_valid_dtype(data, ImageObj.VALID_DTYPES), + ) + obj.set_uniform_coords(float(step), float(step)) + return obj + + +class GenericNDArrayNode(BaseGenericNode): + """Node for numeric HDF5 datasets with three dimensions or more. + + Such datasets (image stacks, spectral cubes...) are listed and described, but + are not imported as a whole: their 1D or 2D views are read with + :func:`sigimax.h5.selection.read_selection`. + """ + + @classmethod + def match(cls, dset): + """Match numeric datasets with three dimensions or more.""" + return ( + super().match(dset) + and dset.dtype.names is None + and dset.shape is not None + and len(dset.shape) >= 3 + and utils.is_supported_num_dtype(dset) + ) + + @property + def icon_name(self): + """Icon name associated to node""" + return "h5array.svg" + + @property + def shape_str(self): + """Return string representation of node shape, if any""" + return " x ".join([str(size) for size in self.dset.shape]) + + @property + def text(self): + """Return node textual representation""" + data, truncated = utils.read_bounded(self.dset, TEXT_PREVIEW_MAX_ITEMS) + return str(data) + (" ..." if truncated else "") + class GenericCompoundNode(BaseGenericNode): """Node for compound/structured HDF5 data that can't convert to numeric arrays.""" @@ -549,4 +612,5 @@ def _format_multiple_records(data, truncated): common.NODE_FACTORY.register(GenericScalarNode, is_generic=True) common.NODE_FACTORY.register(GenericTextNode, is_generic=True) common.NODE_FACTORY.register(GenericArrayNode, is_generic=True) +common.NODE_FACTORY.register(GenericNDArrayNode, is_generic=True) common.NODE_FACTORY.register(GenericCompoundNode, is_generic=True) diff --git a/sigimax/h5/selection.py b/sigimax/h5/selection.py new file mode 100644 index 0000000..d87aa25 --- /dev/null +++ b/sigimax/h5/selection.py @@ -0,0 +1,102 @@ +# Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file. + +""" +SigimaX selections of 1D or 2D views in HDF5 datasets + +Reading a selection only reads the selected elements, so that a view of a large +multidimensional dataset (e.g. a frame of an image stack, or the spectrum of a +pixel in a spectral cube) never loads the whole dataset. +""" + +from __future__ import annotations + +import dataclasses +import math + +import h5py +import numpy as np + +__all__ = ["NDSelection", "preview_step", "read_selection"] + + +@dataclasses.dataclass(frozen=True) +class NDSelection: + """Selection of a 1D or 2D view in an N-dimensional dataset + + Args: + displayed: displayed axes, in display order: ``(axis,)`` for a 1D view, + ``(row_axis, column_axis)`` for a 2D view (a 2D view may be transposed) + indices: index along each axis which is not displayed, in axis order + """ + + displayed: tuple[int, ...] + indices: tuple[int, ...] = () + + def validate(self, shape: tuple[int, ...]) -> None: + """Raise ValueError if the selection does not apply to an array shape + + Args: + shape: shape of the dataset + """ + ndim = len(shape) + if len(self.displayed) not in (1, 2): + raise ValueError("A selection displays one or two axes") + if len(set(self.displayed)) != len(self.displayed) or not all( + 0 <= axis < ndim for axis in self.displayed + ): + raise ValueError(f"Invalid displayed axes {self.displayed} for {ndim}D") + hidden = [axis for axis in range(ndim) if axis not in self.displayed] + if len(self.indices) != len(hidden): + raise ValueError(f"Expected {len(hidden)} indices, got {len(self.indices)}") + for axis, index in zip(hidden, self.indices): + if not 0 <= index < shape[axis]: + raise ValueError(f"Index {index} out of range for axis {axis}") + + def as_dict(self) -> dict[str, list[int]]: + """Return the selection as a JSON-compatible dictionary""" + return {"displayed": list(self.displayed), "indices": list(self.indices)} + + +def preview_step(shape: tuple[int, ...], max_items: int) -> int: + """Return the smallest step keeping a strided view under a number of elements + + Args: + shape: shape of the view + max_items: maximum number of elements of the strided view + + Returns: + Step to apply along each axis of the view + """ + if max_items < 1: + raise ValueError("max_items must be positive") + size = math.prod(shape) + step = max(1, math.floor((size / max_items) ** (1 / max(len(shape), 1)))) + while math.prod(math.ceil(dim / step) for dim in shape) > max_items: + step += 1 + return step + + +def read_selection( + dset: h5py.Dataset, selection: NDSelection, step: int = 1 +) -> np.ndarray: + """Read the selected view of a dataset + + Args: + dset: HDF5 dataset + selection: selected view + step: step along the displayed axes (to read a decimated preview) + + Returns: + Array with one dimension per displayed axis, in display order + """ + selection.validate(dset.shape) + if step < 1: + raise ValueError("step must be positive") + hidden = iter(selection.indices) + region = tuple( + slice(None, None, step) if axis in selection.displayed else next(hidden) + for axis in range(dset.ndim) + ) + data = np.asarray(dset[region]) + order = sorted(selection.displayed) + return data.transpose([order.index(axis) for axis in selection.displayed]) diff --git a/sigimax/locale/fr/LC_MESSAGES/sigimax.po b/sigimax/locale/fr/LC_MESSAGES/sigimax.po index 810d59e..83c9b98 100644 --- a/sigimax/locale/fr/LC_MESSAGES/sigimax.po +++ b/sigimax/locale/fr/LC_MESSAGES/sigimax.po @@ -477,6 +477,9 @@ msgstr "Nom" msgid "Unsupported data" msgstr "Données non prises en charge" +msgid "Loading..." +msgstr "Chargement..." + msgid "Group" msgstr "Groupe" diff --git a/sigimax/tests/hdf5/test_h5_lazy_classification.py b/sigimax/tests/hdf5/test_h5_lazy_classification.py index f58f112..04df27d 100644 --- a/sigimax/tests/hdf5/test_h5_lazy_classification.py +++ b/sigimax/tests/hdf5/test_h5_lazy_classification.py @@ -23,9 +23,11 @@ from sigimax.h5.generic import ( GenericArrayNode, GenericCompoundNode, + GenericNDArrayNode, GenericScalarNode, GenericTextNode, ) +from sigimax.h5.selection import NDSelection, preview_step, read_selection pytestmark = pytest.mark.unit @@ -44,7 +46,7 @@ def fixture_h5fname(tmp_path) -> str: h5file["image"] = np.ones((300, 400), dtype=np.uint16) counts = [[0, 2**32 - 1, 3], [7, 2**31, 4], [1, 2, 5]] h5file["counts"] = np.array(counts, np.uint32) - h5file["cube"] = np.zeros((10, 30, 40)) + h5file["cube"] = np.arange(10 * 30 * 40, dtype=np.float64).reshape(10, 30, 40) records = np.zeros(5000, dtype=[("x", "f8"), ("y", "f4")]) h5file["records"] = records mixed = np.zeros(5000, dtype=[("name", "S8"), ("v", "f8")]) @@ -93,7 +95,9 @@ def test_browsing_reads_no_large_dataset(h5fname: str, full_reads: list) -> None assert isinstance(nodes["/text"], GenericTextNode) assert isinstance(nodes["/records"], GenericArrayNode) assert isinstance(nodes["/mixed"], GenericCompoundNode) - assert "/cube" not in nodes + assert isinstance(nodes["/cube"], GenericNDArrayNode) + assert nodes["/cube"].shape_str == "10 x 30 x 40" + assert not nodes["/cube"].is_supported() assert nodes["/signal"].shape_str == "50000" assert nodes["/signal"].text.endswith(" ...") assert nodes["/records"].shape_str == "5000 x 2" @@ -150,11 +154,73 @@ def match(cls, dset): NODE_FACTORY.unregister(CubeNode) importer = H5Importer(h5fname) try: - assert "/cube" not in [node.id for node in importer.nodes] + assert isinstance(importer.get("/cube"), GenericNDArrayNode) finally: importer.close() assert triggered == ["/cube"] +def test_read_selection(h5fname: str, full_reads: list) -> None: + """Selected views of a cube match NumPy indexing, without reading the cube.""" + with h5py.File(h5fname, "r") as h5file: + dset = h5file["cube"] + cube = np.arange(10 * 30 * 40, dtype=np.float64).reshape(10, 30, 40) + frame = read_selection(dset, NDSelection((1, 2), (4,))) + assert np.array_equal(frame, cube[4]) + transposed = read_selection(dset, NDSelection((2, 1), (4,))) + assert np.array_equal(transposed, cube[4].T) + spectrum = read_selection(dset, NDSelection((0,), (7, 3))) + assert np.array_equal(spectrum, cube[:, 7, 3]) + strided = read_selection(dset, NDSelection((1, 2), (0,)), step=3) + assert np.array_equal(strided, cube[0, ::3, ::3]) + assert NDSelection((0,), (7, 3)).as_dict() == { + "displayed": [0], + "indices": [7, 3], + } + for invalid in ( + NDSelection((0, 1, 2)), + NDSelection((1, 1), (0,)), + NDSelection((1, 2)), + NDSelection((1, 2), (10,)), + ): + with pytest.raises(ValueError): + read_selection(dset, invalid) + assert full_reads == [] + + +@pytest.mark.parametrize( + "shape, max_items, step", + [ + ((100,), 100, 1), + ((101,), 100, 2), + ((4096, 4096), 2**24, 1), + ((300, 400), 1000, 12), + ], + ids=["fits", "1d", "2d-fits", "2d"], +) +def test_preview_step(shape: tuple, max_items: int, step: int) -> None: + """The preview step is the smallest one keeping the view under the limit.""" + assert preview_step(shape, max_items) == step + sizes = [len(range(0, dim, step)) for dim in shape] + assert np.prod(sizes) <= max_items + + +def test_preview_is_decimated(h5fname: str, full_reads: list) -> None: + """Previews of large datasets are read with a step, in physical coordinates.""" + importer = H5Importer(h5fname) + try: + image = importer.get("/image").create_preview_object(1000) + assert image.data.shape == (25, 34) + assert (image.dx, image.dy) == (12.0, 12.0) + signal = importer.get("/signal").create_preview_object(1000) + assert signal.y.size <= 1000 + assert np.array_equal(signal.x, signal.y) + assert full_reads == [] + small = importer.get("/counts") + assert small.create_preview_object(1000) is small.get_native_object() + finally: + importer.close() + + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/sigimax/tests/hdf5/test_h5browser_preview.py b/sigimax/tests/hdf5/test_h5browser_preview.py new file mode 100644 index 0000000..ab73d5b --- /dev/null +++ b/sigimax/tests/hdf5/test_h5browser_preview.py @@ -0,0 +1,78 @@ +# Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file. + +""" +HDF5 browser preview test +------------------------- + +Previews are read in a worker thread: the last selected node is shown, the +previews of nodes selected before are discarded, and closing a file waits for +the preview being read. +""" + +from __future__ import annotations + +import time +import uuid + +import h5py +import numpy as np +import pytest +from guidata.qthelpers import qt_app_context +from plotpy.items import BaseImageItem, CurveItem +from qtpy import QtWidgets as QW + +from sigimax.widgets.h5browser import H5Browser + +pytestmark = pytest.mark.gui + + +def _wait_for_item(plot, item_type: type, timeout: float = 10.0): + """Process events until the plot shows an item of the given type.""" + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + QW.QApplication.processEvents() + items = [item for item in plot.get_items() if isinstance(item, item_type)] + if items: + return items[0] + time.sleep(0.01) + raise TimeoutError(f"No {item_type.__name__} shown in preview") + + +def test_h5browser_preview(tmp_path) -> None: + """The last selected node is previewed, decimated beyond the limit.""" + fname = str(tmp_path / f"{uuid.uuid4()}.h5") + with h5py.File(fname, "w") as h5file: + h5file["image"] = np.ones((300, 400), dtype=np.uint16) + h5file["signal"] = np.arange(5000.0) + with qt_app_context(): + browser = H5Browser() + browser.plotpreview.MAX_ITEMS = 1000 + browser.open_file(fname) + importer = browser.tree.h5importers[0] + preview = browser.plotpreview + preview.update_plot_preview(importer.get("/image")) + preview.update_plot_preview(importer.get("/signal")) + curve = _wait_for_item(preview.curvewidget.get_plot(), CurveItem) + x, _y = curve.get_data() + assert len(x) <= 1000 + QW.QApplication.processEvents() + image_items = [ + item + for item in preview.imagewidget.get_plot().get_items() + if isinstance(item, BaseImageItem) + ] + assert not image_items, "Stale image preview must be discarded" + + preview.update_plot_preview(importer.get("/image")) + image = _wait_for_item(preview.imagewidget.get_plot(), BaseImageItem) + assert image.data.shape == (25, 34) + + preview.update_plot_preview(importer.get("/signal")) + browser.close_file(fname) + QW.QApplication.processEvents() + browser.cleanup() + browser.close() + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/sigimax/widgets/h5browser.py b/sigimax/widgets/h5browser.py index d22f5b3..ca1e655 100644 --- a/sigimax/widgets/h5browser.py +++ b/sigimax/widgets/h5browser.py @@ -18,6 +18,7 @@ import abc import os import os.path as osp +from concurrent.futures import Future, ThreadPoolExecutor from typing import TYPE_CHECKING, Any, Callable from guidata.qthelpers import ( @@ -471,10 +472,24 @@ def set_current_file(self, fname: str) -> None: class PlotPreview(QW.QStackedWidget): - """Plot preview""" + """Plot preview + + Preview data is read in a worker thread, so that the browser stays responsive + while a large dataset is read. Datasets larger than :attr:`MAX_ITEMS` are + decimated. The preview of a node replaces any pending one. + """ + + #: Maximum number of data elements read for a preview + MAX_ITEMS = 2**24 + + SIG_LOADED = QC.Signal(int, object, object) def __init__(self, parent: QW.QWidget) -> None: super().__init__(parent) + self.__generation = 0 + self.__executor = ThreadPoolExecutor(1, thread_name_prefix="h5preview") + self.__pending: Future | None = None + self.SIG_LOADED.connect(self.__show_loaded_object) self.curvewidget = PlotWidget( self, options=PlotOptions( @@ -498,23 +513,59 @@ def __init__(self, parent: QW.QWidget) -> None: def cleanup(self) -> None: """Clean up widget""" + self.wait() for widget in (self.imagewidget, self.curvewidget): widget.get_plot().del_all_items() + def wait(self) -> None: + """Discard the pending preview and wait until its data is no longer read + (before closing HDF5 files)""" + self.__generation += 1 + if self.__pending is not None: + self.__pending.result() + self.__pending = None + + def __load_object(self, generation: int, node: BaseNode) -> None: + """Create the preview object of a node (worker thread)""" + obj, error = None, None + try: + obj = node.create_preview_object(self.MAX_ITEMS) + except Exception as exc: # pylint: disable=broad-except + error = exc + try: + self.SIG_LOADED.emit(generation, obj, error) + except RuntimeError: + pass # Widget deleted while the data was read + + def __show_message(self, text: str) -> None: + """Show a message instead of the preview""" + label = make.label(text, "C", (0, 0), "C") + plot: BasePlot = self.currentWidget().get_plot() + plot.del_all_items() + plot.add_item(label) + plot.replot() + def update_plot_preview(self, node: BaseNode) -> None: """Update plot preview widget""" - try: - obj = node.get_native_object() - except Exception as msg: # pylint: disable=broad-except - qt_handle_error_message(self, msg) + self.__generation += 1 + self.__show_message(_("Loading...")) + self.__pending = self.__executor.submit( + self.__load_object, self.__generation, node + ) + + def __show_loaded_object( + self, generation: int, obj: SignalObj | ImageObj | None, error: Exception + ) -> None: + """Show the preview object, unless another node was selected meanwhile""" + if generation != self.__generation: + return + if error is not None: + self.__show_message(_("Unsupported data")) + qt_handle_error_message(self, error) return if obj is None: # An error occurred while creating the object (invalid data, ...) - label = make.label(_("Unsupported data"), "C", (0, 0), "C") - plot: BasePlot = self.currentWidget().get_plot() - plot.del_all_items() - plot.add_item(label) - plot.replot() + self.__show_message(_("Unsupported data")) return if isinstance(obj, SignalObj): obj: SignalObj @@ -812,6 +863,7 @@ def close_file(self, fname: str) -> None: Args: fname: HDF5 file name """ + self.plotpreview.wait() self.tree.remove_root(fname) self.selector.remove_fname(fname) @@ -836,8 +888,8 @@ def __remove_file(self, fname: str) -> None: def cleanup(self) -> None: """Clean up widget""" - self.tree.cleanup() self.plotpreview.cleanup() + self.tree.cleanup() def get_node(self, item: QW.QTreeWidgetItem | None = None) -> BaseNode: """Return (selected) dataset From df0fd880507a8b7ab96a4f687d024cfbe0046351 Mon Sep 17 00:00:00 2001 From: Pierre Raybaut <1311787+PierreRaybaut@users.noreply.github.com> Date: Wed, 21 Oct 2026 06:48:43 +0200 Subject: [PATCH 04/20] feat(h5): import NeXus NXdata groups as signals or images Assisted-by: Claude Opus 5.5 --- sigimax/h5/__init__.py | 13 +- sigimax/h5/common.py | 5 +- sigimax/h5/nexus.py | 344 +++++++++++++++++++++++ sigimax/locale/fr/LC_MESSAGES/sigimax.po | 4 +- sigimax/mainwindow.py | 5 +- sigimax/tests/hdf5/test_h5_nexus.py | 164 +++++++++++ sigimax/widgets/h5browser.py | 2 +- 7 files changed, 531 insertions(+), 6 deletions(-) create mode 100644 sigimax/h5/nexus.py create mode 100644 sigimax/tests/hdf5/test_h5_nexus.py diff --git a/sigimax/h5/__init__.py b/sigimax/h5/__init__.py index 8b6f422..a23cffe 100644 --- a/sigimax/h5/__init__.py +++ b/sigimax/h5/__init__.py @@ -23,6 +23,17 @@ :members: .. autoclass:: sigimax.h5.common.RootNode :members: + +NeXus ``NXdata`` groups are shown as single nodes, imported as signals or images +with their axes (:mod:`sigimax.h5.nexus`). Views of multidimensional datasets are +read with :func:`sigimax.h5.selection.read_selection`. + +.. autoclass:: sigimax.h5.nexus.NXdataNode +.. autofunction:: sigimax.h5.nexus.describe_nxdata +.. autoclass:: sigimax.h5.selection.NDSelection + :members: +.. autofunction:: sigimax.h5.selection.read_selection +.. autofunction:: sigimax.h5.selection.preview_step """ __all__ = [ @@ -32,5 +43,5 @@ # pylint: disable=unused-import # Registering dynamic I/O features: -from sigimax.h5 import generic # noqa: F401 +from sigimax.h5 import generic, nexus # noqa: F401 from sigimax.h5.common import H5Importer # noqa: F401 diff --git a/sigimax/h5/common.py b/sigimax/h5/common.py index df19a39..c41f75d 100644 --- a/sigimax/h5/common.py +++ b/sigimax/h5/common.py @@ -16,7 +16,7 @@ import numpy as np from guidata.utils.misc import to_string from sigima.io.common.converters import convert_array_to_valid_dtype -from sigima.objects import ImageObj, SignalObj +from sigima.objects import EXPERIMENTAL_SOURCE_KEY, ImageObj, SignalObj from sigimax.config import get_conf @@ -170,10 +170,13 @@ def collect_attributes(self): def __process_metadata(self, obj): """Process metadata from dataset to obj""" + source = obj.metadata.get(EXPERIMENTAL_SOURCE_KEY) obj.reset_metadata_to_defaults() obj.set_metadata_option("HDF5Path", self.h5file.filename) obj.set_metadata_option("HDF5Dataset", self.id) obj.metadata.update(self.metadata) + if source is not None: + obj.metadata[EXPERIMENTAL_SOURCE_KEY] = source @property def object_title(self): diff --git a/sigimax/h5/nexus.py b/sigimax/h5/nexus.py new file mode 100644 index 0000000..f138139 --- /dev/null +++ b/sigimax/h5/nexus.py @@ -0,0 +1,344 @@ +# Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file. + +""" +SigimaX NeXus support: interpretation of ``NXdata`` groups + +A NeXus ``NXdata`` group designates its plottable signal and the axes of each +signal dimension. :class:`NXdataNode` shows such a group as a single node of the +HDF5 browser, which is imported as a signal (1D) or an image (2D) with its axes, +units and uncertainties. Only metadata is read until the object is created. + +Supported conventions (NeXus data model, ``NXdata`` base class): + +- ``@signal`` group attribute, or legacy ``signal=1`` dataset attribute +- ``@axes`` group attribute (``"."`` for a dimension without axis), with optional + ``@_indices`` attributes, or legacy ``axes`` signal attribute +- axis datasets holding one value per point, or bin edges (one more value), which + are converted to bin centers +- ``_errors`` (or legacy ``errors``) uncertainties of a 1D signal +- ``units`` and ``long_name`` dataset attributes +""" + +from __future__ import annotations + +import dataclasses +import re + +import h5py +import numpy as np +from guidata.utils.misc import to_string +from sigima.io.common.converters import convert_array_to_valid_dtype +from sigima.objects import ( + ImageObj, + create_image, + create_signal, + make_experimental_source, + set_experimental_source, +) + +from sigimax.h5 import common, utils + +__all__ = ["NXAxis", "NXdataDescription", "NXdataNode", "describe_nxdata"] + + +def _text(value) -> str: + """Return attribute value as text""" + if isinstance(value, np.ndarray) and value.size == 1: + value = value.flat[0] + return to_string(value) if isinstance(value, (bytes, np.bytes_)) else str(value) + + +def _text_list(value) -> list[str]: + """Return attribute value as a list of texts""" + if isinstance(value, np.ndarray): + return [_text(item) for item in value.flat] + return [_text(value)] + + +@dataclasses.dataclass(frozen=True) +class NXAxis: + """Axis of a signal dimension + + Args: + name: axis dataset name in the ``NXdata`` group + size: number of values of the axis dataset + edges: True if the axis holds bin edges (one value more than points) + units: axis units + label: axis label (``long_name`` attribute, or name) + """ + + name: str + size: int + edges: bool + units: str + label: str + + +@dataclasses.dataclass(frozen=True) +class NXdataDescription: + """Description of an ``NXdata`` group, built from metadata only + + Args: + path: group path + signal: signal dataset name + shape: signal shape + dtype: signal data type + axes: axis of each signal dimension (None if not defined) + errors: uncertainties dataset name, if any + units: signal units + label: signal label + problems: reasons why the signal cannot be imported as described + """ + + path: str + signal: str + shape: tuple[int, ...] + dtype: np.dtype + axes: tuple[NXAxis | None, ...] + errors: str | None + units: str + label: str + problems: tuple[str, ...] + + +def _is_nxdata(group: h5py.Group) -> bool: + """Return True if group is an ``NXdata`` group""" + return _text(group.attrs.get("NX_class", "")) == "NXdata" + + +def _find_signal(group: h5py.Group) -> tuple[str | None, list[str]]: + """Return signal dataset name and axis names declared by an NXdata group""" + if "signal" in group.attrs: + signal = _text(group.attrs["signal"]) + axes = _text_list(group.attrs["axes"]) if "axes" in group.attrs else [] + return signal, axes + for name, item in group.items(): + if isinstance(item, h5py.Dataset) and _text(item.attrs.get("signal", "")) in ( + "1", + "True", + ): + axes = item.attrs.get("axes") + names = ( + re.split(r"[:,\s]+", _text(axes).strip()) if axes is not None else [] + ) + return name, [axis for axis in names if axis] + return None, [] + + +def _axis_dimensions(group: h5py.Group, names: list[str], ndim: int) -> list: + """Return the axis name of each signal dimension""" + axes: list[str | None] = [None] * ndim + for position, name in enumerate(names): + if name == ".": + continue + indices = group.attrs.get(f"{name}_indices") + dims = [position] if indices is None else [int(i) for i in np.ravel(indices)] + if len(dims) == 1 and 0 <= dims[0] < ndim: + axes[dims[0]] = name + return axes + + +def describe_nxdata(group: h5py.Group) -> NXdataDescription | None: + """Describe an ``NXdata`` group, without reading its datasets + + Args: + group: HDF5 group + + Returns: + Description, or None if group is not an ``NXdata`` group with a numeric + signal dataset + """ + if not isinstance(group, h5py.Group) or not _is_nxdata(group): + return None + signal, names = _find_signal(group) + dset = group.get(signal) if signal else None + if ( + not isinstance(dset, h5py.Dataset) + or not dset.shape + or not utils.is_supported_num_dtype(dset) + ): + return None + problems: list[str] = [] + axes: list[NXAxis | None] = [] + for dim, name in enumerate(_axis_dimensions(group, names, dset.ndim)): + axis_dset = group.get(name) if name else None + if name is None: + axes.append(None) + elif not isinstance(axis_dset, h5py.Dataset) or axis_dset.ndim != 1: + problems.append(f"axis '{name}' is not a 1D dataset") + axes.append(None) + elif axis_dset.shape[0] not in (dset.shape[dim], dset.shape[dim] + 1): + problems.append( + f"axis '{name}' has {axis_dset.shape[0]} values " + f"for {dset.shape[dim]} points" + ) + axes.append(None) + else: + axes.append( + NXAxis( + name=name, + size=axis_dset.shape[0], + edges=axis_dset.shape[0] == dset.shape[dim] + 1, + units=_text(axis_dset.attrs.get("units", "")), + label=_text(axis_dset.attrs.get("long_name", name)), + ) + ) + errors = None + for name in (f"{signal}_errors", "errors"): + candidate = group.get(name) + if isinstance(candidate, h5py.Dataset) and candidate.shape == dset.shape: + errors = name + break + return NXdataDescription( + path=group.name, + signal=signal, + shape=dset.shape, + dtype=dset.dtype, + axes=tuple(axes), + errors=errors, + units=_text(dset.attrs.get("units", "")), + label=_text(dset.attrs.get("long_name", signal)), + problems=tuple(problems), + ) + + +def _read_axis(group: h5py.Group, axis: NXAxis) -> np.ndarray: + """Return axis values, at bin centers""" + values = np.asarray(group[axis.name][()], dtype=float) + return (values[:-1] + values[1:]) / 2 if axis.edges else values + + +def _is_uniform(values: np.ndarray) -> bool: + """Return True if values are evenly spaced""" + if values.size < 2: + return True + steps = np.diff(values) + return bool(np.allclose(steps, steps[0], rtol=1e-9, atol=0.0)) + + +class NXdataNode(common.GroupNode): + """Node for NeXus ``NXdata`` groups, imported as a signal or an image""" + + def __init__(self, h5file, dname): + super().__init__(h5file, dname) + self.nxdata = describe_nxdata(self.dset) + + @classmethod + def match(cls, dset): + """Return True if h5 group is an ``NXdata`` group with a numeric signal""" + return describe_nxdata(dset) is not None + + def is_supported(self) -> bool: + """Return True if the signal is 1D or 2D and its axes are valid""" + desc = self.nxdata + return ( + not desc.problems + and len(desc.shape) in (1, 2) + and int(np.prod(desc.shape)) > 1 + ) + + @property + def icon_name(self): + """Icon name associated to node""" + if self.is_supported(): + return "signal.svg" if len(self.nxdata.shape) == 1 else "image.svg" + return "h5group.svg" + + @property + def shape_str(self): + """Return string representation of node shape""" + return " x ".join([str(size) for size in self.nxdata.shape]) + + @property + def dtype_str(self): + """Return string representation of node data type""" + return str(self.nxdata.dtype) + + @property + def text(self): + """Return node textual representation""" + desc = self.nxdata + names = [axis.name if axis else "." for axis in desc.axes] + text = f"NXdata: {desc.signal} ({', '.join(names)})" + if desc.problems: + text += " - " + "; ".join(desc.problems) + return text + + @property + def description(self): + """Return node description""" + return self.text + + def __make_source(self, conversions: list[str]) -> dict: + """Return the experimental source of the imported object""" + desc = self.nxdata + return make_experimental_source( + "nexus", + self.h5file.filename, + entry=desc.path, + selection={ + "signal": desc.signal, + "axes": [axis.name if axis else None for axis in desc.axes], + }, + conversions=conversions or None, + ) + + def create_native_object(self): + """Create signal or image object from the ``NXdata`` group""" + if not self.is_supported(): + return None + desc, group = self.nxdata, self.dset + data = group[desc.signal][()] + conversions = [ + f"axis '{axis.name}': bin edges -> bin centers" + for axis in desc.axes + if axis is not None and axis.edges + ] + if len(desc.shape) == 1: + axis = desc.axes[0] + x = np.arange(data.size, dtype=float) + if axis is not None: + x = _read_axis(group, axis) + dy = None + if desc.errors is not None: + dy = np.asarray(group[desc.errors][()], dtype=float) + y = data if np.iscomplexobj(data) else data.astype(float) + obj = create_signal(self.object_title, x, y, dy=dy) + obj.ylabel, obj.yunit = desc.label, desc.units + if axis is not None: + obj.xlabel, obj.xunit = axis.label, axis.units + else: + obj = create_image( + self.object_title, + convert_array_to_valid_dtype(data, ImageObj.VALID_DTYPES), + ) + obj.zlabel, obj.zunit = desc.label, desc.units + yaxis, xaxis = desc.axes + ny, nx = desc.shape + xc = ( + np.arange(nx, dtype=float) + if xaxis is None + else _read_axis(group, xaxis) + ) + yc = ( + np.arange(ny, dtype=float) + if yaxis is None + else _read_axis(group, yaxis) + ) + if _is_uniform(xc) and _is_uniform(yc): + dx = float(xc[1] - xc[0]) if nx > 1 else 1.0 + dy = float(yc[1] - yc[0]) if ny > 1 else 1.0 + obj.set_uniform_coords(dx, dy, float(xc[0]), float(yc[0])) + else: + obj.set_coords(xc, yc) + if xaxis is not None: + obj.xlabel, obj.xunit = xaxis.label, xaxis.units + if yaxis is not None: + obj.ylabel, obj.yunit = yaxis.label, yaxis.units + if desc.errors is not None: + conversions.append("uncertainties not imported (image)") + set_experimental_source(obj, self.__make_source(conversions)) + return obj + + +common.NODE_FACTORY.register(NXdataNode, is_generic=True) diff --git a/sigimax/locale/fr/LC_MESSAGES/sigimax.po b/sigimax/locale/fr/LC_MESSAGES/sigimax.po index 83c9b98..0afc25a 100644 --- a/sigimax/locale/fr/LC_MESSAGES/sigimax.po +++ b/sigimax/locale/fr/LC_MESSAGES/sigimax.po @@ -173,8 +173,8 @@ msgstr "Enregistrer" msgid "Open" msgstr "Ouvrir" -msgid "HDF5 files (*.h5 *.hdf5 *.hdf *.he5);;All files (*)" -msgstr "Fichiers HDF5 (*.h5 *.hdf5 *.hdf *.he5);;Tous les fichiers (*)" +msgid "HDF5 files (*.h5 *.hdf5 *.hdf *.he5 *.nxs *.nx5);;All files (*)" +msgstr "Fichiers HDF5 (*.h5 *.hdf5 *.hdf *.he5 *.nxs *.nx5);;Tous les fichiers (*)" msgid "Do you want to clear current workspace before importing data from HDF5 files?" msgstr "Voulez-vous effacer l'espace de travail actuel avant d'importer des données depuis des fichiers HDF5 ?" diff --git a/sigimax/mainwindow.py b/sigimax/mainwindow.py index 5a01dbb..b287046 100644 --- a/sigimax/mainwindow.py +++ b/sigimax/mainwindow.py @@ -978,7 +978,10 @@ def open_h5_files( self, _("Open"), basedir, - _("HDF5 files (*.h5 *.hdf5 *.hdf *.he5);;All files (*)"), + _( + "HDF5 files (*.h5 *.hdf5 *.hdf *.he5 *.nxs *.nx5);;" + "All files (*)" + ), ) if not h5files: return diff --git a/sigimax/tests/hdf5/test_h5_nexus.py b/sigimax/tests/hdf5/test_h5_nexus.py new file mode 100644 index 0000000..9ef346a --- /dev/null +++ b/sigimax/tests/hdf5/test_h5_nexus.py @@ -0,0 +1,164 @@ +# Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file. + +""" +NeXus ``NXdata`` unit tests +--------------------------- + +``NXdata`` groups are imported as signals or images with their axes, units and +uncertainties, following the NeXus conventions, and record their origin. +""" + +from __future__ import annotations + +import uuid + +import h5py +import numpy as np +import pytest +from sigima.objects import ImageObj, SignalObj, get_experimental_source + +from sigimax.h5 import H5Importer +from sigimax.h5.nexus import NXdataNode + +pytestmark = pytest.mark.unit + + +def _nxdata(parent: h5py.Group, name: str, signal: str, axes) -> h5py.Group: + """Create an NXdata group""" + group = parent.create_group(name) + group.attrs["NX_class"] = "NXdata" + group.attrs["signal"] = signal + if axes is not None: + group.attrs["axes"] = axes + return group + + +@pytest.fixture(name="nxsfname") +def fixture_nxsfname(tmp_path) -> str: + """NeXus file covering the supported NXdata conventions.""" + fname = str(tmp_path / f"{uuid.uuid4()}.nxs") + with h5py.File(fname, "w") as h5file: + entry = h5file.create_group("entry") + entry.attrs["NX_class"] = "NXentry" + + spectrum = _nxdata(entry, "spectrum", "counts", "energy") + spectrum["counts"] = np.array([5, 9, 30, 8], dtype=np.uint32) + spectrum["counts"].attrs["units"] = "counts" + spectrum["counts"].attrs["long_name"] = "Intensity" + spectrum["counts_errors"] = np.sqrt([5.0, 9.0, 30.0, 8.0]) + spectrum["energy"] = np.array([8.0, 8.1, 8.25, 8.5]) + spectrum["energy"].attrs["units"] = "keV" + + histogram = _nxdata(entry, "histogram", "hits", ["tof"]) + histogram["hits"] = np.array([1.0, 4.0, 2.0]) + histogram["tof"] = np.array([0.0, 2.0, 4.0, 8.0]) + + image = _nxdata(entry, "map", "intensity", np.array([b"y", b"x"])) + image["intensity"] = np.arange(12, dtype=np.uint16).reshape(3, 4) + image["x"] = np.array([0.0, 1.0, 3.0, 6.0]) + image["x"].attrs["units"] = "mm" + image["y"] = np.array([10.0, 20.0, 30.0]) + + indexed = _nxdata(entry, "indexed", "data", np.array([b"x", b"y"])) + indexed.attrs["x_indices"] = 1 + indexed.attrs["y_indices"] = 0 + indexed["data"] = np.ones((2, 3)) + indexed["x"] = np.array([0.5, 1.0, 1.5]) + indexed["y"] = np.array([-1.0, 1.0]) + + legacy = entry.create_group("legacy") + legacy.attrs["NX_class"] = b"NXdata" + legacy["y"] = np.array([1.0, 2.0, 3.0]) + legacy["y"].attrs["signal"] = 1 + legacy["y"].attrs["axes"] = "t" + legacy["t"] = np.array([0.0, 0.1, 0.2]) + + broken = _nxdata(entry, "broken", "counts", "energy") + broken["counts"] = np.ones(4) + broken["energy"] = np.ones(7) + + stack = _nxdata(entry, "stack", "frames", np.array([b".", b"y", b"x"])) + stack["frames"] = np.zeros((5, 3, 4)) + stack["x"] = np.arange(4.0) + stack["y"] = np.arange(3.0) + + plain = entry.create_group("plain") + plain["data"] = np.ones(3) + return fname + + +@pytest.fixture(name="nodes") +def fixture_nodes(nxsfname: str): + """Nodes of the NeXus file, by path.""" + importer = H5Importer(nxsfname) + try: + yield {node.id: node for node in importer.nodes} + finally: + importer.close() + + +def test_nxdata_classification(nodes: dict) -> None: + """NXdata groups are recognized; invalid ones are shown but not importable.""" + for name in ("spectrum", "histogram", "map", "indexed", "legacy"): + node = nodes[f"/entry/{name}"] + assert isinstance(node, NXdataNode) and node.is_supported(), name + assert not isinstance(nodes["/entry/plain"], NXdataNode) + assert not isinstance(nodes["/entry"], NXdataNode) + broken = nodes["/entry/broken"] + assert not broken.is_supported() + assert "axis 'energy' has 7 values for 4 points" in broken.text + stack = nodes["/entry/stack"] + assert not stack.is_supported() + assert stack.shape_str == "5 x 3 x 4" + assert nodes["/entry/spectrum"].icon_name == "signal.svg" + assert nodes["/entry/map"].icon_name == "image.svg" + # Datasets of NXdata groups remain available as raw nodes + assert nodes["/entry/spectrum/counts"].is_supported() + + +def test_nxdata_signal(nodes: dict, nxsfname: str) -> None: + """1D NXdata: axis values, units, labels, uncertainties and origin.""" + obj = nodes["/entry/spectrum"].get_native_object() + assert isinstance(obj, SignalObj) + assert np.array_equal(obj.x, [8.0, 8.1, 8.25, 8.5]) + assert np.array_equal(obj.y, [5.0, 9.0, 30.0, 8.0]) + assert np.allclose(obj.dy, np.sqrt([5.0, 9.0, 30.0, 8.0])) + assert (obj.xlabel, obj.xunit) == ("energy", "keV") + assert (obj.ylabel, obj.yunit) == ("Intensity", "counts") + assert get_experimental_source(obj) == { + "version": 1, + "kind": "nexus", + "location": nxsfname, + "entry": "/entry/spectrum", + "selection": {"signal": "counts", "axes": ["energy"]}, + } + legacy = nodes["/entry/legacy"].get_native_object() + assert np.array_equal(legacy.x, [0.0, 0.1, 0.2]) + assert np.array_equal(legacy.y, [1.0, 2.0, 3.0]) + + +def test_nxdata_bin_edges(nodes: dict) -> None: + """Bin edges are converted to bin centers, and the conversion is recorded.""" + obj = nodes["/entry/histogram"].get_native_object() + assert np.array_equal(obj.x, [1.0, 3.0, 6.0]) + source = get_experimental_source(obj) + assert source["conversions"] == ["axis 'tof': bin edges -> bin centers"] + + +def test_nxdata_image(nodes: dict) -> None: + """2D NXdata: rows follow the first axis, non-uniform axes are kept.""" + obj = nodes["/entry/map"].get_native_object() + assert isinstance(obj, ImageObj) + assert obj.data.dtype == np.uint16 + assert np.array_equal(obj.data, np.arange(12).reshape(3, 4)) + assert not obj.is_uniform_coords + assert np.array_equal(obj.xcoords, [0.0, 1.0, 3.0, 6.0]) + assert np.array_equal(obj.ycoords, [10.0, 20.0, 30.0]) + assert obj.xunit == "mm" + indexed = nodes["/entry/indexed"].get_native_object() + assert indexed.is_uniform_coords + assert (indexed.x0, indexed.dx, indexed.y0, indexed.dy) == (0.5, 0.5, -1.0, 2.0) + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) diff --git a/sigimax/widgets/h5browser.py b/sigimax/widgets/h5browser.py index ca1e655..83bb36c 100644 --- a/sigimax/widgets/h5browser.py +++ b/sigimax/widgets/h5browser.py @@ -776,7 +776,7 @@ def add_file(self, fname: str | None = None) -> None: self, _("Select HDF5 file"), "", - _("HDF5 files (*.h5 *.hdf5 *.hdf *.he5);;All files (*)"), + _("HDF5 files (*.h5 *.hdf5 *.hdf *.he5 *.nxs *.nx5);;All files (*)"), )[0] if fname: self.SIG_ADD_FILENAME.emit(osp.abspath(fname)) From 24c520b77265fb84ed21cafe63be1783abfd6c21 Mon Sep 17 00:00:00 2001 From: Pierre Raybaut <1311787+PierreRaybaut@users.noreply.github.com> Date: Wed, 21 Oct 2026 06:48:50 +0200 Subject: [PATCH 05/20] docs: release notes for HDF5 previews, views and NeXus import Assisted-by: Claude Opus 5.5 --- doc/release_notes/release_1.01.md | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/release_notes/release_1.01.md b/doc/release_notes/release_1.01.md index ea8632f..2a25e94 100644 --- a/doc/release_notes/release_1.01.md +++ b/doc/release_notes/release_1.01.md @@ -34,11 +34,15 @@ ### HDF5 import ### -* Browsing an HDF5 file no longer reads its datasets entirely: generic nodes are classified and described from the dataset shape and data type, and value previews read only the first elements (`TEXT_PREVIEW_MAX_ITEMS`). Compound datasets mixing numeric and string fields are checked on their first records only. Data is read entirely when the object is created, for preview or import. +* Browsing an HDF5 file no longer reads its datasets entirely: generic nodes are classified and described from the dataset shape and data type, and value previews read only the first elements (`TEXT_PREVIEW_MAX_ITEMS`). Compound datasets mixing numeric and string fields are checked on their first records only. Data is read entirely when the object is imported. +* The HDF5 browser reads previews in a worker thread, so that it stays responsive while a large dataset is read: only the last selected node is shown, and closing a file waits for the preview being read. Datasets with more than `PlotPreview.MAX_ITEMS` (2²⁴) elements are previewed with a step along each axis (`BaseNode.create_preview_object`). +* Numeric datasets with three dimensions or more (image stacks, spectral cubes...) are now listed in the HDF5 browser, with their shape and type. They are not imported as a whole: their 1D or 2D views are read with `sigimax.h5.selection.read_selection`, which reads only the selected elements (`NDSelection`: displayed axes, possibly transposed, and an index along each other axis). +* NeXus `NXdata` groups are shown as single nodes, imported as a signal (1D) or an image (2D) with their axes, units, labels and, for signals, uncertainties: `@signal`/`@axes` attributes with `@_indices`, legacy `signal`/`axes` dataset attributes, bin edges converted to bin centers, non-uniform image axes. Invalid axes are reported in the browser instead of being guessed. Imported objects record their origin in the `experimental_source` metadata entry (file, group, signal and axes, conversions). `.nxs` and `.nx5` files are proposed in HDF5 file dialogs. * Dataset attributes with more than `MAX_ATTRIBUTE_ITEMS` (10 000) values are no longer copied to object metadata. * Images with unsigned 32-bit values above 2 147 483 647 are no longer clipped to `int32`: they are converted with Sigima's converter, which keeps every value exactly with Sigima ≥ 1.4.0. `BaseNode.uint32_wng` is kept for compatibility and is always `False`. * `NodeFactory.unregister` removes a node class and its post triggers, so that an application plugin can remove the node classes it registered when it is unloaded. ### Requirements ### -* Sigima 1.3.0 or later is required for the portable annotation model, PlotPy conversion helpers and curve fitting model formulas. \ No newline at end of file +* Sigima 1.3.0 or later is required for the portable annotation model, PlotPy conversion helpers and curve fitting model formulas. +* Sigima 1.4.0 or later is required for the origin of experimental data recorded by NeXus imports and for exact integer conversions. \ No newline at end of file From 222c63b727d7a609962d730784bc08ea3474f212 Mon Sep 17 00:00:00 2001 From: Pierre Raybaut <1311787+PierreRaybaut@users.noreply.github.com> Date: Wed, 21 Oct 2026 08:14:25 +0200 Subject: [PATCH 06/20] feat(h5): declarative data mappings for HDF5 files Assisted-by: Claude Opus 5.5 --- sigimax/h5/__init__.py | 14 +- sigimax/h5/common.py | 36 ++ sigimax/h5/mapping.py | 471 ++++++++++++++++++++++++++ sigimax/h5/nexus.py | 94 +++-- sigimax/tests/hdf5/test_h5_mapping.py | 245 ++++++++++++++ 5 files changed, 835 insertions(+), 25 deletions(-) create mode 100644 sigimax/h5/mapping.py create mode 100644 sigimax/tests/hdf5/test_h5_mapping.py diff --git a/sigimax/h5/__init__.py b/sigimax/h5/__init__.py index a23cffe..a3d1e61 100644 --- a/sigimax/h5/__init__.py +++ b/sigimax/h5/__init__.py @@ -30,6 +30,18 @@ .. autoclass:: sigimax.h5.nexus.NXdataNode .. autofunction:: sigimax.h5.nexus.describe_nxdata +.. autofunction:: sigimax.h5.nexus.describe_signal + +Files without suitable ``NXdata`` groups are described by declarative data +mappings (:mod:`sigimax.h5.mapping`), registered by applications or plugins. + +.. autofunction:: sigimax.h5.mapping.mapping_from_dict +.. autofunction:: sigimax.h5.mapping.load_mapping +.. autofunction:: sigimax.h5.mapping.register_mapping +.. autofunction:: sigimax.h5.mapping.unregister_mapping +.. autofunction:: sigimax.h5.mapping.describe_mapping +.. autoclass:: sigimax.h5.mapping.DataMapping +.. autoclass:: sigimax.h5.mapping.MappedDataNode .. autoclass:: sigimax.h5.selection.NDSelection :members: .. autofunction:: sigimax.h5.selection.read_selection @@ -43,5 +55,5 @@ # pylint: disable=unused-import # Registering dynamic I/O features: -from sigimax.h5 import generic, nexus # noqa: F401 +from sigimax.h5 import generic, mapping, nexus # noqa: F401 from sigimax.h5.common import H5Importer # noqa: F401 diff --git a/sigimax/h5/common.py b/sigimax/h5/common.py index c41f75d..6524d92 100644 --- a/sigimax/h5/common.py +++ b/sigimax/h5/common.py @@ -248,6 +248,7 @@ def __init__(self): self.__generic_classes = [] self.__thirdparty_classes = [] self.__post_triggers = {} + self.__children_providers = [] def add_ignored_datasets(self, names): """Add h5 dataset name to ignore list""" @@ -275,6 +276,34 @@ def unregister(self, cls) -> None: classes.remove(cls) self.__post_triggers.pop(cls, None) + def add_children_provider(self, func: Callable) -> None: + """Add a function returning extra nodes to show under a group node. + Functions take the group node and return a list of nodes whose ids are + not HDF5 paths of the file""" + if func not in self.__children_providers: + self.__children_providers.append(func) + + def remove_children_provider(self, func: Callable) -> None: + """Remove a function added with :meth:`add_children_provider`""" + if func in self.__children_providers: + self.__children_providers.remove(func) + + def get_extra_children(self, node: GroupNode) -> list[BaseNode]: + """Return the extra nodes to show under a group node""" + children = [] + for func in self.__children_providers: + try: + children.extend(func(node)) + except ( + UnicodeDecodeError, + TypeError, + ValueError, + KeyError, + OSError, + ) as exc: + print(f"Warning: Skipping extra nodes of '{node.id}': {exc}") + return children + def get(self, dset): """Return node class that matches h5 dataset""" for name in to_string(dset.name).split("/"): @@ -331,6 +360,13 @@ def collect_children(self, node_dict: dict[str, BaseNode]): # Skip datasets that can't be processed due to various issues print(f"Warning: Skipping dataset '{dset.name}' due to error: {exc}") continue + for child in NODE_FACTORY.get_extra_children(self): + if child.id in node_dict: + print(f"Warning: Skipping node '{child.id}': id already used") + continue + child.collect_attributes() + node_dict[child.id] = child + self.children.append(child) class RootNode(GroupNode): diff --git a/sigimax/h5/mapping.py b/sigimax/h5/mapping.py new file mode 100644 index 0000000..aab19d1 --- /dev/null +++ b/sigimax/h5/mapping.py @@ -0,0 +1,471 @@ +# Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file. + +""" +SigimaX HDF5 data mappings + +Many files do not designate their plottable data with NeXus ``NXdata`` groups, or +designate it incompletely (no axes, several signals). A data mapping describes the +signals and images of such files declaratively: which groups it applies to, and +for each object, its signal dataset, the axis of each dimension, uncertainties, +labels and units. Mappings are JSON-compatible dictionaries, so that they can be +written, shared and versioned without code; no dataset path is coded in SigimaX. + +A group matches a mapping when it has the expected attribute values and contains +the required paths. Each object of a matching group is shown in the HDF5 browser +as an extra node, imported like an ``NXdata`` group. Mapping layout (version 1):: + + { + "version": 1, + "name": "Counters versus time", + "description": "Optional description", + "match": {"attributes": {"NX_class": "NXdata"}, "paths": ["time_1"]}, + "objects": [ + { + "name": "{name}", + "signal": "data_*", + "axes": [{"path": "time_1", "label": "Time", "units": "s"}], + "errors": null, + "label": null, + "units": null + } + ] + } + +Paths are relative to the matching group. The last component of a signal path may +be a pattern (``*``, ``?``, ``[...]``): the object is then repeated for each +matching dataset, and ``{name}`` in its name is replaced by the dataset name. +``axes`` gives one entry per signal dimension (``null``: no axis), as a path or as +an object overriding the axis label and units; without ``axes``, dimensions have +no axis. +""" + +from __future__ import annotations + +import dataclasses +import fnmatch +import json +import re +from typing import Any + +import h5py +from sigima.objects import make_experimental_source + +from sigimax.h5 import common, nexus + +__all__ = [ + "MAPPING_VERSION", + "DataMapping", + "MappedAxis", + "MappedDataNode", + "MappedObject", + "describe_mapping", + "load_mapping", + "mapping_from_dict", + "register_mapping", + "registered_mappings", + "unregister_mapping", +] + +#: Version of the data mapping layout +MAPPING_VERSION = 1 + +_PATTERN_CHARS = re.compile(r"[*?\[]") + + +@dataclasses.dataclass(frozen=True) +class MappedAxis: + """Axis of a mapped signal dimension + + Args: + path: axis dataset path, relative to the matching group + label: axis label (default: ``long_name`` attribute, or dataset name) + units: axis units (default: ``units`` attribute) + """ + + path: str + label: str | None = None + units: str | None = None + + +@dataclasses.dataclass(frozen=True) +class MappedObject: + """Signal or image of a data mapping + + Args: + name: object name, with ``{name}`` replaced by the dataset name when + ``signal`` is a pattern + signal: signal dataset path, relative to the matching group; its last + component may be a pattern + axes: axis of each signal dimension (None: no axis), or None if no + dimension has an axis + errors: uncertainties dataset path (default: ``_errors``, if any) + label: signal label (default: ``long_name`` attribute, or dataset name) + units: signal units (default: ``units`` attribute) + """ + + name: str + signal: str + axes: tuple[MappedAxis | None, ...] | None = None + errors: str | None = None + label: str | None = None + units: str | None = None + + @property + def is_pattern(self) -> bool: + """Return True if the signal path ends with a pattern""" + return bool(_PATTERN_CHARS.search(self.signal.rpartition("/")[2])) + + +@dataclasses.dataclass(frozen=True) +class DataMapping: + """Declarative description of the signals and images of HDF5 groups + + Args: + name: mapping name, shown in the HDF5 browser + objects: mapped objects + attributes: attribute values required on the matching groups + paths: paths required in the matching groups + description: mapping description + """ + + name: str + objects: tuple[MappedObject, ...] + attributes: dict[str, str] = dataclasses.field(default_factory=dict) + paths: tuple[str, ...] = () + description: str = "" + + +def _check_keys(where: str, data: dict, allowed: set[str], required: set[str]): + """Raise if data has unknown or missing keys""" + if not isinstance(data, dict): + raise ValueError(f"{where} must be an object") + unknown = set(data) - allowed + if unknown: + raise ValueError(f"{where}: unknown keys {sorted(unknown)}") + missing = required - set(data) + if missing: + raise ValueError(f"{where}: missing keys {sorted(missing)}") + + +def _text(where: str, value: Any, optional: bool = False) -> str | None: + """Return value if it is a non-empty string, raise otherwise""" + if value is None and optional: + return None + if not isinstance(value, str) or not value.strip(): + raise ValueError(f"{where} must be a non-empty string") + return value + + +def _path(where: str, value: Any, optional: bool = False) -> str | None: + """Return value if it is a relative path, raise otherwise""" + path = _text(where, value, optional) + if path is not None and (path.startswith("/") or "" in path.split("/")): + raise ValueError(f"{where} must be a relative path: {path!r}") + return path + + +def _axis_from_data(where: str, data: Any) -> MappedAxis | None: + """Return mapped axis from its JSON-compatible description""" + if data is None: + return None + if isinstance(data, str): + return MappedAxis(_path(where, data)) + _check_keys(where, data, {"path", "label", "units"}, {"path"}) + return MappedAxis( + _path(f"{where}.path", data["path"]), + _text(f"{where}.label", data.get("label"), optional=True), + _text(f"{where}.units", data.get("units"), optional=True), + ) + + +def _object_from_data(where: str, data: Any) -> MappedObject: + """Return mapped object from its JSON-compatible description""" + keys = {"name", "signal", "axes", "errors", "label", "units"} + _check_keys(where, data, keys, {"name", "signal"}) + axes = data.get("axes") + if axes is not None: + if not isinstance(axes, list): + raise ValueError(f"{where}.axes must be a list") + axes = tuple( + _axis_from_data(f"{where}.axes[{index}]", axis) + for index, axis in enumerate(axes) + ) + obj = MappedObject( + name=_text(f"{where}.name", data["name"]), + signal=_path(f"{where}.signal", data["signal"]), + axes=axes, + errors=_path(f"{where}.errors", data.get("errors"), optional=True), + label=_text(f"{where}.label", data.get("label"), optional=True), + units=_text(f"{where}.units", data.get("units"), optional=True), + ) + if "/" in obj.name: + raise ValueError(f"{where}.name must not contain '/'") + if obj.is_pattern and "{name}" not in obj.name: + raise ValueError(f"{where}.name must contain '{{name}}' for a signal pattern") + if _PATTERN_CHARS.search(obj.signal.rpartition("/")[0]): + raise ValueError(f"{where}.signal: only its last component may be a pattern") + return obj + + +def mapping_from_dict(data: dict[str, Any]) -> DataMapping: + """Return a data mapping from its JSON-compatible description + + Args: + data: mapping description (see module documentation) + + Returns: + Data mapping + + Raises: + ValueError: if the description is invalid + """ + keys = {"version", "name", "description", "match", "objects"} + _check_keys("mapping", data, keys, {"version", "name", "objects"}) + if data["version"] != MAPPING_VERSION: + raise ValueError(f"Unsupported data mapping version: {data['version']!r}") + match = data.get("match", {}) + _check_keys("mapping.match", match, {"attributes", "paths"}, set()) + attributes = match.get("attributes", {}) + if not isinstance(attributes, dict) or not all( + isinstance(key, str) and isinstance(value, str) + for key, value in attributes.items() + ): + raise ValueError("mapping.match.attributes must map names to strings") + paths = match.get("paths", []) + if not isinstance(paths, list): + raise ValueError("mapping.match.paths must be a list") + objects = data["objects"] + if not isinstance(objects, list) or not objects: + raise ValueError("mapping.objects must be a non-empty list") + mapping = DataMapping( + name=_text("mapping.name", data["name"]), + objects=tuple( + _object_from_data(f"mapping.objects[{index}]", obj) + for index, obj in enumerate(objects) + ), + attributes=dict(attributes), + paths=tuple( + _path(f"mapping.match.paths[{index}]", path) + for index, path in enumerate(paths) + ), + description=data.get("description") or "", + ) + if "/" in mapping.name: + raise ValueError("mapping.name must not contain '/'") + return mapping + + +def load_mapping(filename: str) -> DataMapping: + """Load a data mapping from a JSON file + + Args: + filename: JSON file name + + Returns: + Data mapping + + Raises: + ValueError: if the file is not a valid data mapping + """ + try: + with open(filename, encoding="utf-8") as fdesc: + data = json.load(fdesc) + return mapping_from_dict(data) + except ValueError as exc: + raise ValueError(f"Invalid data mapping '{filename}': {exc}") from exc + + +_MAPPINGS: dict[str, DataMapping] = {} + + +def register_mapping(mapping: DataMapping) -> None: + """Register a data mapping, applied to the HDF5 files opened afterwards + + Args: + mapping: data mapping + + Raises: + ValueError: if another mapping with the same name is registered + """ + registered = _MAPPINGS.get(mapping.name) + if registered is not None and registered != mapping: + raise ValueError(f"Another data mapping is named '{mapping.name}'") + _MAPPINGS[mapping.name] = mapping + + +def unregister_mapping(name: str) -> None: + """Unregister a data mapping, if registered + + Args: + name: mapping name + """ + _MAPPINGS.pop(name, None) + + +def registered_mappings() -> list[DataMapping]: + """Return registered data mappings""" + return list(_MAPPINGS.values()) + + +def _matches(mapping: DataMapping, group: h5py.Group) -> bool: + """Return True if group has the attributes and paths required by mapping""" + for key, value in mapping.attributes.items(): + # pylint: disable-next=protected-access + if key not in group.attrs or nexus._text(group.attrs[key]) != value: + return False + return all(group.get(path) is not None for path in mapping.paths) + + +def _natural_key(name: str) -> list: + """Return sort key ordering numbered names numerically""" + return [int(part) if part.isdigit() else part for part in re.split(r"(\d+)", name)] + + +def _expand(obj: MappedObject, group: h5py.Group) -> list[tuple[str, str]]: + """Return name and signal path of each object matching a mapped object""" + if not obj.is_pattern: + return [(obj.name, obj.signal)] + parent, _sep, pattern = obj.signal.rpartition("/") + container = group.get(parent) if parent else group + if not isinstance(container, h5py.Group): + return [] + excluded = {axis.path for axis in obj.axes or () if axis is not None} + excluded.add(obj.errors) + names = sorted( + (name for name in container if fnmatch.fnmatchcase(name, pattern)), + key=_natural_key, + ) + items = [] + for name in names: + path = f"{parent}/{name}" if parent else name + if path not in excluded and not name.endswith("_errors"): + items.append((obj.name.replace("{name}", name), path)) + return items + + +def _describe( + obj: MappedObject, group: h5py.Group, signal: str +) -> nexus.NXdataDescription | None: + """Return the description of a mapped signal""" + dset = group.get(signal) + if not isinstance(dset, h5py.Dataset): + return None + if obj.axes is None: + axes: list[str | None] = [None] * dset.ndim + else: + axes = [None if axis is None else axis.path for axis in obj.axes] + errors = (obj.errors,) if obj.errors else (f"{signal}_errors",) + desc = nexus.describe_signal(group, signal, axes, errors=errors) + if desc is None: + return None + problems = list(desc.problems) + if obj.errors and desc.errors is None: + problems.append(f"errors '{obj.errors}' do not match the signal shape") + axis_descs = list(desc.axes) + for index, axis in enumerate(obj.axes or ()): + if index < len(axis_descs) and axis is not None and axis_descs[index]: + axis_descs[index] = dataclasses.replace( + axis_descs[index], + label=axis.label or axis_descs[index].label, + units=axis.units if axis.units is not None else axis_descs[index].units, + ) + return dataclasses.replace( + desc, + axes=tuple(axis_descs), + label=obj.label or desc.label, + units=obj.units if obj.units is not None else desc.units, + problems=tuple(problems), + ) + + +def describe_mapping( + mapping: DataMapping, group: h5py.Group +) -> list[tuple[str, nexus.NXdataDescription]]: + """Describe the objects of a group with a data mapping, without reading data + + Args: + mapping: data mapping + group: HDF5 group + + Returns: + Name and description of each mapped object found in the group (empty if + the group does not match the mapping) + """ + if not isinstance(group, h5py.Group) or not _matches(mapping, group): + return [] + items = [] + for obj in mapping.objects: + for name, signal in _expand(obj, group): + desc = _describe(obj, group, signal) + if desc is not None: + items.append((name, desc)) + return items + + +class MappedDataNode(nexus.NXdataNode): + """Node of an object described by a data mapping + + Args: + h5file: HDF5 file + dname: path of the matching group + mapping: data mapping + name: object name + nxdata: object description + """ + + def __init__( + self, + h5file, + dname: str, + mapping: DataMapping, + name: str, + nxdata: nexus.NXdataDescription, + ): + super().__init__(h5file, dname, nxdata) + self.mapping = mapping + self.object_name = name + self.text_prefix = mapping.name + + @classmethod + def match(cls, dset): + """Mapped nodes are created by data mappings, not by HDF5 items""" + return False + + @property + def id(self): + """Return node id, distinct from the HDF5 paths of the file""" + return f"{self.dset.name.rstrip('/')}::{self.mapping.name}/{self.object_name}" + + @property + def name(self): + """Return node name""" + return self.object_name + + def make_source(self, conversions: list[str]) -> dict: + """Return the experimental source of the imported object""" + desc = self.nxdata + return make_experimental_source( + "hdf5", + self.h5file.filename, + entry=desc.path, + selection={ + "signal": desc.signal, + "axes": [axis.name if axis else None for axis in desc.axes], + }, + conversions=conversions or None, + details={"mapping": self.mapping.name}, + ) + + +def _mapped_children(node: common.GroupNode) -> list[MappedDataNode]: + """Return the nodes of the mapped objects of a group node""" + if not _MAPPINGS or not isinstance(node.dset, h5py.Group): + return [] + return [ + MappedDataNode(node.h5file, node.dset.name, mapping, name, desc) + for mapping in list(_MAPPINGS.values()) + for name, desc in describe_mapping(mapping, node.dset) + ] + + +common.NODE_FACTORY.add_children_provider(_mapped_children) diff --git a/sigimax/h5/nexus.py b/sigimax/h5/nexus.py index f138139..89f4013 100644 --- a/sigimax/h5/nexus.py +++ b/sigimax/h5/nexus.py @@ -38,13 +38,22 @@ from sigimax.h5 import common, utils -__all__ = ["NXAxis", "NXdataDescription", "NXdataNode", "describe_nxdata"] +__all__ = [ + "NXAxis", + "NXdataDescription", + "NXdataNode", + "describe_nxdata", + "describe_signal", +] def _text(value) -> str: """Return attribute value as text""" if isinstance(value, np.ndarray) and value.size == 1: value = value.flat[0] + if isinstance(value, str) and re.search("[\udc80-\udcff]", value): + # h5py decodes variable-length strings that are not UTF-8 with surrogates + value = value.encode("utf-8", "surrogateescape") return to_string(value) if isinstance(value, (bytes, np.bytes_)) else str(value) @@ -60,7 +69,7 @@ class NXAxis: """Axis of a signal dimension Args: - name: axis dataset name in the ``NXdata`` group + name: axis dataset path, relative to the group size: number of values of the axis dataset edges: True if the axis holds bin edges (one value more than points) units: axis units @@ -80,7 +89,7 @@ class NXdataDescription: Args: path: group path - signal: signal dataset name + signal: signal dataset path, relative to the group shape: signal shape dtype: signal data type axes: axis of each signal dimension (None if not defined) @@ -152,6 +161,31 @@ def describe_nxdata(group: h5py.Group) -> NXdataDescription | None: return None signal, names = _find_signal(group) dset = group.get(signal) if signal else None + if not isinstance(dset, h5py.Dataset) or not dset.shape: + return None + axes = _axis_dimensions(group, names, dset.ndim) + return describe_signal(group, signal, axes, errors=(f"{signal}_errors", "errors")) + + +def describe_signal( + group: h5py.Group, + signal: str, + axes: list[str | None], + errors: tuple[str, ...] = (), +) -> NXdataDescription | None: + """Describe a signal dataset of a group, without reading its datasets + + Args: + group: HDF5 group + signal: signal dataset path, relative to the group + axes: axis dataset path of each signal dimension (None: no axis) + errors: candidate uncertainties dataset paths; the first one with the + signal shape is used + + Returns: + Description, or None if signal is not a numeric dataset + """ + dset = group.get(signal) if ( not isinstance(dset, h5py.Dataset) or not dset.shape @@ -159,45 +193,50 @@ def describe_nxdata(group: h5py.Group) -> NXdataDescription | None: ): return None problems: list[str] = [] - axes: list[NXAxis | None] = [] - for dim, name in enumerate(_axis_dimensions(group, names, dset.ndim)): + if len(axes) != dset.ndim: + problems.append(f"{len(axes)} axes for {dset.ndim} dimensions") + axes = (list(axes) + [None] * dset.ndim)[: dset.ndim] + axis_descs: list[NXAxis | None] = [] + for dim, name in enumerate(axes): axis_dset = group.get(name) if name else None if name is None: - axes.append(None) + axis_descs.append(None) elif not isinstance(axis_dset, h5py.Dataset) or axis_dset.ndim != 1: problems.append(f"axis '{name}' is not a 1D dataset") - axes.append(None) + axis_descs.append(None) elif axis_dset.shape[0] not in (dset.shape[dim], dset.shape[dim] + 1): problems.append( f"axis '{name}' has {axis_dset.shape[0]} values " f"for {dset.shape[dim]} points" ) - axes.append(None) + axis_descs.append(None) else: - axes.append( + axis_descs.append( NXAxis( name=name, size=axis_dset.shape[0], edges=axis_dset.shape[0] == dset.shape[dim] + 1, - units=_text(axis_dset.attrs.get("units", "")), - label=_text(axis_dset.attrs.get("long_name", name)), + units=_text(axis_dset.attrs.get("units", "")).strip(), + label=_text( + axis_dset.attrs.get("long_name", name.split("/")[-1]) + ).strip(), ) ) - errors = None - for name in (f"{signal}_errors", "errors"): + found = None + for name in errors: candidate = group.get(name) if isinstance(candidate, h5py.Dataset) and candidate.shape == dset.shape: - errors = name + found = name break return NXdataDescription( path=group.name, signal=signal, shape=dset.shape, dtype=dset.dtype, - axes=tuple(axes), - errors=errors, - units=_text(dset.attrs.get("units", "")), - label=_text(dset.attrs.get("long_name", signal)), + axes=tuple(axis_descs), + errors=found, + units=_text(dset.attrs.get("units", "")).strip(), + label=_text(dset.attrs.get("long_name", signal.split("/")[-1])).strip(), problems=tuple(problems), ) @@ -217,11 +256,18 @@ def _is_uniform(values: np.ndarray) -> bool: class NXdataNode(common.GroupNode): - """Node for NeXus ``NXdata`` groups, imported as a signal or an image""" + """Node for NeXus ``NXdata`` groups, imported as a signal or an image + + Args: + h5file: HDF5 file + dname: group path + nxdata: signal description (default: described from ``NXdata`` attributes) + """ - def __init__(self, h5file, dname): + def __init__(self, h5file, dname, nxdata: NXdataDescription | None = None): super().__init__(h5file, dname) - self.nxdata = describe_nxdata(self.dset) + self.nxdata = describe_nxdata(self.dset) if nxdata is None else nxdata + self.text_prefix = "NXdata" @classmethod def match(cls, dset): @@ -259,7 +305,7 @@ def text(self): """Return node textual representation""" desc = self.nxdata names = [axis.name if axis else "." for axis in desc.axes] - text = f"NXdata: {desc.signal} ({', '.join(names)})" + text = f"{self.text_prefix}: {desc.signal} ({', '.join(names)})" if desc.problems: text += " - " + "; ".join(desc.problems) return text @@ -269,7 +315,7 @@ def description(self): """Return node description""" return self.text - def __make_source(self, conversions: list[str]) -> dict: + def make_source(self, conversions: list[str]) -> dict: """Return the experimental source of the imported object""" desc = self.nxdata return make_experimental_source( @@ -337,7 +383,7 @@ def create_native_object(self): obj.ylabel, obj.yunit = yaxis.label, yaxis.units if desc.errors is not None: conversions.append("uncertainties not imported (image)") - set_experimental_source(obj, self.__make_source(conversions)) + set_experimental_source(obj, self.make_source(conversions)) return obj diff --git a/sigimax/tests/hdf5/test_h5_mapping.py b/sigimax/tests/hdf5/test_h5_mapping.py new file mode 100644 index 0000000..1c141fd --- /dev/null +++ b/sigimax/tests/hdf5/test_h5_mapping.py @@ -0,0 +1,245 @@ +# Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file. + +""" +HDF5 data mapping unit tests +---------------------------- + +Declarative data mappings describe the signals and images of groups that do not +designate them with NeXus ``NXdata`` groups: they are validated when loaded, apply +only to matching groups, and their objects are imported with their axes, labels, +units and origin. +""" + +from __future__ import annotations + +import json +import uuid + +import h5py +import numpy as np +import pytest +from sigima.objects import ImageObj, SignalObj, get_experimental_source + +from sigimax.h5 import H5Importer +from sigimax.h5.mapping import ( + MappedDataNode, + describe_mapping, + load_mapping, + mapping_from_dict, + register_mapping, + registered_mappings, + unregister_mapping, +) + +pytestmark = pytest.mark.unit + +COUNTERS = { + "version": 1, + "name": "Counters", + "match": {"attributes": {"NX_class": "NXdata"}, "paths": ["time_1"]}, + "objects": [ + { + "name": "{name}", + "signal": "data_*", + "axes": [{"path": "time_1", "label": "Elapsed time"}], + } + ], +} + +MEASUREMENT = { + "version": 1, + "name": "Measurement", + "match": {"attributes": {"NX_class": "NXentry"}, "paths": ["measurement"]}, + "objects": [ + { + "name": "Transmission", + "signal": "measurement/it", + "axes": ["measurement/energy"], + "units": "counts", + }, + { + "name": "Map", + "signal": "measurement/map", + "axes": ["measurement/y", {"path": "measurement/x", "units": "mm"}], + }, + {"name": "Missing", "signal": "measurement/absent"}, + { + "name": "Bad errors", + "signal": "measurement/it", + "axes": ["measurement/energy"], + "errors": "measurement/map", + }, + ], +} + + +@pytest.fixture(name="h5fname") +def fixture_h5fname(tmp_path) -> str: + """HDF5 file with scan counters and a measurement group without NXdata axes""" + fname = str(tmp_path / f"{uuid.uuid4()}.nxs") + with h5py.File(fname, "w") as h5file: + entry = h5file.create_group("entry") + entry.attrs["NX_class"] = "NXentry" + scan = entry.create_group("scan_data") + scan.attrs["NX_class"] = b"NXdata" + for index in (1, 2, 10): + scan[f"data_{index}"] = np.arange(4, dtype=float) * index + scan[f"data_{index}"].attrs["signal"] = index + scan[f"data_{index}"].attrs["units"] = b" " + scan["data_2_errors"] = np.full(4, 0.5) + # Latin-1 units, as variable-length and fixed-length strings + scan["data_2"].attrs["units"] = b"\xb5m" + scan["data_10"].attrs["units"] = np.bytes_(b"\xb5m") + scan["time_1"] = np.array([0.0, 0.5, 1.0, 1.5]) + scan["time_1"].attrs["units"] = "s" + measurement = entry.create_group("measurement") + measurement["energy"] = np.array([8970.0, 8980.0, 8990.0]) + measurement["energy"].attrs["units"] = "eV" + measurement["it"] = np.array([10, 7, 4], dtype=np.int32) + measurement["it"].attrs["units"] = b" " + measurement["map"] = np.arange(6, dtype=np.uint16).reshape(2, 3) + measurement["x"] = np.array([0.0, 1.0, 2.0]) + measurement["y"] = np.array([5.0, 7.0]) + other = h5file.create_group("other") + other.attrs["NX_class"] = "NXdata" + other["data_1"] = np.ones(3) + return fname + + +@pytest.fixture(name="mappings") +def fixture_mappings(): + """Register the test mappings, and unregister them afterwards""" + mappings = [mapping_from_dict(COUNTERS), mapping_from_dict(MEASUREMENT)] + for mapping in mappings: + register_mapping(mapping) + yield mappings + for mapping in mappings: + unregister_mapping(mapping.name) + + +def _mapped_nodes(importer: H5Importer) -> dict[str, MappedDataNode]: + """Return mapped nodes of an importer, by name""" + return { + node.name: node for node in importer.nodes if isinstance(node, MappedDataNode) + } + + +@pytest.mark.parametrize( + ("change", "message"), + [ + ({"version": 2}, "Unsupported data mapping version"), + ({"extra": 1}, "unknown keys"), + ({"name": "a/b"}, "must not contain '/'"), + ({"objects": []}, "non-empty list"), + ({"match": {"paths": ["/abs"]}}, "relative path"), + ({"match": {"attributes": {"NX_class": 1}}}, "map names to strings"), + ({"objects": [{"name": "x", "signal": "data_*"}]}, "'{name}'"), + ({"objects": [{"name": "{name}", "signal": "d*/x"}]}, "last component"), + ({"objects": [{"name": "x", "signal": "s", "axes": "t"}]}, "must be a list"), + ({"objects": [{"name": "x"}]}, "missing keys"), + ], + ids=lambda value: str(value)[:30], +) +def test_invalid_mapping(change: dict, message: str) -> None: + """Invalid descriptions are rejected with an explicit reason""" + with pytest.raises(ValueError, match=message): + mapping_from_dict({**COUNTERS, **change}) + + +def test_load_mapping(tmp_path) -> None: + """Mappings are loaded from JSON files, whose name is given on error""" + good = tmp_path / "good.json" + good.write_text(json.dumps(COUNTERS), encoding="utf-8") + assert load_mapping(str(good)) == mapping_from_dict(COUNTERS) + bad = tmp_path / "bad.json" + bad.write_text("{", encoding="utf-8") + with pytest.raises(ValueError, match="bad.json"): + load_mapping(str(bad)) + + +def test_register_mapping() -> None: + """A name designates one mapping; registering it again is harmless""" + mapping = mapping_from_dict(COUNTERS) + register_mapping(mapping) + try: + register_mapping(mapping_from_dict(COUNTERS)) + other = mapping_from_dict({**COUNTERS, "description": "other"}) + with pytest.raises(ValueError, match="Another data mapping"): + register_mapping(other) + assert registered_mappings().count(mapping) == 1 + finally: + unregister_mapping(mapping.name) + assert mapping not in registered_mappings() + + +def test_describe_mapping(h5fname: str) -> None: + """Only matching groups are described; patterns follow numeric order""" + counters = mapping_from_dict(COUNTERS) + with h5py.File(h5fname, "r") as h5file: + items = describe_mapping(counters, h5file["entry/scan_data"]) + assert [name for name, _desc in items] == ["data_1", "data_2", "data_10"] + desc = dict(items)["data_2"] + assert desc.errors == "data_2_errors" + assert desc.units == dict(items)["data_10"].units == "\u00b5m" + assert desc.axes[0].label == "Elapsed time" and desc.axes[0].units == "s" + assert not describe_mapping(counters, h5file["other"]) + assert not describe_mapping(counters, h5file["entry"]) + + +def test_mapped_nodes(h5fname: str, mappings) -> None: + """Mapped objects are imported with their axes, labels, units and origin""" + importer = H5Importer(h5fname) + try: + nodes = _mapped_nodes(importer) + expected = {"data_1", "data_2", "data_10", "Transmission", "Map", "Bad errors"} + assert set(nodes) == expected + assert not nodes["Bad errors"].is_supported() + assert "do not match the signal shape" in nodes["Bad errors"].text + node = nodes["data_10"] + assert node.id == "/entry/scan_data::Counters/data_10" + assert importer.get(node.id) is node + assert node in importer.get("/entry/scan_data").children + + signal = nodes["data_2"].get_native_object() + assert isinstance(signal, SignalObj) + np.testing.assert_array_equal(signal.x, [0.0, 0.5, 1.0, 1.5]) + np.testing.assert_array_equal(signal.y, [0.0, 2.0, 4.0, 6.0]) + np.testing.assert_array_equal(signal.dy, np.full(4, 0.5)) + assert (signal.xlabel, signal.xunit) == ("Elapsed time", "s") + assert nodes["data_1"].get_native_object().yunit == "" + source = get_experimental_source(signal) + assert source["kind"] == "hdf5" and source["entry"] == "/entry/scan_data" + assert source["selection"] == {"signal": "data_2", "axes": ["time_1"]} + assert source["details"] == {"mapping": "Counters"} + + transmission = nodes["Transmission"].get_native_object() + np.testing.assert_array_equal(transmission.x, [8970.0, 8980.0, 8990.0]) + assert (transmission.ylabel, transmission.yunit) == ("it", "counts") + assert (transmission.xlabel, transmission.xunit) == ("energy", "eV") + + image = nodes["Map"].get_native_object() + assert isinstance(image, ImageObj) + assert (image.x0, image.y0, image.dx, image.dy) == (0.0, 5.0, 1.0, 2.0) + assert (image.xunit, image.ylabel) == ("mm", "y") + finally: + importer.close() + + +def test_mapping_problems(h5fname: str) -> None: + """Objects that cannot be imported as described are shown with the reason""" + mapping = mapping_from_dict(MEASUREMENT) + with h5py.File(h5fname, "r") as h5file: + items = dict(describe_mapping(mapping, h5file["entry"])) + assert "Missing" not in items + assert items["Bad errors"].problems == ( + "errors 'measurement/map' do not match the signal shape", + ) + + +def test_unregistered_mapping(h5fname: str) -> None: + """Files are browsed without mapped nodes when no mapping is registered""" + importer = H5Importer(h5fname) + try: + assert not _mapped_nodes(importer) + finally: + importer.close() From f0d198cf9b05b95ab85edee8cededd30b472abf6 Mon Sep 17 00:00:00 2001 From: Pierre Raybaut <1311787+PierreRaybaut@users.noreply.github.com> Date: Wed, 21 Oct 2026 08:14:26 +0200 Subject: [PATCH 07/20] docs: release notes for HDF5 data mappings Assisted-by: Claude Opus 5.5 --- doc/release_notes/release_1.01.md | 1 + 1 file changed, 1 insertion(+) diff --git a/doc/release_notes/release_1.01.md b/doc/release_notes/release_1.01.md index 2a25e94..3b62dc1 100644 --- a/doc/release_notes/release_1.01.md +++ b/doc/release_notes/release_1.01.md @@ -38,6 +38,7 @@ * The HDF5 browser reads previews in a worker thread, so that it stays responsive while a large dataset is read: only the last selected node is shown, and closing a file waits for the preview being read. Datasets with more than `PlotPreview.MAX_ITEMS` (2²⁴) elements are previewed with a step along each axis (`BaseNode.create_preview_object`). * Numeric datasets with three dimensions or more (image stacks, spectral cubes...) are now listed in the HDF5 browser, with their shape and type. They are not imported as a whole: their 1D or 2D views are read with `sigimax.h5.selection.read_selection`, which reads only the selected elements (`NDSelection`: displayed axes, possibly transposed, and an index along each other axis). * NeXus `NXdata` groups are shown as single nodes, imported as a signal (1D) or an image (2D) with their axes, units, labels and, for signals, uncertainties: `@signal`/`@axes` attributes with `@_indices`, legacy `signal`/`axes` dataset attributes, bin edges converted to bin centers, non-uniform image axes. Invalid axes are reported in the browser instead of being guessed. Imported objects record their origin in the `experimental_source` metadata entry (file, group, signal and axes, conversions). `.nxs` and `.nx5` files are proposed in HDF5 file dialogs. +* Files that do not designate their data with suitable `NXdata` groups (no axes, several counters...) can be described by declarative data mappings (`sigimax.h5.mapping`): JSON-compatible descriptions of the signals and images of the groups having given attributes and paths, with their axes, uncertainties, labels and units, and a name pattern to repeat an object for each counter of a scan. Mapped objects are shown as extra nodes of the HDF5 browser and record the mapping in their `experimental_source`. Applications and plugins register mappings with `register_mapping` (and `unregister_mapping`); `NodeFactory.add_children_provider` lets them add such extra nodes. Blank units and labels are now ignored. * Dataset attributes with more than `MAX_ATTRIBUTE_ITEMS` (10 000) values are no longer copied to object metadata. * Images with unsigned 32-bit values above 2 147 483 647 are no longer clipped to `int32`: they are converted with Sigima's converter, which keeps every value exactly with Sigima ≥ 1.4.0. `BaseNode.uint32_wng` is kept for compatibility and is always `False`. * `NodeFactory.unregister` removes a node class and its post triggers, so that an application plugin can remove the node classes it registered when it is unloaded. From d14e0cf67fd9e727cf96cadcdadfd8d4b87a715d Mon Sep 17 00:00:00 2001 From: Pierre Raybaut <1311787+PierreRaybaut@users.noreply.github.com> Date: Thu, 22 Oct 2026 21:40:37 +0200 Subject: [PATCH 08/20] docs: document NeXus mappings and multidimensional selections Provide complete HDF5 mapping and selection examples, clarify preview limits, expand the API reference, update French translations and require Sigima 1.4. Assisted-by: GPT-6.1 Sol --- .../LC_MESSAGES/release_notes/release_1.01.po | 42 ++++- doc/locale/fr/LC_MESSAGES/requirements.po | 11 +- .../LC_MESSAGES/user_guide/hdf5_workspace.po | 54 +++++- doc/release_notes/release_1.01.md | 11 +- doc/requirements.rst | 2 +- doc/user_guide/hdf5_workspace.rst | 160 ++++++++++++++++++ pyproject.toml | 2 +- requirements.txt | 2 +- sigimax/h5/__init__.py | 2 + 9 files changed, 272 insertions(+), 14 deletions(-) diff --git a/doc/locale/fr/LC_MESSAGES/release_notes/release_1.01.po b/doc/locale/fr/LC_MESSAGES/release_notes/release_1.01.po index 4d5b7a4..dfee39d 100644 --- a/doc/locale/fr/LC_MESSAGES/release_notes/release_1.01.po +++ b/doc/locale/fr/LC_MESSAGES/release_notes/release_1.01.po @@ -6,11 +6,18 @@ #, fuzzy msgid "" msgstr "" +"Project-Id-Version: PROJECT VERSION\n" +"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" +"POT-Creation-Date: 2026-10-22 21:34+0200\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" "Language: fr\n" +"Language-Team: fr \n" "Plural-Forms: nplurals=2; plural=(n > 1);\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" msgid "Version 1.1.0" msgstr "Version 1.1.0" @@ -63,9 +70,38 @@ msgstr "La fenêtre d'ajustement planckien part d'une courbe correspondant au pi msgid "The exponential fit dialog now evaluates the same model as Sigima, and its rate slider covers short time constants whatever the X unit (e.g. 1 ms with X in seconds)." msgstr "La fenêtre d'ajustement exponentiel évalue désormais le même modèle que Sigima, et son curseur de taux couvre les constantes de temps courtes quelle que soit l'unité de X (par exemple 1 ms avec X en secondes)." +msgid "HDF5 import" +msgstr "Import HDF5" + +msgid "Browsing an HDF5 file no longer reads its numeric datasets entirely just to classify them: generic nodes are classified and described from the dataset shape and data type, and value previews read only the first elements (`TEXT_PREVIEW_MAX_ITEMS`). Compound datasets mixing numeric and string fields are checked on their first records only. Full data is read when creating an imported object or an undecimated graphical preview." +msgstr "Parcourir un fichier HDF5 ne lit plus ses jeux de données numériques en entier simplement pour les classifier : les nœuds génériques sont classifiés et décrits à partir de la forme et du type des jeux de données, et les aperçus de valeurs ne lisent que les premiers éléments (`TEXT_PREVIEW_MAX_ITEMS`). Les jeux de données composites mêlant champs numériques et textuels ne sont vérifiés que sur leurs premiers enregistrements. Les données complètes sont lues lors de la création d'un objet importé ou d'un aperçu graphique non sous-échantillonné." + +msgid "The HDF5 browser reads previews in a worker thread, so that it stays responsive while a large dataset is read: only the last selected node is shown, and closing a file waits for the preview being read. Ordinary 1D arrays and 2D images with more than `PlotPreview.MAX_ITEMS` (2²⁴) elements are previewed with a step along each axis. NeXus and mapped objects, compound datasets and signals stored in 2D tables still use complete objects for graphical previews; this is not a universal memory bound." +msgstr "L'explorateur HDF5 lit les aperçus dans un fil d'exécution de travail pour rester réactif pendant la lecture d'un grand jeu de données : seul le dernier nœud sélectionné est affiché, et la fermeture d'un fichier attend la fin de la lecture de l'aperçu. Les tableaux 1D ordinaires et les images 2D de plus de `PlotPreview.MAX_ITEMS` (2²⁴) éléments sont prévisualisés avec un pas le long de chaque axe. Les objets NeXus et décrits par un profil, les jeux de données composites et les signaux stockés dans des tableaux 2D utilisent encore des objets complets pour leurs aperçus graphiques ; ce n'est pas une borne mémoire universelle." + +msgid "Numeric datasets with three dimensions or more (image stacks, spectral cubes...) are now listed in the HDF5 browser, with their shape and type. They are not imported as a whole, and there is no graphical slice selector: developers can read their 1D or 2D views with `sigimax.h5.selection.read_selection`, which reads only the selected elements (`NDSelection`: displayed axes, possibly transposed, and an index along each other axis)." +msgstr "Les jeux de données numériques à trois dimensions ou plus (piles d'images, cubes spectraux...) sont désormais listés dans l'explorateur HDF5 avec leur forme et leur type. Ils ne sont pas importés en entier, et il n'existe pas de sélecteur graphique de coupes : les développeurs peuvent lire leurs vues 1D ou 2D avec `sigimax.h5.selection.read_selection`, qui ne lit que les éléments sélectionnés (`NDSelection` : axes affichés, éventuellement transposés, et un indice sur chaque autre axe)." + +msgid "NeXus `NXdata` groups are shown as single nodes, imported as a signal (1D) or an image (2D) with their axes, units, labels and, for signals, uncertainties: `@signal`/`@axes` attributes with `@_indices`, legacy `signal`/`axes` dataset attributes, bin edges converted to bin centers, non-uniform image axes. Invalid axes are reported in the browser instead of being guessed. Imported objects record their origin in the `experimental_source` metadata entry (file, group, signal and axes, conversions). `.nxs` and `.nx5` files are proposed in HDF5 file dialogs." +msgstr "Les groupes NeXus `NXdata` sont présentés comme des nœuds uniques, importés comme signal (1D) ou image (2D) avec leurs axes, unités, libellés et, pour les signaux, incertitudes : attributs `@signal`/`@axes` avec `@_indices`, anciens attributs de jeux de données `signal`/`axes`, conversion des bords de classes en centres, axes d'image non uniformes. Les axes invalides sont signalés dans l'explorateur au lieu d'être devinés. Les objets importés enregistrent leur origine dans la métadonnée `experimental_source` (fichier, groupe, signal et axes, conversions). Les fichiers `.nxs` et `.nx5` sont proposés dans les dialogues de fichiers HDF5." + +msgid "Files that do not designate their data with suitable `NXdata` groups (no axes, several counters...) can be described by declarative data mappings (`sigimax.h5.mapping`): JSON-compatible descriptions of the signals and images of the groups having given attributes and paths, with their axes, uncertainties, labels and units, and a name pattern to repeat an object for each counter of a scan. Mapped objects are shown as extra nodes of the HDF5 browser and record the mapping in their `experimental_source`. Applications and plugins register mappings with `register_mapping` (and `unregister_mapping`); `NodeFactory.add_children_provider` lets them add such extra nodes. Blank units and labels are now ignored." +msgstr "Les fichiers qui ne désignent pas leurs données par des groupes `NXdata` adaptés (sans axes, plusieurs compteurs...) peuvent être décrits par des profils de correspondance déclaratifs (`sigimax.h5.mapping`) : descriptions compatibles JSON des signaux et images des groupes ayant les attributs et chemins donnés, avec leurs axes, incertitudes, libellés et unités, et un motif de nom pour répéter un objet pour chaque compteur d'un scan. Les objets décrits apparaissent comme des nœuds supplémentaires de l'explorateur HDF5 et enregistrent le profil dans leur `experimental_source`. Les applications et plugins enregistrent les profils avec `register_mapping` (et `unregister_mapping`) ; `NodeFactory.add_children_provider` leur permet d'ajouter ces nœuds supplémentaires. Les unités et libellés vides sont désormais ignorés." + +msgid "Dataset attributes with more than `MAX_ATTRIBUTE_ITEMS` (10 000) values are no longer copied to object metadata." +msgstr "Les attributs de jeux de données contenant plus de `MAX_ATTRIBUTE_ITEMS` (10 000) valeurs ne sont plus copiés dans les métadonnées des objets." + +msgid "Images with unsigned 32-bit values above 2 147 483 647 are no longer clipped to `int32`: they are converted with Sigima's converter, which keeps every value exactly with Sigima ≥ 1.4.0. `BaseNode.uint32_wng` is kept for compatibility and is always `False`." +msgstr "Les images contenant des valeurs entières non signées sur 32 bits supérieures à 2 147 483 647 ne sont plus écrêtées en `int32` : elles sont converties avec le convertisseur de Sigima, qui conserve chaque valeur exactement avec Sigima ≥ 1.4.0. `BaseNode.uint32_wng` est conservé pour compatibilité et vaut toujours `False`." + +msgid "`NodeFactory.unregister` removes a node class and its post triggers, so that an application plugin can remove the node classes it registered when it is unloaded." +msgstr "`NodeFactory.unregister` retire une classe de nœud et ses déclencheurs de post-traitement, afin qu'un plugin applicatif puisse retirer les classes de nœuds qu'il a enregistrées lors de son déchargement." + +msgid "See the [HDF5 guide](../user_guide/hdf5_workspace.rst) for a complete JSON mapping and registration example, NeXus import limits and multidimensional selection examples." +msgstr "Le [guide HDF5](../user_guide/hdf5_workspace.rst) fournit un exemple complet de profil JSON et d'enregistrement, les limites de l'import NeXus et des exemples de sélections multidimensionnelles." + msgid "Requirements" msgstr "Exigences" -msgid "Sigima 1.3.0 or later is required for the portable annotation model, PlotPy conversion helpers and curve fitting model formulas." -msgstr "Sigima 1.3.0 ou une version ultérieure est requise pour le modèle d'annotation portable, les helpers de conversion PlotPy et les formules des modèles d'ajustement de courbes." - +msgid "Sigima 1.4.0 or later is required: the HDF5 import modules use its experimental-source API, and integer conversions use its precision-preserving converter." +msgstr "Sigima 1.4.0 ou ultérieur est requis : les modules d'import HDF5 utilisent son API d'origine expérimentale, et les conversions d'entiers utilisent son convertisseur préservant la précision." diff --git a/doc/locale/fr/LC_MESSAGES/requirements.po b/doc/locale/fr/LC_MESSAGES/requirements.po index 28c214a..52df194 100644 --- a/doc/locale/fr/LC_MESSAGES/requirements.po +++ b/doc/locale/fr/LC_MESSAGES/requirements.po @@ -6,11 +6,18 @@ #, fuzzy msgid "" msgstr "" +"Project-Id-Version: PROJECT VERSION\n" +"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" +"POT-Creation-Date: 2026-10-22 21:34+0200\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" "Language: fr\n" +"Language-Team: fr \n" "Plural-Forms: nplurals=2; plural=(n > 1);\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" msgid "The `sigimax` package requires the following Python modules:" msgstr "Le paquet `sigimax` nécessite les modules Python suivants :" @@ -63,8 +70,8 @@ msgstr "Bibliothèque multiplateforme pour la surveillance des processus et du s msgid "Sigima" msgstr "Sigima" -msgid ">= 1.2.0" -msgstr ">= 1.2.0" +msgid ">= 1.4.0" +msgstr ">= 1.4.0" msgid "Scientific computing engine for 1D signals and 2D images, part of the DataLab open-source platform." msgstr "Moteur de calcul scientifique pour les signaux 1D et les images 2D, faisant partie de la plateforme open-source DataLab." diff --git a/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po b/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po index 6e34087..1d6f19d 100644 --- a/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po +++ b/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po @@ -6,11 +6,18 @@ #, fuzzy msgid "" msgstr "" +"Project-Id-Version: PROJECT VERSION\n" +"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" +"POT-Creation-Date: 2026-10-22 21:34+0200\n" +"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" +"Last-Translator: FULL NAME \n" "Language: fr\n" +"Language-Team: fr \n" "Plural-Forms: nplurals=2; plural=(n > 1);\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" +"Generated-By: Babel 2.18.0\n" msgid "HDF5 Workspace" msgstr "Espace de travail HDF5" @@ -87,9 +94,54 @@ msgstr "``open_h5_files`` et :class:`~sigimax.widgets.h5browser.H5BrowserDialog` msgid "The generic import layer can create :class:`sigima.objects.SignalObj` and :class:`sigima.objects.ImageObj`; a derived application decides how these objects enter its own model." msgstr "La couche d'import générique peut créer des :class:`sigima.objects.SignalObj` et :class:`sigima.objects.ImageObj` ; une application dérivée décide comment ces objets entrent dans son propre modèle." +msgid "NeXus Scientific Objects" +msgstr "Objets scientifiques NeXus" + +msgid "With Sigima 1.4 or later, an ``NXdata`` group is importable as a 1D signal or 2D image with its physical axes, labels and units. Modern group attributes (``signal``, ``axes`` and ``_indices``) and legacy dataset attributes (``signal=1`` and ``axes``) are supported. Selecting a group imports the scientific object; selecting a child dataset still gives access to raw values." +msgstr "Avec Sigima 1.4 ou ultérieur, un groupe ``NXdata`` est importable comme signal 1D ou image 2D avec ses axes physiques, libellés et unités. Les attributs modernes du groupe (``signal``, ``axes`` et ``_indices``) et les anciens attributs des jeux de données (``signal=1`` et ``axes``) sont pris en charge. Sélectionner un groupe importe l'objet scientifique ; sélectionner un jeu de données enfant donne toujours accès aux valeurs brutes." + +msgid "Axis datasets must be 1D and contain either one coordinate per point or one extra value for bin edges. Edges are converted to centers. Image coordinates may be non-uniform. Invalid declared axes are reported and prevent group import; undeclared axes use indices. Signal Y uncertainties are read from ``_errors`` or ``errors`` when their shape matches; image uncertainties are not imported." +msgstr "Les jeux de données des axes doivent être 1D et contenir une coordonnée par point, ou une valeur supplémentaire pour des bords de classes. Les bords sont convertis en centres. Les coordonnées d'image peuvent être non uniformes. Les axes déclarés invalides sont signalés et empêchent l'import du groupe ; les axes non déclarés utilisent les indices. Les incertitudes Y des signaux sont lues dans ``_errors`` ou ``errors`` lorsque leur forme correspond ; les incertitudes des images ne sont pas importées." + +msgid "Imported objects record their file, group, signal, axes and conversions in ``experimental_source`` metadata. This describes the measurement origin, not a reproducible acquisition or a processing history. The application must preserve object metadata in its own workspace serialization." +msgstr "Les objets importés enregistrent leur fichier, groupe, signal, axes et conversions dans les métadonnées ``experimental_source``. Cela décrit l'origine de la mesure, pas une acquisition reproductible ni un historique de traitement. L'application doit préserver les métadonnées des objets dans sa propre sérialisation d'espace de travail." + +msgid "Declarative Data Mappings" +msgstr "Profils de correspondance déclaratifs" + +msgid "For files without suitable ``NXdata`` descriptions, register a mapping before opening the file. A mapping matches groups by attribute values and required paths, then describes the signals or images they contain. Save the following JSON as ``counters.json``:" +msgstr "Pour les fichiers sans descriptions ``NXdata`` adaptées, enregistrez un profil avant d'ouvrir le fichier. Un profil reconnaît les groupes par les valeurs de leurs attributs et des chemins requis, puis décrit les signaux ou images qu'ils contiennent. Enregistrez le JSON suivant dans ``counters.json`` :" + +#, python-brace-format +msgid "Paths are relative to each matching group. Only the last component of ``signal`` may contain a pattern (``*``, ``?`` or ``[...]``); ``{name}`` is then required in the object name and is replaced by each dataset name. Uncertainty datasets ending in ``_errors`` and declared axes are excluded from pattern expansion." +msgstr "Les chemins sont relatifs à chaque groupe reconnu. Seule la dernière composante de ``signal`` peut contenir un motif (``*``, ``?`` ou ``[...]``) ; ``{name}`` est alors obligatoire dans le nom de l'objet et est remplacé par le nom de chaque jeu de données. Les jeux de données d'incertitudes se terminant par ``_errors`` et les axes déclarés sont exclus du développement du motif." + +msgid "``axes`` contains one entry per signal dimension, in array order (rows then columns for an image). Each entry is a path, an object with ``path`` and optional ``label`` and ``units``, or ``null`` to use indices. Omitting ``axes`` uses indices for every dimension. The optional ``errors`` path overrides the default ``_errors`` dataset. Labels and units otherwise come from ``long_name`` and ``units`` attributes, with the dataset name as label fallback. An explicitly requested uncertainty dataset with the wrong shape prevents importing that mapped object." +msgstr "``axes`` contient une entrée par dimension du signal, dans l'ordre du tableau (lignes puis colonnes pour une image). Chaque entrée est un chemin, un objet avec ``path`` et éventuellement ``label`` et ``units``, ou ``null`` pour utiliser les indices. Omettre ``axes`` utilise les indices pour chaque dimension. Le chemin facultatif ``errors`` remplace le jeu de données par défaut ``_errors``. Les libellés et unités viennent sinon des attributs ``long_name`` et ``units``, avec le nom du jeu de données comme libellé par défaut. Un jeu de données d'incertitudes explicitement demandé dont la forme est incorrecte empêche l'import de l'objet décrit." + +msgid "This runnable example uses the JSON file above and creates a small input file:" +msgstr "Cet exemple exécutable utilise le fichier JSON ci-dessus et crée un petit fichier d'entrée :" + +msgid "The registry is process-wide; JSON files are not discovered automatically. A plugin should register its mappings when loaded and unregister them when unloaded. Re-registering an identical mapping is harmless; a different mapping with the same name raises ``ValueError``. Existing open files are not rebuilt when the registry changes. Mapped objects appear as extra browser nodes, leaving raw datasets accessible, and record the mapping name in their experimental source." +msgstr "Le registre est commun au processus ; les fichiers JSON ne sont pas découverts automatiquement. Un plugin doit enregistrer ses profils au chargement et les retirer au déchargement. Réenregistrer un profil identique est sans effet ; un profil différent portant le même nom lève ``ValueError``. Les fichiers déjà ouverts ne sont pas reconstruits lorsque le registre change. Les objets décrits apparaissent comme des nœuds supplémentaires de l'explorateur, en laissant les jeux de données bruts accessibles, et enregistrent le nom du profil dans leur origine expérimentale." + +msgid "Large Files and Multidimensional Views" +msgstr "Grands fichiers et vues multidimensionnelles" + +msgid "Dataset classification uses shape and type metadata; mixed compound datasets are checked on their first records. Text previews are bounded to 100 elements, and attributes above 10 000 elements are skipped. Graphical previews are read in a worker thread, and stale results are not displayed. File closure waits for the preview read to finish." +msgstr "La classification des jeux de données utilise les métadonnées de forme et de type ; les jeux de données composites mixtes sont vérifiés sur leurs premiers enregistrements. Les aperçus textuels sont limités à 100 éléments, et les attributs de plus de 10 000 éléments sont ignorés. Les aperçus graphiques sont lus dans un fil d'exécution de travail, et les résultats périmés ne sont pas affichés. La fermeture d'un fichier attend la fin de la lecture de son aperçu." + +msgid "Ordinary 1D arrays and 2D images exceeding ``2**24`` elements are previewed with a sampling step along each axis. The full data is still read on import. NeXus and mapped objects, compound datasets and signals stored in 2D tables still use complete objects for graphical previews, so this is not a universal memory bound." +msgstr "Les tableaux 1D ordinaires et les images 2D dépassant ``2**24`` éléments sont prévisualisés avec un pas d'échantillonnage le long de chaque axe. Les données complètes restent lues lors de l'import. Les objets NeXus et décrits par un profil, les jeux de données composites et les signaux stockés dans des tableaux 2D utilisent encore des objets complets pour leurs aperçus graphiques : ce n'est donc pas une borne mémoire universelle." + +msgid "Numeric datasets with three or more dimensions are listed but not imported as complete objects. There is no graphical slice selector. Use :class:`sigimax.h5.selection.NDSelection` and :func:`sigimax.h5.selection.read_selection` to read only a selected 1D or 2D view as a NumPy array:" +msgstr "Les jeux de données numériques à trois dimensions ou plus sont listés mais pas importés comme objets complets. Il n'existe pas de sélecteur graphique de coupes. Utilisez :class:`sigimax.h5.selection.NDSelection` et :func:`sigimax.h5.selection.read_selection` pour ne lire qu'une vue 1D ou 2D sélectionnée, sous forme de tableau NumPy :" + +msgid "``displayed`` gives the axes in output order; ``indices`` fixes the other axes in their original order. An optional positive ``step`` samples displayed axes. Invalid axes or indices raise ``ValueError``. This low-level API returns arrays, not calibrated Sigima objects, and does not attach experimental-source metadata." +msgstr "``displayed`` donne les axes dans l'ordre de sortie ; ``indices`` fixe les autres axes dans leur ordre d'origine. Un ``step`` positif facultatif échantillonne les axes affichés. Des axes ou indices invalides lèvent ``ValueError``. Cette API de bas niveau renvoie des tableaux, pas des objets Sigima calibrés, et n'attache pas de métadonnées d'origine expérimentale." + msgid "DataLab" msgstr "DataLab" msgid "DataLab keeps its native workspace format, panel serialization, metadata, ROI, and analysis-result handling. Its ``save_h5_workspace`` override remains the owner of that format. The SigimaX HDF5 GUI and generic import infrastructure do not change DataLab's native HDF5 layout." msgstr "DataLab conserve son format d'espace de travail natif, la sérialisation de ses panneaux, ses métadonnées, ses ROI et la gestion des résultats d'analyse. Sa surcharge de ``save_h5_workspace`` reste propriétaire de ce format. L'interface HDF5 et l'infrastructure d'import générique de SigimaX ne modifient pas la disposition HDF5 native de DataLab." - diff --git a/doc/release_notes/release_1.01.md b/doc/release_notes/release_1.01.md index 3b62dc1..69848db 100644 --- a/doc/release_notes/release_1.01.md +++ b/doc/release_notes/release_1.01.md @@ -34,16 +34,17 @@ ### HDF5 import ### -* Browsing an HDF5 file no longer reads its datasets entirely: generic nodes are classified and described from the dataset shape and data type, and value previews read only the first elements (`TEXT_PREVIEW_MAX_ITEMS`). Compound datasets mixing numeric and string fields are checked on their first records only. Data is read entirely when the object is imported. -* The HDF5 browser reads previews in a worker thread, so that it stays responsive while a large dataset is read: only the last selected node is shown, and closing a file waits for the preview being read. Datasets with more than `PlotPreview.MAX_ITEMS` (2²⁴) elements are previewed with a step along each axis (`BaseNode.create_preview_object`). -* Numeric datasets with three dimensions or more (image stacks, spectral cubes...) are now listed in the HDF5 browser, with their shape and type. They are not imported as a whole: their 1D or 2D views are read with `sigimax.h5.selection.read_selection`, which reads only the selected elements (`NDSelection`: displayed axes, possibly transposed, and an index along each other axis). +* Browsing an HDF5 file no longer reads its numeric datasets entirely just to classify them: generic nodes are classified and described from the dataset shape and data type, and value previews read only the first elements (`TEXT_PREVIEW_MAX_ITEMS`). Compound datasets mixing numeric and string fields are checked on their first records only. Full data is read when creating an imported object or an undecimated graphical preview. +* The HDF5 browser reads previews in a worker thread, so that it stays responsive while a large dataset is read: only the last selected node is shown, and closing a file waits for the preview being read. Ordinary 1D arrays and 2D images with more than `PlotPreview.MAX_ITEMS` (2²⁴) elements are previewed with a step along each axis. NeXus and mapped objects, compound datasets and signals stored in 2D tables still use complete objects for graphical previews; this is not a universal memory bound. +* Numeric datasets with three dimensions or more (image stacks, spectral cubes...) are now listed in the HDF5 browser, with their shape and type. They are not imported as a whole, and there is no graphical slice selector: developers can read their 1D or 2D views with `sigimax.h5.selection.read_selection`, which reads only the selected elements (`NDSelection`: displayed axes, possibly transposed, and an index along each other axis). * NeXus `NXdata` groups are shown as single nodes, imported as a signal (1D) or an image (2D) with their axes, units, labels and, for signals, uncertainties: `@signal`/`@axes` attributes with `@_indices`, legacy `signal`/`axes` dataset attributes, bin edges converted to bin centers, non-uniform image axes. Invalid axes are reported in the browser instead of being guessed. Imported objects record their origin in the `experimental_source` metadata entry (file, group, signal and axes, conversions). `.nxs` and `.nx5` files are proposed in HDF5 file dialogs. * Files that do not designate their data with suitable `NXdata` groups (no axes, several counters...) can be described by declarative data mappings (`sigimax.h5.mapping`): JSON-compatible descriptions of the signals and images of the groups having given attributes and paths, with their axes, uncertainties, labels and units, and a name pattern to repeat an object for each counter of a scan. Mapped objects are shown as extra nodes of the HDF5 browser and record the mapping in their `experimental_source`. Applications and plugins register mappings with `register_mapping` (and `unregister_mapping`); `NodeFactory.add_children_provider` lets them add such extra nodes. Blank units and labels are now ignored. * Dataset attributes with more than `MAX_ATTRIBUTE_ITEMS` (10 000) values are no longer copied to object metadata. * Images with unsigned 32-bit values above 2 147 483 647 are no longer clipped to `int32`: they are converted with Sigima's converter, which keeps every value exactly with Sigima ≥ 1.4.0. `BaseNode.uint32_wng` is kept for compatibility and is always `False`. * `NodeFactory.unregister` removes a node class and its post triggers, so that an application plugin can remove the node classes it registered when it is unloaded. +See the [HDF5 guide](../user_guide/hdf5_workspace.rst) for a complete JSON mapping and registration example, NeXus import limits and multidimensional selection examples. + ### Requirements ### -* Sigima 1.3.0 or later is required for the portable annotation model, PlotPy conversion helpers and curve fitting model formulas. -* Sigima 1.4.0 or later is required for the origin of experimental data recorded by NeXus imports and for exact integer conversions. \ No newline at end of file +* Sigima 1.4.0 or later is required: the HDF5 import modules use its experimental-source API, and integer conversions use its precision-preserving converter. \ No newline at end of file diff --git a/doc/requirements.rst b/doc/requirements.rst index 210d607..7857513 100644 --- a/doc/requirements.rst +++ b/doc/requirements.rst @@ -20,7 +20,7 @@ The `sigimax` package requires the following Python modules: - >= 5.7 - Cross-platform lib for process and system monitoring. * - Sigima - - >= 1.3.0 + - >= 1.4.0 - Scientific computing engine for 1D signals and 2D images, part of the DataLab open-source platform. Optional modules for GUI support (Qt): diff --git a/doc/user_guide/hdf5_workspace.rst b/doc/user_guide/hdf5_workspace.rst index 0b207f1..63515d1 100644 --- a/doc/user_guide/hdf5_workspace.rst +++ b/doc/user_guide/hdf5_workspace.rst @@ -81,6 +81,166 @@ The generic import layer can create :class:`sigima.objects.SignalObj` and :class:`sigima.objects.ImageObj`; a derived application decides how these objects enter its own model. +NeXus Scientific Objects +------------------------ + +With Sigima 1.4 or later, an ``NXdata`` group is importable as a 1D signal or +2D image with its physical axes, labels and units. Modern group attributes +(``signal``, ``axes`` and ``_indices``) and legacy dataset attributes +(``signal=1`` and ``axes``) are supported. Selecting a group imports the +scientific object; selecting a child dataset still gives access to raw values. + +Axis datasets must be 1D and contain either one coordinate per point or one +extra value for bin edges. Edges are converted to centers. Image coordinates +may be non-uniform. Invalid declared axes are reported and prevent group +import; undeclared axes use indices. Signal Y uncertainties are read from +``_errors`` or ``errors`` when their shape matches; image uncertainties +are not imported. + +Imported objects record their file, group, signal, axes and conversions in +``experimental_source`` metadata. This describes the measurement origin, not +a reproducible acquisition or a processing history. The application must +preserve object metadata in its own workspace serialization. + +Declarative Data Mappings +------------------------- + +For files without suitable ``NXdata`` descriptions, register a mapping before +opening the file. A mapping matches groups by attribute values and required +paths, then describes the signals or images they contain. Save the following +JSON as ``counters.json``: + +.. code-block:: json + + { + "version": 1, + "name": "Counters", + "match": { + "attributes": {"NX_class": "NXentry"}, + "paths": ["time"] + }, + "objects": [ + { + "name": "{name}", + "signal": "counts_*", + "axes": [{"path": "time", "label": "Elapsed time", "units": "s"}], + "label": "Counts", + "units": "counts" + } + ] + } + +Paths are relative to each matching group. Only the last component of +``signal`` may contain a pattern (``*``, ``?`` or ``[...]``); ``{name}`` is +then required in the object name and is replaced by each dataset name. +Uncertainty datasets ending in ``_errors`` and declared axes are excluded +from pattern expansion. + +``axes`` contains one entry per signal dimension, in array order (rows then +columns for an image). Each entry is a path, an object with ``path`` and optional +``label`` and ``units``, or ``null`` to use indices. Omitting ``axes`` uses +indices for every dimension. The optional ``errors`` path overrides the +default ``_errors`` dataset. Labels and units otherwise come from +``long_name`` and ``units`` attributes, with the dataset name as label fallback. +An explicitly requested uncertainty dataset with the wrong shape prevents +importing that mapped object. + +This runnable example uses the JSON file above and creates a small input file: + +.. code-block:: python + + from pathlib import Path + from tempfile import TemporaryDirectory + + import h5py + import numpy as np + + from sigimax.h5 import H5Importer + from sigimax.h5.mapping import load_mapping, register_mapping, unregister_mapping + + mapping = load_mapping("counters.json") + with TemporaryDirectory() as directory: + filename = Path(directory) / "scan.h5" + with h5py.File(filename, "w") as h5file: + entry = h5file.create_group("entry") + entry.attrs["NX_class"] = "NXentry" + entry["time"] = [0.0, 0.5, 1.0] + entry["counts_1"] = [10.0, 20.0, 30.0] + entry["counts_1_errors"] = [1.0, 2.0, 3.0] + register_mapping(mapping) + try: + importer = H5Importer(str(filename)) + try: + node = importer.get("/entry::Counters/counts_1") + signal = node.get_native_object() + np.testing.assert_array_equal(signal.x, [0.0, 0.5, 1.0]) + np.testing.assert_array_equal(signal.dy, [1.0, 2.0, 3.0]) + assert signal.xunit == "s" + assert signal.metadata["experimental_source"]["details"] == { + "mapping": "Counters" + } + finally: + importer.close() + finally: + unregister_mapping(mapping.name) + +The registry is process-wide; JSON files are not discovered automatically. +A plugin should register its mappings when loaded and unregister them when +unloaded. Re-registering an identical mapping is harmless; a different mapping +with the same name raises ``ValueError``. Existing open files are not rebuilt +when the registry changes. Mapped objects appear as extra browser nodes, leaving +raw datasets accessible, and record the mapping name in their experimental source. + +Large Files and Multidimensional Views +-------------------------------------- + +Dataset classification uses shape and type metadata; mixed compound datasets +are checked on their first records. Text previews are bounded to 100 elements, +and attributes above 10 000 elements are skipped. Graphical previews are read +in a worker thread, and stale results are not displayed. File closure waits +for the preview read to finish. + +Ordinary 1D arrays and 2D images exceeding ``2**24`` elements are previewed with a +sampling step along each axis. The full data is still read on import. +NeXus and mapped objects, compound datasets and signals stored in 2D tables +still use complete objects for graphical previews, so this is not a universal +memory bound. + +Numeric datasets with three or more dimensions are listed but not imported as +complete objects. There is no graphical slice selector. Use +:class:`sigimax.h5.selection.NDSelection` and +:func:`sigimax.h5.selection.read_selection` to read only a selected 1D or 2D +view as a NumPy array: + +.. code-block:: python + + from pathlib import Path + from tempfile import TemporaryDirectory + + import h5py + import numpy as np + + from sigimax.h5.selection import NDSelection, read_selection + + cube = np.arange(5 * 3 * 4).reshape(5, 3, 4) + with TemporaryDirectory() as directory: + filename = Path(directory) / "cube.h5" + with h5py.File(filename, "w") as h5file: + h5file["frames"] = cube + with h5py.File(filename, "r") as h5file: + dataset = h5file["frames"] + frame = read_selection(dataset, NDSelection((1, 2), (2,))) + spectrum = read_selection(dataset, NDSelection((0,), (1, 3))) + transposed = read_selection(dataset, NDSelection((2, 1), (2,))) + np.testing.assert_array_equal(frame, cube[2]) + np.testing.assert_array_equal(spectrum, cube[:, 1, 3]) + np.testing.assert_array_equal(transposed, cube[2].T) + +``displayed`` gives the axes in output order; ``indices`` fixes the other axes +in their original order. An optional positive ``step`` samples displayed axes. +Invalid axes or indices raise ``ValueError``. This low-level API returns arrays, +not calibrated Sigima objects, and does not attach experimental-source metadata. + DataLab ------- diff --git a/pyproject.toml b/pyproject.toml index 1be4c4d..80adf03 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -40,7 +40,7 @@ dependencies = [ "guidata >= 3.15.0", "PlotPy >= 2.11.0", "psutil >= 5.7", - "Sigima >= 1.3.0", + "Sigima >= 1.4.0", ] dynamic = ["version"] diff --git a/requirements.txt b/requirements.txt index afe330d..fa9933f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ Coverage PlotPy >= 2.11.0 PyQt5 >= 5.15.6 -Sigima >= 1.3.0 +Sigima >= 1.4.0 guidata >= 3.15.0 myst-nb myst_parser diff --git a/sigimax/h5/__init__.py b/sigimax/h5/__init__.py index a3d1e61..5ed7e45 100644 --- a/sigimax/h5/__init__.py +++ b/sigimax/h5/__init__.py @@ -41,6 +41,8 @@ .. autofunction:: sigimax.h5.mapping.unregister_mapping .. autofunction:: sigimax.h5.mapping.describe_mapping .. autoclass:: sigimax.h5.mapping.DataMapping +.. autoclass:: sigimax.h5.mapping.MappedObject +.. autoclass:: sigimax.h5.mapping.MappedAxis .. autoclass:: sigimax.h5.mapping.MappedDataNode .. autoclass:: sigimax.h5.selection.NDSelection :members: From 96ebfc1e053983e06c3a2d77288f2b4923ab36d1 Mon Sep 17 00:00:00 2001 From: Pierre Raybaut <1311787+PierreRaybaut@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:49:22 +0200 Subject: [PATCH 09/20] fix(h5): record dimensions without axis as '.' in experimental_source NeXus and mapped objects recorded a dimension without axis as None, which HDF5 files restore as an empty string (and which Sigima now rejects). Use ".", as NeXus `axes` attributes do. Assisted-by: Claude Opus 5.5 --- .../LC_MESSAGES/user_guide/hdf5_workspace.po | 4 ++-- doc/user_guide/hdf5_workspace.rst | 7 +++--- sigimax/h5/mapping.py | 7 ++++-- sigimax/h5/nexus.py | 7 ++++-- sigimax/tests/hdf5/test_h5_mapping.py | 22 ++++++++++++++++++- sigimax/tests/hdf5/test_h5_nexus.py | 15 +++++++++++++ 6 files changed, 52 insertions(+), 10 deletions(-) diff --git a/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po b/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po index 1d6f19d..6c2a6ba 100644 --- a/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po +++ b/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po @@ -103,8 +103,8 @@ msgstr "Avec Sigima 1.4 ou ultérieur, un groupe ``NXdata`` est importable comme msgid "Axis datasets must be 1D and contain either one coordinate per point or one extra value for bin edges. Edges are converted to centers. Image coordinates may be non-uniform. Invalid declared axes are reported and prevent group import; undeclared axes use indices. Signal Y uncertainties are read from ``_errors`` or ``errors`` when their shape matches; image uncertainties are not imported." msgstr "Les jeux de données des axes doivent être 1D et contenir une coordonnée par point, ou une valeur supplémentaire pour des bords de classes. Les bords sont convertis en centres. Les coordonnées d'image peuvent être non uniformes. Les axes déclarés invalides sont signalés et empêchent l'import du groupe ; les axes non déclarés utilisent les indices. Les incertitudes Y des signaux sont lues dans ``_errors`` ou ``errors`` lorsque leur forme correspond ; les incertitudes des images ne sont pas importées." -msgid "Imported objects record their file, group, signal, axes and conversions in ``experimental_source`` metadata. This describes the measurement origin, not a reproducible acquisition or a processing history. The application must preserve object metadata in its own workspace serialization." -msgstr "Les objets importés enregistrent leur fichier, groupe, signal, axes et conversions dans les métadonnées ``experimental_source``. Cela décrit l'origine de la mesure, pas une acquisition reproductible ni un historique de traitement. L'application doit préserver les métadonnées des objets dans sa propre sérialisation d'espace de travail." +msgid "Imported objects record their file, group, signal, axes and conversions in ``experimental_source`` metadata; a dimension without axis is recorded as ``\".\"``, as in NeXus ``axes`` attributes. This describes the measurement origin, not a reproducible acquisition or a processing history. The application must preserve object metadata in its own workspace serialization." +msgstr "Les objets importés enregistrent leur fichier, groupe, signal, axes et conversions dans les métadonnées ``experimental_source`` ; une dimension sans axe est enregistrée sous la forme ``\".\"``, comme dans les attributs ``axes`` NeXus. Cela décrit l'origine de la mesure, pas une acquisition reproductible ni un historique de traitement. L'application doit préserver les métadonnées des objets dans sa propre sérialisation d'espace de travail." msgid "Declarative Data Mappings" msgstr "Profils de correspondance déclaratifs" diff --git a/doc/user_guide/hdf5_workspace.rst b/doc/user_guide/hdf5_workspace.rst index 63515d1..e6cbbb8 100644 --- a/doc/user_guide/hdf5_workspace.rst +++ b/doc/user_guide/hdf5_workspace.rst @@ -98,9 +98,10 @@ import; undeclared axes use indices. Signal Y uncertainties are read from are not imported. Imported objects record their file, group, signal, axes and conversions in -``experimental_source`` metadata. This describes the measurement origin, not -a reproducible acquisition or a processing history. The application must -preserve object metadata in its own workspace serialization. +``experimental_source`` metadata; a dimension without axis is recorded as +``"."``, as in NeXus ``axes`` attributes. This describes the measurement +origin, not a reproducible acquisition or a processing history. The +application must preserve object metadata in its own workspace serialization. Declarative Data Mappings ------------------------- diff --git a/sigimax/h5/mapping.py b/sigimax/h5/mapping.py index aab19d1..72483fd 100644 --- a/sigimax/h5/mapping.py +++ b/sigimax/h5/mapping.py @@ -442,7 +442,10 @@ def name(self): return self.object_name def make_source(self, conversions: list[str]) -> dict: - """Return the experimental source of the imported object""" + """Return the experimental source of the imported object + + Dimensions without axis are recorded as ``"."``, as in NeXus ``@axes``. + """ desc = self.nxdata return make_experimental_source( "hdf5", @@ -450,7 +453,7 @@ def make_source(self, conversions: list[str]) -> dict: entry=desc.path, selection={ "signal": desc.signal, - "axes": [axis.name if axis else None for axis in desc.axes], + "axes": [axis.name if axis else "." for axis in desc.axes], }, conversions=conversions or None, details={"mapping": self.mapping.name}, diff --git a/sigimax/h5/nexus.py b/sigimax/h5/nexus.py index 89f4013..529c1dd 100644 --- a/sigimax/h5/nexus.py +++ b/sigimax/h5/nexus.py @@ -316,7 +316,10 @@ def description(self): return self.text def make_source(self, conversions: list[str]) -> dict: - """Return the experimental source of the imported object""" + """Return the experimental source of the imported object + + Dimensions without axis are recorded as ``"."``, as in NeXus ``@axes``. + """ desc = self.nxdata return make_experimental_source( "nexus", @@ -324,7 +327,7 @@ def make_source(self, conversions: list[str]) -> dict: entry=desc.path, selection={ "signal": desc.signal, - "axes": [axis.name if axis else None for axis in desc.axes], + "axes": [axis.name if axis else "." for axis in desc.axes], }, conversions=conversions or None, ) diff --git a/sigimax/tests/hdf5/test_h5_mapping.py b/sigimax/tests/hdf5/test_h5_mapping.py index 1c141fd..2b45b97 100644 --- a/sigimax/tests/hdf5/test_h5_mapping.py +++ b/sigimax/tests/hdf5/test_h5_mapping.py @@ -62,6 +62,11 @@ "signal": "measurement/map", "axes": ["measurement/y", {"path": "measurement/x", "units": "mm"}], }, + { + "name": "Columns", + "signal": "measurement/map", + "axes": [None, "measurement/x"], + }, {"name": "Missing", "signal": "measurement/absent"}, { "name": "Bad errors", @@ -191,7 +196,15 @@ def test_mapped_nodes(h5fname: str, mappings) -> None: importer = H5Importer(h5fname) try: nodes = _mapped_nodes(importer) - expected = {"data_1", "data_2", "data_10", "Transmission", "Map", "Bad errors"} + expected = { + "data_1", + "data_2", + "data_10", + "Transmission", + "Map", + "Columns", + "Bad errors", + } assert set(nodes) == expected assert not nodes["Bad errors"].is_supported() assert "do not match the signal shape" in nodes["Bad errors"].text @@ -221,6 +234,13 @@ def test_mapped_nodes(h5fname: str, mappings) -> None: assert isinstance(image, ImageObj) assert (image.x0, image.y0, image.dx, image.dy) == (0.0, 5.0, 1.0, 2.0) assert (image.xunit, image.ylabel) == ("mm", "y") + + # Dimensions without axis are recorded as '.', as in NeXus files + columns = nodes["Columns"].get_native_object() + assert get_experimental_source(columns)["selection"] == { + "signal": "measurement/map", + "axes": [".", "measurement/x"], + } finally: importer.close() diff --git a/sigimax/tests/hdf5/test_h5_nexus.py b/sigimax/tests/hdf5/test_h5_nexus.py index 9ef346a..7d13ac4 100644 --- a/sigimax/tests/hdf5/test_h5_nexus.py +++ b/sigimax/tests/hdf5/test_h5_nexus.py @@ -15,6 +15,7 @@ import h5py import numpy as np import pytest +import sigima.io from sigima.objects import ImageObj, SignalObj, get_experimental_source from sigimax.h5 import H5Importer @@ -66,6 +67,10 @@ def fixture_nxsfname(tmp_path) -> str: indexed["x"] = np.array([0.5, 1.0, 1.5]) indexed["y"] = np.array([-1.0, 1.0]) + partial = _nxdata(entry, "partial", "data", np.array([b".", b"x"])) + partial["data"] = np.ones((2, 3)) + partial["x"] = np.array([0.0, 2.0, 4.0]) + legacy = entry.create_group("legacy") legacy.attrs["NX_class"] = b"NXdata" legacy["y"] = np.array([1.0, 2.0, 3.0]) @@ -160,5 +165,15 @@ def test_nxdata_image(nodes: dict) -> None: assert (indexed.x0, indexed.dx, indexed.y0, indexed.dy) == (0.5, 0.5, -1.0, 2.0) +def test_nxdata_missing_axis_source(nodes: dict, tmp_path) -> None: + """Dimensions without axis are recorded as '.', which files restore.""" + obj = nodes["/entry/partial"].get_native_object() + source = get_experimental_source(obj) + assert source["selection"] == {"signal": "data", "axes": [".", "x"]} + fname = str(tmp_path / "partial.h5ima") + sigima.io.write_image(fname, obj) + assert get_experimental_source(sigima.io.read_image(fname)) == source + + if __name__ == "__main__": pytest.main([__file__, "-v"]) From 9e177650d915c442710cdfced9881c9ce5b6b637 Mon Sep 17 00:00:00 2001 From: Pierre Raybaut <1311787+PierreRaybaut@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:51:12 +0200 Subject: [PATCH 10/20] fix(h5): report invalid NeXus axes and uncertainties instead of failing at import - An `@axes` attribute whose length differs from the signal rank, and `@_indices` attributes designating a missing dimension, several dimensions or a dimension already described, are reported as problems instead of being silently truncated, padded or overwritten. - Axis datasets that do not hold real numbers are reported; such uncertainty datasets are ignored. Both previously made the node look importable and then failed when the object was created. Assisted-by: Claude Opus 5.5 --- .../LC_MESSAGES/user_guide/hdf5_workspace.po | 8 +-- doc/user_guide/hdf5_workspace.rst | 18 ++--- sigimax/h5/mapping.py | 4 +- sigimax/h5/nexus.py | 64 ++++++++++++++---- sigimax/tests/hdf5/test_h5_mapping.py | 6 +- sigimax/tests/hdf5/test_h5_nexus.py | 65 +++++++++++++++++++ 6 files changed, 137 insertions(+), 28 deletions(-) diff --git a/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po b/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po index 6c2a6ba..7205162 100644 --- a/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po +++ b/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po @@ -100,8 +100,8 @@ msgstr "Objets scientifiques NeXus" msgid "With Sigima 1.4 or later, an ``NXdata`` group is importable as a 1D signal or 2D image with its physical axes, labels and units. Modern group attributes (``signal``, ``axes`` and ``_indices``) and legacy dataset attributes (``signal=1`` and ``axes``) are supported. Selecting a group imports the scientific object; selecting a child dataset still gives access to raw values." msgstr "Avec Sigima 1.4 ou ultérieur, un groupe ``NXdata`` est importable comme signal 1D ou image 2D avec ses axes physiques, libellés et unités. Les attributs modernes du groupe (``signal``, ``axes`` et ``_indices``) et les anciens attributs des jeux de données (``signal=1`` et ``axes``) sont pris en charge. Sélectionner un groupe importe l'objet scientifique ; sélectionner un jeu de données enfant donne toujours accès aux valeurs brutes." -msgid "Axis datasets must be 1D and contain either one coordinate per point or one extra value for bin edges. Edges are converted to centers. Image coordinates may be non-uniform. Invalid declared axes are reported and prevent group import; undeclared axes use indices. Signal Y uncertainties are read from ``_errors`` or ``errors`` when their shape matches; image uncertainties are not imported." -msgstr "Les jeux de données des axes doivent être 1D et contenir une coordonnée par point, ou une valeur supplémentaire pour des bords de classes. Les bords sont convertis en centres. Les coordonnées d'image peuvent être non uniformes. Les axes déclarés invalides sont signalés et empêchent l'import du groupe ; les axes non déclarés utilisent les indices. Les incertitudes Y des signaux sont lues dans ``_errors`` ou ``errors`` lorsque leur forme correspond ; les incertitudes des images ne sont pas importées." +msgid "Axis datasets must be 1D, hold real numbers and contain either one coordinate per point or one extra value for bin edges. Edges are converted to centers. Image coordinates may be non-uniform. The ``axes`` attribute must give one entry per signal dimension, and ``_indices`` attributes must designate distinct existing dimensions. Invalid declared axes are reported and prevent group import; undeclared axes use indices. Signal Y uncertainties are read from ``_errors`` or ``errors`` when they hold real numbers and their shape matches; image uncertainties are not imported." +msgstr "Les jeux de données des axes doivent être 1D, contenir des nombres réels et contenir une coordonnée par point, ou une valeur supplémentaire pour des bords de classes. Les bords sont convertis en centres. Les coordonnées d'image peuvent être non uniformes. L'attribut ``axes`` doit donner une entrée par dimension du signal, et les attributs ``_indices`` doivent désigner des dimensions existantes et distinctes. Les axes déclarés invalides sont signalés et empêchent l'import du groupe ; les axes non déclarés utilisent les indices. Les incertitudes Y des signaux sont lues dans ``_errors`` ou ``errors`` lorsqu'elles contiennent des nombres réels et que leur forme correspond ; les incertitudes des images ne sont pas importées." msgid "Imported objects record their file, group, signal, axes and conversions in ``experimental_source`` metadata; a dimension without axis is recorded as ``\".\"``, as in NeXus ``axes`` attributes. This describes the measurement origin, not a reproducible acquisition or a processing history. The application must preserve object metadata in its own workspace serialization." msgstr "Les objets importés enregistrent leur fichier, groupe, signal, axes et conversions dans les métadonnées ``experimental_source`` ; une dimension sans axe est enregistrée sous la forme ``\".\"``, comme dans les attributs ``axes`` NeXus. Cela décrit l'origine de la mesure, pas une acquisition reproductible ni un historique de traitement. L'application doit préserver les métadonnées des objets dans sa propre sérialisation d'espace de travail." @@ -116,8 +116,8 @@ msgstr "Pour les fichiers sans descriptions ``NXdata`` adaptées, enregistrez un msgid "Paths are relative to each matching group. Only the last component of ``signal`` may contain a pattern (``*``, ``?`` or ``[...]``); ``{name}`` is then required in the object name and is replaced by each dataset name. Uncertainty datasets ending in ``_errors`` and declared axes are excluded from pattern expansion." msgstr "Les chemins sont relatifs à chaque groupe reconnu. Seule la dernière composante de ``signal`` peut contenir un motif (``*``, ``?`` ou ``[...]``) ; ``{name}`` est alors obligatoire dans le nom de l'objet et est remplacé par le nom de chaque jeu de données. Les jeux de données d'incertitudes se terminant par ``_errors`` et les axes déclarés sont exclus du développement du motif." -msgid "``axes`` contains one entry per signal dimension, in array order (rows then columns for an image). Each entry is a path, an object with ``path`` and optional ``label`` and ``units``, or ``null`` to use indices. Omitting ``axes`` uses indices for every dimension. The optional ``errors`` path overrides the default ``_errors`` dataset. Labels and units otherwise come from ``long_name`` and ``units`` attributes, with the dataset name as label fallback. An explicitly requested uncertainty dataset with the wrong shape prevents importing that mapped object." -msgstr "``axes`` contient une entrée par dimension du signal, dans l'ordre du tableau (lignes puis colonnes pour une image). Chaque entrée est un chemin, un objet avec ``path`` et éventuellement ``label`` et ``units``, ou ``null`` pour utiliser les indices. Omettre ``axes`` utilise les indices pour chaque dimension. Le chemin facultatif ``errors`` remplace le jeu de données par défaut ``_errors``. Les libellés et unités viennent sinon des attributs ``long_name`` et ``units``, avec le nom du jeu de données comme libellé par défaut. Un jeu de données d'incertitudes explicitement demandé dont la forme est incorrecte empêche l'import de l'objet décrit." +msgid "``axes`` contains one entry per signal dimension, in array order (rows then columns for an image). Each entry is a path, an object with ``path`` and optional ``label`` and ``units``, or ``null`` to use indices. Omitting ``axes`` uses indices for every dimension. The optional ``errors`` path overrides the default ``_errors`` dataset. Labels and units otherwise come from ``long_name`` and ``units`` attributes, with the dataset name as label fallback. An explicitly requested uncertainty dataset that does not hold real numbers or has the wrong shape prevents importing that mapped object." +msgstr "``axes`` contient une entrée par dimension du signal, dans l'ordre du tableau (lignes puis colonnes pour une image). Chaque entrée est un chemin, un objet avec ``path`` et éventuellement ``label`` et ``units``, ou ``null`` pour utiliser les indices. Omettre ``axes`` utilise les indices pour chaque dimension. Le chemin facultatif ``errors`` remplace le jeu de données par défaut ``_errors``. Les libellés et unités viennent sinon des attributs ``long_name`` et ``units``, avec le nom du jeu de données comme libellé par défaut. Un jeu de données d'incertitudes explicitement demandé qui ne contient pas de nombres réels ou dont la forme est incorrecte empêche l'import de l'objet décrit." msgid "This runnable example uses the JSON file above and creates a small input file:" msgstr "Cet exemple exécutable utilise le fichier JSON ci-dessus et crée un petit fichier d'entrée :" diff --git a/doc/user_guide/hdf5_workspace.rst b/doc/user_guide/hdf5_workspace.rst index e6cbbb8..f4dc8ca 100644 --- a/doc/user_guide/hdf5_workspace.rst +++ b/doc/user_guide/hdf5_workspace.rst @@ -90,12 +90,14 @@ With Sigima 1.4 or later, an ``NXdata`` group is importable as a 1D signal or (``signal=1`` and ``axes``) are supported. Selecting a group imports the scientific object; selecting a child dataset still gives access to raw values. -Axis datasets must be 1D and contain either one coordinate per point or one -extra value for bin edges. Edges are converted to centers. Image coordinates -may be non-uniform. Invalid declared axes are reported and prevent group -import; undeclared axes use indices. Signal Y uncertainties are read from -``_errors`` or ``errors`` when their shape matches; image uncertainties -are not imported. +Axis datasets must be 1D, hold real numbers and contain either one coordinate +per point or one extra value for bin edges. Edges are converted to centers. +Image coordinates may be non-uniform. The ``axes`` attribute must give one +entry per signal dimension, and ``_indices`` attributes must designate +distinct existing dimensions. Invalid declared axes are reported and prevent +group import; undeclared axes use indices. Signal Y uncertainties are read from +``_errors`` or ``errors`` when they hold real numbers and their shape +matches; image uncertainties are not imported. Imported objects record their file, group, signal, axes and conversions in ``experimental_source`` metadata; a dimension without axis is recorded as @@ -143,8 +145,8 @@ columns for an image). Each entry is a path, an object with ``path`` and optiona indices for every dimension. The optional ``errors`` path overrides the default ``_errors`` dataset. Labels and units otherwise come from ``long_name`` and ``units`` attributes, with the dataset name as label fallback. -An explicitly requested uncertainty dataset with the wrong shape prevents -importing that mapped object. +An explicitly requested uncertainty dataset that does not hold real numbers or +has the wrong shape prevents importing that mapped object. This runnable example uses the JSON file above and creates a small input file: diff --git a/sigimax/h5/mapping.py b/sigimax/h5/mapping.py index 72483fd..0398613 100644 --- a/sigimax/h5/mapping.py +++ b/sigimax/h5/mapping.py @@ -360,7 +360,9 @@ def _describe( return None problems = list(desc.problems) if obj.errors and desc.errors is None: - problems.append(f"errors '{obj.errors}' do not match the signal shape") + problems.append( + f"errors '{obj.errors}' is not a numeric dataset with the signal shape" + ) axis_descs = list(desc.axes) for index, axis in enumerate(obj.axes or ()): if index < len(axis_descs) and axis is not None and axis_descs[index]: diff --git a/sigimax/h5/nexus.py b/sigimax/h5/nexus.py index 529c1dd..50e5bef 100644 --- a/sigimax/h5/nexus.py +++ b/sigimax/h5/nexus.py @@ -12,9 +12,10 @@ - ``@signal`` group attribute, or legacy ``signal=1`` dataset attribute - ``@axes`` group attribute (``"."`` for a dimension without axis), with optional - ``@_indices`` attributes, or legacy ``axes`` signal attribute -- axis datasets holding one value per point, or bin edges (one more value), which - are converted to bin centers + ``@_indices`` attributes, or legacy ``axes`` signal attribute; ``@axes`` + must give one entry per signal dimension +- axis datasets holding one real value per point, or bin edges (one more value), + which are converted to bin centers - ``_errors`` (or legacy ``errors``) uncertainties of a 1D signal - ``units`` and ``long_name`` dataset attributes """ @@ -134,17 +135,39 @@ def _find_signal(group: h5py.Group) -> tuple[str | None, list[str]]: return None, [] -def _axis_dimensions(group: h5py.Group, names: list[str], ndim: int) -> list: - """Return the axis name of each signal dimension""" +def _axis_dimensions( + group: h5py.Group, names: list[str], ndim: int +) -> tuple[list[str | None], list[str]]: + """Return the axis name of each signal dimension, and the problems found""" axes: list[str | None] = [None] * ndim + problems: list[str] = [] + if names and len(names) != ndim: + problems.append(f"{len(names)} axes for {ndim} dimensions") for position, name in enumerate(names): if name == ".": continue indices = group.attrs.get(f"{name}_indices") - dims = [position] if indices is None else [int(i) for i in np.ravel(indices)] - if len(dims) == 1 and 0 <= dims[0] < ndim: + if indices is None: + if position >= ndim: + continue # Already reported as an axis count mismatch + dims = [position] + else: + try: + dims = [int(index) for index in np.ravel(indices)] + except (TypeError, ValueError): + problems.append(f"axis '{name}' has invalid indices") + continue + if len(dims) != 1: + problems.append(f"axis '{name}' spans {len(dims)} dimensions") + elif not 0 <= dims[0] < ndim: + problems.append(f"axis '{name}' refers to missing dimension {dims[0]}") + elif axes[dims[0]] is not None: + problems.append( + f"axes '{axes[dims[0]]}' and '{name}' describe dimension {dims[0]}" + ) + else: axes[dims[0]] = name - return axes + return axes, problems def describe_nxdata(group: h5py.Group) -> NXdataDescription | None: @@ -163,8 +186,16 @@ def describe_nxdata(group: h5py.Group) -> NXdataDescription | None: dset = group.get(signal) if signal else None if not isinstance(dset, h5py.Dataset) or not dset.shape: return None - axes = _axis_dimensions(group, names, dset.ndim) - return describe_signal(group, signal, axes, errors=(f"{signal}_errors", "errors")) + axes, problems = _axis_dimensions(group, names, dset.ndim) + desc = describe_signal(group, signal, axes, errors=(f"{signal}_errors", "errors")) + if desc is None or not problems: + return desc + return dataclasses.replace(desc, problems=tuple(problems) + desc.problems) + + +def _is_real_number(dset: h5py.Dataset) -> bool: + """Return True if dataset holds real numbers (integers or floats)""" + return dset.dtype.kind in "iuf" def describe_signal( @@ -179,8 +210,8 @@ def describe_signal( group: HDF5 group signal: signal dataset path, relative to the group axes: axis dataset path of each signal dimension (None: no axis) - errors: candidate uncertainties dataset paths; the first one with the - signal shape is used + errors: candidate uncertainties dataset paths; the first numeric one with + the signal shape is used Returns: Description, or None if signal is not a numeric dataset @@ -204,6 +235,9 @@ def describe_signal( elif not isinstance(axis_dset, h5py.Dataset) or axis_dset.ndim != 1: problems.append(f"axis '{name}' is not a 1D dataset") axis_descs.append(None) + elif not _is_real_number(axis_dset): + problems.append(f"axis '{name}' does not hold real numbers") + axis_descs.append(None) elif axis_dset.shape[0] not in (dset.shape[dim], dset.shape[dim] + 1): problems.append( f"axis '{name}' has {axis_dset.shape[0]} values " @@ -225,7 +259,11 @@ def describe_signal( found = None for name in errors: candidate = group.get(name) - if isinstance(candidate, h5py.Dataset) and candidate.shape == dset.shape: + if ( + isinstance(candidate, h5py.Dataset) + and candidate.shape == dset.shape + and _is_real_number(candidate) + ): found = name break return NXdataDescription( diff --git a/sigimax/tests/hdf5/test_h5_mapping.py b/sigimax/tests/hdf5/test_h5_mapping.py index 2b45b97..7d60628 100644 --- a/sigimax/tests/hdf5/test_h5_mapping.py +++ b/sigimax/tests/hdf5/test_h5_mapping.py @@ -207,7 +207,9 @@ def test_mapped_nodes(h5fname: str, mappings) -> None: } assert set(nodes) == expected assert not nodes["Bad errors"].is_supported() - assert "do not match the signal shape" in nodes["Bad errors"].text + assert "is not a numeric dataset with the signal shape" in ( + nodes["Bad errors"].text + ) node = nodes["data_10"] assert node.id == "/entry/scan_data::Counters/data_10" assert importer.get(node.id) is node @@ -252,7 +254,7 @@ def test_mapping_problems(h5fname: str) -> None: items = dict(describe_mapping(mapping, h5file["entry"])) assert "Missing" not in items assert items["Bad errors"].problems == ( - "errors 'measurement/map' do not match the signal shape", + "errors 'measurement/map' is not a numeric dataset with the signal shape", ) diff --git a/sigimax/tests/hdf5/test_h5_nexus.py b/sigimax/tests/hdf5/test_h5_nexus.py index 7d13ac4..804d615 100644 --- a/sigimax/tests/hdf5/test_h5_nexus.py +++ b/sigimax/tests/hdf5/test_h5_nexus.py @@ -10,7 +10,9 @@ from __future__ import annotations +import contextlib import uuid +from typing import Callable import h5py import numpy as np @@ -102,6 +104,19 @@ def fixture_nodes(nxsfname: str): importer.close() +@contextlib.contextmanager +def _nodes_of(tmp_path, build: Callable[[h5py.File], None]): + """Create a file with a builder function, and yield its nodes by path.""" + fname = str(tmp_path / f"{uuid.uuid4()}.nxs") + with h5py.File(fname, "w") as h5file: + build(h5file) + importer = H5Importer(fname) + try: + yield {node.id: node for node in importer.nodes} + finally: + importer.close() + + def test_nxdata_classification(nodes: dict) -> None: """NXdata groups are recognized; invalid ones are shown but not importable.""" for name in ("spectrum", "histogram", "map", "indexed", "legacy"): @@ -175,5 +190,55 @@ def test_nxdata_missing_axis_source(nodes: dict, tmp_path) -> None: assert get_experimental_source(sigima.io.read_image(fname)) == source +def test_nxdata_invalid_axes(tmp_path) -> None: + """Axes that do not describe the signal dimensions, and axes or + uncertainties that do not hold real numbers, are reported.""" + + def build(h5file: h5py.File) -> None: + short = _nxdata(h5file, "short", "data", ["x"]) + short["data"] = np.ones((3, 4)) + short["x"] = np.arange(4.0) + long = _nxdata(h5file, "long", "data", ["y", "x", "z"]) + long["data"] = np.ones((3, 4)) + long["x"], long["y"], long["z"] = np.arange(4.0), np.arange(3.0), [0.0, 1.0] + twice = _nxdata(h5file, "twice", "data", ["x", "y"]) + twice.attrs["x_indices"] = twice.attrs["y_indices"] = 0 + twice["data"] = np.ones((3, 4)) + twice["x"], twice["y"] = np.arange(3.0), np.arange(3.0) + missing = _nxdata(h5file, "missing", "data", ["x"]) + missing.attrs["x_indices"] = 3 + missing["data"] = np.ones(4) + missing["x"] = np.arange(4.0) + spanning = _nxdata(h5file, "spanning", "data", ["x", "."]) + spanning.attrs["x_indices"] = [0, 1] + spanning["data"] = np.ones((3, 4)) + spanning["x"] = np.arange(3.0) + text = _nxdata(h5file, "text", "data", ["x"]) + text["data"] = np.ones(3) + text["x"] = np.array([b"a", b"b", b"c"]) + text_errors = _nxdata(h5file, "text_errors", "data", ["x"]) + text_errors["data"] = np.ones(3) + text_errors["x"] = np.arange(3.0) + text_errors["data_errors"] = np.array([b"a", b"b", b"c"]) + + with _nodes_of(tmp_path, build) as nodes: + expected = { + "short": "1 axes for 2 dimensions", + "long": "3 axes for 2 dimensions", + "twice": "axes 'x' and 'y' describe dimension 0", + "missing": "axis 'x' refers to missing dimension 3", + "spanning": "axis 'x' spans 2 dimensions", + "text": "axis 'x' does not hold real numbers", + } + for name, problem in expected.items(): + node = nodes[f"/{name}"] + assert not node.is_supported(), name + assert problem in node.nxdata.problems, name + assert problem in node.text, name + node = nodes["/text_errors"] + assert node.is_supported() and node.nxdata.errors is None + assert isinstance(node.get_native_object(), SignalObj) + + if __name__ == "__main__": pytest.main([__file__, "-v"]) From 589aafa2fd5e61a7e7d0b2b972479ef6be43d31f Mon Sep 17 00:00:00 2001 From: Pierre Raybaut <1311787+PierreRaybaut@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:52:43 +0200 Subject: [PATCH 11/20] fix(h5): detect uniform float32 axes and import decreasing image axes - Uniform axes are detected within the precision of the data type they were stored with: float32 axes and axes with a large origin were imported as non-uniform coordinates. The pixel size is computed from the axis ends. - Decreasing image axes are reversed with the image, and the conversion is recorded in experimental_source: PlotPy failed to show such images ("x must be an increasing 1D array"). Image axes that are not strictly monotonic (e.g. constant) raise a ValueError instead of producing a degenerate image. Assisted-by: Claude Opus 5.5 --- .../LC_MESSAGES/user_guide/hdf5_workspace.po | 4 +- doc/user_guide/hdf5_workspace.rst | 4 +- sigimax/h5/nexus.py | 91 +++++++++++++------ sigimax/tests/hdf5/test_h5_nexus.py | 55 +++++++++++ 4 files changed, 125 insertions(+), 29 deletions(-) diff --git a/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po b/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po index 7205162..d223123 100644 --- a/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po +++ b/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po @@ -100,8 +100,8 @@ msgstr "Objets scientifiques NeXus" msgid "With Sigima 1.4 or later, an ``NXdata`` group is importable as a 1D signal or 2D image with its physical axes, labels and units. Modern group attributes (``signal``, ``axes`` and ``_indices``) and legacy dataset attributes (``signal=1`` and ``axes``) are supported. Selecting a group imports the scientific object; selecting a child dataset still gives access to raw values." msgstr "Avec Sigima 1.4 ou ultérieur, un groupe ``NXdata`` est importable comme signal 1D ou image 2D avec ses axes physiques, libellés et unités. Les attributs modernes du groupe (``signal``, ``axes`` et ``_indices``) et les anciens attributs des jeux de données (``signal=1`` et ``axes``) sont pris en charge. Sélectionner un groupe importe l'objet scientifique ; sélectionner un jeu de données enfant donne toujours accès aux valeurs brutes." -msgid "Axis datasets must be 1D, hold real numbers and contain either one coordinate per point or one extra value for bin edges. Edges are converted to centers. Image coordinates may be non-uniform. The ``axes`` attribute must give one entry per signal dimension, and ``_indices`` attributes must designate distinct existing dimensions. Invalid declared axes are reported and prevent group import; undeclared axes use indices. Signal Y uncertainties are read from ``_errors`` or ``errors`` when they hold real numbers and their shape matches; image uncertainties are not imported." -msgstr "Les jeux de données des axes doivent être 1D, contenir des nombres réels et contenir une coordonnée par point, ou une valeur supplémentaire pour des bords de classes. Les bords sont convertis en centres. Les coordonnées d'image peuvent être non uniformes. L'attribut ``axes`` doit donner une entrée par dimension du signal, et les attributs ``_indices`` doivent désigner des dimensions existantes et distinctes. Les axes déclarés invalides sont signalés et empêchent l'import du groupe ; les axes non déclarés utilisent les indices. Les incertitudes Y des signaux sont lues dans ``_errors`` ou ``errors`` lorsqu'elles contiennent des nombres réels et que leur forme correspond ; les incertitudes des images ne sont pas importées." +msgid "Axis datasets must be 1D, hold real numbers and contain either one coordinate per point or one extra value for bin edges. Edges are converted to centers. Image coordinates may be non-uniform; a decreasing image axis is reversed, with the image, and this conversion is recorded, while an image axis that is not strictly monotonic prevents import. The ``axes`` attribute must give one entry per signal dimension, and ``_indices`` attributes must designate distinct existing dimensions. Invalid declared axes are reported and prevent group import; undeclared axes use indices. Signal Y uncertainties are read from ``_errors`` or ``errors`` when they hold real numbers and their shape matches; image uncertainties are not imported." +msgstr "Les jeux de données des axes doivent être 1D, contenir des nombres réels et contenir une coordonnée par point, ou une valeur supplémentaire pour des bords de classes. Les bords sont convertis en centres. Les coordonnées d'image peuvent être non uniformes ; un axe d'image décroissant est inversé, avec l'image, et cette conversion est enregistrée, tandis qu'un axe d'image qui n'est pas strictement monotone empêche l'import. L'attribut ``axes`` doit donner une entrée par dimension du signal, et les attributs ``_indices`` doivent désigner des dimensions existantes et distinctes. Les axes déclarés invalides sont signalés et empêchent l'import du groupe ; les axes non déclarés utilisent les indices. Les incertitudes Y des signaux sont lues dans ``_errors`` ou ``errors`` lorsqu'elles contiennent des nombres réels et que leur forme correspond ; les incertitudes des images ne sont pas importées." msgid "Imported objects record their file, group, signal, axes and conversions in ``experimental_source`` metadata; a dimension without axis is recorded as ``\".\"``, as in NeXus ``axes`` attributes. This describes the measurement origin, not a reproducible acquisition or a processing history. The application must preserve object metadata in its own workspace serialization." msgstr "Les objets importés enregistrent leur fichier, groupe, signal, axes et conversions dans les métadonnées ``experimental_source`` ; une dimension sans axe est enregistrée sous la forme ``\".\"``, comme dans les attributs ``axes`` NeXus. Cela décrit l'origine de la mesure, pas une acquisition reproductible ni un historique de traitement. L'application doit préserver les métadonnées des objets dans sa propre sérialisation d'espace de travail." diff --git a/doc/user_guide/hdf5_workspace.rst b/doc/user_guide/hdf5_workspace.rst index f4dc8ca..515204f 100644 --- a/doc/user_guide/hdf5_workspace.rst +++ b/doc/user_guide/hdf5_workspace.rst @@ -92,7 +92,9 @@ scientific object; selecting a child dataset still gives access to raw values. Axis datasets must be 1D, hold real numbers and contain either one coordinate per point or one extra value for bin edges. Edges are converted to centers. -Image coordinates may be non-uniform. The ``axes`` attribute must give one +Image coordinates may be non-uniform; a decreasing image axis is reversed, +with the image, and this conversion is recorded, while an image axis that is +not strictly monotonic prevents import. The ``axes`` attribute must give one entry per signal dimension, and ``_indices`` attributes must designate distinct existing dimensions. Invalid declared axes are reported and prevent group import; undeclared axes use indices. Signal Y uncertainties are read from diff --git a/sigimax/h5/nexus.py b/sigimax/h5/nexus.py index 50e5bef..dc5a986 100644 --- a/sigimax/h5/nexus.py +++ b/sigimax/h5/nexus.py @@ -15,7 +15,8 @@ ``@_indices`` attributes, or legacy ``axes`` signal attribute; ``@axes`` must give one entry per signal dimension - axis datasets holding one real value per point, or bin edges (one more value), - which are converted to bin centers + which are converted to bin centers; decreasing image axes are reversed, with + the image, as images need increasing coordinates - ``_errors`` (or legacy ``errors``) uncertainties of a 1D signal - ``units`` and ``long_name`` dataset attributes """ @@ -279,18 +280,55 @@ def describe_signal( ) -def _read_axis(group: h5py.Group, axis: NXAxis) -> np.ndarray: - """Return axis values, at bin centers""" - values = np.asarray(group[axis.name][()], dtype=float) - return (values[:-1] + values[1:]) / 2 if axis.edges else values +def _read_axis(group: h5py.Group, axis: NXAxis) -> tuple[np.ndarray, np.dtype]: + """Return axis values, at bin centers, and the data type of the axis dataset""" + dset = group[axis.name] + values = np.asarray(dset[()], dtype=float) + if axis.edges: + values = (values[:-1] + values[1:]) / 2 + return values, dset.dtype -def _is_uniform(values: np.ndarray) -> bool: - """Return True if values are evenly spaced""" - if values.size < 2: +def _is_uniform(values: np.ndarray, dtype: np.dtype) -> bool: + """Return True if monotonic values are evenly spaced, within the precision of + the data type they were stored with""" + if values.size < 3: return True + eps = np.finfo(dtype if dtype.kind == "f" else float).eps + tolerance = 4 * float(eps) * max(abs(values[0]), abs(values[-1])) + expected = np.linspace(values[0], values[-1], values.size) + return bool(np.all(np.abs(values - expected) <= tolerance)) + + +def _is_decreasing(values: np.ndarray, name: str) -> bool: + """Return True if axis values are strictly decreasing, False if they are + strictly increasing + + Raises: + ValueError: if values are not strictly monotonic + """ steps = np.diff(values) - return bool(np.allclose(steps, steps[0], rtol=1e-9, atol=0.0)) + if np.all(steps > 0): + return False + if np.all(steps < 0): + return True + raise ValueError(f"Axis '{name}' is not strictly monotonic") + + +def _image_axis( + group: h5py.Group, axis: NXAxis | None, size: int +) -> tuple[np.ndarray, np.dtype, bool]: + """Return image axis values in increasing order, the data type they were + stored with, and True if they were reversed + + Raises: + ValueError: if axis values are not strictly monotonic + """ + if axis is None: + return np.arange(size, dtype=float), np.dtype(float), False + values, dtype = _read_axis(group, axis) + reverse = _is_decreasing(values, axis.name) + return (values[::-1] if reverse else values), dtype, reverse class NXdataNode(common.GroupNode): @@ -385,7 +423,7 @@ def create_native_object(self): axis = desc.axes[0] x = np.arange(data.size, dtype=float) if axis is not None: - x = _read_axis(group, axis) + x, _dtype = _read_axis(group, axis) dy = None if desc.errors is not None: dy = np.asarray(group[desc.errors][()], dtype=float) @@ -395,26 +433,27 @@ def create_native_object(self): if axis is not None: obj.xlabel, obj.xunit = axis.label, axis.units else: + yaxis, xaxis = desc.axes + ny, nx = desc.shape + # PlotPy images need increasing coordinates: decreasing axes are reversed + xc, xdtype, xreversed = _image_axis(group, xaxis, nx) + yc, ydtype, yreversed = _image_axis(group, yaxis, ny) + for axis, reverse, dim in ((xaxis, xreversed, 1), (yaxis, yreversed, 0)): + if reverse: + data = np.flip(data, axis=dim) + conversions.append( + f"axis '{axis.name}': reversed to increasing order" + ) obj = create_image( self.object_title, - convert_array_to_valid_dtype(data, ImageObj.VALID_DTYPES), + convert_array_to_valid_dtype( + np.ascontiguousarray(data), ImageObj.VALID_DTYPES + ), ) obj.zlabel, obj.zunit = desc.label, desc.units - yaxis, xaxis = desc.axes - ny, nx = desc.shape - xc = ( - np.arange(nx, dtype=float) - if xaxis is None - else _read_axis(group, xaxis) - ) - yc = ( - np.arange(ny, dtype=float) - if yaxis is None - else _read_axis(group, yaxis) - ) - if _is_uniform(xc) and _is_uniform(yc): - dx = float(xc[1] - xc[0]) if nx > 1 else 1.0 - dy = float(yc[1] - yc[0]) if ny > 1 else 1.0 + if _is_uniform(xc, xdtype) and _is_uniform(yc, ydtype): + dx = float(xc[-1] - xc[0]) / (nx - 1) if nx > 1 else 1.0 + dy = float(yc[-1] - yc[0]) / (ny - 1) if ny > 1 else 1.0 obj.set_uniform_coords(dx, dy, float(xc[0]), float(yc[0])) else: obj.set_coords(xc, yc) diff --git a/sigimax/tests/hdf5/test_h5_nexus.py b/sigimax/tests/hdf5/test_h5_nexus.py index 804d615..bdb6771 100644 --- a/sigimax/tests/hdf5/test_h5_nexus.py +++ b/sigimax/tests/hdf5/test_h5_nexus.py @@ -20,6 +20,7 @@ import sigima.io from sigima.objects import ImageObj, SignalObj, get_experimental_source +from sigimax.adapters_plotpy import create_adapter_from_object from sigimax.h5 import H5Importer from sigimax.h5.nexus import NXdataNode @@ -240,5 +241,59 @@ def build(h5file: h5py.File) -> None: assert isinstance(node.get_native_object(), SignalObj) +def _image_group(h5file: h5py.File, name: str, x, y=None) -> None: + """Create an NXdata image group with an X axis, and optionally a Y axis.""" + ny = 2 if y is None else len(y) + group = _nxdata(h5file, name, "data", ["." if y is None else "y", "x"]) + group["data"] = np.arange(ny * len(x), dtype=np.uint16).reshape(ny, len(x)) + group["x"] = x + if y is not None: + group["y"] = y + + +def test_nxdata_image_axes(tmp_path) -> None: + """Uniform axes are detected within the precision of their data type, and + decreasing axes are reversed (PlotPy images need increasing coordinates).""" + + def build(h5file: h5py.File) -> None: + f32 = np.linspace(0.0, 1.0, 11, dtype=np.float32) + _image_group(h5file, "float32", f32, f32) + _image_group(h5file, "offset", np.linspace(1000.0, 1000.1, 1001)) + _image_group(h5file, "decreasing", [3.0, 2.0, 1.0, 0.0], [10.0, 5.0]) + _image_group(h5file, "decreasing_nu", [6.0, 3.0, 1.0, 0.0]) + _image_group(h5file, "constant", np.zeros(4)) + _image_group(h5file, "unordered", [0.0, 2.0, 1.0, 3.0]) + + with _nodes_of(tmp_path, build) as nodes: + image = nodes["/float32"].get_native_object() + assert image.is_uniform_coords + assert np.isclose(image.dx, 0.1) and np.isclose(image.dy, 0.1) + + image = nodes["/offset"].get_native_object() + assert image.is_uniform_coords + assert image.x0 == 1000.0 + assert np.isclose(image.dx, 1e-4, rtol=1e-12) + + image = nodes["/decreasing"].get_native_object() + assert image.is_uniform_coords + assert (image.x0, image.dx, image.y0, image.dy) == (0.0, 1.0, 5.0, 5.0) + assert np.array_equal(image.data, np.arange(8).reshape(2, 4)[::-1, ::-1]) + assert get_experimental_source(image)["conversions"] == [ + "axis 'x': reversed to increasing order", + "axis 'y': reversed to increasing order", + ] + create_adapter_from_object(image).make_item() + + image = nodes["/decreasing_nu"].get_native_object() + assert not image.is_uniform_coords + assert np.array_equal(image.xcoords, [0.0, 1.0, 3.0, 6.0]) + assert np.array_equal(image.data, np.arange(8).reshape(2, 4)[:, ::-1]) + create_adapter_from_object(image).make_item() + + for name in ("constant", "unordered"): + with pytest.raises(ValueError, match="not strictly monotonic"): + nodes[f"/{name}"].get_native_object() + + if __name__ == "__main__": pytest.main([__file__, "-v"]) From 6fd3103bc38cf02c33d110b172a96a8b13122098 Mon Sep 17 00:00:00 2001 From: Pierre Raybaut <1311787+PierreRaybaut@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:54:53 +0200 Subject: [PATCH 12/20] fix(h5): keep complex64 signals and large-integer warnings on HDF5 import BaseNode.set_signal_data cast every dataset that was not float64 or complex128 to float64 before Sigima's converter: complex64 1D datasets lost their imaginary part (ComplexWarning), and integers beyond 2**53 were rounded without Sigima's precision warning. NXdata signals bypassed the converter too. - New common.to_signal_data helper, used by generic and NXdata signals: float64 for real data (as before), complex128 for complex data, through Sigima's converter. - 2D tables with complex X values or uncertainties are reported as unsupported instead of being truncated. - Release notes: NeXus axis handling (previous commits) and this fix. Assisted-by: Claude Opus 5.5 --- .../LC_MESSAGES/release_notes/release_1.01.po | 7 +- doc/release_notes/release_1.01.md | 3 +- sigimax/h5/common.py | 29 +++++- sigimax/h5/nexus.py | 2 +- sigimax/tests/hdf5/test_h5_signal_dtypes.py | 94 +++++++++++++++++++ 5 files changed, 126 insertions(+), 9 deletions(-) create mode 100644 sigimax/tests/hdf5/test_h5_signal_dtypes.py diff --git a/doc/locale/fr/LC_MESSAGES/release_notes/release_1.01.po b/doc/locale/fr/LC_MESSAGES/release_notes/release_1.01.po index dfee39d..b63ed2d 100644 --- a/doc/locale/fr/LC_MESSAGES/release_notes/release_1.01.po +++ b/doc/locale/fr/LC_MESSAGES/release_notes/release_1.01.po @@ -82,8 +82,8 @@ msgstr "L'explorateur HDF5 lit les aperçus dans un fil d'exécution de travail msgid "Numeric datasets with three dimensions or more (image stacks, spectral cubes...) are now listed in the HDF5 browser, with their shape and type. They are not imported as a whole, and there is no graphical slice selector: developers can read their 1D or 2D views with `sigimax.h5.selection.read_selection`, which reads only the selected elements (`NDSelection`: displayed axes, possibly transposed, and an index along each other axis)." msgstr "Les jeux de données numériques à trois dimensions ou plus (piles d'images, cubes spectraux...) sont désormais listés dans l'explorateur HDF5 avec leur forme et leur type. Ils ne sont pas importés en entier, et il n'existe pas de sélecteur graphique de coupes : les développeurs peuvent lire leurs vues 1D ou 2D avec `sigimax.h5.selection.read_selection`, qui ne lit que les éléments sélectionnés (`NDSelection` : axes affichés, éventuellement transposés, et un indice sur chaque autre axe)." -msgid "NeXus `NXdata` groups are shown as single nodes, imported as a signal (1D) or an image (2D) with their axes, units, labels and, for signals, uncertainties: `@signal`/`@axes` attributes with `@_indices`, legacy `signal`/`axes` dataset attributes, bin edges converted to bin centers, non-uniform image axes. Invalid axes are reported in the browser instead of being guessed. Imported objects record their origin in the `experimental_source` metadata entry (file, group, signal and axes, conversions). `.nxs` and `.nx5` files are proposed in HDF5 file dialogs." -msgstr "Les groupes NeXus `NXdata` sont présentés comme des nœuds uniques, importés comme signal (1D) ou image (2D) avec leurs axes, unités, libellés et, pour les signaux, incertitudes : attributs `@signal`/`@axes` avec `@_indices`, anciens attributs de jeux de données `signal`/`axes`, conversion des bords de classes en centres, axes d'image non uniformes. Les axes invalides sont signalés dans l'explorateur au lieu d'être devinés. Les objets importés enregistrent leur origine dans la métadonnée `experimental_source` (fichier, groupe, signal et axes, conversions). Les fichiers `.nxs` et `.nx5` sont proposés dans les dialogues de fichiers HDF5." +msgid "NeXus `NXdata` groups are shown as single nodes, imported as a signal (1D) or an image (2D) with their axes, units, labels and, for signals, uncertainties: `@signal`/`@axes` attributes with `@_indices`, legacy `signal`/`axes` dataset attributes, bin edges converted to bin centers, non-uniform image axes. Uniform axes are detected within the precision of their data type (e.g. `float32` axes), and decreasing image axes are reversed with the image. Invalid axes (axis count differing from the signal rank, axes designating a missing or already described dimension, axes not holding real numbers, image axes not strictly monotonic) are reported instead of being guessed. Imported objects record their origin in the `experimental_source` metadata entry (file, group, signal and axes, `\".\"` for a dimension without axis, conversions). `.nxs` and `.nx5` files are proposed in HDF5 file dialogs." +msgstr "Les groupes NeXus `NXdata` sont présentés comme des nœuds uniques, importés comme signal (1D) ou image (2D) avec leurs axes, unités, libellés et, pour les signaux, incertitudes : attributs `@signal`/`@axes` avec `@_indices`, anciens attributs de jeux de données `signal`/`axes`, conversion des bords de classes en centres, axes d'image non uniformes. Les axes uniformes sont détectés à la précision de leur type de données (par exemple des axes `float32`), et les axes d'image décroissants sont inversés avec l'image. Les axes invalides (nombre d'axes différent du rang du signal, axes désignant une dimension absente ou déjà décrite, axes ne contenant pas de nombres réels, axes d'image non strictement monotones) sont signalés au lieu d'être devinés. Les objets importés enregistrent leur origine dans la métadonnée `experimental_source` (fichier, groupe, signal et axes, `\".\"` pour une dimension sans axe, conversions). Les fichiers `.nxs` et `.nx5` sont proposés dans les dialogues de fichiers HDF5." msgid "Files that do not designate their data with suitable `NXdata` groups (no axes, several counters...) can be described by declarative data mappings (`sigimax.h5.mapping`): JSON-compatible descriptions of the signals and images of the groups having given attributes and paths, with their axes, uncertainties, labels and units, and a name pattern to repeat an object for each counter of a scan. Mapped objects are shown as extra nodes of the HDF5 browser and record the mapping in their `experimental_source`. Applications and plugins register mappings with `register_mapping` (and `unregister_mapping`); `NodeFactory.add_children_provider` lets them add such extra nodes. Blank units and labels are now ignored." msgstr "Les fichiers qui ne désignent pas leurs données par des groupes `NXdata` adaptés (sans axes, plusieurs compteurs...) peuvent être décrits par des profils de correspondance déclaratifs (`sigimax.h5.mapping`) : descriptions compatibles JSON des signaux et images des groupes ayant les attributs et chemins donnés, avec leurs axes, incertitudes, libellés et unités, et un motif de nom pour répéter un objet pour chaque compteur d'un scan. Les objets décrits apparaissent comme des nœuds supplémentaires de l'explorateur HDF5 et enregistrent le profil dans leur `experimental_source`. Les applications et plugins enregistrent les profils avec `register_mapping` (et `unregister_mapping`) ; `NodeFactory.add_children_provider` leur permet d'ajouter ces nœuds supplémentaires. Les unités et libellés vides sont désormais ignorés." @@ -94,6 +94,9 @@ msgstr "Les attributs de jeux de données contenant plus de `MAX_ATTRIBUTE_ITEMS msgid "Images with unsigned 32-bit values above 2 147 483 647 are no longer clipped to `int32`: they are converted with Sigima's converter, which keeps every value exactly with Sigima ≥ 1.4.0. `BaseNode.uint32_wng` is kept for compatibility and is always `False`." msgstr "Les images contenant des valeurs entières non signées sur 32 bits supérieures à 2 147 483 647 ne sont plus écrêtées en `int32` : elles sont converties avec le convertisseur de Sigima, qui conserve chaque valeur exactement avec Sigima ≥ 1.4.0. `BaseNode.uint32_wng` est conservé pour compatibilité et vaut toujours `False`." +msgid "1D datasets of single-precision complex numbers (`complex64`) are now imported as complex signals: only their real part was kept (with a `ComplexWarning`). Integer signals beyond 2⁵³ now emit Sigima's precision warning instead of being rounded silently, and 2D tables with complex X values or uncertainties are reported as unsupported instead of being truncated." +msgstr "Les jeux de données 1D de nombres complexes en simple précision (`complex64`) sont désormais importés comme des signaux complexes : seule leur partie réelle était conservée (avec un `ComplexWarning`). Les signaux entiers au-delà de 2⁵³ émettent désormais l'avertissement de précision de Sigima au lieu d'être arrondis silencieusement, et les tableaux 2D dont les valeurs X ou les incertitudes sont complexes sont signalés comme non pris en charge au lieu d'être tronqués." + msgid "`NodeFactory.unregister` removes a node class and its post triggers, so that an application plugin can remove the node classes it registered when it is unloaded." msgstr "`NodeFactory.unregister` retire une classe de nœud et ses déclencheurs de post-traitement, afin qu'un plugin applicatif puisse retirer les classes de nœuds qu'il a enregistrées lors de son déchargement." diff --git a/doc/release_notes/release_1.01.md b/doc/release_notes/release_1.01.md index 69848db..a208174 100644 --- a/doc/release_notes/release_1.01.md +++ b/doc/release_notes/release_1.01.md @@ -37,10 +37,11 @@ * Browsing an HDF5 file no longer reads its numeric datasets entirely just to classify them: generic nodes are classified and described from the dataset shape and data type, and value previews read only the first elements (`TEXT_PREVIEW_MAX_ITEMS`). Compound datasets mixing numeric and string fields are checked on their first records only. Full data is read when creating an imported object or an undecimated graphical preview. * The HDF5 browser reads previews in a worker thread, so that it stays responsive while a large dataset is read: only the last selected node is shown, and closing a file waits for the preview being read. Ordinary 1D arrays and 2D images with more than `PlotPreview.MAX_ITEMS` (2²⁴) elements are previewed with a step along each axis. NeXus and mapped objects, compound datasets and signals stored in 2D tables still use complete objects for graphical previews; this is not a universal memory bound. * Numeric datasets with three dimensions or more (image stacks, spectral cubes...) are now listed in the HDF5 browser, with their shape and type. They are not imported as a whole, and there is no graphical slice selector: developers can read their 1D or 2D views with `sigimax.h5.selection.read_selection`, which reads only the selected elements (`NDSelection`: displayed axes, possibly transposed, and an index along each other axis). -* NeXus `NXdata` groups are shown as single nodes, imported as a signal (1D) or an image (2D) with their axes, units, labels and, for signals, uncertainties: `@signal`/`@axes` attributes with `@_indices`, legacy `signal`/`axes` dataset attributes, bin edges converted to bin centers, non-uniform image axes. Invalid axes are reported in the browser instead of being guessed. Imported objects record their origin in the `experimental_source` metadata entry (file, group, signal and axes, conversions). `.nxs` and `.nx5` files are proposed in HDF5 file dialogs. +* NeXus `NXdata` groups are shown as single nodes, imported as a signal (1D) or an image (2D) with their axes, units, labels and, for signals, uncertainties: `@signal`/`@axes` attributes with `@_indices`, legacy `signal`/`axes` dataset attributes, bin edges converted to bin centers, non-uniform image axes. Uniform axes are detected within the precision of their data type (e.g. `float32` axes), and decreasing image axes are reversed with the image. Invalid axes (axis count differing from the signal rank, axes designating a missing or already described dimension, axes not holding real numbers, image axes not strictly monotonic) are reported instead of being guessed. Imported objects record their origin in the `experimental_source` metadata entry (file, group, signal and axes, `"."` for a dimension without axis, conversions). `.nxs` and `.nx5` files are proposed in HDF5 file dialogs. * Files that do not designate their data with suitable `NXdata` groups (no axes, several counters...) can be described by declarative data mappings (`sigimax.h5.mapping`): JSON-compatible descriptions of the signals and images of the groups having given attributes and paths, with their axes, uncertainties, labels and units, and a name pattern to repeat an object for each counter of a scan. Mapped objects are shown as extra nodes of the HDF5 browser and record the mapping in their `experimental_source`. Applications and plugins register mappings with `register_mapping` (and `unregister_mapping`); `NodeFactory.add_children_provider` lets them add such extra nodes. Blank units and labels are now ignored. * Dataset attributes with more than `MAX_ATTRIBUTE_ITEMS` (10 000) values are no longer copied to object metadata. * Images with unsigned 32-bit values above 2 147 483 647 are no longer clipped to `int32`: they are converted with Sigima's converter, which keeps every value exactly with Sigima ≥ 1.4.0. `BaseNode.uint32_wng` is kept for compatibility and is always `False`. +* 1D datasets of single-precision complex numbers (`complex64`) are now imported as complex signals: only their real part was kept (with a `ComplexWarning`). Integer signals beyond 2⁵³ now emit Sigima's precision warning instead of being rounded silently, and 2D tables with complex X values or uncertainties are reported as unsupported instead of being truncated. * `NodeFactory.unregister` removes a node class and its post triggers, so that an application plugin can remove the node classes it registered when it is unloaded. See the [HDF5 guide](../user_guide/hdf5_workspace.rst) for a complete JSON mapping and registration example, NeXus import limits and multidimensional selection examples. diff --git a/sigimax/h5/common.py b/sigimax/h5/common.py index 6524d92..336d23c 100644 --- a/sigimax/h5/common.py +++ b/sigimax/h5/common.py @@ -54,6 +54,22 @@ def data_to_xy(data: np.ndarray) -> list[np.ndarray]: return x, y, dx, dy +def to_signal_data(data: np.ndarray) -> np.ndarray: + """Return signal values as float64, or as complex128 for complex data. + + Integers are converted with Sigima's converter, which warns when values cannot + be represented exactly. + + Args: + data: numeric array read from a dataset + + Returns: + Array with a data type supported by signals + """ + valid = (np.complex128,) if np.iscomplexobj(data) else (np.float64,) + return convert_array_to_valid_dtype(np.asarray(data), valid) + + class BaseNode(metaclass=abc.ABCMeta): """Object representing a HDF5 node""" @@ -191,15 +207,18 @@ def object_title(self): return title def set_signal_data(self, obj: SignalObj) -> None: - """Set signal data (handles various issues)""" - data = self.data - if data.dtype not in (float, np.complex128): - data = np.array(data, dtype=float) - data = convert_array_to_valid_dtype(data, SignalObj.VALID_DTYPES) + """Set signal data (handles various issues) + + Raises: + ValueError: if a 2D table has complex X values or uncertainties + """ + data = to_signal_data(self.data) if len(data.shape) == 1: obj.set_xydata(np.arange(data.size), data) else: x, y, dx, dy = data_to_xy(data) + if any(np.iscomplexobj(array) for array in (x, dx, dy)): + raise ValueError("Complex X values or uncertainties are not supported") obj.set_xydata(x, y, dx, dy) def set_image_data(self, obj: ImageObj) -> None: diff --git a/sigimax/h5/nexus.py b/sigimax/h5/nexus.py index dc5a986..7869314 100644 --- a/sigimax/h5/nexus.py +++ b/sigimax/h5/nexus.py @@ -427,7 +427,7 @@ def create_native_object(self): dy = None if desc.errors is not None: dy = np.asarray(group[desc.errors][()], dtype=float) - y = data if np.iscomplexobj(data) else data.astype(float) + y = common.to_signal_data(data) obj = create_signal(self.object_title, x, y, dy=dy) obj.ylabel, obj.yunit = desc.label, desc.units if axis is not None: diff --git a/sigimax/tests/hdf5/test_h5_signal_dtypes.py b/sigimax/tests/hdf5/test_h5_signal_dtypes.py new file mode 100644 index 0000000..43e4768 --- /dev/null +++ b/sigimax/tests/hdf5/test_h5_signal_dtypes.py @@ -0,0 +1,94 @@ +# Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file. + +""" +HDF5 signal data type unit tests +-------------------------------- + +Signals imported from HDF5 datasets and NeXus ``NXdata`` groups keep the imaginary +part of complex data, and integers that cannot be represented exactly as floats +emit a warning instead of being rounded silently. +""" + +from __future__ import annotations + +import uuid +import warnings + +import h5py +import numpy as np +import pytest +from sigima.objects import SignalObj + +from sigimax.h5 import H5Importer + +pytestmark = pytest.mark.unit + +LARGE = 2**60 + 1 + +# NumPy >= 1.25 moved ComplexWarning to numpy.exceptions (and NumPy 2 removed the +# old location) +ComplexWarning = getattr(getattr(np, "exceptions", np), "ComplexWarning") + + +@pytest.fixture(name="nodes") +def fixture_nodes(tmp_path): + """Nodes of a file with complex and large-integer signals, by path.""" + fname = str(tmp_path / f"{uuid.uuid4()}.h5") + with h5py.File(fname, "w") as h5file: + h5file["complex"] = (np.arange(5) + 1j * np.arange(5)).astype(np.complex64) + h5file["large"] = np.array([LARGE, 1, 2], dtype=np.int64) + h5file["counts"] = np.array([1, 2, 3], dtype=np.int32) + h5file["table"] = (np.arange(10) + 1j).astype(np.complex64).reshape(2, 5) + for name, data in ( + ("nx_complex", h5file["complex"][()]), + ("nx_large", h5file["large"][()]), + ): + group = h5file.create_group(name) + group.attrs["NX_class"] = "NXdata" + group.attrs["signal"] = "data" + group["data"] = data + importer = H5Importer(fname) + try: + yield {node.id: node for node in importer.nodes} + finally: + importer.close() + + +@pytest.mark.parametrize("path", ["/complex", "/nx_complex"]) +def test_complex64_signal(nodes: dict, path: str) -> None: + """Complex single-precision signals keep their imaginary part.""" + with warnings.catch_warnings(): + warnings.simplefilter("error", ComplexWarning) + obj = nodes[path].get_native_object() + assert isinstance(obj, SignalObj) + assert obj.y.dtype == np.complex128 + assert np.array_equal(obj.y, np.arange(5) + 1j * np.arange(5)) + + +@pytest.mark.parametrize("path", ["/large", "/nx_large"]) +def test_large_integer_signal(nodes: dict, path: str) -> None: + """Integers beyond 2**53 are converted with a warning.""" + with pytest.warns(UserWarning, match="cannot be represented exactly"): + obj = nodes[path].get_native_object() + assert obj.y.dtype == np.float64 + assert obj.y[0] == float(LARGE) + + +def test_integer_signal_dtype(nodes: dict) -> None: + """Integer signals are still imported as float64.""" + with warnings.catch_warnings(): + warnings.simplefilter("error") + obj = nodes["/counts"].get_native_object() + assert obj.y.dtype == np.float64 + assert np.array_equal(obj.y, [1.0, 2.0, 3.0]) + + +def test_complex_table_is_not_supported(nodes: dict) -> None: + """Tables with complex X values are not imported, instead of being truncated.""" + with warnings.catch_warnings(): + warnings.simplefilter("error", ComplexWarning) + assert nodes["/table"].get_native_object() is None + + +if __name__ == "__main__": + pytest.main([__file__, "-v"]) From 0549ac2e4458043a62d7c6f57bcd65bf5d5cbaa3 Mon Sep 17 00:00:00 2001 From: Pierre Raybaut <1311787+PierreRaybaut@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:56:53 +0200 Subject: [PATCH 13/20] fix(h5browser): discard stale previews and wait for them before import - Previews of nodes selected while another one was read were all read in turn: they are now cancelled, and skipped by the worker if they already left the queue. - Closing the browser dialog (OK, Cancel or window close) waits for the preview being read: SigimaX and DataLab read the selected nodes right after, which accessed the HDF5 file concurrently with the preview thread. - PlotPreview.wait waits for every previous read (it only waited for the last request), and cleanup stops the worker thread, which is created again when needed. Assisted-by: Claude Opus 5.5 --- .../LC_MESSAGES/release_notes/release_1.01.po | 4 +- .../LC_MESSAGES/user_guide/hdf5_workspace.po | 4 +- doc/release_notes/release_1.01.md | 2 +- doc/user_guide/hdf5_workspace.rst | 3 +- sigimax/tests/hdf5/test_h5browser_preview.py | 80 ++++++++++++++++++- sigimax/widgets/h5browser.py | 36 +++++++-- 6 files changed, 113 insertions(+), 16 deletions(-) diff --git a/doc/locale/fr/LC_MESSAGES/release_notes/release_1.01.po b/doc/locale/fr/LC_MESSAGES/release_notes/release_1.01.po index b63ed2d..631445f 100644 --- a/doc/locale/fr/LC_MESSAGES/release_notes/release_1.01.po +++ b/doc/locale/fr/LC_MESSAGES/release_notes/release_1.01.po @@ -76,8 +76,8 @@ msgstr "Import HDF5" msgid "Browsing an HDF5 file no longer reads its numeric datasets entirely just to classify them: generic nodes are classified and described from the dataset shape and data type, and value previews read only the first elements (`TEXT_PREVIEW_MAX_ITEMS`). Compound datasets mixing numeric and string fields are checked on their first records only. Full data is read when creating an imported object or an undecimated graphical preview." msgstr "Parcourir un fichier HDF5 ne lit plus ses jeux de données numériques en entier simplement pour les classifier : les nœuds génériques sont classifiés et décrits à partir de la forme et du type des jeux de données, et les aperçus de valeurs ne lisent que les premiers éléments (`TEXT_PREVIEW_MAX_ITEMS`). Les jeux de données composites mêlant champs numériques et textuels ne sont vérifiés que sur leurs premiers enregistrements. Les données complètes sont lues lors de la création d'un objet importé ou d'un aperçu graphique non sous-échantillonné." -msgid "The HDF5 browser reads previews in a worker thread, so that it stays responsive while a large dataset is read: only the last selected node is shown, and closing a file waits for the preview being read. Ordinary 1D arrays and 2D images with more than `PlotPreview.MAX_ITEMS` (2²⁴) elements are previewed with a step along each axis. NeXus and mapped objects, compound datasets and signals stored in 2D tables still use complete objects for graphical previews; this is not a universal memory bound." -msgstr "L'explorateur HDF5 lit les aperçus dans un fil d'exécution de travail pour rester réactif pendant la lecture d'un grand jeu de données : seul le dernier nœud sélectionné est affiché, et la fermeture d'un fichier attend la fin de la lecture de l'aperçu. Les tableaux 1D ordinaires et les images 2D de plus de `PlotPreview.MAX_ITEMS` (2²⁴) éléments sont prévisualisés avec un pas le long de chaque axe. Les objets NeXus et décrits par un profil, les jeux de données composites et les signaux stockés dans des tableaux 2D utilisent encore des objets complets pour leurs aperçus graphiques ; ce n'est pas une borne mémoire universelle." +msgid "The HDF5 browser reads previews in a worker thread, so that it stays responsive while a large dataset is read: only the last selected node is shown, the previews of nodes selected meanwhile are not read, and closing a file or the browser dialog waits for the preview being read (imported objects are then read without concurrent access to the file). Ordinary 1D arrays and 2D images with more than `PlotPreview.MAX_ITEMS` (2²⁴) elements are previewed with a step along each axis. NeXus and mapped objects, compound datasets and signals stored in 2D tables still use complete objects for graphical previews; this is not a universal memory bound." +msgstr "L'explorateur HDF5 lit les aperçus dans un fil d'exécution de travail pour rester réactif pendant la lecture d'un grand jeu de données : seul le dernier nœud sélectionné est affiché, les aperçus des nœuds sélectionnés entre-temps ne sont pas lus, et la fermeture d'un fichier ou du dialogue de l'explorateur attend la fin de la lecture de l'aperçu (les objets importés sont ensuite lus sans accès concurrent au fichier). Les tableaux 1D ordinaires et les images 2D de plus de `PlotPreview.MAX_ITEMS` (2²⁴) éléments sont prévisualisés avec un pas le long de chaque axe. Les objets NeXus et décrits par un profil, les jeux de données composites et les signaux stockés dans des tableaux 2D utilisent encore des objets complets pour leurs aperçus graphiques ; ce n'est pas une borne mémoire universelle." msgid "Numeric datasets with three dimensions or more (image stacks, spectral cubes...) are now listed in the HDF5 browser, with their shape and type. They are not imported as a whole, and there is no graphical slice selector: developers can read their 1D or 2D views with `sigimax.h5.selection.read_selection`, which reads only the selected elements (`NDSelection`: displayed axes, possibly transposed, and an index along each other axis)." msgstr "Les jeux de données numériques à trois dimensions ou plus (piles d'images, cubes spectraux...) sont désormais listés dans l'explorateur HDF5 avec leur forme et leur type. Ils ne sont pas importés en entier, et il n'existe pas de sélecteur graphique de coupes : les développeurs peuvent lire leurs vues 1D ou 2D avec `sigimax.h5.selection.read_selection`, qui ne lit que les éléments sélectionnés (`NDSelection` : axes affichés, éventuellement transposés, et un indice sur chaque autre axe)." diff --git a/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po b/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po index d223123..df0aa47 100644 --- a/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po +++ b/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po @@ -128,8 +128,8 @@ msgstr "Le registre est commun au processus ; les fichiers JSON ne sont pas déc msgid "Large Files and Multidimensional Views" msgstr "Grands fichiers et vues multidimensionnelles" -msgid "Dataset classification uses shape and type metadata; mixed compound datasets are checked on their first records. Text previews are bounded to 100 elements, and attributes above 10 000 elements are skipped. Graphical previews are read in a worker thread, and stale results are not displayed. File closure waits for the preview read to finish." -msgstr "La classification des jeux de données utilise les métadonnées de forme et de type ; les jeux de données composites mixtes sont vérifiés sur leurs premiers enregistrements. Les aperçus textuels sont limités à 100 éléments, et les attributs de plus de 10 000 éléments sont ignorés. Les aperçus graphiques sont lus dans un fil d'exécution de travail, et les résultats périmés ne sont pas affichés. La fermeture d'un fichier attend la fin de la lecture de son aperçu." +msgid "Dataset classification uses shape and type metadata; mixed compound datasets are checked on their first records. Text previews are bounded to 100 elements, and attributes above 10 000 elements are skipped. Graphical previews are read in a worker thread; previews of items selected meanwhile are not read, and stale results are not displayed. Closing a file or the browser dialog waits for the preview read to finish." +msgstr "La classification des jeux de données utilise les métadonnées de forme et de type ; les jeux de données composites mixtes sont vérifiés sur leurs premiers enregistrements. Les aperçus textuels sont limités à 100 éléments, et les attributs de plus de 10 000 éléments sont ignorés. Les aperçus graphiques sont lus dans un fil d'exécution de travail ; les aperçus des éléments sélectionnés entre-temps ne sont pas lus, et les résultats périmés ne sont pas affichés. La fermeture d'un fichier ou du dialogue de l'explorateur attend la fin de la lecture de l'aperçu." msgid "Ordinary 1D arrays and 2D images exceeding ``2**24`` elements are previewed with a sampling step along each axis. The full data is still read on import. NeXus and mapped objects, compound datasets and signals stored in 2D tables still use complete objects for graphical previews, so this is not a universal memory bound." msgstr "Les tableaux 1D ordinaires et les images 2D dépassant ``2**24`` éléments sont prévisualisés avec un pas d'échantillonnage le long de chaque axe. Les données complètes restent lues lors de l'import. Les objets NeXus et décrits par un profil, les jeux de données composites et les signaux stockés dans des tableaux 2D utilisent encore des objets complets pour leurs aperçus graphiques : ce n'est donc pas une borne mémoire universelle." diff --git a/doc/release_notes/release_1.01.md b/doc/release_notes/release_1.01.md index a208174..266ad49 100644 --- a/doc/release_notes/release_1.01.md +++ b/doc/release_notes/release_1.01.md @@ -35,7 +35,7 @@ ### HDF5 import ### * Browsing an HDF5 file no longer reads its numeric datasets entirely just to classify them: generic nodes are classified and described from the dataset shape and data type, and value previews read only the first elements (`TEXT_PREVIEW_MAX_ITEMS`). Compound datasets mixing numeric and string fields are checked on their first records only. Full data is read when creating an imported object or an undecimated graphical preview. -* The HDF5 browser reads previews in a worker thread, so that it stays responsive while a large dataset is read: only the last selected node is shown, and closing a file waits for the preview being read. Ordinary 1D arrays and 2D images with more than `PlotPreview.MAX_ITEMS` (2²⁴) elements are previewed with a step along each axis. NeXus and mapped objects, compound datasets and signals stored in 2D tables still use complete objects for graphical previews; this is not a universal memory bound. +* The HDF5 browser reads previews in a worker thread, so that it stays responsive while a large dataset is read: only the last selected node is shown, the previews of nodes selected meanwhile are not read, and closing a file or the browser dialog waits for the preview being read (imported objects are then read without concurrent access to the file). Ordinary 1D arrays and 2D images with more than `PlotPreview.MAX_ITEMS` (2²⁴) elements are previewed with a step along each axis. NeXus and mapped objects, compound datasets and signals stored in 2D tables still use complete objects for graphical previews; this is not a universal memory bound. * Numeric datasets with three dimensions or more (image stacks, spectral cubes...) are now listed in the HDF5 browser, with their shape and type. They are not imported as a whole, and there is no graphical slice selector: developers can read their 1D or 2D views with `sigimax.h5.selection.read_selection`, which reads only the selected elements (`NDSelection`: displayed axes, possibly transposed, and an index along each other axis). * NeXus `NXdata` groups are shown as single nodes, imported as a signal (1D) or an image (2D) with their axes, units, labels and, for signals, uncertainties: `@signal`/`@axes` attributes with `@_indices`, legacy `signal`/`axes` dataset attributes, bin edges converted to bin centers, non-uniform image axes. Uniform axes are detected within the precision of their data type (e.g. `float32` axes), and decreasing image axes are reversed with the image. Invalid axes (axis count differing from the signal rank, axes designating a missing or already described dimension, axes not holding real numbers, image axes not strictly monotonic) are reported instead of being guessed. Imported objects record their origin in the `experimental_source` metadata entry (file, group, signal and axes, `"."` for a dimension without axis, conversions). `.nxs` and `.nx5` files are proposed in HDF5 file dialogs. * Files that do not designate their data with suitable `NXdata` groups (no axes, several counters...) can be described by declarative data mappings (`sigimax.h5.mapping`): JSON-compatible descriptions of the signals and images of the groups having given attributes and paths, with their axes, uncertainties, labels and units, and a name pattern to repeat an object for each counter of a scan. Mapped objects are shown as extra nodes of the HDF5 browser and record the mapping in their `experimental_source`. Applications and plugins register mappings with `register_mapping` (and `unregister_mapping`); `NodeFactory.add_children_provider` lets them add such extra nodes. Blank units and labels are now ignored. diff --git a/doc/user_guide/hdf5_workspace.rst b/doc/user_guide/hdf5_workspace.rst index 515204f..ded23c6 100644 --- a/doc/user_guide/hdf5_workspace.rst +++ b/doc/user_guide/hdf5_workspace.rst @@ -202,7 +202,8 @@ Large Files and Multidimensional Views Dataset classification uses shape and type metadata; mixed compound datasets are checked on their first records. Text previews are bounded to 100 elements, and attributes above 10 000 elements are skipped. Graphical previews are read -in a worker thread, and stale results are not displayed. File closure waits +in a worker thread; previews of items selected meanwhile are not read, and +stale results are not displayed. Closing a file or the browser dialog waits for the preview read to finish. Ordinary 1D arrays and 2D images exceeding ``2**24`` elements are previewed with a diff --git a/sigimax/tests/hdf5/test_h5browser_preview.py b/sigimax/tests/hdf5/test_h5browser_preview.py index ab73d5b..765535a 100644 --- a/sigimax/tests/hdf5/test_h5browser_preview.py +++ b/sigimax/tests/hdf5/test_h5browser_preview.py @@ -5,12 +5,13 @@ ------------------------- Previews are read in a worker thread: the last selected node is shown, the -previews of nodes selected before are discarded, and closing a file waits for -the preview being read. +previews of nodes selected before are discarded without being read, and closing +a file or the browser dialog waits for the preview being read. """ from __future__ import annotations +import threading import time import uuid @@ -20,8 +21,9 @@ from guidata.qthelpers import qt_app_context from plotpy.items import BaseImageItem, CurveItem from qtpy import QtWidgets as QW +from sigima.objects import create_signal -from sigimax.widgets.h5browser import H5Browser +from sigimax.widgets.h5browser import H5Browser, H5BrowserDialog, PlotPreview pytestmark = pytest.mark.gui @@ -74,5 +76,77 @@ def test_h5browser_preview(tmp_path) -> None: browser.close() +class _SpyNode: + """Node whose preview is read when the test releases it.""" + + def __init__(self, release: threading.Event) -> None: + self.release = release + self.started = threading.Event() + self.reads = 0 + self.done = False + + def create_preview_object(self, max_items: int): # pylint: disable=unused-argument + """Return a small signal, once released.""" + self.reads += 1 + self.started.set() + assert self.release.wait(10.0) + self.done = True + return create_signal("Spy", np.arange(3.0), np.arange(3.0)) + + +def _wait_until(condition, timeout: float = 10.0) -> None: + """Process events until condition is true.""" + deadline = time.monotonic() + timeout + while not condition(): + if time.monotonic() > deadline: + raise TimeoutError("Condition not met") + QW.QApplication.processEvents() + time.sleep(0.01) + + +def test_stale_previews_are_not_read() -> None: + """Nodes selected while a preview is read are skipped: only the last one is + read after it, and the worker thread is stopped on cleanup.""" + with qt_app_context(): + preview = PlotPreview(None) + threads = set(threading.enumerate()) + release = threading.Event() + nodes = [_SpyNode(release) for _index in range(5)] + for node in nodes: + preview.update_plot_preview(node) + assert nodes[0].started.wait(10.0) + workers = set(threading.enumerate()) - threads + release.set() + _wait_until(lambda: nodes[-1].done) + assert [node.reads for node in nodes] == [1, 0, 0, 0, 1] + preview.cleanup() + assert not any(thread.is_alive() for thread in workers) + # The preview may be used again after cleanup + node = _SpyNode(release) + preview.update_plot_preview(node) + _wait_until(lambda: node.done) + preview.cleanup() + preview.close() + + +@pytest.mark.parametrize("method", ["accept", "reject"]) +def test_dialog_waits_for_preview(method: str) -> None: + """Closing the dialog waits for the preview being read, so that the caller + does not read the file concurrently.""" + with qt_app_context(): + dialog = H5BrowserDialog(None) + release = threading.Event() + node = _SpyNode(release) + dialog.browser.plotpreview.update_plot_preview(node) + assert node.started.wait(10.0) + timer = threading.Timer(0.2, release.set) + timer.start() + getattr(dialog, method)() + assert node.done + timer.join() + dialog.cleanup() + dialog.close() + + if __name__ == "__main__": pytest.main([__file__, "-v"]) diff --git a/sigimax/widgets/h5browser.py b/sigimax/widgets/h5browser.py index 83bb36c..e69439e 100644 --- a/sigimax/widgets/h5browser.py +++ b/sigimax/widgets/h5browser.py @@ -475,8 +475,11 @@ class PlotPreview(QW.QStackedWidget): """Plot preview Preview data is read in a worker thread, so that the browser stays responsive - while a large dataset is read. Datasets larger than :attr:`MAX_ITEMS` are - decimated. The preview of a node replaces any pending one. + while a large dataset is read. Nodes may provide a decimated preview object + for datasets larger than :attr:`MAX_ITEMS` (see + :meth:`sigimax.h5.common.BaseNode.create_preview_object`). The preview of a + node replaces any pending one: previews of nodes selected meanwhile are not + read. """ #: Maximum number of data elements read for a preview @@ -487,7 +490,7 @@ class PlotPreview(QW.QStackedWidget): def __init__(self, parent: QW.QWidget) -> None: super().__init__(parent) self.__generation = 0 - self.__executor = ThreadPoolExecutor(1, thread_name_prefix="h5preview") + self.__executor: ThreadPoolExecutor | None = None self.__pending: Future | None = None self.SIG_LOADED.connect(self.__show_loaded_object) self.curvewidget = PlotWidget( @@ -512,21 +515,30 @@ def __init__(self, parent: QW.QWidget) -> None: self.addWidget(self.imagewidget) def cleanup(self) -> None: - """Clean up widget""" + """Clean up widget, and stop its worker thread""" self.wait() + if self.__executor is not None: + self.__executor.shutdown(wait=True) + self.__executor = None for widget in (self.imagewidget, self.curvewidget): widget.get_plot().del_all_items() def wait(self) -> None: - """Discard the pending preview and wait until its data is no longer read - (before closing HDF5 files)""" + """Discard the pending previews and wait until no data is read + (before closing HDF5 files or importing objects)""" self.__generation += 1 if self.__pending is not None: - self.__pending.result() + self.__pending.cancel() self.__pending = None + if self.__executor is not None: + # The single worker runs calls in order: once this one is done, the + # previous ones are done or were discarded + self.__executor.submit(int).result() def __load_object(self, generation: int, node: BaseNode) -> None: """Create the preview object of a node (worker thread)""" + if generation != self.__generation: + return # Another node was selected meanwhile: do not read this one obj, error = None, None try: obj = node.create_preview_object(self.MAX_ITEMS) @@ -548,7 +560,11 @@ def __show_message(self, text: str) -> None: def update_plot_preview(self, node: BaseNode) -> None: """Update plot preview widget""" self.__generation += 1 + if self.__pending is not None: + self.__pending.cancel() self.__show_message(_("Loading...")) + if self.__executor is None: + self.__executor = ThreadPoolExecutor(1, thread_name_prefix="h5preview") self.__pending = self.__executor.submit( self.__load_object, self.__generation, node ) @@ -988,6 +1004,12 @@ def accept(self) -> None: self.nodes = self.browser.tree.get_nodes() QW.QDialog.accept(self) + def done(self, result: int) -> None: + """Close dialog, once the preview being read is done: the selected nodes + are then read by the caller, and the files closed""" + self.browser.plotpreview.wait() + super().done(result) + def is_empty(self) -> bool: """Return True if tree is empty""" return self.browser.tree.is_empty() From d2da190c18d4709a0ffdeb2e27c6e8763a777325 Mon Sep 17 00:00:00 2001 From: Pierre Raybaut <1311787+PierreRaybaut@users.noreply.github.com> Date: Fri, 23 Oct 2026 07:59:27 +0200 Subject: [PATCH 14/20] feat(h5): decimated previews for NeXus and mapped objects NeXus NXdata groups and mapped objects were previewed with their complete native object, whatever their size. Above PlotPreview.MAX_ITEMS elements, their preview now reads one element out of a step along each dimension (signal, uncertainties and axis values), with the same axis handling as the import (bin centers, reversed decreasing image axes). Import still reads every value. Assisted-by: Claude Opus 5.5 --- .../LC_MESSAGES/release_notes/release_1.01.po | 4 +- .../LC_MESSAGES/user_guide/hdf5_workspace.po | 4 +- doc/release_notes/release_1.01.md | 2 +- doc/user_guide/hdf5_workspace.rst | 10 +-- sigimax/h5/nexus.py | 63 +++++++++++++++---- sigimax/tests/hdf5/test_h5_nexus.py | 38 +++++++++++ 6 files changed, 98 insertions(+), 23 deletions(-) diff --git a/doc/locale/fr/LC_MESSAGES/release_notes/release_1.01.po b/doc/locale/fr/LC_MESSAGES/release_notes/release_1.01.po index 631445f..1bde036 100644 --- a/doc/locale/fr/LC_MESSAGES/release_notes/release_1.01.po +++ b/doc/locale/fr/LC_MESSAGES/release_notes/release_1.01.po @@ -76,8 +76,8 @@ msgstr "Import HDF5" msgid "Browsing an HDF5 file no longer reads its numeric datasets entirely just to classify them: generic nodes are classified and described from the dataset shape and data type, and value previews read only the first elements (`TEXT_PREVIEW_MAX_ITEMS`). Compound datasets mixing numeric and string fields are checked on their first records only. Full data is read when creating an imported object or an undecimated graphical preview." msgstr "Parcourir un fichier HDF5 ne lit plus ses jeux de données numériques en entier simplement pour les classifier : les nœuds génériques sont classifiés et décrits à partir de la forme et du type des jeux de données, et les aperçus de valeurs ne lisent que les premiers éléments (`TEXT_PREVIEW_MAX_ITEMS`). Les jeux de données composites mêlant champs numériques et textuels ne sont vérifiés que sur leurs premiers enregistrements. Les données complètes sont lues lors de la création d'un objet importé ou d'un aperçu graphique non sous-échantillonné." -msgid "The HDF5 browser reads previews in a worker thread, so that it stays responsive while a large dataset is read: only the last selected node is shown, the previews of nodes selected meanwhile are not read, and closing a file or the browser dialog waits for the preview being read (imported objects are then read without concurrent access to the file). Ordinary 1D arrays and 2D images with more than `PlotPreview.MAX_ITEMS` (2²⁴) elements are previewed with a step along each axis. NeXus and mapped objects, compound datasets and signals stored in 2D tables still use complete objects for graphical previews; this is not a universal memory bound." -msgstr "L'explorateur HDF5 lit les aperçus dans un fil d'exécution de travail pour rester réactif pendant la lecture d'un grand jeu de données : seul le dernier nœud sélectionné est affiché, les aperçus des nœuds sélectionnés entre-temps ne sont pas lus, et la fermeture d'un fichier ou du dialogue de l'explorateur attend la fin de la lecture de l'aperçu (les objets importés sont ensuite lus sans accès concurrent au fichier). Les tableaux 1D ordinaires et les images 2D de plus de `PlotPreview.MAX_ITEMS` (2²⁴) éléments sont prévisualisés avec un pas le long de chaque axe. Les objets NeXus et décrits par un profil, les jeux de données composites et les signaux stockés dans des tableaux 2D utilisent encore des objets complets pour leurs aperçus graphiques ; ce n'est pas une borne mémoire universelle." +msgid "The HDF5 browser reads previews in a worker thread, so that it stays responsive while a large dataset is read: only the last selected node is shown, the previews of nodes selected meanwhile are not read, and closing a file or the browser dialog waits for the preview being read (imported objects are then read without concurrent access to the file). Ordinary 1D arrays and 2D images, NeXus and mapped objects with more than `PlotPreview.MAX_ITEMS` (2²⁴) elements are previewed with a step along each axis. Compound datasets and signals stored in 2D tables still use complete objects for graphical previews; this is not a universal memory bound." +msgstr "L'explorateur HDF5 lit les aperçus dans un fil d'exécution de travail pour rester réactif pendant la lecture d'un grand jeu de données : seul le dernier nœud sélectionné est affiché, les aperçus des nœuds sélectionnés entre-temps ne sont pas lus, et la fermeture d'un fichier ou du dialogue de l'explorateur attend la fin de la lecture de l'aperçu (les objets importés sont ensuite lus sans accès concurrent au fichier). Les tableaux 1D ordinaires et les images 2D, les objets NeXus et décrits par un profil de plus de `PlotPreview.MAX_ITEMS` (2²⁴) éléments sont prévisualisés avec un pas le long de chaque axe. Les jeux de données composites et les signaux stockés dans des tableaux 2D utilisent encore des objets complets pour leurs aperçus graphiques ; ce n'est pas une borne mémoire universelle." msgid "Numeric datasets with three dimensions or more (image stacks, spectral cubes...) are now listed in the HDF5 browser, with their shape and type. They are not imported as a whole, and there is no graphical slice selector: developers can read their 1D or 2D views with `sigimax.h5.selection.read_selection`, which reads only the selected elements (`NDSelection`: displayed axes, possibly transposed, and an index along each other axis)." msgstr "Les jeux de données numériques à trois dimensions ou plus (piles d'images, cubes spectraux...) sont désormais listés dans l'explorateur HDF5 avec leur forme et leur type. Ils ne sont pas importés en entier, et il n'existe pas de sélecteur graphique de coupes : les développeurs peuvent lire leurs vues 1D ou 2D avec `sigimax.h5.selection.read_selection`, qui ne lit que les éléments sélectionnés (`NDSelection` : axes affichés, éventuellement transposés, et un indice sur chaque autre axe)." diff --git a/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po b/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po index df0aa47..c6b8949 100644 --- a/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po +++ b/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po @@ -131,8 +131,8 @@ msgstr "Grands fichiers et vues multidimensionnelles" msgid "Dataset classification uses shape and type metadata; mixed compound datasets are checked on their first records. Text previews are bounded to 100 elements, and attributes above 10 000 elements are skipped. Graphical previews are read in a worker thread; previews of items selected meanwhile are not read, and stale results are not displayed. Closing a file or the browser dialog waits for the preview read to finish." msgstr "La classification des jeux de données utilise les métadonnées de forme et de type ; les jeux de données composites mixtes sont vérifiés sur leurs premiers enregistrements. Les aperçus textuels sont limités à 100 éléments, et les attributs de plus de 10 000 éléments sont ignorés. Les aperçus graphiques sont lus dans un fil d'exécution de travail ; les aperçus des éléments sélectionnés entre-temps ne sont pas lus, et les résultats périmés ne sont pas affichés. La fermeture d'un fichier ou du dialogue de l'explorateur attend la fin de la lecture de l'aperçu." -msgid "Ordinary 1D arrays and 2D images exceeding ``2**24`` elements are previewed with a sampling step along each axis. The full data is still read on import. NeXus and mapped objects, compound datasets and signals stored in 2D tables still use complete objects for graphical previews, so this is not a universal memory bound." -msgstr "Les tableaux 1D ordinaires et les images 2D dépassant ``2**24`` éléments sont prévisualisés avec un pas d'échantillonnage le long de chaque axe. Les données complètes restent lues lors de l'import. Les objets NeXus et décrits par un profil, les jeux de données composites et les signaux stockés dans des tableaux 2D utilisent encore des objets complets pour leurs aperçus graphiques : ce n'est donc pas une borne mémoire universelle." +msgid "Ordinary 1D arrays and 2D images, NeXus and mapped objects exceeding ``2**24`` elements are previewed with a sampling step along each axis. The full data is still read on import. Compound datasets and signals stored in 2D tables still use complete objects for graphical previews, so this is not a universal memory bound." +msgstr "Les tableaux 1D ordinaires et les images 2D, les objets NeXus et décrits par un profil dépassant ``2**24`` éléments sont prévisualisés avec un pas d'échantillonnage le long de chaque axe. Les données complètes restent lues lors de l'import. Les jeux de données composites et les signaux stockés dans des tableaux 2D utilisent encore des objets complets pour leurs aperçus graphiques : ce n'est donc pas une borne mémoire universelle." msgid "Numeric datasets with three or more dimensions are listed but not imported as complete objects. There is no graphical slice selector. Use :class:`sigimax.h5.selection.NDSelection` and :func:`sigimax.h5.selection.read_selection` to read only a selected 1D or 2D view as a NumPy array:" msgstr "Les jeux de données numériques à trois dimensions ou plus sont listés mais pas importés comme objets complets. Il n'existe pas de sélecteur graphique de coupes. Utilisez :class:`sigimax.h5.selection.NDSelection` et :func:`sigimax.h5.selection.read_selection` pour ne lire qu'une vue 1D ou 2D sélectionnée, sous forme de tableau NumPy :" diff --git a/doc/release_notes/release_1.01.md b/doc/release_notes/release_1.01.md index 266ad49..b0904d7 100644 --- a/doc/release_notes/release_1.01.md +++ b/doc/release_notes/release_1.01.md @@ -35,7 +35,7 @@ ### HDF5 import ### * Browsing an HDF5 file no longer reads its numeric datasets entirely just to classify them: generic nodes are classified and described from the dataset shape and data type, and value previews read only the first elements (`TEXT_PREVIEW_MAX_ITEMS`). Compound datasets mixing numeric and string fields are checked on their first records only. Full data is read when creating an imported object or an undecimated graphical preview. -* The HDF5 browser reads previews in a worker thread, so that it stays responsive while a large dataset is read: only the last selected node is shown, the previews of nodes selected meanwhile are not read, and closing a file or the browser dialog waits for the preview being read (imported objects are then read without concurrent access to the file). Ordinary 1D arrays and 2D images with more than `PlotPreview.MAX_ITEMS` (2²⁴) elements are previewed with a step along each axis. NeXus and mapped objects, compound datasets and signals stored in 2D tables still use complete objects for graphical previews; this is not a universal memory bound. +* The HDF5 browser reads previews in a worker thread, so that it stays responsive while a large dataset is read: only the last selected node is shown, the previews of nodes selected meanwhile are not read, and closing a file or the browser dialog waits for the preview being read (imported objects are then read without concurrent access to the file). Ordinary 1D arrays and 2D images, NeXus and mapped objects with more than `PlotPreview.MAX_ITEMS` (2²⁴) elements are previewed with a step along each axis. Compound datasets and signals stored in 2D tables still use complete objects for graphical previews; this is not a universal memory bound. * Numeric datasets with three dimensions or more (image stacks, spectral cubes...) are now listed in the HDF5 browser, with their shape and type. They are not imported as a whole, and there is no graphical slice selector: developers can read their 1D or 2D views with `sigimax.h5.selection.read_selection`, which reads only the selected elements (`NDSelection`: displayed axes, possibly transposed, and an index along each other axis). * NeXus `NXdata` groups are shown as single nodes, imported as a signal (1D) or an image (2D) with their axes, units, labels and, for signals, uncertainties: `@signal`/`@axes` attributes with `@_indices`, legacy `signal`/`axes` dataset attributes, bin edges converted to bin centers, non-uniform image axes. Uniform axes are detected within the precision of their data type (e.g. `float32` axes), and decreasing image axes are reversed with the image. Invalid axes (axis count differing from the signal rank, axes designating a missing or already described dimension, axes not holding real numbers, image axes not strictly monotonic) are reported instead of being guessed. Imported objects record their origin in the `experimental_source` metadata entry (file, group, signal and axes, `"."` for a dimension without axis, conversions). `.nxs` and `.nx5` files are proposed in HDF5 file dialogs. * Files that do not designate their data with suitable `NXdata` groups (no axes, several counters...) can be described by declarative data mappings (`sigimax.h5.mapping`): JSON-compatible descriptions of the signals and images of the groups having given attributes and paths, with their axes, uncertainties, labels and units, and a name pattern to repeat an object for each counter of a scan. Mapped objects are shown as extra nodes of the HDF5 browser and record the mapping in their `experimental_source`. Applications and plugins register mappings with `register_mapping` (and `unregister_mapping`); `NodeFactory.add_children_provider` lets them add such extra nodes. Blank units and labels are now ignored. diff --git a/doc/user_guide/hdf5_workspace.rst b/doc/user_guide/hdf5_workspace.rst index ded23c6..5c12e46 100644 --- a/doc/user_guide/hdf5_workspace.rst +++ b/doc/user_guide/hdf5_workspace.rst @@ -206,11 +206,11 @@ in a worker thread; previews of items selected meanwhile are not read, and stale results are not displayed. Closing a file or the browser dialog waits for the preview read to finish. -Ordinary 1D arrays and 2D images exceeding ``2**24`` elements are previewed with a -sampling step along each axis. The full data is still read on import. -NeXus and mapped objects, compound datasets and signals stored in 2D tables -still use complete objects for graphical previews, so this is not a universal -memory bound. +Ordinary 1D arrays and 2D images, NeXus and mapped objects exceeding ``2**24`` +elements are previewed with a sampling step along each axis. The full data is +still read on import. Compound datasets and signals stored in 2D tables still +use complete objects for graphical previews, so this is not a universal memory +bound. Numeric datasets with three or more dimensions are listed but not imported as complete objects. There is no graphical slice selector. Use diff --git a/sigimax/h5/nexus.py b/sigimax/h5/nexus.py index 7869314..475461f 100644 --- a/sigimax/h5/nexus.py +++ b/sigimax/h5/nexus.py @@ -6,7 +6,8 @@ A NeXus ``NXdata`` group designates its plottable signal and the axes of each signal dimension. :class:`NXdataNode` shows such a group as a single node of the HDF5 browser, which is imported as a signal (1D) or an image (2D) with its axes, -units and uncertainties. Only metadata is read until the object is created. +units and uncertainties. Only metadata is read until the object is created, and +previews of large groups read one element out of a step along each dimension. Supported conventions (NeXus data model, ``NXdata`` base class): @@ -24,6 +25,7 @@ from __future__ import annotations import dataclasses +import math import re import h5py @@ -32,6 +34,7 @@ from sigima.io.common.converters import convert_array_to_valid_dtype from sigima.objects import ( ImageObj, + SignalObj, create_image, create_signal, make_experimental_source, @@ -39,6 +42,7 @@ ) from sigimax.h5 import common, utils +from sigimax.h5.selection import preview_step __all__ = [ "NXAxis", @@ -316,18 +320,25 @@ def _is_decreasing(values: np.ndarray, name: str) -> bool: def _image_axis( - group: h5py.Group, axis: NXAxis | None, size: int + group: h5py.Group, axis: NXAxis | None, size: int, step: int = 1 ) -> tuple[np.ndarray, np.dtype, bool]: """Return image axis values in increasing order, the data type they were stored with, and True if they were reversed + Args: + group: NXdata group + axis: axis description, or None for a dimension without axis + size: number of points along the dimension + step: step between the returned values (to read a decimated preview) + Raises: ValueError: if axis values are not strictly monotonic """ if axis is None: - return np.arange(size, dtype=float), np.dtype(float), False + return np.arange(0, size, step, dtype=float), np.dtype(float), False values, dtype = _read_axis(group, axis) reverse = _is_decreasing(values, axis.name) + values = values[::step] return (values[::-1] if reverse else values), dtype, reverse @@ -412,8 +423,35 @@ def create_native_object(self): """Create signal or image object from the ``NXdata`` group""" if not self.is_supported(): return None + obj, conversions = self._build_object(1) + set_experimental_source(obj, self.make_source(conversions)) + return obj + + def create_preview_object(self, max_items: int): + """Return the native object, or a decimated one if the signal has more + than ``max_items`` elements (only the read elements of its axes are kept) + + Args: + max_items: maximum number of signal elements to read + """ + if not self.is_supported(): + return None + shape = self.nxdata.shape + if math.prod(shape) <= max_items: + return self.get_native_object() + obj, _conversions = self._build_object(preview_step(shape, max_items)) + return obj + + def _build_object(self, step: int) -> tuple[SignalObj | ImageObj, list[str]]: + """Create signal or image object from the ``NXdata`` group, reading one + element out of ``step`` along each dimension + + Returns: + Object and conversions applied to the source values + """ desc, group = self.nxdata, self.dset - data = group[desc.signal][()] + dset = group[desc.signal] + data = dset[(slice(None, None, step),) * dset.ndim] conversions = [ f"axis '{axis.name}': bin edges -> bin centers" for axis in desc.axes @@ -421,12 +459,12 @@ def create_native_object(self): ] if len(desc.shape) == 1: axis = desc.axes[0] - x = np.arange(data.size, dtype=float) + x = np.arange(0, desc.shape[0], step, dtype=float) if axis is not None: - x, _dtype = _read_axis(group, axis) + x = _read_axis(group, axis)[0][::step] dy = None if desc.errors is not None: - dy = np.asarray(group[desc.errors][()], dtype=float) + dy = np.asarray(group[desc.errors][::step], dtype=float) y = common.to_signal_data(data) obj = create_signal(self.object_title, x, y, dy=dy) obj.ylabel, obj.yunit = desc.label, desc.units @@ -436,8 +474,8 @@ def create_native_object(self): yaxis, xaxis = desc.axes ny, nx = desc.shape # PlotPy images need increasing coordinates: decreasing axes are reversed - xc, xdtype, xreversed = _image_axis(group, xaxis, nx) - yc, ydtype, yreversed = _image_axis(group, yaxis, ny) + xc, xdtype, xreversed = _image_axis(group, xaxis, nx, step) + yc, ydtype, yreversed = _image_axis(group, yaxis, ny, step) for axis, reverse, dim in ((xaxis, xreversed, 1), (yaxis, yreversed, 0)): if reverse: data = np.flip(data, axis=dim) @@ -452,8 +490,8 @@ def create_native_object(self): ) obj.zlabel, obj.zunit = desc.label, desc.units if _is_uniform(xc, xdtype) and _is_uniform(yc, ydtype): - dx = float(xc[-1] - xc[0]) / (nx - 1) if nx > 1 else 1.0 - dy = float(yc[-1] - yc[0]) / (ny - 1) if ny > 1 else 1.0 + dx = float(xc[-1] - xc[0]) / (xc.size - 1) if xc.size > 1 else 1.0 + dy = float(yc[-1] - yc[0]) / (yc.size - 1) if yc.size > 1 else 1.0 obj.set_uniform_coords(dx, dy, float(xc[0]), float(yc[0])) else: obj.set_coords(xc, yc) @@ -463,8 +501,7 @@ def create_native_object(self): obj.ylabel, obj.yunit = yaxis.label, yaxis.units if desc.errors is not None: conversions.append("uncertainties not imported (image)") - set_experimental_source(obj, self.make_source(conversions)) - return obj + return obj, conversions common.NODE_FACTORY.register(NXdataNode, is_generic=True) diff --git a/sigimax/tests/hdf5/test_h5_nexus.py b/sigimax/tests/hdf5/test_h5_nexus.py index bdb6771..120abd8 100644 --- a/sigimax/tests/hdf5/test_h5_nexus.py +++ b/sigimax/tests/hdf5/test_h5_nexus.py @@ -23,6 +23,7 @@ from sigimax.adapters_plotpy import create_adapter_from_object from sigimax.h5 import H5Importer from sigimax.h5.nexus import NXdataNode +from sigimax.h5.selection import preview_step pytestmark = pytest.mark.unit @@ -295,5 +296,42 @@ def build(h5file: h5py.File) -> None: nodes[f"/{name}"].get_native_object() +def test_nxdata_decimated_preview(tmp_path) -> None: + """Previews of large NXdata groups read one element out of a step, with the + matching axis values; small groups are previewed with the native object.""" + + def build(h5file: h5py.File) -> None: + image = _nxdata(h5file, "image", "data", ["y", "x"]) + image["data"] = np.arange(300 * 400, dtype=np.float32).reshape(300, 400) + image["x"] = np.linspace(0.0, 39.9, 400) + image["y"] = np.linspace(29.9, 0.0, 300) # Decreasing: reversed + spectrum = _nxdata(h5file, "spectrum", "counts", "energy") + spectrum["counts"] = np.arange(5000.0) + spectrum["counts_errors"] = np.full(5000, 0.5) + spectrum["energy"] = np.linspace(8.0, 9.0, 5001) # Bin edges + + with _nodes_of(tmp_path, build) as nodes: + preview = nodes["/image"].create_preview_object(1000) + step = preview_step((300, 400), 1000) + assert preview.data.shape == (25, 34) and preview.data.size <= 1000 + assert preview.is_uniform_coords + assert np.isclose(preview.dx, 0.1 * step) and preview.x0 == 0.0 + # Rows 0, step... 24 * step are read, then reversed with the Y axis + assert np.isclose(preview.y0, 29.9 - 24 * step * 0.1) + assert np.isclose(preview.dy, 0.1 * step) + assert preview.data[0, 0] == 24 * step * 400 + assert preview.data[-1, 1] == step + + node = nodes["/spectrum"] + preview = node.create_preview_object(1000) + native = node.get_native_object() + assert preview.y.size <= 1000 + step = preview_step((5000,), 1000) + assert np.allclose(preview.x, native.x[::step]) + assert np.array_equal(preview.y, native.y[::step]) + assert np.array_equal(preview.dy, native.dy[::step]) + assert node.create_preview_object(5000) is native + + if __name__ == "__main__": pytest.main([__file__, "-v"]) From a854d676d901373a3f21aba4ef50a749cdac0443 Mon Sep 17 00:00:00 2001 From: Pierre Raybaut <1311787+PierreRaybaut@users.noreply.github.com> Date: Fri, 23 Oct 2026 09:33:03 +0200 Subject: [PATCH 15/20] fix(h5): skip broken links when opening HDF5 files Broken soft links and external links to missing files (e.g. synchrotron master files without their detector files) are returned as None by h5py: opening such a file failed with an AttributeError, so it could be neither browsed nor imported. Broken links are now skipped, with a console warning telling the link target. Assisted-by: Claude Opus 5.5 --- sigimax/h5/common.py | 16 ++++++++- .../tests/hdf5/test_h5_lazy_classification.py | 33 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/sigimax/h5/common.py b/sigimax/h5/common.py index 336d23c..8ae5e4c 100644 --- a/sigimax/h5/common.py +++ b/sigimax/h5/common.py @@ -365,7 +365,21 @@ def icon_name(self): def collect_children(self, node_dict: dict[str, BaseNode]): """Construct tree""" - for dset in self.dset.values(): + for name in self.dset: + dset = self.dset.get(name) + if dset is None: + # Broken link: soft link to a missing object, or external link to + # a missing file (e.g. detector data of synchrotron master files) + link = self.dset.get(name, getlink=True) + if isinstance(link, h5py.ExternalLink): + reason = f"broken external link to '{link.filename}:{link.path}'" + elif isinstance(link, h5py.SoftLink): + reason = f"broken soft link to '{link.path}'" + else: + reason = "object cannot be opened" + path = f"{self.dset.name.rstrip('/')}/{name}" + print(f"Warning: Skipping '{path}': {reason}") + continue try: child_cls = NODE_FACTORY.get(dset) if child_cls is not None: diff --git a/sigimax/tests/hdf5/test_h5_lazy_classification.py b/sigimax/tests/hdf5/test_h5_lazy_classification.py index 04df27d..0338c28 100644 --- a/sigimax/tests/hdf5/test_h5_lazy_classification.py +++ b/sigimax/tests/hdf5/test_h5_lazy_classification.py @@ -130,6 +130,39 @@ def test_uint32_image_is_exact(h5fname: str) -> None: importer.close() +def test_broken_links_are_skipped(tmp_path, capsys) -> None: + """Files with broken soft or external links (e.g. synchrotron master files + without their detector files) are opened: the broken links are skipped.""" + fname = str(tmp_path / f"{uuid.uuid4()}.nxs") + with h5py.File(fname, "w") as h5file: + entry = h5file.create_group("entry") + entry["signal"] = np.arange(10.0) + entry["dangling_soft"] = h5py.SoftLink("/entry/missing") + entry["dangling_external"] = h5py.ExternalLink("missing.h5", "/data") + h5file["root_link"] = h5py.SoftLink("/missing") + detector = entry.create_group("detector") + detector.attrs.update({"NX_class": "NXdata", "signal": "frames"}) + detector["frames"] = h5py.ExternalLink("missing_detector.h5", "/frames") + importer = H5Importer(fname) + try: + ids = {node.id for node in importer.nodes} + assert {"/entry", "/entry/signal", "/entry/detector"} <= ids + assert not ids & { + "/entry/dangling_soft", + "/entry/dangling_external", + "/root_link", + "/entry/detector/frames", + } + assert importer.get("/entry/signal").get_native_object().y.size == 10 + assert not importer.get("/entry/detector").is_supported() + finally: + importer.close() + output = capsys.readouterr().out + assert "'/entry/dangling_soft': broken soft link to '/entry/missing'" in output + assert "'/root_link': broken soft link" in output + assert "broken external link to 'missing.h5:/data'" in output + + def test_unregister_node_class(h5fname: str) -> None: """A node class registered by an application can be removed again.""" From 5b720f6ad0c6d28fa4acce5689eab99de14081f8 Mon Sep 17 00:00:00 2001 From: Pierre Raybaut <1311787+PierreRaybaut@users.noreply.github.com> Date: Fri, 23 Oct 2026 09:33:04 +0200 Subject: [PATCH 16/20] perf(h5): bisect the decimated preview step The step was incremented one by one from a geometric estimate, which took about 100 000 iterations for very elongated arrays (e.g. 3 x 1e8). The smallest step is now found by bisection (the view size decreases with the step), with the same results. Assisted-by: Claude Opus 5.5 --- sigimax/h5/selection.py | 20 ++++++++++++++----- .../tests/hdf5/test_h5_lazy_classification.py | 9 ++++++++- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/sigimax/h5/selection.py b/sigimax/h5/selection.py index d87aa25..c060d26 100644 --- a/sigimax/h5/selection.py +++ b/sigimax/h5/selection.py @@ -69,11 +69,21 @@ def preview_step(shape: tuple[int, ...], max_items: int) -> int: """ if max_items < 1: raise ValueError("max_items must be positive") - size = math.prod(shape) - step = max(1, math.floor((size / max_items) ** (1 / max(len(shape), 1)))) - while math.prod(math.ceil(dim / step) for dim in shape) > max_items: - step += 1 - return step + + def view_size(step: int) -> int: + """Return the number of elements of the view read with step""" + return math.prod(math.ceil(dim / step) for dim in shape) + + # The view size decreases with the step: bisect for the smallest step (with + # one element per axis at the largest dimension, the view fits) + low, high = 1, max((1, *shape)) + while low < high: + middle = (low + high) // 2 + if view_size(middle) > max_items: + low = middle + 1 + else: + high = middle + return low def read_selection( diff --git a/sigimax/tests/hdf5/test_h5_lazy_classification.py b/sigimax/tests/hdf5/test_h5_lazy_classification.py index 0338c28..666b2ee 100644 --- a/sigimax/tests/hdf5/test_h5_lazy_classification.py +++ b/sigimax/tests/hdf5/test_h5_lazy_classification.py @@ -228,14 +228,21 @@ def test_read_selection(h5fname: str, full_reads: list) -> None: ((101,), 100, 2), ((4096, 4096), 2**24, 1), ((300, 400), 1000, 12), + ((3, 100_000_000), 1000, 100_000), + ((2, 10**10), 2**24, 597), + ((0, 400), 10, 1), + ((), 1, 1), ], - ids=["fits", "1d", "2d-fits", "2d"], + ids=["fits", "1d", "2d-fits", "2d", "asymmetric", "huge", "empty", "scalar"], ) def test_preview_step(shape: tuple, max_items: int, step: int) -> None: """The preview step is the smallest one keeping the view under the limit.""" assert preview_step(shape, max_items) == step sizes = [len(range(0, dim, step)) for dim in shape] assert np.prod(sizes) <= max_items + if step > 1: + sizes = [len(range(0, dim, step - 1)) for dim in shape] + assert np.prod(sizes) > max_items def test_preview_is_decimated(h5fname: str, full_reads: list) -> None: From 8f1e8fd0b0ba1c90f95d716d4be81604ec47dbfd Mon Sep 17 00:00:00 2001 From: Pierre Raybaut <1311787+PierreRaybaut@users.noreply.github.com> Date: Fri, 23 Oct 2026 09:33:05 +0200 Subject: [PATCH 17/20] fix(h5browser): clear the preview when an unsupported node is selected Selecting a node that cannot be previewed (a group, for example) while a preview was read showed that preview for the newly selected node. The pending preview is now discarded and the plot cleared. The pending preview is also discarded, without waiting, before the node description is read: the description no longer waits for the read of a preview that is not needed anymore. Assisted-by: Claude Opus 5.5 --- sigimax/tests/hdf5/test_h5browser_preview.py | 51 +++++++++++++++++++- sigimax/widgets/h5browser.py | 26 ++++++++-- 2 files changed, 71 insertions(+), 6 deletions(-) diff --git a/sigimax/tests/hdf5/test_h5browser_preview.py b/sigimax/tests/hdf5/test_h5browser_preview.py index 765535a..8c1b67a 100644 --- a/sigimax/tests/hdf5/test_h5browser_preview.py +++ b/sigimax/tests/hdf5/test_h5browser_preview.py @@ -5,8 +5,9 @@ ------------------------- Previews are read in a worker thread: the last selected node is shown, the -previews of nodes selected before are discarded without being read, and closing -a file or the browser dialog waits for the preview being read. +previews of nodes selected before are discarded without being read, selecting a +node that cannot be previewed clears the plot, and closing a file or the browser +dialog waits for the preview being read. """ from __future__ import annotations @@ -20,6 +21,7 @@ import pytest from guidata.qthelpers import qt_app_context from plotpy.items import BaseImageItem, CurveItem +from qtpy import QtCore as QC from qtpy import QtWidgets as QW from sigima.objects import create_signal @@ -129,6 +131,51 @@ def test_stale_previews_are_not_read() -> None: preview.close() +def _find_item(tree, node_id: str) -> QW.QTreeWidgetItem: + """Return the tree item of a node.""" + for item in tree.find_all_items(): + if item.data(0, QC.Qt.UserRole) == node_id: + return item + raise KeyError(node_id) + + +def _curves(plot) -> list[CurveItem]: + """Return the curves shown in a plot.""" + return [item for item in plot.get_items() if isinstance(item, CurveItem)] + + +def test_unsupported_node_clears_preview(tmp_path) -> None: + """Selecting a node that cannot be previewed clears the plot, and the preview + being read for the node selected before is not shown.""" + fname = str(tmp_path / f"{uuid.uuid4()}.h5") + with h5py.File(fname, "w") as h5file: + h5file.create_group("group")["signal"] = np.arange(10.0) + with qt_app_context(): + browser = H5Browser() + browser.open_file(fname) + plot = browser.plotpreview.curvewidget.get_plot() + browser.tree.clicked(_find_item(browser.tree, "/group/signal")) + _wait_for_item(plot, CurveItem) + browser.tree.clicked(_find_item(browser.tree, "/group")) + assert not _curves(plot) + assert browser.groupandattrs.group.table.item(0, 0).text() == "/group" + + release = threading.Event() + node = _SpyNode(release) + browser.plotpreview.update_plot_preview(node) + assert node.started.wait(10.0) + browser.tree.clicked(_find_item(browser.tree, "/group")) + release.set() + _wait_until(lambda: node.done) + deadline = time.monotonic() + 0.2 + while time.monotonic() < deadline: + QW.QApplication.processEvents() + time.sleep(0.01) + assert not _curves(plot), "Discarded preview must not be shown" + browser.cleanup() + browser.close() + + @pytest.mark.parametrize("method", ["accept", "reject"]) def test_dialog_waits_for_preview(method: str) -> None: """Closing the dialog waits for the preview being read, so that the caller diff --git a/sigimax/widgets/h5browser.py b/sigimax/widgets/h5browser.py index e69439e..c9d28e2 100644 --- a/sigimax/widgets/h5browser.py +++ b/sigimax/widgets/h5browser.py @@ -523,13 +523,26 @@ def cleanup(self) -> None: for widget in (self.imagewidget, self.curvewidget): widget.get_plot().del_all_items() - def wait(self) -> None: - """Discard the pending previews and wait until no data is read - (before closing HDF5 files or importing objects)""" + def discard(self) -> None: + """Discard the pending previews, without waiting: the preview being read, + if any, is not shown and the previews not started yet are not read""" self.__generation += 1 if self.__pending is not None: self.__pending.cancel() self.__pending = None + + def clear_preview(self) -> None: + """Discard the pending previews and clear the plot, without waiting (for + nodes that cannot be previewed)""" + self.discard() + plot: BasePlot = self.currentWidget().get_plot() + plot.del_all_items() + plot.replot() + + def wait(self) -> None: + """Discard the pending previews and wait until no data is read + (before closing HDF5 files or importing objects)""" + self.discard() if self.__executor is not None: # The single worker runs calls in order: once this one is done, the # previous ones are done or were discarded @@ -928,9 +941,14 @@ def __item_selected_on_tree(self, item: QW.QTreeWidgetItem) -> None: """ # View the selected item node = self.get_node(item) + # Discard the previous preview first: its pending read is not started, + # so that reading the node description below does not wait for it + self.plotpreview.discard() + self.groupandattrs.update_from_node(node) if node.is_supported(): self.plotpreview.update_plot_preview(node) - self.groupandattrs.update_from_node(node) + else: + self.plotpreview.clear_preview() # Update the file selector combo box with block_signals(self.selector.combo): # Avoid triggering current file changed signal, which would result in From f39fd67e2bf32c737e88e0a29da76bb17c01f158 Mon Sep 17 00:00:00 2001 From: Pierre Raybaut <1311787+PierreRaybaut@users.noreply.github.com> Date: Fri, 23 Oct 2026 09:33:05 +0200 Subject: [PATCH 18/20] refactor(h5): address Pylint messages of the HDF5 import modules - Ignore NumPy's dynamically populated `finfo` class, as Sigima does. - Split the compound branch of `GenericArrayNode.numeric_layout` and the signal/image construction of `NXdataNode._build_object`. - Share `NXdataNode.make_source` with mapped nodes (`SOURCE_KIND`, `source_details`) instead of duplicating it. - Tests: use `usefixtures` for the mapping registration fixture and `contextlib.closing` for the importer. Assisted-by: Claude Opus 5.5 --- .pylintrc | 2 + sigimax/h5/generic.py | 23 ++- sigimax/h5/mapping.py | 24 +--- sigimax/h5/nexus.py | 147 +++++++++++++------- sigimax/tests/hdf5/test_h5_mapping.py | 3 +- sigimax/tests/hdf5/test_h5_signal_dtypes.py | 6 +- 6 files changed, 131 insertions(+), 74 deletions(-) diff --git a/.pylintrc b/.pylintrc index e481c44..8977422 100644 --- a/.pylintrc +++ b/.pylintrc @@ -4,6 +4,8 @@ max-line-length=88 [TYPECHECK] +# NumPy's floating-point limits are populated dynamically. +ignored-classes=finfo ignored-modules=qtpy.QtWidgets,qtpy.QtCore,qtpy.QtGui,plotpy._scaler [MESSAGES CONTROL] diff --git a/sigimax/h5/generic.py b/sigimax/h5/generic.py index 4a729e3..53f0c2a 100644 --- a/sigimax/h5/generic.py +++ b/sigimax/h5/generic.py @@ -303,10 +303,25 @@ def numeric_layout(cls, dset) -> tuple[tuple[int, ...], np.dtype] | None: dtype, shape = dset.dtype, dset.shape if not shape: return None - if dtype.names is None: - if utils.is_supported_num_dtype(dset) and len(shape) in (1, 2): - return shape, dtype - return None + if dtype.names is not None: + return cls._compound_layout(dset) + if utils.is_supported_num_dtype(dset) and len(shape) in (1, 2): + return shape, dtype + return None + + @classmethod + def _compound_layout(cls, dset) -> tuple[tuple[int, ...], np.dtype] | None: + """Return shape and dtype of the numeric array read from a compound + dataset (see :meth:`numeric_layout`). + + Args: + dset: HDF5 compound dataset + + Returns: + Shape and dtype of the 1D or 2D numeric array, or None if the dataset + does not provide one + """ + dtype, shape = dset.dtype, dset.shape fields = {name: dtype.fields[name][0] for name in dtype.names} numeric = [ name for name, fdt in fields.items() if np.issubdtype(fdt, np.number) diff --git a/sigimax/h5/mapping.py b/sigimax/h5/mapping.py index 0398613..8fc3b61 100644 --- a/sigimax/h5/mapping.py +++ b/sigimax/h5/mapping.py @@ -48,7 +48,6 @@ from typing import Any import h5py -from sigima.objects import make_experimental_source from sigimax.h5 import common, nexus @@ -415,6 +414,8 @@ class MappedDataNode(nexus.NXdataNode): nxdata: object description """ + SOURCE_KIND = "hdf5" + def __init__( self, h5file, @@ -443,23 +444,10 @@ def name(self): """Return node name""" return self.object_name - def make_source(self, conversions: list[str]) -> dict: - """Return the experimental source of the imported object - - Dimensions without axis are recorded as ``"."``, as in NeXus ``@axes``. - """ - desc = self.nxdata - return make_experimental_source( - "hdf5", - self.h5file.filename, - entry=desc.path, - selection={ - "signal": desc.signal, - "axes": [axis.name if axis else "." for axis in desc.axes], - }, - conversions=conversions or None, - details={"mapping": self.mapping.name}, - ) + def source_details(self) -> dict: + """Return the details of the experimental source of the imported object: + the name of the data mapping""" + return {"mapping": self.mapping.name} def _mapped_children(node: common.GroupNode) -> list[MappedDataNode]: diff --git a/sigimax/h5/nexus.py b/sigimax/h5/nexus.py index 475461f..168aff9 100644 --- a/sigimax/h5/nexus.py +++ b/sigimax/h5/nexus.py @@ -342,6 +342,27 @@ def _image_axis( return (values[::-1] if reverse else values), dtype, reverse +def _set_image_coords( + obj: ImageObj, + xaxis: tuple[np.ndarray, np.dtype], + yaxis: tuple[np.ndarray, np.dtype], +) -> None: + """Set image coordinates: uniform if axes are evenly spaced, else non-uniform + + Args: + obj: image object + xaxis: X axis values in increasing order, and their stored data type + yaxis: Y axis values in increasing order, and their stored data type + """ + (xc, xdtype), (yc, ydtype) = xaxis, yaxis + if _is_uniform(xc, xdtype) and _is_uniform(yc, ydtype): + dx = float(xc[-1] - xc[0]) / (xc.size - 1) if xc.size > 1 else 1.0 + dy = float(yc[-1] - yc[0]) / (yc.size - 1) if yc.size > 1 else 1.0 + obj.set_uniform_coords(dx, dy, float(xc[0]), float(yc[0])) + else: + obj.set_coords(xc, yc) + + class NXdataNode(common.GroupNode): """Node for NeXus ``NXdata`` groups, imported as a signal or an image @@ -351,6 +372,9 @@ class NXdataNode(common.GroupNode): nxdata: signal description (default: described from ``NXdata`` attributes) """ + #: Kind of the experimental source of the imported objects + SOURCE_KIND = "nexus" + def __init__(self, h5file, dname, nxdata: NXdataDescription | None = None): super().__init__(h5file, dname) self.nxdata = describe_nxdata(self.dset) if nxdata is None else nxdata @@ -402,6 +426,11 @@ def description(self): """Return node description""" return self.text + def source_details(self) -> dict | None: + """Return the details of the experimental source of the imported object + (None: no details)""" + return None + def make_source(self, conversions: list[str]) -> dict: """Return the experimental source of the imported object @@ -409,7 +438,7 @@ def make_source(self, conversions: list[str]) -> dict: """ desc = self.nxdata return make_experimental_source( - "nexus", + self.SOURCE_KIND, self.h5file.filename, entry=desc.path, selection={ @@ -417,6 +446,7 @@ def make_source(self, conversions: list[str]) -> dict: "axes": [axis.name if axis else "." for axis in desc.axes], }, conversions=conversions or None, + details=self.source_details(), ) def create_native_object(self): @@ -449,8 +479,8 @@ def _build_object(self, step: int) -> tuple[SignalObj | ImageObj, list[str]]: Returns: Object and conversions applied to the source values """ - desc, group = self.nxdata, self.dset - dset = group[desc.signal] + desc = self.nxdata + dset = self.dset[desc.signal] data = dset[(slice(None, None, step),) * dset.ndim] conversions = [ f"axis '{axis.name}': bin edges -> bin centers" @@ -458,50 +488,73 @@ def _build_object(self, step: int) -> tuple[SignalObj | ImageObj, list[str]]: if axis is not None and axis.edges ] if len(desc.shape) == 1: - axis = desc.axes[0] - x = np.arange(0, desc.shape[0], step, dtype=float) - if axis is not None: - x = _read_axis(group, axis)[0][::step] - dy = None - if desc.errors is not None: - dy = np.asarray(group[desc.errors][::step], dtype=float) - y = common.to_signal_data(data) - obj = create_signal(self.object_title, x, y, dy=dy) - obj.ylabel, obj.yunit = desc.label, desc.units - if axis is not None: - obj.xlabel, obj.xunit = axis.label, axis.units - else: - yaxis, xaxis = desc.axes - ny, nx = desc.shape - # PlotPy images need increasing coordinates: decreasing axes are reversed - xc, xdtype, xreversed = _image_axis(group, xaxis, nx, step) - yc, ydtype, yreversed = _image_axis(group, yaxis, ny, step) - for axis, reverse, dim in ((xaxis, xreversed, 1), (yaxis, yreversed, 0)): - if reverse: - data = np.flip(data, axis=dim) - conversions.append( - f"axis '{axis.name}': reversed to increasing order" - ) - obj = create_image( - self.object_title, - convert_array_to_valid_dtype( - np.ascontiguousarray(data), ImageObj.VALID_DTYPES - ), - ) - obj.zlabel, obj.zunit = desc.label, desc.units - if _is_uniform(xc, xdtype) and _is_uniform(yc, ydtype): - dx = float(xc[-1] - xc[0]) / (xc.size - 1) if xc.size > 1 else 1.0 - dy = float(yc[-1] - yc[0]) / (yc.size - 1) if yc.size > 1 else 1.0 - obj.set_uniform_coords(dx, dy, float(xc[0]), float(yc[0])) - else: - obj.set_coords(xc, yc) - if xaxis is not None: - obj.xlabel, obj.xunit = xaxis.label, xaxis.units - if yaxis is not None: - obj.ylabel, obj.yunit = yaxis.label, yaxis.units - if desc.errors is not None: - conversions.append("uncertainties not imported (image)") - return obj, conversions + return self._build_signal(data, step), conversions + return self._build_image(data, step, conversions), conversions + + def _build_signal(self, data: np.ndarray, step: int) -> SignalObj: + """Create the signal object of a 1D ``NXdata`` group + + Args: + data: signal values, read with step + step: step between the read elements + + Returns: + Signal object + """ + desc, group = self.nxdata, self.dset + axis = desc.axes[0] + x = np.arange(0, desc.shape[0], step, dtype=float) + if axis is not None: + x = _read_axis(group, axis)[0][::step] + dy = None + if desc.errors is not None: + dy = np.asarray(group[desc.errors][::step], dtype=float) + y = common.to_signal_data(data) + obj = create_signal(self.object_title, x, y, dy=dy) + obj.ylabel, obj.yunit = desc.label, desc.units + if axis is not None: + obj.xlabel, obj.xunit = axis.label, axis.units + return obj + + def _build_image( + self, data: np.ndarray, step: int, conversions: list[str] + ) -> ImageObj: + """Create the image object of a 2D ``NXdata`` group + + Args: + data: image values, read with step along both dimensions + step: step between the read elements + conversions: conversions applied to the source values, completed with + the ones applied to the image + + Returns: + Image object + """ + desc, group = self.nxdata, self.dset + yaxis, xaxis = desc.axes + ny, nx = desc.shape + # PlotPy images need increasing coordinates: decreasing axes are reversed + xc, xdtype, xreversed = _image_axis(group, xaxis, nx, step) + yc, ydtype, yreversed = _image_axis(group, yaxis, ny, step) + for axis, reverse, dim in ((xaxis, xreversed, 1), (yaxis, yreversed, 0)): + if reverse: + data = np.flip(data, axis=dim) + conversions.append(f"axis '{axis.name}': reversed to increasing order") + obj = create_image( + self.object_title, + convert_array_to_valid_dtype( + np.ascontiguousarray(data), ImageObj.VALID_DTYPES + ), + ) + obj.zlabel, obj.zunit = desc.label, desc.units + _set_image_coords(obj, (xc, xdtype), (yc, ydtype)) + if xaxis is not None: + obj.xlabel, obj.xunit = xaxis.label, xaxis.units + if yaxis is not None: + obj.ylabel, obj.yunit = yaxis.label, yaxis.units + if desc.errors is not None: + conversions.append("uncertainties not imported (image)") + return obj common.NODE_FACTORY.register(NXdataNode, is_generic=True) diff --git a/sigimax/tests/hdf5/test_h5_mapping.py b/sigimax/tests/hdf5/test_h5_mapping.py index 7d60628..0398765 100644 --- a/sigimax/tests/hdf5/test_h5_mapping.py +++ b/sigimax/tests/hdf5/test_h5_mapping.py @@ -191,7 +191,8 @@ def test_describe_mapping(h5fname: str) -> None: assert not describe_mapping(counters, h5file["entry"]) -def test_mapped_nodes(h5fname: str, mappings) -> None: +@pytest.mark.usefixtures("mappings") +def test_mapped_nodes(h5fname: str) -> None: """Mapped objects are imported with their axes, labels, units and origin""" importer = H5Importer(h5fname) try: diff --git a/sigimax/tests/hdf5/test_h5_signal_dtypes.py b/sigimax/tests/hdf5/test_h5_signal_dtypes.py index 43e4768..4316154 100644 --- a/sigimax/tests/hdf5/test_h5_signal_dtypes.py +++ b/sigimax/tests/hdf5/test_h5_signal_dtypes.py @@ -11,6 +11,7 @@ from __future__ import annotations +import contextlib import uuid import warnings @@ -47,11 +48,8 @@ def fixture_nodes(tmp_path): group.attrs["NX_class"] = "NXdata" group.attrs["signal"] = "data" group["data"] = data - importer = H5Importer(fname) - try: + with contextlib.closing(H5Importer(fname)) as importer: yield {node.id: node for node in importer.nodes} - finally: - importer.close() @pytest.mark.parametrize("path", ["/complex", "/nx_complex"]) From bc5ff14f3ee668a072c218d5608d7027b65c0b70 Mon Sep 17 00:00:00 2001 From: Pierre Raybaut <1311787+PierreRaybaut@users.noreply.github.com> Date: Fri, 23 Oct 2026 09:33:06 +0200 Subject: [PATCH 19/20] docs: document broken links and preview clearing, complete French catalogs - Release notes and HDF5 guide: broken links are skipped, and selecting a node that cannot be previewed clears the preview. - French translations: new and updated entries, the Main window and Qt helpers entries of the 1.1 release notes, and the Sigima >= 1.4.0 requirement of the installation guide. Assisted-by: Claude Opus 5.5 --- .../LC_MESSAGES/release_notes/release_1.01.po | 25 ++++++++----------- doc/locale/fr/LC_MESSAGES/requirements.po | 7 ------ .../LC_MESSAGES/user_guide/hdf5_workspace.po | 12 +++------ .../fr/LC_MESSAGES/user_guide/installation.po | 11 ++++---- doc/release_notes/release_1.01.md | 3 ++- doc/user_guide/hdf5_workspace.rst | 6 +++-- 6 files changed, 26 insertions(+), 38 deletions(-) diff --git a/doc/locale/fr/LC_MESSAGES/release_notes/release_1.01.po b/doc/locale/fr/LC_MESSAGES/release_notes/release_1.01.po index 1bde036..1df3c75 100644 --- a/doc/locale/fr/LC_MESSAGES/release_notes/release_1.01.po +++ b/doc/locale/fr/LC_MESSAGES/release_notes/release_1.01.po @@ -6,18 +6,11 @@ #, fuzzy msgid "" msgstr "" -"Project-Id-Version: PROJECT VERSION\n" -"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2026-10-22 21:34+0200\n" -"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" -"Last-Translator: FULL NAME \n" "Language: fr\n" -"Language-Team: fr \n" "Plural-Forms: nplurals=2; plural=(n > 1);\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" -"Generated-By: Babel 2.18.0\n" msgid "Version 1.1.0" msgstr "Version 1.1.0" @@ -38,22 +31,22 @@ msgid "Canonical annotation identifiers, metadata, extensions, and persistent lo msgstr "Les identifiants d'annotation canoniques, les métadonnées, les extensions et l'état de verrouillage persistant sont conservés lors des allers-retours d'édition PlotPy. Les entrées d'annotation opaques spécifiques à l'application continuent de coexister avec les annotations graphiques." msgid "Main window" -msgstr "" +msgstr "Fenêtre principale" msgid "With Qt 5, the dock widget sizes saved on exit are now restored at startup: the window is sized and laid out before its state is restored, so tabified docks no longer shrink to the default window size and leave the extra width to the central widget." -msgstr "" +msgstr "Avec Qt 5, les tailles des panneaux ancrables enregistrées à la fermeture sont désormais restaurées au démarrage : la fenêtre est dimensionnée et mise en page avant la restauration de son état, si bien que les panneaux regroupés en onglets ne se réduisent plus à la taille par défaut de la fenêtre en laissant la largeur supplémentaire au widget central." msgid "Closing the main window (unless it is only hidden, see `hide_on_close`) now releases it as the instance returned by `get_instance`, which creates a new window afterwards: the closed window, with its widgets and data, is no longer kept in memory until another main window is created." msgstr "La fermeture de la fenêtre principale (sauf si elle est seulement masquée, voir `hide_on_close`) la libère désormais en tant qu'instance renvoyée par `get_instance`, qui crée ensuite une nouvelle fenêtre : la fenêtre fermée, avec ses widgets et ses données, n'est plus conservée en mémoire jusqu'à la création d'une autre fenêtre principale." msgid "Qt helpers" -msgstr "" +msgstr "Utilitaires Qt" msgid "`block_signals` now restores the previous signal-blocking state of the widget and of its children instead of unblocking them unconditionally: nested blocking contexts and widgets already blocked by the caller keep their state." -msgstr "" +msgstr "`block_signals` restaure désormais l'état précédent de blocage des signaux du widget et de ses enfants au lieu de les débloquer systématiquement : les contextes de blocage imbriqués et les widgets déjà bloqués par l'appelant conservent leur état." msgid "`sigimax_app_context` no longer schedules the unattended/screenshot close timer when the Qt event loop is not executed (`exec_loop=False`): the pending timer previously closed unrelated top-level widgets created later, such as the window of a subsequent test." -msgstr "" +msgstr "`sigimax_app_context` ne programme plus le minuteur de fermeture des modes sans surveillance et capture d'écran lorsque la boucle d'événements Qt n'est pas exécutée (`exec_loop=False`) : ce minuteur en attente fermait auparavant des widgets de premier niveau sans rapport créés ensuite, comme la fenêtre d'un test suivant." msgid "Curve fitting dialogs" msgstr "Fenêtres d'ajustement de courbes" @@ -76,8 +69,8 @@ msgstr "Import HDF5" msgid "Browsing an HDF5 file no longer reads its numeric datasets entirely just to classify them: generic nodes are classified and described from the dataset shape and data type, and value previews read only the first elements (`TEXT_PREVIEW_MAX_ITEMS`). Compound datasets mixing numeric and string fields are checked on their first records only. Full data is read when creating an imported object or an undecimated graphical preview." msgstr "Parcourir un fichier HDF5 ne lit plus ses jeux de données numériques en entier simplement pour les classifier : les nœuds génériques sont classifiés et décrits à partir de la forme et du type des jeux de données, et les aperçus de valeurs ne lisent que les premiers éléments (`TEXT_PREVIEW_MAX_ITEMS`). Les jeux de données composites mêlant champs numériques et textuels ne sont vérifiés que sur leurs premiers enregistrements. Les données complètes sont lues lors de la création d'un objet importé ou d'un aperçu graphique non sous-échantillonné." -msgid "The HDF5 browser reads previews in a worker thread, so that it stays responsive while a large dataset is read: only the last selected node is shown, the previews of nodes selected meanwhile are not read, and closing a file or the browser dialog waits for the preview being read (imported objects are then read without concurrent access to the file). Ordinary 1D arrays and 2D images, NeXus and mapped objects with more than `PlotPreview.MAX_ITEMS` (2²⁴) elements are previewed with a step along each axis. Compound datasets and signals stored in 2D tables still use complete objects for graphical previews; this is not a universal memory bound." -msgstr "L'explorateur HDF5 lit les aperçus dans un fil d'exécution de travail pour rester réactif pendant la lecture d'un grand jeu de données : seul le dernier nœud sélectionné est affiché, les aperçus des nœuds sélectionnés entre-temps ne sont pas lus, et la fermeture d'un fichier ou du dialogue de l'explorateur attend la fin de la lecture de l'aperçu (les objets importés sont ensuite lus sans accès concurrent au fichier). Les tableaux 1D ordinaires et les images 2D, les objets NeXus et décrits par un profil de plus de `PlotPreview.MAX_ITEMS` (2²⁴) éléments sont prévisualisés avec un pas le long de chaque axe. Les jeux de données composites et les signaux stockés dans des tableaux 2D utilisent encore des objets complets pour leurs aperçus graphiques ; ce n'est pas une borne mémoire universelle." +msgid "The HDF5 browser reads previews in a worker thread, so that it stays responsive while a large dataset is read: only the last selected node is shown, the previews of nodes selected meanwhile are not read, selecting a node that cannot be previewed (a group, for example) clears the preview, and closing a file or the browser dialog waits for the preview being read (imported objects are then read without concurrent access to the file). Ordinary 1D arrays and 2D images, NeXus and mapped objects with more than `PlotPreview.MAX_ITEMS` (2²⁴) elements are previewed with a step along each axis. Compound datasets and signals stored in 2D tables still use complete objects for graphical previews; this is not a universal memory bound." +msgstr "L'explorateur HDF5 lit les aperçus dans un fil d'exécution de travail pour rester réactif pendant la lecture d'un grand jeu de données : seul le dernier nœud sélectionné est affiché, les aperçus des nœuds sélectionnés entre-temps ne sont pas lus, la sélection d'un nœud qui ne peut pas être prévisualisé (un groupe, par exemple) efface l'aperçu, et la fermeture d'un fichier ou du dialogue de l'explorateur attend la fin de la lecture de l'aperçu (les objets importés sont ensuite lus sans accès concurrent au fichier). Les tableaux 1D ordinaires et les images 2D, les objets NeXus et décrits par un profil de plus de `PlotPreview.MAX_ITEMS` (2²⁴) éléments sont prévisualisés avec un pas le long de chaque axe. Les jeux de données composites et les signaux stockés dans des tableaux 2D utilisent encore des objets complets pour leurs aperçus graphiques ; ce n'est pas une borne mémoire universelle." msgid "Numeric datasets with three dimensions or more (image stacks, spectral cubes...) are now listed in the HDF5 browser, with their shape and type. They are not imported as a whole, and there is no graphical slice selector: developers can read their 1D or 2D views with `sigimax.h5.selection.read_selection`, which reads only the selected elements (`NDSelection`: displayed axes, possibly transposed, and an index along each other axis)." msgstr "Les jeux de données numériques à trois dimensions ou plus (piles d'images, cubes spectraux...) sont désormais listés dans l'explorateur HDF5 avec leur forme et leur type. Ils ne sont pas importés en entier, et il n'existe pas de sélecteur graphique de coupes : les développeurs peuvent lire leurs vues 1D ou 2D avec `sigimax.h5.selection.read_selection`, qui ne lit que les éléments sélectionnés (`NDSelection` : axes affichés, éventuellement transposés, et un indice sur chaque autre axe)." @@ -100,6 +93,9 @@ msgstr "Les jeux de données 1D de nombres complexes en simple précision (`comp msgid "`NodeFactory.unregister` removes a node class and its post triggers, so that an application plugin can remove the node classes it registered when it is unloaded." msgstr "`NodeFactory.unregister` retire une classe de nœud et ses déclencheurs de post-traitement, afin qu'un plugin applicatif puisse retirer les classes de nœuds qu'il a enregistrées lors de son déchargement." +msgid "Files containing broken links (soft links to a missing object, or external links to a missing file, as in synchrotron master files whose detector files are not available) can now be opened: the broken links are skipped with a warning in the console, instead of preventing the whole file from being browsed or imported." +msgstr "Les fichiers contenant des liens rompus (liens symboliques vers un objet absent, ou liens externes vers un fichier absent, comme dans les fichiers maîtres de synchrotron dont les fichiers de détecteur ne sont pas disponibles) peuvent désormais être ouverts : les liens rompus sont ignorés avec un avertissement dans la console, au lieu d'empêcher l'exploration ou l'import de tout le fichier." + msgid "See the [HDF5 guide](../user_guide/hdf5_workspace.rst) for a complete JSON mapping and registration example, NeXus import limits and multidimensional selection examples." msgstr "Le [guide HDF5](../user_guide/hdf5_workspace.rst) fournit un exemple complet de profil JSON et d'enregistrement, les limites de l'import NeXus et des exemples de sélections multidimensionnelles." @@ -108,3 +104,4 @@ msgstr "Exigences" msgid "Sigima 1.4.0 or later is required: the HDF5 import modules use its experimental-source API, and integer conversions use its precision-preserving converter." msgstr "Sigima 1.4.0 ou ultérieur est requis : les modules d'import HDF5 utilisent son API d'origine expérimentale, et les conversions d'entiers utilisent son convertisseur préservant la précision." + diff --git a/doc/locale/fr/LC_MESSAGES/requirements.po b/doc/locale/fr/LC_MESSAGES/requirements.po index 52df194..85bd5b2 100644 --- a/doc/locale/fr/LC_MESSAGES/requirements.po +++ b/doc/locale/fr/LC_MESSAGES/requirements.po @@ -6,18 +6,11 @@ #, fuzzy msgid "" msgstr "" -"Project-Id-Version: PROJECT VERSION\n" -"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2026-10-22 21:34+0200\n" -"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" -"Last-Translator: FULL NAME \n" "Language: fr\n" -"Language-Team: fr \n" "Plural-Forms: nplurals=2; plural=(n > 1);\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" -"Generated-By: Babel 2.18.0\n" msgid "The `sigimax` package requires the following Python modules:" msgstr "Le paquet `sigimax` nécessite les modules Python suivants :" diff --git a/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po b/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po index c6b8949..c65ecf2 100644 --- a/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po +++ b/doc/locale/fr/LC_MESSAGES/user_guide/hdf5_workspace.po @@ -6,18 +6,11 @@ #, fuzzy msgid "" msgstr "" -"Project-Id-Version: PROJECT VERSION\n" -"Report-Msgid-Bugs-To: EMAIL@ADDRESS\n" -"POT-Creation-Date: 2026-10-22 21:34+0200\n" -"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" -"Last-Translator: FULL NAME \n" "Language: fr\n" -"Language-Team: fr \n" "Plural-Forms: nplurals=2; plural=(n > 1);\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=utf-8\n" "Content-Transfer-Encoding: 8bit\n" -"Generated-By: Babel 2.18.0\n" msgid "HDF5 Workspace" msgstr "Espace de travail HDF5" @@ -128,8 +121,8 @@ msgstr "Le registre est commun au processus ; les fichiers JSON ne sont pas déc msgid "Large Files and Multidimensional Views" msgstr "Grands fichiers et vues multidimensionnelles" -msgid "Dataset classification uses shape and type metadata; mixed compound datasets are checked on their first records. Text previews are bounded to 100 elements, and attributes above 10 000 elements are skipped. Graphical previews are read in a worker thread; previews of items selected meanwhile are not read, and stale results are not displayed. Closing a file or the browser dialog waits for the preview read to finish." -msgstr "La classification des jeux de données utilise les métadonnées de forme et de type ; les jeux de données composites mixtes sont vérifiés sur leurs premiers enregistrements. Les aperçus textuels sont limités à 100 éléments, et les attributs de plus de 10 000 éléments sont ignorés. Les aperçus graphiques sont lus dans un fil d'exécution de travail ; les aperçus des éléments sélectionnés entre-temps ne sont pas lus, et les résultats périmés ne sont pas affichés. La fermeture d'un fichier ou du dialogue de l'explorateur attend la fin de la lecture de l'aperçu." +msgid "Dataset classification uses shape and type metadata; mixed compound datasets are checked on their first records. Text previews are bounded to 100 elements, and attributes above 10 000 elements are skipped. Graphical previews are read in a worker thread; previews of items selected meanwhile are not read, and stale results are not displayed. Selecting an item that cannot be previewed, such as a group, clears the preview. Closing a file or the browser dialog waits for the preview read to finish. Broken soft or external links (e.g. to missing detector files) are skipped, with a console warning, when a file is opened." +msgstr "La classification des jeux de données utilise les métadonnées de forme et de type ; les jeux de données composites mixtes sont vérifiés sur leurs premiers enregistrements. Les aperçus textuels sont limités à 100 éléments, et les attributs de plus de 10 000 éléments sont ignorés. Les aperçus graphiques sont lus dans un fil d'exécution de travail ; les aperçus des éléments sélectionnés entre-temps ne sont pas lus, et les résultats périmés ne sont pas affichés. La sélection d'un élément qui ne peut pas être prévisualisé, comme un groupe, efface l'aperçu. La fermeture d'un fichier ou du dialogue de l'explorateur attend la fin de la lecture de l'aperçu. Les liens symboliques ou externes rompus (par exemple vers des fichiers de détecteur absents) sont ignorés, avec un avertissement dans la console, à l'ouverture d'un fichier." msgid "Ordinary 1D arrays and 2D images, NeXus and mapped objects exceeding ``2**24`` elements are previewed with a sampling step along each axis. The full data is still read on import. Compound datasets and signals stored in 2D tables still use complete objects for graphical previews, so this is not a universal memory bound." msgstr "Les tableaux 1D ordinaires et les images 2D, les objets NeXus et décrits par un profil dépassant ``2**24`` éléments sont prévisualisés avec un pas d'échantillonnage le long de chaque axe. Les données complètes restent lues lors de l'import. Les jeux de données composites et les signaux stockés dans des tableaux 2D utilisent encore des objets complets pour leurs aperçus graphiques : ce n'est donc pas une borne mémoire universelle." @@ -145,3 +138,4 @@ msgstr "DataLab" msgid "DataLab keeps its native workspace format, panel serialization, metadata, ROI, and analysis-result handling. Its ``save_h5_workspace`` override remains the owner of that format. The SigimaX HDF5 GUI and generic import infrastructure do not change DataLab's native HDF5 layout." msgstr "DataLab conserve son format d'espace de travail natif, la sérialisation de ses panneaux, ses métadonnées, ses ROI et la gestion des résultats d'analyse. Sa surcharge de ``save_h5_workspace`` reste propriétaire de ce format. L'interface HDF5 et l'infrastructure d'import générique de SigimaX ne modifient pas la disposition HDF5 native de DataLab." + diff --git a/doc/locale/fr/LC_MESSAGES/user_guide/installation.po b/doc/locale/fr/LC_MESSAGES/user_guide/installation.po index 3eb3e0f..4da9386 100644 --- a/doc/locale/fr/LC_MESSAGES/user_guide/installation.po +++ b/doc/locale/fr/LC_MESSAGES/user_guide/installation.po @@ -42,15 +42,15 @@ msgstr "Impatient d'essayer la prochaine version de SigimaX ? Vous pouvez égale msgid "Conda package" msgstr "Paquet Conda" +msgid ":octicon:`info;1em;sd-text-info` :bdg-info-line:`GNU/Linux` :bdg-info-line:`Windows` :bdg-info-line:`macOS`" +msgstr ":octicon:`info;1em;sd-text-info` :bdg-info-line:`GNU/Linux` :bdg-info-line:`Windows` :bdg-info-line:`macOS`" + msgid "To install ``sigimax`` package from the `conda-forge` channel (https://anaconda.org/conda-forge/sigimax), run the following command:" msgstr "Pour installer le paquet ``sigimax`` depuis le canal ``conda-forge`` (https://anaconda.org/conda-forge/sigimax), exécutez la commande suivante :" msgid "Package manager ``pip``" msgstr "Gestionnaire de paquets ``pip``" -msgid ":octicon:`info;1em;sd-text-info` :bdg-info-line:`GNU/Linux` :bdg-info-line:`Windows` :bdg-info-line:`macOS`" -msgstr ":octicon:`info;1em;sd-text-info` :bdg-info-line:`GNU/Linux` :bdg-info-line:`Windows` :bdg-info-line:`macOS`" - msgid "SigimaX's package ``sigimax`` is available on the Python Package Index (PyPI) at: https://pypi.python.org/pypi/sigimax." msgstr "Le paquet ``sigimax`` de SigimaX est disponible sur le Python Package Index (PyPI) à l'adresse : https://pypi.python.org/pypi/sigimax." @@ -144,8 +144,8 @@ msgstr "Bibliothèque multiplateforme pour la surveillance des processus et du s msgid "Sigima" msgstr "Sigima" -msgid ">= 1.2.0" -msgstr ">= 1.2.0" +msgid ">= 1.4.0" +msgstr ">= 1.4.0" msgid "Scientific computing engine for 1D signals and 2D images, part of the DataLab open-source platform." msgstr "Moteur de calcul scientifique pour les signaux 1D et les images 2D, partie de la plateforme open-source DataLab." @@ -254,3 +254,4 @@ msgstr "pytest-xvfb" msgid "A pytest plugin to run Xvfb (or Xephyr/Xvnc) for tests." msgstr "Un plugin pytest pour exécuter Xvfb (ou Xephyr/Xvnc) pour les tests." + diff --git a/doc/release_notes/release_1.01.md b/doc/release_notes/release_1.01.md index b0904d7..9fbda08 100644 --- a/doc/release_notes/release_1.01.md +++ b/doc/release_notes/release_1.01.md @@ -35,7 +35,7 @@ ### HDF5 import ### * Browsing an HDF5 file no longer reads its numeric datasets entirely just to classify them: generic nodes are classified and described from the dataset shape and data type, and value previews read only the first elements (`TEXT_PREVIEW_MAX_ITEMS`). Compound datasets mixing numeric and string fields are checked on their first records only. Full data is read when creating an imported object or an undecimated graphical preview. -* The HDF5 browser reads previews in a worker thread, so that it stays responsive while a large dataset is read: only the last selected node is shown, the previews of nodes selected meanwhile are not read, and closing a file or the browser dialog waits for the preview being read (imported objects are then read without concurrent access to the file). Ordinary 1D arrays and 2D images, NeXus and mapped objects with more than `PlotPreview.MAX_ITEMS` (2²⁴) elements are previewed with a step along each axis. Compound datasets and signals stored in 2D tables still use complete objects for graphical previews; this is not a universal memory bound. +* The HDF5 browser reads previews in a worker thread, so that it stays responsive while a large dataset is read: only the last selected node is shown, the previews of nodes selected meanwhile are not read, selecting a node that cannot be previewed (a group, for example) clears the preview, and closing a file or the browser dialog waits for the preview being read (imported objects are then read without concurrent access to the file). Ordinary 1D arrays and 2D images, NeXus and mapped objects with more than `PlotPreview.MAX_ITEMS` (2²⁴) elements are previewed with a step along each axis. Compound datasets and signals stored in 2D tables still use complete objects for graphical previews; this is not a universal memory bound. * Numeric datasets with three dimensions or more (image stacks, spectral cubes...) are now listed in the HDF5 browser, with their shape and type. They are not imported as a whole, and there is no graphical slice selector: developers can read their 1D or 2D views with `sigimax.h5.selection.read_selection`, which reads only the selected elements (`NDSelection`: displayed axes, possibly transposed, and an index along each other axis). * NeXus `NXdata` groups are shown as single nodes, imported as a signal (1D) or an image (2D) with their axes, units, labels and, for signals, uncertainties: `@signal`/`@axes` attributes with `@_indices`, legacy `signal`/`axes` dataset attributes, bin edges converted to bin centers, non-uniform image axes. Uniform axes are detected within the precision of their data type (e.g. `float32` axes), and decreasing image axes are reversed with the image. Invalid axes (axis count differing from the signal rank, axes designating a missing or already described dimension, axes not holding real numbers, image axes not strictly monotonic) are reported instead of being guessed. Imported objects record their origin in the `experimental_source` metadata entry (file, group, signal and axes, `"."` for a dimension without axis, conversions). `.nxs` and `.nx5` files are proposed in HDF5 file dialogs. * Files that do not designate their data with suitable `NXdata` groups (no axes, several counters...) can be described by declarative data mappings (`sigimax.h5.mapping`): JSON-compatible descriptions of the signals and images of the groups having given attributes and paths, with their axes, uncertainties, labels and units, and a name pattern to repeat an object for each counter of a scan. Mapped objects are shown as extra nodes of the HDF5 browser and record the mapping in their `experimental_source`. Applications and plugins register mappings with `register_mapping` (and `unregister_mapping`); `NodeFactory.add_children_provider` lets them add such extra nodes. Blank units and labels are now ignored. @@ -43,6 +43,7 @@ * Images with unsigned 32-bit values above 2 147 483 647 are no longer clipped to `int32`: they are converted with Sigima's converter, which keeps every value exactly with Sigima ≥ 1.4.0. `BaseNode.uint32_wng` is kept for compatibility and is always `False`. * 1D datasets of single-precision complex numbers (`complex64`) are now imported as complex signals: only their real part was kept (with a `ComplexWarning`). Integer signals beyond 2⁵³ now emit Sigima's precision warning instead of being rounded silently, and 2D tables with complex X values or uncertainties are reported as unsupported instead of being truncated. * `NodeFactory.unregister` removes a node class and its post triggers, so that an application plugin can remove the node classes it registered when it is unloaded. +* Files containing broken links (soft links to a missing object, or external links to a missing file, as in synchrotron master files whose detector files are not available) can now be opened: the broken links are skipped with a warning in the console, instead of preventing the whole file from being browsed or imported. See the [HDF5 guide](../user_guide/hdf5_workspace.rst) for a complete JSON mapping and registration example, NeXus import limits and multidimensional selection examples. diff --git a/doc/user_guide/hdf5_workspace.rst b/doc/user_guide/hdf5_workspace.rst index 5c12e46..b695e5a 100644 --- a/doc/user_guide/hdf5_workspace.rst +++ b/doc/user_guide/hdf5_workspace.rst @@ -203,8 +203,10 @@ Dataset classification uses shape and type metadata; mixed compound datasets are checked on their first records. Text previews are bounded to 100 elements, and attributes above 10 000 elements are skipped. Graphical previews are read in a worker thread; previews of items selected meanwhile are not read, and -stale results are not displayed. Closing a file or the browser dialog waits -for the preview read to finish. +stale results are not displayed. Selecting an item that cannot be previewed, +such as a group, clears the preview. Closing a file or the browser dialog waits +for the preview read to finish. Broken soft or external links (e.g. to missing +detector files) are skipped, with a console warning, when a file is opened. Ordinary 1D arrays and 2D images, NeXus and mapped objects exceeding ``2**24`` elements are previewed with a sampling step along each axis. The full data is From 974811d59912b5acbeb00cf39c7a26df16d4d536 Mon Sep 17 00:00:00 2001 From: Pierre Raybaut <1311787+PierreRaybaut@users.noreply.github.com> Date: Fri, 23 Oct 2026 10:29:14 +0200 Subject: [PATCH 20/20] fix(h5): fall back to the dataset name for blank NeXus labels A long_name attribute that is empty or made of spaces gave an empty label to the signal or axis, instead of the dataset name. Assisted-by: Claude Opus 5.5 --- sigimax/h5/nexus.py | 7 +++---- sigimax/tests/hdf5/test_h5_nexus.py | 18 ++++++++++++++++++ 2 files changed, 21 insertions(+), 4 deletions(-) diff --git a/sigimax/h5/nexus.py b/sigimax/h5/nexus.py index 168aff9..a4a0596 100644 --- a/sigimax/h5/nexus.py +++ b/sigimax/h5/nexus.py @@ -256,9 +256,8 @@ def describe_signal( size=axis_dset.shape[0], edges=axis_dset.shape[0] == dset.shape[dim] + 1, units=_text(axis_dset.attrs.get("units", "")).strip(), - label=_text( - axis_dset.attrs.get("long_name", name.split("/")[-1]) - ).strip(), + label=_text(axis_dset.attrs.get("long_name", "")).strip() + or name.split("/")[-1], ) ) found = None @@ -279,7 +278,7 @@ def describe_signal( axes=tuple(axis_descs), errors=found, units=_text(dset.attrs.get("units", "")).strip(), - label=_text(dset.attrs.get("long_name", signal.split("/")[-1])).strip(), + label=_text(dset.attrs.get("long_name", "")).strip() or signal.split("/")[-1], problems=tuple(problems), ) diff --git a/sigimax/tests/hdf5/test_h5_nexus.py b/sigimax/tests/hdf5/test_h5_nexus.py index 120abd8..e693e76 100644 --- a/sigimax/tests/hdf5/test_h5_nexus.py +++ b/sigimax/tests/hdf5/test_h5_nexus.py @@ -159,6 +159,24 @@ def test_nxdata_signal(nodes: dict, nxsfname: str) -> None: assert np.array_equal(legacy.y, [1.0, 2.0, 3.0]) +def test_nxdata_blank_labels(tmp_path) -> None: + """Blank ``long_name`` attributes fall back to the dataset names.""" + + def build(h5file: h5py.File) -> None: + group = _nxdata(h5file, "spectrum", "counts", ["energy"]) + group["counts"] = np.arange(4.0) + group["counts"].attrs["long_name"] = b" " + group["energy"] = np.arange(4.0) + group["energy"].attrs["long_name"] = "" + + with _nodes_of(tmp_path, build) as nodes: + node = nodes["/spectrum"] + assert node.nxdata.label == "counts" + assert node.nxdata.axes[0].label == "energy" + obj = node.get_native_object() + assert (obj.xlabel, obj.ylabel) == ("energy", "counts") + + def test_nxdata_bin_edges(nodes: dict) -> None: """Bin edges are converted to bin centers, and the conversion is recorded.""" obj = nodes["/entry/histogram"].get_native_object()