From c3c57f0010765cbf4b2612bc6c644ce5fb42527f Mon Sep 17 00:00:00 2001 From: Pierre Raybaut <1311787+PierreRaybaut@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:41:54 +0100 Subject: [PATCH 1/5] feat(h5): import multidimensional data, legacy NeXus scans and compressed data Datasets and NeXus signals of three dimensions or more are imported through a view, one signal or image per frame: the HDF5 browser shows a view selector (displayed axes, frame range along another axis, fixed indices), previews the first frame, and imports the frames once enabled. The default view follows the NeXus interpretation of the signal. Legacy NeXus axes are read from the axis and primary attributes of axis datasets, and the other signals of a group (auxiliary_signals attribute, or legacy signal attributes as all the counters and detectors of a scan) are shown as nodes of their own, with the axes that fit their dimensions. The browser selects the default NXdata group of a file, the origin of imported objects records the title and times of the entry, and mapping axes may describe the first dimensions only. Compressed detector data is read with hdf5plugin, now a dependency, and data compressed with an unavailable filter is reported instead of failing. Assisted-by: Claude Opus 5.5 --- pyproject.toml | 1 + requirements.txt | 1 + sigimax/h5/__init__.py | 23 +- sigimax/h5/common.py | 15 + sigimax/h5/generic.py | 98 ++++- sigimax/h5/mapping.py | 15 +- sigimax/h5/nexus.py | 432 +++++++++++++++++++-- sigimax/h5/selection.py | 118 +++++- sigimax/h5/utils.py | 22 ++ sigimax/h5/views.py | 226 +++++++++++ sigimax/tests/hdf5/test_h5_views.py | 296 ++++++++++++++ sigimax/tests/hdf5/test_h5browser_views.py | 89 +++++ sigimax/widgets/h5browser.py | 338 +++++++++++++++- 13 files changed, 1614 insertions(+), 60 deletions(-) create mode 100644 sigimax/h5/views.py create mode 100644 sigimax/tests/hdf5/test_h5_views.py create mode 100644 sigimax/tests/hdf5/test_h5browser_views.py diff --git a/pyproject.toml b/pyproject.toml index a2cdd09..7b282bd 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -37,6 +37,7 @@ classifiers = [ requires-python = ">=3.10, <4" dependencies = [ "guidata >= 3.15.0", + "hdf5plugin >= 4.0", "PlotPy >= 2.11.0", "psutil >= 5.7", "Sigima >= 1.4.0", diff --git a/requirements.txt b/requirements.txt index fa9933f..6e147ca 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,6 +3,7 @@ PlotPy >= 2.11.0 PyQt5 >= 5.15.6 Sigima >= 1.4.0 guidata >= 3.15.0 +hdf5plugin >= 4.0 myst-nb myst_parser psutil >= 5.7 diff --git a/sigimax/h5/__init__.py b/sigimax/h5/__init__.py index 618cf53..db44d51 100644 --- a/sigimax/h5/__init__.py +++ b/sigimax/h5/__init__.py @@ -25,12 +25,19 @@ :members: NeXus ``NXdata`` groups are shown as single nodes, imported as signals or images -with their axes (:mod:`sigimax.h5.nexus`). Views of multidimensional datasets are -read with :func:`sigimax.h5.selection.read_selection`. +with their axes (:mod:`sigimax.h5.nexus`); their auxiliary signals are shown as +nodes of their own. Datasets of three dimensions or more are imported through +views, one signal or image per frame (:mod:`sigimax.h5.views`), and views of +multidimensional datasets are read with +:func:`sigimax.h5.selection.read_selection`. .. autoclass:: sigimax.h5.nexus.NXdataNode +.. autoclass:: sigimax.h5.nexus.AuxiliarySignalNode .. autofunction:: sigimax.h5.nexus.describe_nxdata +.. autofunction:: sigimax.h5.nexus.describe_auxiliary_signals .. autofunction:: sigimax.h5.nexus.describe_signal +.. autofunction:: sigimax.h5.nexus.default_nxdata_path +.. autofunction:: sigimax.h5.nexus.entry_information .. autodata:: sigimax.h5.nexus.UNIFORM_AXIS_TOLERANCE Files without suitable ``NXdata`` groups are described by declarative data @@ -47,8 +54,16 @@ .. autoclass:: sigimax.h5.mapping.MappedDataNode .. autoclass:: sigimax.h5.selection.NDSelection :members: +.. autoclass:: sigimax.h5.selection.NDView + :members: +.. autofunction:: sigimax.h5.selection.default_view .. autofunction:: sigimax.h5.selection.read_selection .. autofunction:: sigimax.h5.selection.preview_step +.. autoclass:: sigimax.h5.views.NDViewMixin + :members: +.. autoclass:: sigimax.h5.views.FrameNode +.. autofunction:: sigimax.h5.views.expand_views +.. autofunction:: sigimax.h5.utils.unavailable_filters """ __all__ = [ @@ -57,6 +72,10 @@ # pylint: disable=unused-import +# Compression filters of detector data (bitshuffle/LZ4, Blosc, Zstandard...), +# available to h5py once registered by hdf5plugin +import hdf5plugin # noqa: F401 + # Registering dynamic I/O features: from sigimax.h5 import generic, mapping, nexus # noqa: F401 from sigimax.h5.common import H5Importer # noqa: F401 diff --git a/sigimax/h5/common.py b/sigimax/h5/common.py index 8cc7991..38a953e 100644 --- a/sigimax/h5/common.py +++ b/sigimax/h5/common.py @@ -131,6 +131,11 @@ def is_supported(self) -> bool: """Return True if node is associated to supported data""" return False + def can_preview(self) -> bool: + """Return True if the node can be previewed (by default, if it is + supported: multidimensional nodes are previewed before being enabled)""" + return self.is_supported() + def create_native_object(self): """Create native object, if supported""" return None @@ -197,11 +202,21 @@ def __process_metadata(self, obj): @property def object_title(self): """Return signal/image object title""" + return self.make_title() + + def make_title(self, suffix: str = "") -> str: + """Return the title of an object read from the node + + Args: + suffix: text following the node name or path (e.g. frame indices), + before the file name + """ conf = get_conf() if conf.h5_fullpath_in_title.get(): title = self.id else: title = self.name + title += suffix if conf.h5_fname_in_title.get(): title += f" ({osp.basename(self.h5file.filename)})" return title diff --git a/sigimax/h5/generic.py b/sigimax/h5/generic.py index 53f0c2a..072014a 100644 --- a/sigimax/h5/generic.py +++ b/sigimax/h5/generic.py @@ -14,10 +14,18 @@ import numpy as np from guidata.utils.misc import to_string from sigima.io.common.converters import convert_array_to_valid_dtype -from sigima.objects import ImageObj, SignalObj, create_image, create_signal +from sigima.objects import ( + ImageObj, + SignalObj, + create_image, + create_signal, + make_experimental_source, + set_experimental_source, +) from sigimax.h5 import common, utils from sigimax.h5.selection import NDSelection, preview_step, read_selection +from sigimax.h5.views import NDViewMixin #: Maximum number of elements read to show the textual preview of an array TEXT_PREVIEW_MAX_ITEMS = 100 @@ -25,6 +33,12 @@ #: Number of compound records read to decide if their string fields matter COMPOUND_SAMPLE_SIZE = 1000 + +def _unavailable_filters_text(missing: list[str]) -> str: + """Return the text of a dataset that cannot be read for want of filters""" + return "" + + # ============================================================================= # Encoding and Data Reading Utilities # ============================================================================= @@ -438,7 +452,9 @@ def layout(self) -> tuple[tuple[int, ...], np.dtype]: def is_supported(self) -> bool: """Return True if node is associated to supported data""" - return int(np.prod(self.layout[0])) > 1 + return int(np.prod(self.layout[0])) > 1 and not utils.unavailable_filters( + self.dset + ) @property def __is_signal(self): @@ -466,6 +482,9 @@ def dtype_str(self): @property def text(self): """Return node textual representation""" + missing = utils.unavailable_filters(self.dset) + if missing: + return _unavailable_filters_text(missing) data, truncated = utils.read_bounded(self.dset, TEXT_PREVIEW_MAX_ITEMS) if self.dset.dtype.names is not None: data = self.extract_numeric_from_compound(data) @@ -511,14 +530,18 @@ def create_preview_object(self, max_items: int): return obj -class GenericNDArrayNode(BaseGenericNode): +class GenericNDArrayNode(BaseGenericNode, NDViewMixin): """Node for numeric HDF5 datasets with three dimensions or more. - Such datasets (image stacks, spectral cubes...) are listed and described, but - are not imported as a whole: their 1D or 2D views are read with - :func:`sigimax.h5.selection.read_selection`. + Such datasets (image stacks, spectral cubes...) are listed and described, and + imported through a view (:class:`~sigimax.h5.views.NDViewMixin`): one signal + or image per frame, read with :func:`sigimax.h5.selection.read_selection`. """ + def __init__(self, h5file, dname): + super().__init__(h5file, dname) + self.init_view() + @classmethod def match(cls, dset): """Match numeric datasets with three dimensions or more.""" @@ -530,9 +553,31 @@ def match(cls, dset): and utils.is_supported_num_dtype(dset) ) + @property + def nd_shape(self) -> tuple[int, ...] | None: + """Return the shape of the dataset (None if it cannot be read)""" + if utils.unavailable_filters(self.dset): + return None + return self.dset.shape + + @property + def interpretation(self) -> str: + """Return the NeXus interpretation of the dataset (empty if not given)""" + return to_string(self.dset.attrs.get("interpretation", "")).strip() + + def is_supported(self) -> bool: + """Return True if the frames of the view are to be imported""" + return self.import_view + + def can_preview(self) -> bool: + """Return True if the dataset can be viewed""" + return self.view is not None + @property def icon_name(self): """Icon name associated to node""" + if self.import_view: + return "image.svg" if len(self.view.displayed) == 2 else "signal.svg" return "h5array.svg" @property @@ -543,9 +588,50 @@ def shape_str(self): @property def text(self): """Return node textual representation""" + missing = utils.unavailable_filters(self.dset) + if missing: + return _unavailable_filters_text(missing) data, truncated = utils.read_bounded(self.dset, TEXT_PREVIEW_MAX_ITEMS) return str(data) + (" ..." if truncated else "") + def create_preview_object(self, max_items: int): + """Return the first frame of the view, read with a step if it has more + than ``max_items`` elements""" + return self.preview_view_object(max_items) + + def create_view_object( + self, selection: NDSelection, title: str, max_items: int | None = None + ): + """Create the signal or image of a frame, with pixel indices as + coordinates + + Args: + selection: frame selection + title: object title + max_items: maximum number of elements to read (None: read the whole + frame and record the origin of the object) + """ + sizes = tuple(self.dset.shape[axis] for axis in selection.displayed) + step = self.view_step(sizes, max_items) + data = read_selection(self.dset, selection, step) + if len(sizes) == 1: + x = np.arange(0, sizes[0], step, dtype=float) + obj = create_signal(title, x, common.to_signal_data(data)) + else: + obj = create_image( + title, convert_array_to_valid_dtype(data, ImageObj.VALID_DTYPES) + ) + obj.set_uniform_coords(float(step), float(step)) + if max_items is None: + source = make_experimental_source( + "hdf5", + self.h5file.filename, + entry=self.dset.name, + selection={"view": selection.as_dict()}, + ) + set_experimental_source(obj, source) + return obj + class GenericCompoundNode(BaseGenericNode): """Node for compound/structured HDF5 data that can't convert to numeric arrays.""" diff --git a/sigimax/h5/mapping.py b/sigimax/h5/mapping.py index 8fc3b61..55811cd 100644 --- a/sigimax/h5/mapping.py +++ b/sigimax/h5/mapping.py @@ -34,9 +34,12 @@ Paths are relative to the matching group. The last component of a signal path may be a pattern (``*``, ``?``, ``[...]``): the object is then repeated for each matching dataset, and ``{name}`` in its name is replaced by the dataset name. -``axes`` gives one entry per signal dimension (``null``: no axis), as a path or as -an object overriding the axis label and units; without ``axes``, dimensions have -no axis. +``axes`` gives the axis of the first signal dimensions (``null``: no axis), as a +path or as an object overriding the axis label and units; the following +dimensions have no axis, so that a pattern may match signals of several ranks +(e.g. counters and detector images of a scan, with the scan axis first). Without +``axes``, dimensions have no axis. Objects of three dimensions or more are +imported through a view (:mod:`sigimax.h5.views`). """ from __future__ import annotations @@ -95,8 +98,8 @@ class MappedObject: ``signal`` is a pattern signal: signal dataset path, relative to the matching group; its last component may be a pattern - axes: axis of each signal dimension (None: no axis), or None if no - dimension has an axis + axes: axis of the first signal dimensions (None: no axis), the following + dimensions having no axis, or None if no dimension has an axis errors: uncertainties dataset path (default: ``_errors``, if any) label: signal label (default: ``long_name`` attribute, or dataset name) units: signal units (default: ``units`` attribute) @@ -353,6 +356,8 @@ def _describe( axes: list[str | None] = [None] * dset.ndim else: axes = [None if axis is None else axis.path for axis in obj.axes] + # Axes describe the first dimensions: the following ones have no axis + axes += [None] * (dset.ndim - len(axes)) errors = (obj.errors,) if obj.errors else (f"{signal}_errors",) desc = nexus.describe_signal(group, signal, axes, errors=errors) if desc is None: diff --git a/sigimax/h5/nexus.py b/sigimax/h5/nexus.py index 5366084..d75647a 100644 --- a/sigimax/h5/nexus.py +++ b/sigimax/h5/nexus.py @@ -6,8 +6,11 @@ A NeXus ``NXdata`` group designates its plottable signal and the axes of each signal dimension. :class:`NXdataNode` shows such a group as a single node of the HDF5 browser, which is imported as a signal (1D) or an image (2D) with its axes, -units and uncertainties. Only metadata is read until the object is created, and -previews of large groups read one element out of a step along each dimension. +units and uncertainties, or, for signals of three dimensions or more, through a +view: one signal or image per frame (:mod:`sigimax.h5.views`). Auxiliary signals +of the group are shown as nodes of their own (:class:`AuxiliarySignalNode`). +Only metadata is read until the object is created, and previews of large groups +read one element out of a step along each dimension. Supported conventions (NeXus data model, ``NXdata`` base class): @@ -15,6 +18,12 @@ - ``@axes`` group attribute (``"."`` for a dimension without axis), with optional ``@_indices`` attributes, or legacy ``axes`` signal attribute; ``@axes`` must give one entry per signal dimension +- without ``@axes``, legacy ``axis`` attributes of axis datasets (dimension number, + from 1), the dataset with the lowest ``primary`` attribute being preferred for + each dimension; an axis applies to a dimension only if its size fits +- auxiliary signals: ``@auxiliary_signals`` group attribute, and the other + datasets having a legacy ``signal`` attribute (whatever its value), with the + axes that fit their dimensions - axis datasets holding one real value per point, or bin edges (one more value), which are converted to bin centers; decreasing image axes are reversed, with the image, as images need increasing coordinates @@ -22,7 +31,14 @@ uniform coordinates, their maximum deviation being recorded as a conversion; other image axes keep their stored coordinates - ``_errors`` (or legacy ``errors``) uncertainties of a 1D signal -- ``units`` and ``long_name`` dataset attributes +- ``units`` and ``long_name`` dataset attributes, ``interpretation`` signal + attribute (default view of multidimensional signals) +- ``@default`` attributes of the file and its entries, designating the default + ``NXdata`` group (:func:`default_nxdata_path`) +- ``title``, ``start_time`` and ``end_time`` of the ``NXentry`` group, recorded in + the origin of the imported objects + +Signals compressed with a filter that HDF5 cannot apply are reported as problems. """ from __future__ import annotations @@ -45,15 +61,20 @@ ) from sigimax.h5 import common, utils -from sigimax.h5.selection import preview_step +from sigimax.h5.selection import NDSelection, preview_step, read_selection +from sigimax.h5.views import NDViewMixin __all__ = [ "UNIFORM_AXIS_TOLERANCE", + "AuxiliarySignalNode", "NXAxis", "NXdataDescription", "NXdataNode", + "default_nxdata_path", + "describe_auxiliary_signals", "describe_nxdata", "describe_signal", + "entry_information", ] #: Maximum deviation of image axis values from even spacing for the axis to be @@ -131,6 +152,53 @@ def _is_nxdata(group: h5py.Group) -> bool: return _text(group.attrs.get("NX_class", "")) == "NXdata" +def _attr_rank(item: h5py.Dataset, name: str) -> int | None: + """Return a positive integer attribute (legacy ``signal``, ``axis`` or + ``primary``), or None if it is missing or invalid; ``True`` stands for 1""" + if name not in item.attrs: + return None + text = _text(item.attrs[name]).strip() + if text == "True": + return 1 + try: + value = float(text) + except ValueError: + return None + return int(value) if value.is_integer() and value >= 1 else None + + +def _natural_key(name: str) -> list: + """Return sort key ordering numbered names numerically""" + return [int(part) if part.isdigit() else part for part in re.split(r"(\d+)", name)] + + +def _legacy_axes(group: h5py.Group, shape: tuple[int, ...]) -> list[str | None]: + """Return the axis of each signal dimension designated by legacy ``axis`` + attributes (dimension number, from 1), or None + + An axis applies to a dimension only if it holds one real value per point (or + bin edges); among the axes of a dimension, the one with the lowest ``primary`` + attribute is preferred. + """ + candidates: dict[int, list[tuple[float, str]]] = {} + for name, item in group.items(): + if not isinstance(item, h5py.Dataset) or "axis" not in item.attrs: + continue + number = _attr_rank(item, "axis") + if number is None or number > len(shape) or item.ndim != 1: + continue + size = shape[number - 1] + if not _is_real_number(item) or item.shape[0] not in (size, size + 1): + continue + primary = _attr_rank(item, "primary") + rank = math.inf if primary is None else primary + candidates.setdefault(number - 1, []).append((rank, name)) + return [ + min(candidates[dim])[1] if dim in candidates else None + for dim in range(len(shape)) + ] + + def _find_signal(group: h5py.Group) -> tuple[str | None, list[str]]: """Return signal dataset name and axis names declared by an NXdata group""" if "signal" in group.attrs: @@ -201,13 +269,64 @@ def describe_nxdata(group: h5py.Group) -> NXdataDescription | None: dset = group.get(signal) if signal else None if not isinstance(dset, h5py.Dataset) or not dset.shape: return None - axes, problems = _axis_dimensions(group, names, dset.ndim) + if names: + axes, problems = _axis_dimensions(group, names, dset.ndim) + else: + axes, problems = _legacy_axes(group, dset.shape), [] desc = describe_signal(group, signal, axes, errors=(f"{signal}_errors", "errors")) if desc is None or not problems: return desc return dataclasses.replace(desc, problems=tuple(problems) + desc.problems) +def describe_auxiliary_signals( + group: h5py.Group, main: NXdataDescription | None = None +) -> list[NXdataDescription]: + """Describe the auxiliary signals of an ``NXdata`` group, without reading its + datasets + + Auxiliary signals are those of the ``@auxiliary_signals`` group attribute, + then the datasets having a legacy ``signal`` attribute, by attribute value and + name, except the signal of the group. They have the axes of the signal if they + have its shape, otherwise the axes designated by legacy ``axis`` attributes + that fit their dimensions. + + Args: + group: HDF5 group + main: description of the group (default: described from its attributes) + + Returns: + Descriptions of the numeric auxiliary signals + """ + main = describe_nxdata(group) if main is None else main + if main is None: + return [] + names: list[str] = [] + if "auxiliary_signals" in group.attrs: + names.extend(_text_list(group.attrs["auxiliary_signals"])) + legacy = [] + for name, item in group.items(): + if isinstance(item, h5py.Dataset): + rank = _attr_rank(item, "signal") + if rank is not None: + legacy.append((rank, _natural_key(name), name)) + names.extend(name for *_key, name in sorted(legacy) if name not in names) + main_axes = [axis.name if axis is not None else None for axis in main.axes] + descs = [] + for name in names: + dset = group.get(name) + if name == main.signal or not isinstance(dset, h5py.Dataset) or not dset.shape: + continue + if dset.shape == main.shape and not main.problems: + axes = main_axes + else: + axes = _legacy_axes(group, dset.shape) + desc = describe_signal(group, name, axes, errors=(f"{name}_errors",)) + if desc is not None: + descs.append(desc) + return descs + + def _is_real_number(dset: h5py.Dataset) -> bool: """Return True if dataset holds real numbers (integers or floats)""" return dset.dtype.kind in "iuf" @@ -239,6 +358,9 @@ def describe_signal( ): return None problems: list[str] = [] + missing = utils.unavailable_filters(dset) + if missing: + problems.append("compressed with unavailable filter: " + ", ".join(missing)) if len(axes) != dset.ndim: problems.append(f"{len(axes)} axes for {dset.ndim} dimensions") axes = (list(axes) + [None] * dset.ndim)[: dset.ndim] @@ -417,8 +539,81 @@ def _set_image_coords( ) -class NXdataNode(common.GroupNode): - """Node for NeXus ``NXdata`` groups, imported as a signal or an image +def _entry_group(group: h5py.Group) -> h5py.Group | None: + """Return the ``NXentry`` group containing a group, if any""" + while group.name != "/": + group = group.parent + if _text(group.attrs.get("NX_class", "")) == "NXentry": + return group + return None + + +def _entry_field(entry: h5py.Group, name: str) -> str: + """Return a text field of an ``NXentry`` group (empty if missing)""" + item = entry.get(name) + if not isinstance(item, h5py.Dataset) or item.size != 1: + return "" + if item.dtype.kind not in "SUO": + return "" + return _text(item[()]).strip() + + +def entry_information(group: h5py.Group) -> tuple[dict[str, str], dict[str, str]]: + """Return the title and the start and end times of the ``NXentry`` group + containing a group, as ``experimental_source`` details and timestamps + + Args: + group: HDF5 group + + Returns: + Details (``{"entry_title": ...}``) and timestamps (``{"start": ..., + "end": ...}``), without the missing fields + """ + entry = _entry_group(group) + if entry is None: + return {}, {} + title = _entry_field(entry, "title") + details = {"entry_title": title} if title else {} + timestamps = { + key: value + for key, value in ( + ("start", _entry_field(entry, "start_time")), + ("end", _entry_field(entry, "end_time")), + ) + if value + } + return details, timestamps + + +def default_nxdata_path(h5file: h5py.File) -> str | None: + """Return the path of the default ``NXdata`` group of a NeXus file + + The ``@default`` attribute of the file designates its default entry, whose + ``@default`` attribute designates its default ``NXdata`` group (or a subentry, + designating its own). + + Args: + h5file: HDF5 file + + Returns: + Group path, or None if the file does not designate an ``NXdata`` group + """ + group: h5py.Group = h5file + for _level in range(8): + if "default" not in group.attrs: + return None + child = group.get(_text(group.attrs["default"]).strip()) + if not isinstance(child, h5py.Group): + return None + if _is_nxdata(child): + return child.name + group = child + return None + + +class NXdataNode(common.GroupNode, NDViewMixin): + """Node for NeXus ``NXdata`` groups, imported as a signal or an image, or + through a view for signals of three dimensions or more Args: h5file: HDF5 file @@ -433,24 +628,44 @@ def __init__(self, h5file, dname, nxdata: NXdataDescription | None = None): super().__init__(h5file, dname) self.nxdata = describe_nxdata(self.dset) if nxdata is None else nxdata self.text_prefix = "NXdata" + self.init_view() @classmethod def match(cls, dset): """Return True if h5 group is an ``NXdata`` group with a numeric signal""" return describe_nxdata(dset) is not None + @property + def nd_shape(self) -> tuple[int, ...] | None: + """Return the signal shape (None if the signal cannot be imported)""" + return None if self.nxdata.problems else self.nxdata.shape + + @property + def interpretation(self) -> str: + """Return the ``interpretation`` attribute of the signal (empty if not + given)""" + dset = self.dset.get(self.nxdata.signal) + return _text(dset.attrs.get("interpretation", "")).strip() + def is_supported(self) -> bool: - """Return True if the signal is 1D or 2D and its axes are valid""" + """Return True if the signal is 1D or 2D and its axes are valid, or if the + view of a multidimensional signal is enabled""" desc = self.nxdata - return ( - not desc.problems - and len(desc.shape) in (1, 2) - and int(np.prod(desc.shape)) > 1 - ) + if desc.problems: + return False + if len(desc.shape) >= 3: + return self.import_view + return int(np.prod(desc.shape)) > 1 + + def can_preview(self) -> bool: + """Return True if the signal is supported or can be viewed""" + return self.view is not None or self.is_supported() @property def icon_name(self): """Icon name associated to node""" + if len(self.nxdata.shape) >= 3 and self.import_view: + return "image.svg" if len(self.view.displayed) == 2 else "signal.svg" if self.is_supported(): return "signal.svg" if len(self.nxdata.shape) == 1 else "image.svg" return "h5group.svg" @@ -485,27 +700,45 @@ def source_details(self) -> dict | None: (None: no details)""" return None - def make_source(self, conversions: list[str]) -> dict: + def make_source( + self, conversions: list[str], selection: NDSelection | None = None + ) -> dict: """Return the experimental source of the imported object - Dimensions without axis are recorded as ``"."``, as in NeXus ``@axes``. + Dimensions without axis are recorded as ``"."``, as in NeXus ``@axes``. The + title and times of the entry are recorded in the details and timestamps. + + Args: + conversions: conversions applied to the source values + selection: frame of a multidimensional signal (None: whole signal), + recorded as ``view``, with the coordinates of the frame along the + hidden axes as ``positions`` """ desc = self.nxdata + chosen = { + "signal": desc.signal, + "axes": [axis.name if axis else "." for axis in desc.axes], + } + if selection is not None: + chosen["view"] = selection.as_dict() + positions = self.frame_positions(selection) + if positions: + chosen["positions"] = positions + details, timestamps = entry_information(self.dset) + details.update(self.source_details() or {}) return make_experimental_source( self.SOURCE_KIND, self.h5file.filename, entry=desc.path, - selection={ - "signal": desc.signal, - "axes": [axis.name if axis else "." for axis in desc.axes], - }, + selection=chosen, + timestamps=timestamps or None, conversions=conversions or None, - details=self.source_details(), + details=details or None, ) def create_native_object(self): """Create signal or image object from the ``NXdata`` group""" - if not self.is_supported(): + if not self.is_supported() or len(self.nxdata.shape) >= 3: return None obj, conversions = self._build_object(1) set_experimental_source(obj, self.make_source(conversions)) @@ -513,11 +746,14 @@ def create_native_object(self): def create_preview_object(self, max_items: int): """Return the native object, or a decimated one if the signal has more - than ``max_items`` elements (only the read elements of its axes are kept) + than ``max_items`` elements (only the read elements of its axes are kept); + for multidimensional signals, the first frame of the view Args: max_items: maximum number of signal elements to read """ + if len(self.nxdata.shape) >= 3 and self.view is not None: + return self.preview_view_object(max_items) if not self.is_supported(): return None shape = self.nxdata.shape @@ -526,6 +762,60 @@ def create_preview_object(self, max_items: int): obj, _conversions = self._build_object(preview_step(shape, max_items)) return obj + def axis_name(self, dim: int) -> str: + """Return the name of the axis of a dimension (empty if it has none)""" + axis = self.nxdata.axes[dim] + return "" if axis is None else axis.name + + def frame_positions(self, selection: NDSelection) -> dict[str, float]: + """Return the coordinates of a frame along the hidden axes which have an + axis, by axis name (bin centers for bin edges)""" + positions = {} + ndim = len(self.nxdata.shape) + hidden = [dim for dim in range(ndim) if dim not in selection.displayed] + for dim, index in zip(hidden, selection.indices): + axis = self.nxdata.axes[dim] + if axis is None: + continue + dset = self.dset[axis.name] + values = np.asarray(dset[index : index + 2 if axis.edges else index + 1]) + positions[axis.name] = float(np.mean(values, dtype=float)) + return positions + + def create_view_object( + self, selection: NDSelection, title: str, max_items: int | None = None + ) -> SignalObj | ImageObj: + """Create the signal or image of a frame of a multidimensional signal, with + the axes of its displayed dimensions + + Args: + selection: frame selection + title: object title + max_items: maximum number of elements to read (None: read the whole + frame and record the origin of the object) + """ + desc, group = self.nxdata, self.dset + sizes = tuple(desc.shape[dim] for dim in selection.displayed) + step = self.view_step(sizes, max_items) + data = read_selection(group[desc.signal], selection, step) + axes = tuple(desc.axes[dim] for dim in selection.displayed) + conversions = [ + f"axis '{axis.name}': bin edges -> bin centers" + for axis in axes + if axis is not None and axis.edges + ] + if len(sizes) == 1: + dy = None + if desc.errors is not None: + errors = read_selection(group[desc.errors], selection, step) + dy = np.asarray(errors, dtype=float) + obj = self._build_signal(data, step, axes[0], sizes[0], dy, title) + else: + obj = self._build_image(data, step, conversions, axes, sizes, title) + if max_items is None: + set_experimental_source(obj, self.make_source(conversions, selection)) + return obj + def _build_object(self, step: int) -> tuple[SignalObj | ImageObj, list[str]]: """Create signal or image object from the ``NXdata`` group, reading one element out of ``step`` along each dimension @@ -542,51 +832,73 @@ def _build_object(self, step: int) -> tuple[SignalObj | ImageObj, list[str]]: if axis is not None and axis.edges ] if len(desc.shape) == 1: - return self._build_signal(data, step), conversions - return self._build_image(data, step, conversions), conversions - - def _build_signal(self, data: np.ndarray, step: int) -> SignalObj: - """Create the signal object of a 1D ``NXdata`` group + dy = None + if desc.errors is not None: + dy = np.asarray(self.dset[desc.errors][::step], dtype=float) + obj = self._build_signal(data, step, desc.axes[0], desc.shape[0], dy) + return obj, conversions + obj = self._build_image(data, step, conversions, desc.axes, desc.shape) + return obj, conversions + + def _build_signal( + self, + data: np.ndarray, + step: int, + axis: NXAxis | None, + size: int, + dy: np.ndarray | None = None, + title: str | None = None, + ) -> SignalObj: + """Create a signal object from 1D values of the ``NXdata`` group Args: data: signal values, read with step step: step between the read elements + axis: axis of the values (None: no axis) + size: number of values along the dimension + dy: uncertainties, read with step (None: no uncertainties) + title: object title (default: node title) Returns: Signal object """ desc, group = self.nxdata, self.dset - axis = desc.axes[0] - x = np.arange(0, desc.shape[0], step, dtype=float) + x = np.arange(0, size, step, dtype=float) if axis is not None: x = _read_axis(group, axis, step)[0] - dy = None - if desc.errors is not None: - dy = np.asarray(group[desc.errors][::step], dtype=float) y = common.to_signal_data(data) - obj = create_signal(self.object_title, x, y, dy=dy) + obj = create_signal(title or self.object_title, x, y, dy=dy) obj.ylabel, obj.yunit = desc.label, desc.units if axis is not None: obj.xlabel, obj.xunit = axis.label, axis.units return obj def _build_image( - self, data: np.ndarray, step: int, conversions: list[str] + self, + data: np.ndarray, + step: int, + conversions: list[str], + axes: tuple[NXAxis | None, ...], + sizes: tuple[int, ...], + title: str | None = None, ) -> ImageObj: - """Create the image object of a 2D ``NXdata`` group + """Create an image object from 2D values of the ``NXdata`` group Args: data: image values, read with step along both dimensions step: step between the read elements conversions: conversions applied to the source values, completed with the ones applied to the image + axes: axes of the rows and of the columns (None: no axis) + sizes: number of rows and columns + title: object title (default: node title) Returns: Image object """ desc, group = self.nxdata, self.dset - yaxis, xaxis = desc.axes - ny, nx = desc.shape + yaxis, xaxis = axes + ny, nx = sizes # PlotPy images need increasing coordinates: decreasing axes are reversed xc, xdtype, xreversed = _image_axis(group, xaxis, nx, step) yc, ydtype, yreversed = _image_axis(group, yaxis, ny, step) @@ -595,7 +907,7 @@ def _build_image( data = np.flip(data, axis=dim) conversions.append(f"axis '{axis.name}': reversed to increasing order") obj = create_image( - self.object_title, + title or self.object_title, convert_array_to_valid_dtype( np.ascontiguousarray(data), ImageObj.VALID_DTYPES ), @@ -616,4 +928,48 @@ def _build_image( return obj +class AuxiliarySignalNode(NXdataNode): + """Node of an auxiliary signal of an ``NXdata`` group, imported with the axes + that fit its dimensions + + Args: + h5file: HDF5 file + dname: path of the ``NXdata`` group + nxdata: auxiliary signal description + """ + + def __init__(self, h5file, dname: str, nxdata: NXdataDescription): + super().__init__(h5file, dname, nxdata) + self.text_prefix = "NXdata auxiliary signal" + + @classmethod + def match(cls, dset): + """Auxiliary signal nodes are created with their group, not by HDF5 items""" + return False + + @property + def id(self): + """Return node id, distinct from the HDF5 paths of the file""" + return f"{self.dset.name.rstrip('/')}::{self.nxdata.signal}" + + @property + def name(self): + """Return node name""" + return self.nxdata.signal + + def collect_children(self, node_dict, ancestors=()): + """Auxiliary signals have no children""" + + +def _auxiliary_children(node: common.GroupNode) -> list[AuxiliarySignalNode]: + """Return the nodes of the auxiliary signals of an ``NXdata`` group node""" + if type(node) is not NXdataNode: # pylint: disable=unidiomatic-typecheck + return [] + return [ + AuxiliarySignalNode(node.h5file, node.dset.name, desc) + for desc in describe_auxiliary_signals(node.dset, node.nxdata) + ] + + common.NODE_FACTORY.register(NXdataNode, is_generic=True) +common.NODE_FACTORY.add_children_provider(_auxiliary_children) diff --git a/sigimax/h5/selection.py b/sigimax/h5/selection.py index c060d26..1d01db9 100644 --- a/sigimax/h5/selection.py +++ b/sigimax/h5/selection.py @@ -16,7 +16,7 @@ import h5py import numpy as np -__all__ = ["NDSelection", "preview_step", "read_selection"] +__all__ = ["NDSelection", "NDView", "default_view", "preview_step", "read_selection"] @dataclasses.dataclass(frozen=True) @@ -57,6 +57,122 @@ def as_dict(self) -> dict[str, list[int]]: return {"displayed": list(self.displayed), "indices": list(self.indices)} +@dataclasses.dataclass(frozen=True) +class NDView: + """View of a multidimensional dataset, imported as one object per frame + + A frame is a 1D or 2D view (:class:`NDSelection`): frames are taken along one + axis which is not displayed, from a first to a last index with a step, the + other hidden axes being fixed. + + Args: + displayed: displayed axes, in display order: ``(axis,)`` for signals, + ``(row_axis, column_axis)`` for images + indices: index along each axis which is not displayed, in axis order (for + the frame axis: index of the first frame) + frame_axis: axis along which frames are taken (None: a single frame) + last: index of the last frame along the frame axis (included) + step: step between frames + """ + + displayed: tuple[int, ...] + indices: tuple[int, ...] = () + frame_axis: int | None = None + last: int = 0 + step: int = 1 + + def hidden_axes(self, ndim: int) -> list[int]: + """Return the axes which are not displayed, in axis order""" + return [axis for axis in range(ndim) if axis not in self.displayed] + + def frame_indices(self, ndim: int) -> range: + """Return the indices of the frames along the frame axis""" + if self.frame_axis is None: + return range(1) + first = self.indices[self.hidden_axes(ndim).index(self.frame_axis)] + return range(first, self.last + 1, self.step) + + def validate(self, shape: tuple[int, ...]) -> None: + """Raise ValueError if the view does not apply to an array shape + + Args: + shape: shape of the dataset + """ + ndim = len(shape) + NDSelection(self.displayed, self.indices).validate(shape) + if self.frame_axis is None: + return + if self.frame_axis not in self.hidden_axes(ndim): + raise ValueError(f"Frame axis {self.frame_axis} is displayed or missing") + if self.step < 1: + raise ValueError("The step between frames must be positive") + first = self.indices[self.hidden_axes(ndim).index(self.frame_axis)] + if not first <= self.last < shape[self.frame_axis]: + raise ValueError( + f"Last frame {self.last} out of range [{first}, " + f"{shape[self.frame_axis] - 1}]" + ) + + def selections(self, shape: tuple[int, ...]) -> list[NDSelection]: + """Return the selection of each frame + + Args: + shape: shape of the dataset + + Raises: + ValueError: if the view does not apply to the shape + """ + self.validate(shape) + if self.frame_axis is None: + return [NDSelection(self.displayed, self.indices)] + position = self.hidden_axes(len(shape)).index(self.frame_axis) + selections = [] + for index in self.frame_indices(len(shape)): + indices = list(self.indices) + indices[position] = index + selections.append(NDSelection(self.displayed, tuple(indices))) + return selections + + def as_dict(self) -> dict[str, object]: + """Return the view as a JSON-compatible dictionary""" + result: dict[str, object] = { + "displayed": list(self.displayed), + "indices": list(self.indices), + } + if self.frame_axis is not None: + result.update(frame_axis=self.frame_axis, last=self.last, step=self.step) + return result + + +def default_view(shape: tuple[int, ...], interpretation: str = "") -> NDView: + """Return the default view of a multidimensional dataset: every frame along the + axis preceding the displayed ones + + Args: + shape: shape of the dataset (two dimensions or more) + interpretation: NeXus ``interpretation`` of the data: ``"spectrum"`` + displays the last axis as signals, ``"rgba-image"`` displays the first + two axes as images, one per channel (last axis); otherwise the last two + axes are displayed as images + + Raises: + ValueError: if the dataset has less than two dimensions + """ + ndim = len(shape) + if ndim < 2: + raise ValueError("A view needs a dataset with two dimensions or more") + if interpretation == "spectrum": + displayed, frame_axis = (ndim - 1,), ndim - 2 + elif interpretation == "rgba-image" and ndim >= 3: + displayed, frame_axis = (ndim - 3, ndim - 2), ndim - 1 + elif ndim == 2: + return NDView((0, 1)) + else: + displayed, frame_axis = (ndim - 2, ndim - 1), ndim - 3 + indices = tuple(0 for axis in range(ndim) if axis not in displayed) + return NDView(displayed, indices, frame_axis, shape[frame_axis] - 1) + + def preview_step(shape: tuple[int, ...], max_items: int) -> int: """Return the smallest step keeping a strided view under a number of elements diff --git a/sigimax/h5/utils.py b/sigimax/h5/utils.py index 312fa0b..f57d809 100644 --- a/sigimax/h5/utils.py +++ b/sigimax/h5/utils.py @@ -110,6 +110,28 @@ def get_size(dset: h5py.Dataset) -> int: return 0 if dset.shape is None else int(np.prod(dset.shape, dtype=np.int64)) +def unavailable_filters(dset: h5py.Dataset) -> list[str]: + """Return the compression filters of a dataset that HDF5 cannot apply + + Such datasets cannot be read. Filters of detector data (bitshuffle/LZ4, + Blosc...) are made available by hdf5plugin, imported with :mod:`sigimax.h5`. + + Args: + dset: HDF5 dataset + + Returns: + Name and identifier of each unavailable filter, e.g. ``"bitshuffle (32008)"`` + """ + plist = dset.id.get_create_plist() + names = [] + for index in range(plist.get_nfilters()): + code, _flags, _values, name = plist.get_filter(index) + if not h5py.h5z.filter_avail(code): + text = to_string(name).strip() or "filter" + names.append(f"{text} ({code})") + return names + + def read_bounded(dset: h5py.Dataset, max_items: int) -> tuple[Any, bool]: """Read the first elements of a dataset, along each axis diff --git a/sigimax/h5/views.py b/sigimax/h5/views.py new file mode 100644 index 0000000..3190c90 --- /dev/null +++ b/sigimax/h5/views.py @@ -0,0 +1,226 @@ +# Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file. + +""" +SigimaX views of multidimensional HDF5 datasets + +Datasets with three dimensions or more (image stacks, spectral cubes...) are +imported through a view (:class:`~sigimax.h5.selection.NDView`): one signal or +image per frame. Nodes of such datasets derive from :class:`NDViewMixin`: a +default view is proposed (and previewed), and the node is imported once its view +is enabled (:meth:`NDViewMixin.set_view`). When importing, each enabled node is +replaced by one :class:`FrameNode` per frame (:func:`expand_views`), so that +applications import frames as they import other nodes. +""" + +from __future__ import annotations + +import math + +from sigimax.h5 import common +from sigimax.h5.selection import NDSelection, NDView, default_view, preview_step + +__all__ = ["FrameNode", "NDViewMixin", "expand_views", "frame_index_text"] + + +def frame_index_text(selection: NDSelection, ndim: int) -> str: + """Return the indices of a frame, NumPy style, e.g. ``[3, :, :]`` + + Args: + selection: frame selection + ndim: number of dimensions of the dataset + """ + hidden = iter(selection.indices) + words = [ + ":" if axis in selection.displayed else str(next(hidden)) + for axis in range(ndim) + ] + return "[" + ", ".join(words) + "]" + + +class NDViewMixin: + """Mixin of the nodes of multidimensional datasets, imported through a view + + Subclasses call :meth:`init_view` once their shape is known, and implement + :attr:`nd_shape` and :meth:`create_view_object`. + + Attributes: + view: current view (None if the dataset cannot be viewed) + import_view: True if the frames of the view are to be imported + """ + + view: NDView | None = None + import_view: bool = False + + @property + def nd_shape(self) -> tuple[int, ...] | None: + """Return the shape of the multidimensional data (None: no view)""" + raise NotImplementedError + + @property + def interpretation(self) -> str: + """Return the NeXus interpretation of the data (empty if not given)""" + return "" + + def init_view(self) -> None: + """Set the default view, not enabled, for data of three dimensions or + more""" + shape = self.nd_shape + self.view = None + if shape is not None and len(shape) >= 3 and math.prod(shape) > 0: + self.view = default_view(shape, self.interpretation) + self.import_view = False + + def set_view(self, view: NDView | None, enabled: bool = True) -> None: + """Set the view, and whether its frames are to be imported + + Args: + view: view (None: default view) + enabled: True if the frames of the view are to be imported + + Raises: + ValueError: if the node cannot be viewed, or the view does not apply to + its shape + """ + shape = self.nd_shape + if shape is None or len(shape) < 3: + raise ValueError("Only data with three dimensions or more is viewed") + if view is None: + view = default_view(shape, self.interpretation) + view.validate(shape) + self.view, self.import_view = view, enabled + + def frame_count(self) -> int: + """Return the number of frames of the view (0 if there is no view)""" + if self.view is None: + return 0 + return len(self.view.frame_indices(len(self.nd_shape))) + + def frame_positions( # pylint: disable=unused-argument + self, selection: NDSelection + ) -> dict[str, float]: + """Return the coordinates of a frame along the hidden axes which have + coordinates, by axis name (none by default)""" + return {} + + def axis_name(self, dim: int) -> str: # pylint: disable=unused-argument + """Return the name of the axis of a dimension (empty if it has none)""" + return "" + + def axis_label(self, dim: int) -> str: + """Return a label describing a dimension: number, axis name and size""" + name = self.axis_name(dim) + size = self.nd_shape[dim] + return f"{dim} ({name}, {size})" if name else f"{dim} ({size})" + + def frame_title(self, selection: NDSelection) -> str: + """Return the title of the object of a frame: node name, frame indices and + coordinates along the hidden axes, if any""" + suffix = frame_index_text(selection, len(self.nd_shape)) + positions = self.frame_positions(selection) + if positions: + text = ", ".join(f"{name}={value:g}" for name, value in positions.items()) + suffix += f" ({text})" + return self.make_title(suffix) + + def frame_nodes(self) -> list[FrameNode]: + """Return one node per frame of the view""" + if self.view is None: + return [] + return [FrameNode(self, sel) for sel in self.view.selections(self.nd_shape)] + + def preview_view_object(self, max_items: int): + """Return the object of the first frame of the view, read with a step if + it has more than ``max_items`` elements""" + selection = self.view.selections(self.nd_shape)[0] + return self.create_view_object( + selection, self.frame_title(selection), max_items + ) + + def create_view_object( + self, selection: NDSelection, title: str, max_items: int | None = None + ): + """Create the signal or image of a frame + + Args: + selection: frame selection + title: object title + max_items: maximum number of elements to read (None: read the whole + frame and record the origin of the object) + """ + raise NotImplementedError + + @staticmethod + def view_step(sizes: tuple[int, ...], max_items: int | None) -> int: + """Return the step to read a frame of the given sizes""" + if max_items is None or math.prod(sizes) <= max_items: + return 1 + return preview_step(sizes, max_items) + + +class FrameNode(common.BaseNode): + """Node of one frame of the view of a multidimensional node, created when the + frames are imported + + Args: + parent: multidimensional node + selection: frame selection + """ + + def __init__(self, parent: NDViewMixin, selection: NDSelection) -> None: + super().__init__(parent.h5file, parent.dset.name) + self.parent = parent + self.selection = selection + self.metadata = dict(parent.metadata) + + @property + def id(self): + """Return node id, distinct from the HDF5 paths of the file""" + return self.parent.id + frame_index_text( + self.selection, len(self.parent.nd_shape) + ) + + @property + def name(self): + """Return node name""" + return self.parent.name + frame_index_text( + self.selection, len(self.parent.nd_shape) + ) + + @property + def object_title(self): + """Return the title of the object of the frame""" + return self.parent.frame_title(self.selection) + + def is_supported(self) -> bool: + """Frames are always imported""" + return True + + def create_native_object(self): + """Create the signal or image of the frame""" + return self.parent.create_view_object(self.selection, self.object_title) + + def create_preview_object(self, max_items: int): + """Return the frame, read with a step if it has more than ``max_items`` + elements""" + return self.parent.create_view_object( + self.selection, self.object_title, max_items + ) + + +def expand_views(nodes: list[common.BaseNode]) -> list[common.BaseNode]: + """Return nodes to import, multidimensional nodes being replaced by their + frames + + Args: + nodes: nodes selected for import + + Returns: + Nodes to import + """ + expanded: list[common.BaseNode] = [] + for node in nodes: + if isinstance(node, NDViewMixin) and node.import_view: + expanded.extend(node.frame_nodes()) + elif not isinstance(node, NDViewMixin) or node.is_supported(): + expanded.append(node) + return expanded diff --git a/sigimax/tests/hdf5/test_h5_views.py b/sigimax/tests/hdf5/test_h5_views.py new file mode 100644 index 0000000..2a02033 --- /dev/null +++ b/sigimax/tests/hdf5/test_h5_views.py @@ -0,0 +1,296 @@ +# Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file. + +""" +Multidimensional views and NeXus conventions unit tests +-------------------------------------------------------- + +Datasets of three dimensions or more are imported through a view, one signal or +image per frame. Legacy NeXus ``axis``/``primary`` attributes designate axes by +dimension, auxiliary signals are shown as nodes of their own, entries give the +title and times recorded in the origin, ``@default`` designates the default +``NXdata`` group, and signals compressed with unavailable filters are reported. +""" + +from __future__ import annotations + +import contextlib +import uuid +from typing import Callable + +import h5py +import hdf5plugin +import numpy as np +import pytest +from sigima.objects import ImageObj, SignalObj, get_experimental_source + +from sigimax.h5 import H5Importer +from sigimax.h5.generic import GenericNDArrayNode +from sigimax.h5.mapping import ( + describe_mapping, + mapping_from_dict, +) +from sigimax.h5.nexus import ( + AuxiliarySignalNode, + NXdataNode, + default_nxdata_path, + entry_information, +) +from sigimax.h5.selection import NDSelection, NDView, default_view +from sigimax.h5.views import FrameNode, expand_views + +pytestmark = pytest.mark.unit + + +@contextlib.contextmanager +def _nodes_of(tmp_path, build: Callable[[h5py.File], None]): + """Create a file with a builder function, and yield its nodes by id.""" + fname = str(tmp_path / f"{uuid.uuid4()}.nxs") + with h5py.File(fname, "w") as h5file: + build(h5file) + importer = H5Importer(fname) + try: + yield {node.id: node for node in importer.nodes} + finally: + importer.close() + + +def test_view_frames() -> None: + """Views give one selection per frame, and are validated.""" + view = NDView((1, 2), (1,), 0, 5, 2) + assert view.hidden_axes(3) == [0] + assert list(view.frame_indices(3)) == [1, 3, 5] + assert view.selections((6, 4, 5)) == [ + NDSelection((1, 2), (index,)) for index in (1, 3, 5) + ] + assert view.as_dict() == { + "displayed": [1, 2], + "indices": [1], + "frame_axis": 0, + "last": 5, + "step": 2, + } + spectra = NDView((2,), (0, 3), 0, 1) + assert spectra.selections((2, 4, 8)) == [ + NDSelection((2,), (0, 3)), + NDSelection((2,), (1, 3)), + ] + for invalid, message in ( + (NDView((1, 2), (0,), 0, 6), "Last frame 6 out of range"), + (NDView((1, 2), (0,), 1, 0), "is displayed or missing"), + (NDView((1, 2), (0,), 0, 1, 0), "must be positive"), + ): + with pytest.raises(ValueError, match=message): + invalid.validate((6, 4, 5)) + + +def test_default_views() -> None: + """Default views follow the NeXus interpretation of the data.""" + assert default_view((5, 3, 4)) == NDView((1, 2), (0,), 0, 4) + assert default_view((2, 5, 3, 4)) == NDView((2, 3), (0, 0), 1, 4) + assert default_view((5, 8), "spectrum") == NDView((1,), (0,), 0, 4) + assert default_view((6, 4, 3), "rgba-image") == NDView((0, 1), (0,), 2, 2) + with pytest.raises(ValueError, match="two dimensions or more"): + default_view((5,)) + + +def _step_scan(h5file: h5py.File) -> None: + """Scan written as by SOLEIL: legacy attributes, counters and detectors of + several ranks all designated as signals, time stamps with axis 0""" + entry = h5file.create_group("scan_0494") + entry.attrs["NX_class"] = "NXentry" + entry["title"] = np.array([b"step scan"]) + entry["start_time"] = "2021-03-16T15:06:05+01:00" + scan = entry.create_group("scan_data") + scan.attrs["NX_class"] = "NXdata" + scan["actuator_1_1"] = np.array([10.0, 10.5, 11.0]) + scan["actuator_1_1"].attrs.update(axis=np.int32(1), primary=np.int32(1)) + scan["actuator_1_1"].attrs["units"] = "deg" + scan["trajectory_1_1"] = np.array([10.0, 10.5, 11.0]) + scan["trajectory_1_1"].attrs.update(axis=np.int32(1), primary=np.int32(2)) + scan["sensors_rel_timestamps"] = np.array([0.0, 1.0, 2.0]) + scan["sensors_rel_timestamps"].attrs["axis"] = np.int32(0) + scan["data_01"] = np.array([1.0, 2.0, 3.0]) + scan["data_02"] = np.arange(3 * 8, dtype=np.int32).reshape(3, 8) + scan["data_03"] = np.arange(3 * 4 * 5, dtype=np.uint16).reshape(3, 4, 5) + scan["data_03"].attrs["interpretation"] = "image" + for name in ("data_01", "data_02", "data_03"): + scan[name].attrs["signal"] = np.int32(1) + + +def test_legacy_axes_and_auxiliary_signals(tmp_path) -> None: + """Legacy axis attributes give the axis of each dimension that fits, the + primary one first; other signals are auxiliary signal nodes.""" + with _nodes_of(tmp_path, _step_scan) as nodes: + main = nodes["/scan_0494/scan_data"] + assert main.__class__ is NXdataNode and main.is_supported() + assert main.text == "NXdata: data_01 (actuator_1_1)" + signal = main.get_native_object() + assert isinstance(signal, SignalObj) + np.testing.assert_array_equal(signal.x, [10.0, 10.5, 11.0]) + assert (signal.xlabel, signal.xunit) == ("actuator_1_1", "deg") + source = get_experimental_source(signal) + assert source["details"] == {"entry_title": "step scan"} + assert source["timestamps"] == {"start": "2021-03-16T15:06:05+01:00"} + image = nodes["/scan_0494/scan_data::data_02"] + assert isinstance(image, AuxiliarySignalNode) and image.is_supported() + assert image.text == "NXdata auxiliary signal: data_02 (actuator_1_1, .)" + obj = image.get_native_object() + assert isinstance(obj, ImageObj) and obj.data.shape == (3, 8) + assert (obj.y0, obj.dy, obj.ylabel) == (10.0, 0.5, "actuator_1_1") + assert get_experimental_source(obj)["selection"]["signal"] == "data_02" + stack = nodes["/scan_0494/scan_data::data_03"] + assert not stack.is_supported() and stack.can_preview() + assert stack.view == NDView((1, 2), (0,), 0, 2) + + +def test_auxiliary_signals_attribute(tmp_path) -> None: + """Auxiliary signals of the @auxiliary_signals attribute have the signal + axes.""" + + def build(h5file: h5py.File) -> None: + group = h5file.create_group("data") + group.attrs.update(NX_class="NXdata", signal="I", axes=["energy"]) + group.attrs["auxiliary_signals"] = ["I0", "missing"] + group["energy"] = np.array([1.0, 2.0, 4.0]) + group["I"] = np.ones(3) + group["I0"] = np.full(3, 2.0) + + with _nodes_of(tmp_path, build) as nodes: + signal = nodes["/data::I0"].get_native_object() + np.testing.assert_array_equal(signal.x, [1.0, 2.0, 4.0]) + np.testing.assert_array_equal(signal.y, [2.0, 2.0, 2.0]) + assert "/data::missing" not in nodes + + +def test_nxdata_frames(tmp_path) -> None: + """Frames of a multidimensional NXdata signal are images with the axes of the + displayed dimensions, titled and located with the frame axis values.""" + + def build(h5file: h5py.File) -> None: + group = h5file.create_group("stack") + group.attrs.update(NX_class="NXdata", signal="frames") + group.attrs["axes"] = np.array([b"theta", b".", b"x"]) + group["frames"] = np.arange(4 * 3 * 2, dtype=np.uint16).reshape(4, 3, 2) + group["theta"] = np.array([0.0, 0.1, 0.2, 0.3]) + group["x"] = np.array([5.0, 7.0]) + + with _nodes_of(tmp_path, build) as nodes: + node = nodes["/stack"] + assert not node.is_supported() and node.create_native_object() is None + assert not expand_views([node]) + node.set_view(NDView((1, 2), (1,), 0, 3, 2)) + assert node.is_supported() and node.frame_count() == 2 + frames = expand_views([node]) + assert all(isinstance(frame, FrameNode) for frame in frames) + first, second = (frame.get_native_object() for frame in frames) + assert first.title.startswith("stack[1, :, :] (theta=0.1)") + np.testing.assert_array_equal(first.data, np.arange(6, 12).reshape(3, 2)) + assert (first.x0, first.dx, first.xlabel) == (5.0, 2.0, "x") + np.testing.assert_array_equal(second.data, np.arange(18, 24).reshape(3, 2)) + source = get_experimental_source(second) + assert source["selection"]["view"] == {"displayed": [1, 2], "indices": [3]} + assert source["selection"]["positions"] == {"theta": 0.3} + node.set_view(NDView((0,), (0, 1), 1, 2)) + spectrum = expand_views([node])[1].get_native_object() + np.testing.assert_array_equal(spectrum.x, [0.0, 0.1, 0.2, 0.3]) + np.testing.assert_array_equal(spectrum.y, [3, 9, 15, 21]) + preview = node.create_preview_object(2) + assert isinstance(preview, SignalObj) and preview.y.size == 2 + assert get_experimental_source(preview) is None + + +def test_generic_frames(tmp_path) -> None: + """Frames of a plain dataset have pixel coordinates.""" + + def build(h5file: h5py.File) -> None: + h5file["cube"] = np.arange(2 * 3 * 4, dtype=float).reshape(2, 3, 4) + + with _nodes_of(tmp_path, build) as nodes: + node = nodes["/cube"] + assert isinstance(node, GenericNDArrayNode) and node.can_preview() + assert node.view == NDView((1, 2), (0,), 0, 1) + node.set_view(None) + images = [frame.get_native_object() for frame in node.frame_nodes()] + assert [image.title.split()[0] for image in images] == [ + "cube[0,", + "cube[1,", + ] + assert images[0].title.startswith("cube[0, :, :]") + np.testing.assert_array_equal(images[1].data, np.arange(12, 24).reshape(3, 4)) + source = get_experimental_source(images[1]) + assert source["kind"] == "hdf5" and source["entry"] == "/cube" + assert node.create_preview_object(4).data.shape == (2, 2) + + +def test_entry_and_default(tmp_path) -> None: + """The default NXdata group follows the @default attributes.""" + + def build(h5file: h5py.File) -> None: + h5file.attrs["default"] = "entry" + entry = h5file.create_group("entry") + entry.attrs.update(NX_class="NXentry", default="data") + entry["end_time"] = b"2026-10-27T10:00:00" + group = entry.create_group("data") + group.attrs.update(NX_class="NXdata", signal="y") + group["y"] = np.ones(3) + + with _nodes_of(tmp_path, build) as nodes: + node = nodes["/entry/data"] + assert default_nxdata_path(node.h5file) == "/entry/data" + assert entry_information(node.dset) == ({}, {"end": "2026-10-27T10:00:00"}) + assert default_nxdata_path(nodes["/"].h5file) == "/entry/data" + + +def test_compression_filters(tmp_path, monkeypatch: pytest.MonkeyPatch) -> None: + """Compressed detector data is read with hdf5plugin; data compressed with an + unavailable filter is reported instead of failing.""" + data = np.arange(64 * 32, dtype=np.uint32).reshape(64, 32) + fname = str(tmp_path / "eiger.nxs") + with h5py.File(fname, "w") as h5file: + group = h5file.create_group("eiger") + group.attrs.update(NX_class="NXdata", signal="data") + group.create_dataset("data", data=data, **hdf5plugin.Bitshuffle(cname="lz4")) + importer = H5Importer(fname) + try: + node = importer.get("/eiger") + np.testing.assert_array_equal(node.get_native_object().data, data) + finally: + importer.close() + available = h5py.h5z.filter_avail + monkeypatch.setattr( + h5py.h5z, "filter_avail", lambda code: code != 32008 and available(code) + ) + importer = H5Importer(fname) + try: + node = importer.get("/eiger") + assert not node.is_supported() + assert "compressed with unavailable filter: bitshuffle" in node.text + raw = importer.get("/eiger/data") + assert not raw.is_supported() + assert raw.text.startswith(" None: + """Mapping axes describe the first dimensions: signals of several ranks share + one mapped object.""" + mapping = mapping_from_dict( + { + "version": 1, + "name": "Scan", + "match": {"paths": ["actuator_1_1"]}, + "objects": [ + {"name": "{name}", "signal": "data_*", "axes": ["actuator_1_1"]} + ], + } + ) + with _nodes_of(tmp_path, _step_scan) as nodes: + group = nodes["/scan_0494/scan_data"].dset + described = dict(describe_mapping(mapping, group)) + assert [axis.name if axis else None for axis in described["data_02"].axes] == [ + "actuator_1_1", + None, + ] + assert all(not desc.problems for desc in described.values()) + assert described["data_03"].shape == (3, 4, 5) diff --git a/sigimax/tests/hdf5/test_h5browser_views.py b/sigimax/tests/hdf5/test_h5browser_views.py new file mode 100644 index 0000000..d833d0a --- /dev/null +++ b/sigimax/tests/hdf5/test_h5browser_views.py @@ -0,0 +1,89 @@ +# Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file. + +""" +HDF5 browser view selector test +------------------------------- + +The default NXdata group of a NeXus file is selected when the file is opened. +Multidimensional datasets are shown with a view selector: their frames are +previewed, and imported once enabled, which makes their item checkable; the +item expands into one node per frame when importing. +""" + +from __future__ import annotations + +import uuid + +import h5py +import numpy as np +import pytest +from guidata.qthelpers import qt_app_context +from qtpy import QtCore as QC +from qtpy import QtWidgets as QW + +from sigimax.h5.views import FrameNode +from sigimax.widgets.h5browser import H5BrowserDialog + +pytestmark = pytest.mark.gui + + +def _write_file(fname: str) -> None: + """Write a NeXus file with a default 1D NXdata group and an image stack""" + with h5py.File(fname, "w") as h5file: + h5file.attrs["default"] = "entry" + entry = h5file.create_group("entry") + entry.attrs.update(NX_class="NXentry", default="data") + data = entry.create_group("data") + data.attrs.update(NX_class="NXdata", signal="y") + data["y"] = np.arange(5.0) + stack = entry.create_group("stack") + stack.attrs.update(NX_class="NXdata", signal="frames") + stack.attrs["axes"] = np.array([b"theta", b".", b"."]) + stack["frames"] = np.arange(4 * 3 * 2, dtype=np.uint16).reshape(4, 3, 2) + stack["theta"] = np.array([0.0, 0.1, 0.2, 0.3]) + + +def test_view_selector(tmp_path) -> None: + """Frames of a stack are imported once enabled in the view selector.""" + fname = str(tmp_path / f"{uuid.uuid4()}.nxs") + _write_file(fname) + with qt_app_context(): + dialog = H5BrowserDialog() + try: + dialog.open_file(fname) + browser, tree = dialog.browser, dialog.browser.tree + assert tree.currentItem().data(0, QC.Qt.UserRole) == "/entry/data" + selector = browser.viewselector + assert selector.node is None + item = tree.find_node_item(tree.topLevelItem(0), "/entry/stack") + assert not item.flags() & QC.Qt.ItemIsUserCheckable + tree.clicked(item) + node = selector.node + assert node is tree.get_node(item) and not selector.isHidden() + assert node.frame_count() == 4 + selector.last_spin.setValue(1) + assert node.frame_count() == 2 and not node.import_view + selector.import_check.setChecked(True) + assert item.checkState(0) == QC.Qt.Checked and node.import_view + frames = tree.get_nodes() + assert [type(frame) for frame in frames] == [FrameNode, FrameNode] + assert [frame.name for frame in frames] == [ + "stack[0, :, :]", + "stack[1, :, :]", + ] + # Signals along the first axis: views are images of pairs of axes, + # then signals along each axis + selector.view_combo.setCurrentIndex(3) + assert node.view.displayed == (0,) and node.frame_count() == 3 + assert dialog.bbox.button(QW.QDialogButtonBox.Ok).isEnabled() + item.setCheckState(0, QC.Qt.Unchecked) + assert not node.import_view and not selector.import_check.isChecked() + assert not item.flags() & QC.Qt.ItemIsUserCheckable + assert not tree.get_nodes() + data_item = tree.find_node_item(tree.topLevelItem(0), "/entry/data") + tree.clicked(data_item) + assert selector.node is None and selector.isHidden() + finally: + dialog.browser.plotpreview.wait() + dialog.cleanup() + dialog.close() diff --git a/sigimax/widgets/h5browser.py b/sigimax/widgets/h5browser.py index c9d28e2..2850940 100644 --- a/sigimax/widgets/h5browser.py +++ b/sigimax/widgets/h5browser.py @@ -42,7 +42,11 @@ from sigimax.adapters_plotpy import CURVESTYLES, create_adapter_from_object from sigimax.config import _, get_conf +from sigimax.env import execenv from sigimax.h5 import H5Importer +from sigimax.h5.nexus import default_nxdata_path +from sigimax.h5.selection import NDView +from sigimax.h5.views import FrameNode, NDViewMixin, expand_views from sigimax.utils.qthelpers import block_signals, qt_handle_error_message __all__ = [ @@ -51,8 +55,12 @@ "H5BrowserDialog", "H5FileSelector", "H5TreeWidget", + "ViewSelector", ] +#: Number of frames of multidimensional datasets imported without confirmation +MAX_FRAMES_WITHOUT_CONFIRMATION = 100 + if TYPE_CHECKING: from plotpy.plot import BasePlot @@ -227,11 +235,15 @@ def __init__(self, parent: QW.QWidget) -> None: self.fnames: list[str] = [] self.h5importers: list[H5Importer] = [] - def add_root(self, fname: str) -> None: + def add_root(self, fname: str) -> QW.QTreeWidgetItem | None: """Add HDF5 root (new file) Args: fname: HDF5 file name + + Returns: + Item of the default NeXus ``NXdata`` group of the file (``@default`` + attributes), expanded, or None """ self.fnames.append(osp.abspath(fname)) importer = H5Importer(fname) @@ -246,6 +258,56 @@ def add_root(self, fname: str) -> None: for index in range(rootitem.childCount()): child = rootitem.child(index) self.collapseItem(child) + path = default_nxdata_path(importer.h5file) + if path is None: + return None + item = self.find_node_item(rootitem, path) + if item is not None: + parent = item.parent() + while parent is not None: + self.expandItem(parent) + parent = parent.parent() + return item + + def find_node_item( + self, rootitem: QW.QTreeWidgetItem, node_id: str + ) -> QW.QTreeWidgetItem | None: + """Return the item of a node of a file, by node id + + Args: + rootitem: root item of the file + node_id: node id + """ + items = [rootitem] + while items: + item = items.pop() + if item.data(0, QC.Qt.UserRole) == node_id: + return item + items.extend(item.child(index) for index in range(item.childCount())) + return None + + def set_item_importable(self, item: QW.QTreeWidgetItem, state: bool) -> None: + """Make the item of a multidimensional node checkable and checked, once + the import of its frames is enabled, or neither of them + + Args: + item: tree item + state: True if the frames of the node are to be imported + """ + node = self.get_node(item) + with block_signals(self): + if state: + item.setFlags( + QC.Qt.ItemIsEnabled + | QC.Qt.ItemIsSelectable + | QC.Qt.ItemIsUserCheckable + ) + item.setCheckState(0, QC.Qt.Checked) + else: + item.setData(0, QC.Qt.CheckStateRole, None) + item.setFlags(QC.Qt.ItemIsEnabled) + item.setIcon(0, get_icon(node.icon_name)) + self.itemChanged.emit(item, 0) def remove_root(self, fname: str) -> None: """Remove HDF5 root @@ -303,6 +365,9 @@ def get_node(self, item: QW.QTreeWidgetItem) -> BaseNode: def get_nodes(self, only_checked_items: bool = True) -> list[BaseNode]: """Get all nodes associated to checked items + Multidimensional nodes are replaced by the nodes of the frames of their + view (see :func:`sigimax.h5.views.expand_views`). + Args: only_checked_items: If True, only checked items are returned @@ -317,7 +382,7 @@ def get_nodes(self, only_checked_items: bool = True) -> list[BaseNode]: if item is not self.topLevelItem(0): node = self.get_node(item) datasets.append(node) - return datasets + return expand_views(datasets) def activated(self, item: QW.QTreeWidgetItem) -> None: """Double-click event""" @@ -613,6 +678,207 @@ def __show_loaded_object( self.setCurrentWidget(widget) +class ViewSelector(QW.QGroupBox): + """View of the multidimensional dataset selected in the HDF5 browser + + The view shows frames of the dataset: signals along one axis or images of two + axes, taken along another axis from a first to a last index with a step, the + other axes being fixed. Its first frame is previewed; the frames are imported + once "Import these frames" is checked. + """ + + #: Emitted when the view is changed (to update the preview) + SIG_VIEW_CHANGED = QC.Signal() + #: Emitted when the import of the frames is enabled or disabled + SIG_IMPORT_CHANGED = QC.Signal(bool) + + def __init__(self, parent: QW.QWidget) -> None: + super().__init__(_("Multidimensional data"), parent) + self.node: NDViewMixin | None = None + self.__views: list[tuple[int, ...]] = [] + self.__hidden: list[int] = [] + self.view_combo = QW.QComboBox(self) + self.frame_combo = QW.QComboBox(self) + self.first_spin = QW.QSpinBox(self) + self.last_spin = QW.QSpinBox(self) + self.step_spin = QW.QSpinBox(self) + self.step_spin.setMinimum(1) + self.fixed_spins: dict[int, QW.QSpinBox] = {} + self.fixed_layout = QW.QHBoxLayout() + self.count_label = QW.QLabel(self) + self.import_check = QW.QCheckBox(_("Import these frames"), self) + frames = QW.QHBoxLayout() + for label, spin in ( + (_("First"), self.first_spin), + (_("Last"), self.last_spin), + (_("Step"), self.step_spin), + ): + frames.addWidget(QW.QLabel(label, self)) + frames.addWidget(spin) + frames.addStretch() + layout = QW.QFormLayout(self) + layout.addRow(_("View"), self.view_combo) + layout.addRow(_("Frames along"), self.frame_combo) + layout.addRow("", frames) + layout.addRow(_("Fixed indices"), self.fixed_layout) + layout.addRow(self.count_label, self.import_check) + self.view_combo.currentIndexChanged.connect(self.__view_combo_changed) + self.frame_combo.currentIndexChanged.connect(self.__frame_combo_changed) + for spin in (self.first_spin, self.last_spin, self.step_spin): + spin.valueChanged.connect(self.__view_edited) + self.import_check.toggled.connect(self.__import_toggled) + self.setVisible(False) + + @property + def ndim(self) -> int: + """Return the number of dimensions of the dataset""" + return len(self.node.nd_shape) + + def set_node(self, node: BaseNode | None) -> None: + """Show the view of a node, or hide the selector if the node has no view + + Args: + node: HDF5 node + """ + if not isinstance(node, NDViewMixin) or node.view is None: + self.node = None + self.setVisible(False) + return + self.node = node + view = node.view + self.__views = [ + (row, column) + for row in range(self.ndim) + for column in range(row + 1, self.ndim) + ] + [(dim,) for dim in range(self.ndim)] + if view.displayed not in self.__views: + self.__views.append(view.displayed) + with block_signals(self.view_combo): + self.view_combo.clear() + for displayed in self.__views: + self.view_combo.addItem(self.__view_text(displayed)) + self.view_combo.setCurrentIndex(self.__views.index(view.displayed)) + self.__set_hidden_axes(view) + with block_signals(self.import_check): + self.import_check.setChecked(node.import_view) + self.__update_count() + self.setVisible(True) + + def __view_text(self, displayed: tuple[int, ...]) -> str: + """Return the text of a view of the view combo box""" + labels = [self.node.axis_label(dim) for dim in displayed] + if len(displayed) == 1: + return _("Signals along axis %s") % labels[0] + return _("Images: rows along axis %s, columns along axis %s") % tuple(labels) + + def __set_hidden_axes(self, view: NDView) -> None: + """Set the frame axis combo box and the fixed index spin boxes from a view""" + shape = self.node.nd_shape + self.__hidden = view.hidden_axes(self.ndim) + indices = dict(zip(self.__hidden, view.indices)) + with block_signals(self.frame_combo): + self.frame_combo.clear() + for dim in self.__hidden: + self.frame_combo.addItem(_("axis %s") % self.node.axis_label(dim)) + frame_axis = view.frame_axis + if frame_axis is None: + frame_axis = self.__hidden[0] + self.frame_combo.setCurrentIndex(self.__hidden.index(frame_axis)) + for spin in self.fixed_spins.values(): + self.fixed_layout.removeWidget(spin) + spin.deleteLater() + self.fixed_spins = {} + for dim in self.__hidden: + if dim == frame_axis: + continue + spin = QW.QSpinBox(self) + spin.setRange(0, shape[dim] - 1) + spin.setValue(indices[dim]) + spin.setPrefix(f"{dim}: ") + spin.valueChanged.connect(self.__view_edited) + self.fixed_spins[dim] = spin + self.fixed_layout.addWidget(spin) + last = view.last if view.frame_axis is not None else shape[frame_axis] - 1 + self.__set_range(shape[frame_axis], indices[frame_axis], last, view.step) + + def __set_range(self, size: int, first: int, last: int, step: int) -> None: + """Set the frame range spin boxes""" + spins = (self.first_spin, self.last_spin, self.step_spin) + for spin in spins: + spin.blockSignals(True) + try: + self.first_spin.setRange(0, size - 1) + self.last_spin.setRange(0, size - 1) + self.step_spin.setRange(1, max(1, size)) + self.first_spin.setValue(first) + self.last_spin.setValue(last) + self.step_spin.setValue(step) + finally: + for spin in spins: + spin.blockSignals(False) + + def current_view(self) -> NDView: + """Return the view set in the selector""" + displayed = self.__views[self.view_combo.currentIndex()] + frame_axis = self.__hidden[self.frame_combo.currentIndex()] + indices = tuple( + self.first_spin.value() + if dim == frame_axis + else self.fixed_spins[dim].value() + for dim in self.__hidden + ) + last = max(self.last_spin.value(), self.first_spin.value()) + return NDView(displayed, indices, frame_axis, last, self.step_spin.value()) + + def __view_combo_changed(self, index: int) -> None: + """Displayed axes changed: every frame along the first hidden axis""" + displayed = self.__views[index] + hidden = [dim for dim in range(self.ndim) if dim not in displayed] + last = self.node.nd_shape[hidden[0]] - 1 + view = NDView(displayed, tuple(0 for _dim in hidden), hidden[0], last) + self.__set_hidden_axes(view) + self.__view_edited() + + def __frame_combo_changed(self, index: int) -> None: + """Frame axis changed: every frame along the new axis, the former frame axis + being fixed at the first frame""" + displayed = self.__views[self.view_combo.currentIndex()] + frame_axis = self.__hidden[index] + previous = {dim: spin.value() for dim, spin in self.fixed_spins.items()} + indices = tuple( + 0 if dim == frame_axis else previous.get(dim, self.first_spin.value()) + for dim in self.__hidden + ) + last = self.node.nd_shape[frame_axis] - 1 + view = NDView(displayed, indices, frame_axis, last) + self.__set_hidden_axes(view) + self.__view_edited() + + def __update_count(self) -> None: + """Show the number of frames of the view""" + count = self.node.frame_count() + kind = _("images") if len(self.node.view.displayed) == 2 else _("signals") + self.count_label.setText(_("%d frame(s) as %s") % (count, kind)) + + def __view_edited(self) -> None: + """Apply the view set in the selector to the node""" + if self.node is None: + return + try: + self.node.set_view(self.current_view(), self.node.import_view) + except ValueError: + return + self.__update_count() + self.SIG_VIEW_CHANGED.emit() + + def __import_toggled(self, state: bool) -> None: + """Enable or disable the import of the frames""" + if self.node is None: + return + self.node.set_view(self.current_view(), state) + self.SIG_IMPORT_CHANGED.emit(state) + + class TablePreview(QW.QWidget): """Table preview @@ -833,6 +1099,7 @@ class H5Browser(QW.QSplitter): def __init__(self, parent: QW.QWidget | None = None) -> None: super().__init__(parent) + self.__current_item: QW.QTreeWidgetItem | None = None if parent is None: self.setWindowIcon(get_icon(get_conf().app_logo_path.get())) self.selector = H5FileSelector(self) @@ -854,10 +1121,19 @@ def __init__(self, parent: QW.QWidget | None = None) -> None: preview.setOrientation(QC.Qt.Vertical) self.addWidget(preview) self.plotpreview = PlotPreview(self) - preview.addWidget(self.plotpreview) + self.viewselector = ViewSelector(self) + self.viewselector.SIG_VIEW_CHANGED.connect(self.__view_changed) + self.viewselector.SIG_IMPORT_CHANGED.connect(self.__view_import_changed) + plotandview = QW.QWidget(self) + plotandview.setLayout(QW.QVBoxLayout()) + plotandview.layout().setContentsMargins(0, 0, 0, 0) + plotandview.layout().addWidget(self.plotpreview) + plotandview.layout().addWidget(self.viewselector) + preview.addWidget(plotandview) self.groupandattrs = GroupAndAttributes(self, self.show_array) preview.addWidget(self.groupandattrs) preview.setSizes([int(self.size().height() / 2)] * 2) + self.tree.itemChanged.connect(self.__item_changed) def __create_toolbar(self) -> QW.QToolBar: """Create toolbar with tree actions @@ -878,13 +1154,17 @@ def __create_toolbar(self) -> QW.QToolBar: return toolbar def open_file(self, fname: str) -> None: - """Open HDF5 file + """Open HDF5 file, and select its default NeXus ``NXdata`` group, if any Args: fname: HDF5 file name """ - self.tree.add_root(fname) + default_item = self.tree.add_root(fname) self.selector.add_fname(fname) + if default_item is not None: + self.tree.setCurrentItem(default_item) + self.tree.scrollToItem(default_item) + self.__item_selected_on_tree(default_item) def close_file(self, fname: str) -> None: """Close HDF5 file @@ -893,6 +1173,8 @@ def close_file(self, fname: str) -> None: fname: HDF5 file name """ self.plotpreview.wait() + self.__current_item = None + self.viewselector.set_node(None) self.tree.remove_root(fname) self.selector.remove_fname(fname) @@ -941,11 +1223,13 @@ def __item_selected_on_tree(self, item: QW.QTreeWidgetItem) -> None: """ # View the selected item node = self.get_node(item) + self.__current_item = item # Discard the previous preview first: its pending read is not started, # so that reading the node description below does not wait for it self.plotpreview.discard() self.groupandattrs.update_from_node(node) - if node.is_supported(): + self.viewselector.set_node(node) + if node.can_preview(): self.plotpreview.update_plot_preview(node) else: self.plotpreview.clear_preview() @@ -967,6 +1251,28 @@ def __selector_current_file_changed(self, fname: str) -> None: if fname: self.tree.set_current_file(fname) + def __view_changed(self) -> None: + """View of the current multidimensional node changed: preview it""" + self.plotpreview.update_plot_preview(self.viewselector.node) + + def __view_import_changed(self, state: bool) -> None: + """Import of the frames of the current multidimensional node enabled or + disabled: make its item checkable and checked, or not""" + if self.__current_item is not None: + self.tree.set_item_importable(self.__current_item, state) + + def __item_changed(self, item: QW.QTreeWidgetItem, _column: int = 0) -> None: + """Item check state changed: an unchecked multidimensional node is not + imported any more""" + node = self.tree.get_node(item) + if not isinstance(node, NDViewMixin) or not node.import_view: + return + if item.checkState(0) == QC.Qt.Unchecked: + node.import_view = False + self.tree.set_item_importable(item, False) + if self.viewselector.node is node: + self.viewselector.set_node(node) + def show_array(self) -> None: """Show array""" node = self.get_node() @@ -1018,8 +1324,24 @@ def __init__( self.refresh_buttons() def accept(self) -> None: - """Accept changes""" - self.nodes = self.browser.tree.get_nodes() + """Accept changes, once confirmed if many frames of multidimensional + datasets are to be imported""" + nodes = self.browser.tree.get_nodes() + frames = sum(isinstance(node, FrameNode) for node in nodes) + if ( + frames > MAX_FRAMES_WITHOUT_CONFIRMATION + and not execenv.unattended + and QW.QMessageBox.question( + self, + self.windowTitle(), + _("%d frames of multidimensional datasets will be imported.") % frames + + "\n" + + _("Do you want to continue?"), + ) + != QW.QMessageBox.Yes + ): + return + self.nodes = nodes QW.QDialog.accept(self) def done(self, result: int) -> None: From c4f4212ad16d0ba14d401f58c39bb35afdf61619 Mon Sep 17 00:00:00 2001 From: Pierre Raybaut <1311787+PierreRaybaut@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:41:55 +0100 Subject: [PATCH 2/5] docs(h5): document multidimensional views and legacy NeXus scans User guide, release notes and French translations of the view selector. Assisted-by: Claude Opus 5.5 --- doc/release_notes/release_1.01.md | 12 ++-- doc/user_guide/hdf5_workspace.rst | 85 ++++++++++++++++++++++-- sigimax/locale/fr/LC_MESSAGES/sigimax.po | 55 +++++++++++++++ 3 files changed, 141 insertions(+), 11 deletions(-) diff --git a/doc/release_notes/release_1.01.md b/doc/release_notes/release_1.01.md index f82e7f0..47116bd 100644 --- a/doc/release_notes/release_1.01.md +++ b/doc/release_notes/release_1.01.md @@ -41,9 +41,12 @@ * Browsing an HDF5 file no longer reads its numeric datasets entirely just to classify them: generic nodes are classified and described from the dataset shape and data type, and value previews read only the first elements (`TEXT_PREVIEW_MAX_ITEMS`). Compound datasets mixing numeric and string fields are checked on their first records only. Full data is read when creating an imported object or an undecimated graphical preview. * The HDF5 browser reads previews in a worker thread, so that it stays responsive while a large dataset is read: only the last selected node is shown, the previews of nodes selected meanwhile are not read, selecting a node that cannot be previewed (a group, for example) clears the preview, and closing a file or the browser dialog waits for the preview being read (imported objects are then read without concurrent access to the file). Ordinary 1D arrays and 2D images, NeXus and mapped objects with more than `PlotPreview.MAX_ITEMS` (2²⁴) elements are previewed with a step along each axis; the axes of NeXus images are still read entirely, to check that they are strictly monotonic as on import (they hold far fewer values than the image). Compound datasets and signals stored in 2D tables still use complete objects for graphical previews; this is not a universal memory bound. -* Numeric datasets with three dimensions or more (image stacks, spectral cubes...) are now listed in the HDF5 browser, with their shape and type. They are not imported as a whole, and there is no graphical slice selector: developers can read their 1D or 2D views with `sigimax.h5.selection.read_selection`, which reads only the selected elements (`NDSelection`: displayed axes, possibly transposed, and an index along each other axis). -* NeXus `NXdata` groups are shown as single nodes, imported as a signal (1D) or an image (2D) with their axes, units, labels and, for signals, uncertainties: `@signal`/`@axes` attributes with `@_indices`, legacy `signal`/`axes` dataset attributes, bin edges converted to bin centers, non-uniform image axes. Image axes deviating from even spacing by at most 2.5 units in the last place of their data type (`UNIFORM_AXIS_TOLERANCE`) are imported as uniform coordinates, so that evenly spaced `float32` axes are recognized, and their maximum deviation is recorded among the conversions; other image axes keep their stored coordinates. Decreasing image axes are reversed with the image. Invalid axes (axis count differing from the signal rank, axes designating a missing or already described dimension, axes not holding real numbers, image axes not strictly monotonic) are reported instead of being guessed. Imported objects record their origin in the `experimental_source` metadata entry (file, group, signal and axes, `"."` for a dimension without axis, conversions). `.nxs` and `.nx5` files are proposed in HDF5 file dialogs. -* Files that do not designate their data with suitable `NXdata` groups (no axes, several counters...) can be described by declarative data mappings (`sigimax.h5.mapping`): JSON-compatible descriptions of the signals and images of the groups having given attributes and paths, with their axes, uncertainties, labels and units, and a name pattern to repeat an object for each counter of a scan. Mapped objects are shown as extra nodes of the HDF5 browser and record the mapping in their `experimental_source`. Applications and plugins register mappings with `register_mapping` (and `unregister_mapping`); `NodeFactory.add_children_provider` lets them add such extra nodes. Blank units and labels are now ignored. +* Numeric datasets with three dimensions or more (image stacks, spectral cubes...) are now listed in the HDF5 browser, with their shape and type, and imported through a view: one signal or image per frame. When such a dataset (or NeXus signal) is selected, a view selector proposes the displayed axes (images of two axes, or signals along one axis), the axis along which frames are taken with their first, last and step, and the index of the other axes; the first frame is previewed, and the frames are imported once "Import these frames" is checked, the browser asking for confirmation beyond `MAX_FRAMES_WITHOUT_CONFIRMATION` (100) frames. The default view follows the NeXus `interpretation` of the signal (`spectrum`: signals along the last axis, `rgba-image`: one image per channel, otherwise images of the last two axes). Frames of NeXus signals keep the axes of their displayed dimensions, and their title and origin give the frame indices and the coordinates of the frame along the other axes. Developers set views with `NDView`, `default_view` and `set_view`, and `expand_views` replaces a node by the nodes of its frames (`sigimax.h5.views`); `read_selection` reads only the selected elements (`NDSelection`: displayed axes, possibly transposed, and an index along each other axis). +* NeXus `NXdata` groups are shown as single nodes, imported as a signal (1D) or an image (2D) with their axes, units, labels and, for signals, uncertainties: `@signal`/`@axes` attributes with `@_indices`, legacy `signal`/`axes` dataset attributes, bin edges converted to bin centers, non-uniform image axes. Image axes deviating from even spacing by at most 2.5 units in the last place of their data type (`UNIFORM_AXIS_TOLERANCE`) are imported as uniform coordinates, so that evenly spaced `float32` axes are recognized, and their maximum deviation is recorded among the conversions; other image axes keep their stored coordinates. Decreasing image axes are reversed with the image. Invalid axes (axis count differing from the signal rank, axes designating a missing or already described dimension, axes not holding real numbers, image axes not strictly monotonic) are reported instead of being guessed. Imported objects record their origin in the `experimental_source` metadata entry (file, group, signal and axes, `"."` for a dimension without axis, conversions, and the title, start and end times of the entry). `.nxs` and `.nx5` files are proposed in HDF5 file dialogs. +* NeXus files written with the legacy conventions (e.g. scans of synchrotron SOLEIL) are imported with their axes: without `@axes`, the `axis` attribute of axis datasets gives the dimension they describe (from 1), the dataset with the lowest `primary` attribute being preferred, and an axis applies to the dimensions whose size fits. The other signals of an `NXdata` group (`@auxiliary_signals` attribute, or datasets having a legacy `signal` attribute, as counters and detectors of a scan) are shown as nodes of their own, with the axes that fit their dimensions: for example, a 1D counter is a signal versus the scanned motor, a detector writing a spectrum per point is an image with the motor along Y, and a detector writing an image per point is a stack of images, one per motor position. +* The HDF5 browser selects and expands the default `NXdata` group of a NeXus file (`@default` attributes of the file and of its entry) when the file is opened (`default_nxdata_path`). +* Compressed detector data (bitshuffle/LZ4 of Eiger detectors, Blosc, Zstandard...) is now read: SigimaX requires hdf5plugin, which provides these HDF5 filters. Datasets compressed with a filter that is still unavailable are reported in the browser (filter name and number) instead of failing when read. +* Files that do not designate their data with suitable `NXdata` groups (no axes, several counters...) can be described by declarative data mappings (`sigimax.h5.mapping`): JSON-compatible descriptions of the signals and images of the groups having given attributes and paths, with their axes, uncertainties, labels and units, and a name pattern to repeat an object for each counter of a scan. The `axes` of a mapped object describe its first dimensions, the following ones having no axis, so that one pattern applies to counters, spectra and images of a scan, with the scan axis first. Mapped objects are shown as extra nodes of the HDF5 browser and record the mapping in their `experimental_source`. Applications and plugins register mappings with `register_mapping` (and `unregister_mapping`); `NodeFactory.add_children_provider` lets them add such extra nodes. Blank units and labels are now ignored. * Dataset attributes with more than `MAX_ATTRIBUTE_ITEMS` (10 000) values are no longer copied to object metadata. * Images with unsigned 32-bit values above 2 147 483 647 are no longer clipped to `int32`: they are converted with Sigima's converter, which keeps every value exactly with Sigima ≥ 1.4.0. `BaseNode.uint32_wng` is kept for compatibility and is always `False`. * 1D datasets of single-precision complex numbers (`complex64`) are now imported as complex signals: only their real part was kept (with a `ComplexWarning`). Integer signals beyond 2⁵³ now emit Sigima's precision warning instead of being rounded silently, and 2D tables with complex X values or uncertainties are reported as unsupported instead of being truncated. @@ -55,4 +58,5 @@ See the [HDF5 guide](../user_guide/hdf5_workspace.rst) for a complete JSON mappi ### Requirements ### -* Sigima 1.4.0 or later is required: the HDF5 import modules use its experimental-source API, integer conversions use its precision-preserving converter, and the tooltips of rectangular ROIs use its pixel-range conversions. \ No newline at end of file +* Sigima 1.4.0 or later is required: the HDF5 import modules use its experimental-source API, integer conversions use its precision-preserving converter, and the tooltips of rectangular ROIs use its pixel-range conversions. +* hdf5plugin 4.0 or later is required, for the HDF5 compression filters of detector data. \ No newline at end of file diff --git a/doc/user_guide/hdf5_workspace.rst b/doc/user_guide/hdf5_workspace.rst index 6c8e65d..0cc52bd 100644 --- a/doc/user_guide/hdf5_workspace.rst +++ b/doc/user_guide/hdf5_workspace.rst @@ -90,6 +90,27 @@ With Sigima 1.4 or later, an ``NXdata`` group is importable as a 1D signal or (``signal=1`` and ``axes``) are supported. Selecting a group imports the scientific object; selecting a child dataset still gives access to raw values. +Files written with the legacy NeXus conventions, such as the scans of +synchrotron SOLEIL, designate axes by dimension: without ``axes`` attribute, the +``axis`` attribute of an axis dataset gives the dimension it describes (from 1), +the dataset with the lowest ``primary`` attribute being preferred, and an axis +applies to a dimension only if its size fits. The other signals of a group, given +by its ``auxiliary_signals`` attribute or by a legacy ``signal`` attribute of +their dataset (whatever its value, as all the counters and detectors of a scan), +are shown as nodes of their own, below the group, with the axes that fit their +dimensions: a 1D counter is a signal versus the scanned motor, a detector writing +a spectrum per point an image with the motor along Y, and a detector writing an +image per point a stack of images (see below). + +When a NeXus file is opened, the browser selects its default ``NXdata`` group, +designated by the ``default`` attributes of the file and of its entry +(``sigimax.h5.nexus.default_nxdata_path``). + +Compressed detector data (bitshuffle/LZ4, Blosc, Zstandard...) is read with the +HDF5 filters of hdf5plugin, a dependency of SigimaX. A dataset compressed with a +filter that is still unavailable is reported in the browser, with the filter name +and number, instead of failing when read. + Axis datasets must be 1D, hold real numbers and contain either one coordinate per point or one extra value for bin edges. Edges are converted to centers. Image coordinates may be non-uniform; a decreasing image axis is reversed, @@ -108,7 +129,9 @@ real numbers and their shape matches; image uncertainties are not imported. Imported objects record their file, group, signal, axes and conversions in ``experimental_source`` metadata; a dimension without axis is recorded as -``"."``, as in NeXus ``axes`` attributes. This describes the measurement +``"."``, as in NeXus ``axes`` attributes. The title of the ``NXentry`` group is +recorded in the details (``entry_title``), and its start and end times in the +timestamps. This describes the measurement origin, not a reproducible acquisition or a processing history. The application must preserve object metadata in its own workspace serialization. @@ -146,9 +169,11 @@ then required in the object name and is replaced by each dataset name. Uncertainty datasets ending in ``_errors`` and declared axes are excluded from pattern expansion. -``axes`` contains one entry per signal dimension, in array order (rows then +``axes`` gives the axes of the first signal dimensions, in array order (rows then columns for an image). Each entry is a path, an object with ``path`` and optional -``label`` and ``units``, or ``null`` to use indices. Omitting ``axes`` uses +``label`` and ``units``, or ``null`` to use indices. The following dimensions use +indices, so that a pattern may match signals of several ranks, e.g. the counters, +spectra and images of a scan, with the scan axis first. Omitting ``axes`` uses indices for every dimension. The optional ``errors`` path overrides the default ``_errors`` dataset. Labels and units otherwise come from ``long_name`` and ``units`` attributes, with the dataset name as label fallback. @@ -221,11 +246,57 @@ still read on import. Compound datasets and signals stored in 2D tables still use complete objects for graphical previews, so this is not a universal memory bound. -Numeric datasets with three or more dimensions are listed but not imported as -complete objects. There is no graphical slice selector. Use +Numeric datasets and NeXus signals with three or more dimensions are imported +through a view, as one signal or image per frame. When such an item is selected, +the browser shows a view selector: the displayed axes (images of two axes, or +signals along one axis), the axis along which frames are taken, with the first +and last frame and a step, and the index of the other axes. The first frame of +the view is previewed. Checking "Import these frames" makes the item checkable +and checked; unchecking the item cancels the import. The browser asks for +confirmation before importing more than 100 frames +(``sigimax.widgets.h5browser.MAX_FRAMES_WITHOUT_CONFIRMATION``). The default view +displays the last two axes as images, with every frame along the preceding axis, +or follows the NeXus ``interpretation`` attribute of the signal: ``spectrum`` +displays signals along the last axis, ``rgba-image`` one image per channel. The +frames of NeXus signals keep the axes of their displayed dimensions; their title +and ``experimental_source`` give the frame indices (``view``) and the coordinates +of the frame along the other axes (``positions``). + +Applications set views with :class:`sigimax.h5.selection.NDView`, and replace a +node by the nodes of its frames with :func:`sigimax.h5.views.expand_views`, which +the browser applies to the selected nodes: + +.. code-block:: python + + from pathlib import Path + from tempfile import TemporaryDirectory + + import h5py + import numpy as np + + from sigimax.h5 import H5Importer + from sigimax.h5.selection import NDView + from sigimax.h5.views import expand_views + + cube = np.arange(5 * 3 * 4).reshape(5, 3, 4) + with TemporaryDirectory() as directory: + filename = Path(directory) / "stack.h5" + with h5py.File(filename, "w") as h5file: + h5file["frames"] = cube + importer = H5Importer(str(filename)) + try: + node = importer.get("/frames") + # Images of axes 1 and 2, frames 1 and 3 along axis 0 + node.set_view(NDView((1, 2), (1,), frame_axis=0, last=3, step=2)) + images = [frame.get_native_object() for frame in expand_views([node])] + np.testing.assert_array_equal(images[1].data, cube[3]) + finally: + importer.close() + +To read a view as a NumPy array, without creating Sigima objects, use :class:`sigimax.h5.selection.NDSelection` and -:func:`sigimax.h5.selection.read_selection` to read only a selected 1D or 2D -view as a NumPy array: +:func:`sigimax.h5.selection.read_selection`, which read only the selected +elements: .. code-block:: python diff --git a/sigimax/locale/fr/LC_MESSAGES/sigimax.po b/sigimax/locale/fr/LC_MESSAGES/sigimax.po index 0afc25a..2127ba5 100644 --- a/sigimax/locale/fr/LC_MESSAGES/sigimax.po +++ b/sigimax/locale/fr/LC_MESSAGES/sigimax.po @@ -675,3 +675,58 @@ msgstr "Veuillez sélectionner le fichier à importer." msgid "Example Wizard" msgstr "Assistant d'exemple" +#: sigimax/widgets/h5browser.py +msgid "Multidimensional data" +msgstr "Données multidimensionnelles" + +#: sigimax/widgets/h5browser.py +msgid "Import these frames" +msgstr "Importer ces trames" + +#: sigimax/widgets/h5browser.py +msgid "First" +msgstr "Première" + +#: sigimax/widgets/h5browser.py +msgid "Last" +msgstr "Dernière" + +#: sigimax/widgets/h5browser.py +msgid "Step" +msgstr "Pas" + +#: sigimax/widgets/h5browser.py +msgid "View" +msgstr "Vue" + +#: sigimax/widgets/h5browser.py +msgid "Frames along" +msgstr "Trames selon" + +#: sigimax/widgets/h5browser.py +msgid "Fixed indices" +msgstr "Indices fixés" + +#: sigimax/widgets/h5browser.py +msgid "Signals along axis %s" +msgstr "Signaux selon l'axe %s" + +#: sigimax/widgets/h5browser.py +msgid "Images: rows along axis %s, columns along axis %s" +msgstr "Images : lignes selon l'axe %s, colonnes selon l'axe %s" + +#: sigimax/widgets/h5browser.py +msgid "axis %s" +msgstr "axe %s" + +#: sigimax/widgets/h5browser.py +msgid "%d frame(s) as %s" +msgstr "%d trame(s) en %s" + +#: sigimax/widgets/h5browser.py +msgid "%d frames of multidimensional datasets will be imported." +msgstr "%d trames de jeux de données multidimensionnels vont être importées." + +#: sigimax/widgets/h5browser.py +msgid "Do you want to continue?" +msgstr "Voulez-vous continuer ?" From 711219a11331e27da6a41df462a6603aba30a3e2 Mon Sep 17 00:00:00 2001 From: Pierre Raybaut <1311787+PierreRaybaut@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:08:24 +0100 Subject: [PATCH 3/5] fix(h5): read the NeXus interpretation on the NXdata group too The default view of a multidimensional signal follows its interpretation attribute, written on the signal dataset or, by some writers, on its group. Assisted-by: Claude Opus 5.5 --- sigimax/h5/nexus.py | 10 +++++++--- sigimax/tests/hdf5/test_h5_views.py | 17 +++++++++++++++++ 2 files changed, 24 insertions(+), 3 deletions(-) diff --git a/sigimax/h5/nexus.py b/sigimax/h5/nexus.py index d75647a..0eb075e 100644 --- a/sigimax/h5/nexus.py +++ b/sigimax/h5/nexus.py @@ -642,10 +642,14 @@ def nd_shape(self) -> tuple[int, ...] | None: @property def interpretation(self) -> str: - """Return the ``interpretation`` attribute of the signal (empty if not - given)""" + """Return the ``interpretation`` attribute of the signal, or of the group + (empty if not given)""" dset = self.dset.get(self.nxdata.signal) - return _text(dset.attrs.get("interpretation", "")).strip() + for item in (dset, self.dset): + text = _text(item.attrs.get("interpretation", "")).strip() + if text: + return text + return "" def is_supported(self) -> bool: """Return True if the signal is 1D or 2D and its axes are valid, or if the diff --git a/sigimax/tests/hdf5/test_h5_views.py b/sigimax/tests/hdf5/test_h5_views.py index 2a02033..755d762 100644 --- a/sigimax/tests/hdf5/test_h5_views.py +++ b/sigimax/tests/hdf5/test_h5_views.py @@ -93,6 +93,23 @@ def test_default_views() -> None: default_view((5,)) +def test_interpretation(tmp_path) -> None: + """The interpretation of a signal is read on its dataset or on its group.""" + + def build(h5file: h5py.File) -> None: + rgb = h5file.create_group("rgb") + rgb.attrs.update(NX_class="NXdata", signal="img", interpretation="rgba-image") + rgb["img"] = np.zeros((6, 4, 3), dtype=np.uint8) + spectra = h5file.create_group("spectra") + spectra.attrs.update(NX_class="NXdata", signal="counts") + spectra["counts"] = np.zeros((2, 3, 16)) + spectra["counts"].attrs["interpretation"] = "spectrum" + + with _nodes_of(tmp_path, build) as nodes: + assert nodes["/rgb"].view == NDView((0, 1), (0,), 2, 2) + assert nodes["/spectra"].view == NDView((2,), (0, 0), 1, 2) + + def _step_scan(h5file: h5py.File) -> None: """Scan written as by SOLEIL: legacy attributes, counters and detectors of several ranks all designated as signals, time stamps with axis 0""" From 603982729ea760a6aaba26e7bcc38fbbe1bd889c Mon Sep 17 00:00:00 2001 From: Pierre Raybaut <1311787+PierreRaybaut@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:12:55 +0100 Subject: [PATCH 4/5] perf(h5): query the unavailable filters of a dataset once The filters of generic dataset nodes are queried once and cached, instead of at each support check and text preview while the browser tree is built. Assisted-by: Claude Opus 5.5 --- sigimax/h5/generic.py | 25 +++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/sigimax/h5/generic.py b/sigimax/h5/generic.py index 072014a..b1a6bc6 100644 --- a/sigimax/h5/generic.py +++ b/sigimax/h5/generic.py @@ -8,6 +8,7 @@ from __future__ import annotations +import functools import math import h5py @@ -191,6 +192,12 @@ def match(cls, dset): """Return True if h5 dataset matches this node pattern.""" return isinstance(dset, h5py.Dataset) + @functools.cached_property + def missing_filters(self) -> list[str]: + """Compression filters of the dataset that HDF5 cannot apply (see + :func:`sigimax.h5.utils.unavailable_filters`)""" + return utils.unavailable_filters(self.dset) + @property def icon_name(self): """Icon name associated to node.""" @@ -452,9 +459,7 @@ def layout(self) -> tuple[tuple[int, ...], np.dtype]: def is_supported(self) -> bool: """Return True if node is associated to supported data""" - return int(np.prod(self.layout[0])) > 1 and not utils.unavailable_filters( - self.dset - ) + return int(np.prod(self.layout[0])) > 1 and not self.missing_filters @property def __is_signal(self): @@ -482,9 +487,8 @@ def dtype_str(self): @property def text(self): """Return node textual representation""" - missing = utils.unavailable_filters(self.dset) - if missing: - return _unavailable_filters_text(missing) + if self.missing_filters: + return _unavailable_filters_text(self.missing_filters) data, truncated = utils.read_bounded(self.dset, TEXT_PREVIEW_MAX_ITEMS) if self.dset.dtype.names is not None: data = self.extract_numeric_from_compound(data) @@ -556,9 +560,7 @@ def match(cls, dset): @property def nd_shape(self) -> tuple[int, ...] | None: """Return the shape of the dataset (None if it cannot be read)""" - if utils.unavailable_filters(self.dset): - return None - return self.dset.shape + return None if self.missing_filters else self.dset.shape @property def interpretation(self) -> str: @@ -588,9 +590,8 @@ def shape_str(self): @property def text(self): """Return node textual representation""" - missing = utils.unavailable_filters(self.dset) - if missing: - return _unavailable_filters_text(missing) + if self.missing_filters: + return _unavailable_filters_text(self.missing_filters) data, truncated = utils.read_bounded(self.dset, TEXT_PREVIEW_MAX_ITEMS) return str(data) + (" ..." if truncated else "") From 466de626b712ad0b785417dfcbbb828045efb28c Mon Sep 17 00:00:00 2001 From: Pierre Raybaut <1311787+PierreRaybaut@users.noreply.github.com> Date: Tue, 27 Oct 2026 12:17:17 +0100 Subject: [PATCH 5/5] fix(h5): compare tree check states with Qt enums (PyQt6) With PyQt6, check states are enums that are not equal to integers: the browser returned unchecked items as checked nodes, and checking whether an item was checked raised a TypeError. Assisted-by: Claude Opus 5.5 --- sigimax/widgets/h5browser.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sigimax/widgets/h5browser.py b/sigimax/widgets/h5browser.py index 2850940..e018ff4 100644 --- a/sigimax/widgets/h5browser.py +++ b/sigimax/widgets/h5browser.py @@ -377,7 +377,7 @@ def get_nodes(self, only_checked_items: bool = True) -> list[BaseNode]: datasets = [] for item in self.find_all_items(): if item.flags() & QC.Qt.ItemIsUserCheckable: - if only_checked_items and item.checkState(0) == 0: + if only_checked_items and item.checkState(0) == QC.Qt.Unchecked: continue if item is not self.topLevelItem(0): node = self.get_node(item) @@ -404,7 +404,7 @@ def is_empty(self) -> bool: def is_any_item_checked(self) -> bool: """Return True if any item is checked""" for item in self.find_all_items(): - if item.checkState(0) > 0: + if item.checkState(0) != QC.Qt.Unchecked: return True return False