HDF5 browser: multidimensional views and legacy NeXus scans (tests and docs)

.github/workflows/test_pyqt5.yml

@@ -77,8 +77,9 @@ jobs:
            pip install -e ../DataLab-Capsule
            # Install tomli for TOML parsing (safe if already present)
            pip install tomli
-           # Extract dependencies and save to file, then install
-           python -c "import tomli; f=open('pyproject.toml','rb'); data=tomli.load(f); deps=[d for d in data['project']['dependencies'] if not any(p in d for p in ['guidata','PlotPy','Sigima', 'SigimaX', 'datalab-capsule'])]; open('deps.txt','w').write('\n'.join(deps))"
+           # Extract dependencies of DataLab and SigimaX (installed with --no-deps
+           # above), save them to file, then install
+           python -c "import tomli; deps=[d for p in ('pyproject.toml', '../SigimaX/pyproject.toml') for d in tomli.load(open(p,'rb'))['project']['dependencies'] if not any(n in d for n in ['guidata','PlotPy','Sigima', 'SigimaX', 'datalab-capsule'])]; open('deps.txt','w').write('\n'.join(deps))"
            pip install -r deps.txt
            # Install DataLab without dependencies
            pip install --no-deps .
@@ -105,8 +106,9 @@ jobs:
            pip install -e ../DataLab-Capsule
            # Install tomli for TOML parsing (safe if already present)
            pip install tomli
-           # Extract dependencies and save to file, then install
-           python -c "import tomli; f=open('pyproject.toml','rb'); data=tomli.load(f); deps=[d for d in data['project']['dependencies'] if not any(p in d for p in ['guidata','PlotPy','Sigima', 'SigimaX', 'datalab-capsule'])]; open('deps.txt','w').write('\n'.join(deps))"
+           # Extract dependencies of DataLab and SigimaX (installed with --no-deps
+           # above), save them to file, then install
+           python -c "import tomli; deps=[d for p in ('pyproject.toml', '../SigimaX/pyproject.toml') for d in tomli.load(open(p,'rb'))['project']['dependencies'] if not any(n in d for n in ['guidata','PlotPy','Sigima', 'SigimaX', 'datalab-capsule'])]; open('deps.txt','w').write('\n'.join(deps))"
            pip install -r deps.txt
            # Install DataLab without dependencies
            pip install --no-deps .
@@ -186,9 +188,10 @@ jobs:
         python -m pip install -e ../SigimaX --no-deps
         python -m pip install -e ../DataLab-Capsule
 
-        # Extract dependency names from pyproject.toml (excluding Sigima and
-        # SigimaX, installed from local clones above) and install latest versions
-        python -m pip install -U --upgrade-strategy eager $(python -c "import tomllib, re; print(' '.join(re.sub(r'[\[\]<>=!~,.\s].*$', '', d).strip() for d in tomllib.loads(open('pyproject.toml', 'rb').read().decode())['project']['dependencies'] if not any(p in d for p in ['Sigima', 'SigimaX', 'datalab-capsule'])))")
+        # Extract dependency names from the pyproject.toml of DataLab and SigimaX
+        # (excluding Sigima and SigimaX, installed from local clones above) and
+        # install latest versions
+        python -m pip install -U --upgrade-strategy eager $(python -c "import tomllib, re; print(' '.join(sorted({re.sub(r'[\[\]<>=!~,.\s].*$', '', d).strip() for p in ('pyproject.toml', '../SigimaX/pyproject.toml') for d in tomllib.loads(open(p, 'rb').read().decode())['project']['dependencies'] if not any(n in d for n in ['Sigima', 'SigimaX', 'datalab-capsule'])})))")
 
     - name: Lint with Ruff (latest)
       run: ruff check --output-format=github datalab

.github/workflows/test_pyqt6.yml

@@ -77,8 +77,9 @@ jobs:
            pip install -e ../DataLab-Capsule
            # Install tomli for TOML parsing (safe if already present)
            pip install tomli
-           # Extract dependencies and save to file, then install
-           python -c "import tomli; f=open('pyproject.toml','rb'); data=tomli.load(f); deps=[d for d in data['project']['dependencies'] if not any(p in d for p in ['guidata','PlotPy','Sigima', 'SigimaX', 'datalab-capsule'])]; open('deps.txt','w').write('\n'.join(deps))"
+           # Extract dependencies of DataLab and SigimaX (installed with --no-deps
+           # above), save them to file, then install
+           python -c "import tomli; deps=[d for p in ('pyproject.toml', '../SigimaX/pyproject.toml') for d in tomli.load(open(p,'rb'))['project']['dependencies'] if not any(n in d for n in ['guidata','PlotPy','Sigima', 'SigimaX', 'datalab-capsule'])]; open('deps.txt','w').write('\n'.join(deps))"
            pip install -r deps.txt
            # Install DataLab without dependencies
            pip install --no-deps .
@@ -105,8 +106,9 @@ jobs:
            pip install -e ../DataLab-Capsule
            # Install tomli for TOML parsing (safe if already present)
            pip install tomli
-           # Extract dependencies and save to file, then install
-           python -c "import tomli; f=open('pyproject.toml','rb'); data=tomli.load(f); deps=[d for d in data['project']['dependencies'] if not any(p in d for p in ['guidata','PlotPy','Sigima', 'SigimaX', 'datalab-capsule'])]; open('deps.txt','w').write('\n'.join(deps))"
+           # Extract dependencies of DataLab and SigimaX (installed with --no-deps
+           # above), save them to file, then install
+           python -c "import tomli; deps=[d for p in ('pyproject.toml', '../SigimaX/pyproject.toml') for d in tomli.load(open(p,'rb'))['project']['dependencies'] if not any(n in d for n in ['guidata','PlotPy','Sigima', 'SigimaX', 'datalab-capsule'])]; open('deps.txt','w').write('\n'.join(deps))"
            pip install -r deps.txt
            # Install DataLab without dependencies
            pip install --no-deps .
@@ -166,9 +168,10 @@ jobs:
         python -m pip install -e ../SigimaX --no-deps
         python -m pip install -e ../DataLab-Capsule
 
-        # Extract dependency names from pyproject.toml (excluding Sigima and
-        # SigimaX, installed from local clones above) and install latest versions
-        python -m pip install -U --upgrade-strategy eager $(python -c "import tomllib, re; print(' '.join(re.sub(r'[\[\]<>=!~,.\s].*$', '', d).strip() for d in tomllib.loads(open('pyproject.toml', 'rb').read().decode())['project']['dependencies'] if not any(p in d for p in ['Sigima', 'SigimaX', 'datalab-capsule'])))")
+        # Extract dependency names from the pyproject.toml of DataLab and SigimaX
+        # (excluding Sigima and SigimaX, installed from local clones above) and
+        # install latest versions
+        python -m pip install -U --upgrade-strategy eager $(python -c "import tomllib, re; print(' '.join(sorted({re.sub(r'[\[\]<>=!~,.\s].*$', '', d).strip() for p in ('pyproject.toml', '../SigimaX/pyproject.toml') for d in tomllib.loads(open(p, 'rb').read().decode())['project']['dependencies'] if not any(n in d for n in ['Sigima', 'SigimaX', 'datalab-capsule'])})))")
 
     - name: Lint with Ruff (latest)
       run: ruff check --output-format=github datalab

datalab/tests/features/hdf5/nexus_views_app_test.py

@@ -0,0 +1,105 @@
+# Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file.
+
+"""
+NeXus scans and multidimensional views test
+--------------------------------------------
+
+A scan written with the legacy NeXus conventions (as recent SOLEIL scans) is
+imported with its axes: its counter versus the scanned motor and its detector
+spectra as an image with the motor along Y. The frames of an image stack whose
+view is enabled in the HDF5 browser are imported as images titled and located
+by the motor position, and their origin survives a workspace round trip.
+"""
+
+# guitest: show
+
+from __future__ import annotations
+
+import json
+import os.path as osp
+
+import h5py
+import numpy as np
+import pytest
+from sigima.objects import get_experimental_source
+from sigimax.h5.selection import NDView
+from sigimax.widgets.h5browser import H5BrowserDialog
+
+from datalab.tests import datalab_test_app_context, helpers
+
+
+def _write_scan(fname: str) -> None:
+    """Write a step scan: legacy axis attributes, a counter, a detector writing a
+    spectrum per point and a detector writing an image per point"""
+    with h5py.File(fname, "w") as h5file:
+        entry = h5file.create_group("scan_0494")
+        entry.attrs["NX_class"] = "NXentry"
+        entry["title"] = "LaB6 calibration"
+        scan = entry.create_group("scan_data")
+        scan.attrs["NX_class"] = "NXdata"
+        scan["actuator_1_1"] = np.array([-8.0, -7.5, -7.0])
+        scan["actuator_1_1"].attrs.update(axis=np.int32(1), primary=np.int32(1))
+        scan["actuator_1_1"].attrs["units"] = "deg"
+        scan["counter"] = np.array([5.0, 9.0, 4.0])
+        scan["spectra"] = np.arange(3 * 16, dtype=np.int32).reshape(3, 16)
+        scan["frames"] = np.arange(3 * 4 * 5, dtype=np.uint16).reshape(3, 4, 5)
+        for name in ("counter", "spectra", "frames"):
+            scan[name].attrs["signal"] = np.int32(1)
+
+
+def _nexus_objects(panel, signal: str) -> list:
+    """Return the objects of a panel imported from a signal of an NXdata group"""
+    return [
+        obj
+        for obj in panel.objmodel.get_all_objects()
+        if (get_experimental_source(obj) or {}).get("selection", {}).get("signal")
+        == signal
+    ]
+
+
+def test_scan_frames_and_workspace(monkeypatch: pytest.MonkeyPatch) -> None:
+    """Legacy scan counters and detectors, then frames of a stack enabled in the
+    browser, and workspace round trip"""
+    original_open = H5BrowserDialog.open_file
+
+    def open_and_enable_frames(dialog: H5BrowserDialog, fname: str) -> None:
+        """Open the file, and enable the import of the first two frames"""
+        original_open(dialog, fname)
+        tree = dialog.browser.tree
+        item = tree.find_node_item(tree.topLevelItem(0), "/scan_0494/scan_data::frames")
+        tree.get_node(item).set_view(NDView((1, 2), (0,), 0, 1))
+        tree.set_item_importable(item, True)
+
+    monkeypatch.setattr(H5BrowserDialog, "open_file", open_and_enable_frames)
+    with helpers.WorkdirRestoringTempDir() as tmpdir:
+        nxsname = osp.join(tmpdir, "scan.nxs")
+        _write_scan(nxsname)
+        wsname = osp.join(tmpdir, "workspace.h5")
+        with datalab_test_app_context(console=False, exec_loop=False) as win:
+            win.h5inputoutput.import_files([nxsname], True, False)
+            (counter,) = _nexus_objects(win.signalpanel, "counter")
+            np.testing.assert_array_equal(counter.x, [-8.0, -7.5, -7.0])
+            assert counter.xunit == "deg"
+            source = get_experimental_source(counter)
+            assert source["details"] == {"entry_title": "LaB6 calibration"}
+            (spectra,) = _nexus_objects(win.imagepanel, "spectra")
+            assert spectra.data.shape == (3, 16)
+            assert (spectra.y0, spectra.dy) == (-8.0, 0.5)
+            frames = _nexus_objects(win.imagepanel, "frames")
+            assert len(frames) == 2
+            assert frames[1].title.startswith("frames[1, :, :] (actuator_1_1=-7.5)")
+            np.testing.assert_array_equal(
+                frames[1].data, np.arange(20, 40).reshape(4, 5)
+            )
+            win.save_h5_workspace(wsname)
+
+            win.load_h5_workspace([wsname], reset_all=True)
+            frame = _nexus_objects(win.imagepanel, "frames")[1]
+            source = get_experimental_source(frame)
+            assert source["selection"]["view"] == {"displayed": [1, 2], "indices": [1]}
+            assert source["selection"]["positions"] == {"actuator_1_1": -7.5}
+            json.dumps(frame.metadata["experimental_source"])
+
+
+if __name__ == "__main__":
+    pytest.main([__file__])

doc/features/common/h5browser.rst

@@ -51,9 +51,26 @@ neighbouring datasets. Images need increasing coordinates: a decreasing image
 axis is reversed together with the image, and this conversion is recorded,
 while an image axis that is not strictly monotonic prevents the import.
 
+Files written with the legacy NeXus conventions, such as the scans of
+synchrotron SOLEIL, are imported with their axes too: an axis dataset may give
+the dimension it describes with its ``axis`` attribute, the one with the lowest
+``primary`` attribute being preferred. The other measured datasets of a group,
+such as all the counters and detectors of a scan, appear as additional items
+below the group, each with the axes that fit its dimensions: a counter is a
+signal versus the scanned motor, a detector writing a spectrum at each point an
+image with the motor along Y, and a detector writing an image at each point a
+stack of images (see below).
+
+When a NeXus file designates a default measurement (``default`` attributes), the
+browser selects and previews it when the file is opened. Compressed detector
+data, such as the bitshuffle/LZ4 data of Eiger detectors, is read; a dataset
+compressed with an unavailable filter is reported in the tree instead of failing.
+
 Files with several counters or incomplete NeXus descriptions can be interpreted
 by a plugin-provided data mapping. Each mapped object appears as an additional
-item with its own axes, labels and units. Mappings must be registered before
+item with its own axes, labels and units; the axes of a mapped object may
+describe its first dimensions only, so that one mapping applies to the counters
+and detectors of a scan. Mappings must be registered before
 opening the file; DataLab does not automatically discover JSON files beside it.
 Plugin authors can follow the
 `SigimaX HDF5 import guide <https://sigimax.readthedocs.io/en/latest/user_guide/hdf5_workspace.html>`_.
@@ -77,10 +94,30 @@ limit: compound datasets and signals stored as 2D tables still use complete
 objects for their graphical previews. Closing a file or the browser waits for
 its preview read to end.
 
+Multidimensional datasets
+-------------------------
+
 Datasets with three or more dimensions, such as image stacks and spectral
-cubes, are listed but cannot be imported as complete objects. There is no
-graphical slice selector yet. Developers can read 1D or 2D views without loading
-the whole cube using the selection API described in the SigimaX guide.
+cubes, are imported as one image or signal per frame. Selecting such an item
+shows a view selector below the preview:
+
+  * **View**: images of two axes, or signals along one axis
+  * **Frames along**: the axis along which frames are taken, with the **First**
+    and **Last** frame and a **Step**
+  * **Fixed indices**: the index of the other axes, if any
+  * **Import these frames**: makes the item checkable and checked; unchecking
+    the item cancels the import
+
+The first frame of the view is previewed, and the number of frames is shown.
+Images of the last two axes, with every frame along the preceding axis, are
+proposed by default, unless the NeXus ``interpretation`` attribute of the
+dataset designates spectra (signals along the last axis) or RGB images (one
+image per channel). DataLab asks for confirmation before importing more than
+100 frames. The frames of NeXus datasets keep the axes of their displayed
+dimensions; their title gives the frame indices and, if the other axes have
+coordinates, the position of the frame (e.g. ``frames[3, :, :] (theta=0.3)``).
+Developers can also read views without creating objects, using the selection
+API described in the SigimaX guide.
 
 Numerical values and experimental origin
 ---------------------------------------
@@ -94,8 +131,9 @@ signals keep their imaginary part, including single-precision ones.
 
 NeXus and mapped objects carry an ``experimental_source`` metadata entry
 describing the source file, group, measured dataset, axes (``"."`` for a
-dimension without axis) and recorded conversions. Mapped objects also identify
-their mapping. This entry describes
+dimension without axis) and recorded conversions, with the title, start and end
+times of the NeXus entry, and, for frames, the frame indices and position.
+Mapped objects also identify their mapping. This entry describes
 where the measurement came from; it is not the processing provenance shown in
 the Properties panel and does not make the experiment reproducible.