From 4c0b9abd9616a7cb6d6c944e0a1bd451bdc1c721 Mon Sep 17 00:00:00 2001 From: Pierre Raybaut <1311787+PierreRaybaut@users.noreply.github.com> Date: Tue, 27 Oct 2026 11:23:46 +0100 Subject: [PATCH 1/4] fix: read real SPEC and XDI files as reference readers do SPEC: MCA spectra are grouped by device (@A1, @A2...) with their own calibration and channels, instead of mixing devices and reading the device number as a first channel. Labels separated by single spaces are split according to the column count, repeated labels get a suffix, empty motor lines and None values (Bluesky) are accepted, and other '@' lines are ignored. A scan that cannot be read is skipped with the error instead of stopping the import or File > Open. MCA spectra are imported as one image per device (one spectrum per row), mesh scans as images of the two motors, and scan headers are kept as metadata. XDI: files are read as UTF-8, invalid data tables are refused with their line as the XDI library does, invalid header lines are recorded, comments and all fields are kept as metadata, angles are converted to energies with the lattice spacing, and outer scans give one signal per block. The formats are removed and registered again with Sigima's public API. Assisted-by: Claude Opus 5.5 --- src/datalab_synchrotron/adapters/desktop.py | 87 ++-- src/datalab_synchrotron/core/__init__.py | 21 +- src/datalab_synchrotron/core/convert.py | 532 ++++++++++++++++---- src/datalab_synchrotron/core/spec.py | 226 +++++++-- src/datalab_synchrotron/core/xdi.py | 212 ++++++-- src/datalab_synchrotron/formats.py | 58 ++- tests/integration/test_desktop_adapter.py | 93 +++- tests/support/spec_files.py | 61 ++- tests/unit/test_formats.py | 32 +- tests/unit/test_scan_import.py | 48 +- tests/unit/test_spec.py | 147 +++++- tests/unit/test_xdi.py | 125 ++++- 12 files changed, 1376 insertions(+), 266 deletions(-) diff --git a/src/datalab_synchrotron/adapters/desktop.py b/src/datalab_synchrotron/adapters/desktop.py index ac2922f..a9acafd 100644 --- a/src/datalab_synchrotron/adapters/desktop.py +++ b/src/datalab_synchrotron/adapters/desktop.py @@ -22,10 +22,11 @@ check_scan_selection, import_scan, is_spec_file, + mesh_shape, select_scans, summarize_skipped, ) -from ..formats import unregister_formats +from ..formats import register_formats, unregister_formats from ..mappings import register_mappings, unregister_mappings from ..workflow import ( ABSORPTION_RECIPE, @@ -36,7 +37,7 @@ from ..workflow import EXAMPLES as WORKFLOW_EXAMPLES from ..workflow import RECIPES as WORKFLOW_RECIPES -#: Number of signals added to the workspace at once by the SPEC import +#: Number of objects added to the workspace at once by the SPEC import ADD_BATCH_SIZE = 200 PLUGIN_ICON = "datalab_synchrotron:icons/synchrotron.svg" @@ -45,10 +46,17 @@ def _import_parameters(spec: SpecFile) -> gds.DataSet: - """Return the import parameters of a SPEC file, with its columns as choices.""" + """Return the import parameters of a SPEC file, with its columns and MCA devices + as choices. + + The MCA and mesh items are hidden if no scan has MCA spectra or is a mesh scan. + """ labels: list[str] = [] + devices: list[str] = [] for info in spec.scans: labels.extend(label for label in info.labels if label not in labels) + devices.extend(device for device in info.mca_devices if device not in devices) + has_mesh = any(mesh_shape(info) is not None for info in spec.scans) # Default columns: those of the most frequent scan structure structures = collections.Counter(info.labels for info in spec.scans if info.labels) first = structures.most_common(1)[0][0] if structures else ("",) @@ -67,7 +75,17 @@ class SpecImportParameters(gds.DataSet): counters = gds.MultipleChoiceItem( _("Counters"), choices, default=[first[-1]] ).vertical(3) - mca = gds.BoolItem(_("MCA spectra"), default=False) + mca = gds.MultipleChoiceItem( + _("MCA spectra"), + [(device, device) for device in devices] or [("", "-")], + default=[], + help=_("One image per device and scan: one spectrum per row"), + ).set_prop("display", hide=not devices) + mesh = gds.BoolItem( + _("Mesh scans as images"), + default=True, + help=_("Counters of mesh scans as images of the two motors"), + ).set_prop("display", hide=not has_mesh) return SpecImportParameters(title=osp.basename(spec.path)) @@ -124,7 +142,9 @@ def open_cu_foil_example(self) -> PluginExample | None: return self.launch_example(CU_FOIL_EXAMPLE.id) def register_hooks(self) -> None: - """Apply the shipped HDF5 data mappings.""" + """Register the file formats (again, after an unload) and apply the + shipped HDF5 data mappings.""" + register_formats() register_mappings() def unregister_hooks(self) -> None: @@ -160,24 +180,25 @@ def import_spec_scans(self, filename: str | None = None) -> int: return 0 keys = select_scans(spec, param.scans) counters = list(param.counters) - check_scan_selection(param.x, counters, param.mca) + devices = [device for device in param.mca if device] + check_scan_selection(param.x, counters, devices) except ValueError as exc: self.show_error(str(exc)) return 0 count, skipped, cancelled = self._add_scans( - spec, keys, param.x, counters, param.mca + spec, keys, param.x, counters, devices, param.mesh ) report = summarize_skipped(skipped) if cancelled: self.show_info( - _("Import cancelled: %d signal(s) imported.") % count + _("Import cancelled: %d object(s) imported.") % count + "".join("\n" + line for line in report) ) elif count == 0: - self.show_error(_("No signal was imported:") + "\n" + "\n".join(report)) + self.show_error(_("Nothing was imported:") + "\n" + "\n".join(report)) elif skipped: self.show_warning( - _("Selected columns were not imported from %d of %d scans:") + _("Selected data was not imported from %d of %d scans:") % (len(skipped), len(keys)) + "\n" + "\n".join(report) @@ -190,12 +211,15 @@ def _add_scans( keys: list[str], x: str, counters: list[str], - include_mca: bool, + mca_devices: list[str], + mesh_as_images: bool, ) -> tuple[int, list[SkippedScan], bool]: - """Read and add scans one at a time, in a group each, until cancelled. + """Read and add scans one at a time, until cancelled: signals in a group of + the signal panel, images in a group of the image panel, named after the + scan. Returns: - Number of added signals, skipped scans, and True if cancelled + Number of added objects, skipped scans, and True if cancelled """ count, skipped = 0, [] title = _("Importing SPEC scans") @@ -209,26 +233,31 @@ def _add_scans( QW.QApplication.processEvents() if progress.wasCanceled(): return count, skipped, True - group, reason = import_scan(spec, key, x, counters, include_mca) + group, reason = import_scan( + spec, key, x, counters, mca_devices, mesh_as_images + ) if reason is not None: skipped.append(reason) if group is None: continue - group_id = self.proxy.add_group(group.title, panel="signal") - signals = group.signals - for start in range(0, len(signals), ADD_BATCH_SIZE): - progress.setLabelText( - _("Adding signals of scan %s (%d of %d)") - % (key, start, len(signals)) - ) - QW.QApplication.processEvents() - if progress.wasCanceled(): - return count, skipped, True - batch = signals[start : start + ADD_BATCH_SIZE] - self.signalpanel.add_objects( - batch, group_id=group_id, set_current=False - ) - count += len(batch) + for panel_name, panel, objects in ( + ("signal", self.signalpanel, group.signals), + ("image", self.imagepanel, group.images), + ): + if not objects: + continue + group_id = self.proxy.add_group(group.title, panel=panel_name) + for start in range(0, len(objects), ADD_BATCH_SIZE): + progress.setLabelText( + _("Adding objects of scan %s (%d of %d)") + % (key, start, len(objects)) + ) + QW.QApplication.processEvents() + if progress.wasCanceled(): + return count, skipped, True + batch = objects[start : start + ADD_BATCH_SIZE] + panel.add_objects(batch, group_id=group_id, set_current=False) + count += len(batch) progress.setValue(len(keys)) return count, skipped, False diff --git a/src/datalab_synchrotron/core/__init__.py b/src/datalab_synchrotron/core/__init__.py index 98f40b7..2b0e008 100644 --- a/src/datalab_synchrotron/core/__init__.py +++ b/src/datalab_synchrotron/core/__init__.py @@ -1,19 +1,26 @@ """Host-independent readers of experimental data formats, and XAS analysis.""" from .convert import ( + HC_EV_ANGSTROM, + MeshShape, ScanGroup, ScanImport, SkippedScan, check_scan_selection, import_scan, - mca_signals, + mca_images, + mesh_images, + mesh_shape, plan_scan_import, + scan_metadata, scan_signals, select_scans, summarize_skipped, + xdi_energy, + xdi_metadata, xdi_signals, ) -from .spec import SpecFile, SpecScan, SpecScanInfo, is_spec_file +from .spec import SpecFile, SpecScan, SpecScanInfo, is_spec_file, mca_device from .xas import ( MAX_EDGE_KEV, Absorption, @@ -28,11 +35,13 @@ from .xdi import XdiColumn, XdiFile, is_xdi_file, read_xdi __all__ = [ + "HC_EV_ANGSTROM", "MAX_EDGE_KEV", "Absorption", "AbsorptionMode", "EdgeNormalization", "EnergyUnit", + "MeshShape", "ScanGroup", "ScanImport", "SkippedScan", @@ -48,12 +57,18 @@ "import_scan", "is_spec_file", "is_xdi_file", - "mca_signals", + "mca_device", + "mca_images", + "mesh_images", + "mesh_shape", "normalize_edge", "plan_scan_import", "read_xdi", + "scan_metadata", "scan_signals", "select_scans", "summarize_skipped", + "xdi_energy", + "xdi_metadata", "xdi_signals", ] diff --git a/src/datalab_synchrotron/core/convert.py b/src/datalab_synchrotron/core/convert.py index f6d6a0c..6cb4f42 100644 --- a/src/datalab_synchrotron/core/convert.py +++ b/src/datalab_synchrotron/core/convert.py @@ -1,4 +1,10 @@ -"""Conversion of SPEC scans and XDI files to signals recording their origin.""" +"""Conversion of SPEC scans and XDI files to signals and images recording their +origin. + +The header information of the files (SPEC scan headers, XDI fields and comments) +is kept as object metadata, while the ``experimental_source`` metadata entry +identifies where the data comes from. +""" from __future__ import annotations @@ -9,47 +15,125 @@ import numpy as np from sigima.objects import ( + ImageObj, SignalObj, + create_image, create_signal, make_experimental_source, set_experimental_source, ) -from .spec import SpecFile, SpecScan -from .xdi import XdiFile +from .spec import SpecFile, SpecScan, SpecScanInfo +from .xdi import XdiColumn, XdiFile __all__ = [ + "HC_EV_ANGSTROM", + "MeshShape", "ScanGroup", "ScanImport", "SkippedScan", "check_scan_selection", "import_scan", - "mca_signals", + "mca_images", + "mesh_images", + "mesh_shape", "plan_scan_import", + "scan_metadata", "scan_signals", "select_scans", "summarize_skipped", + "xdi_energy", + "xdi_metadata", "xdi_signals", ] +#: Planck constant times the speed of light, in eV.Å (exact SI values) +HC_EV_ANGSTROM = 12398.419843320026 + +#: Relative tolerance on the steps of evenly spaced image coordinates +_UNIFORM_RTOL = 1e-9 + -def _source(spec: SpecFile, scan: SpecScan, selection: dict, conversions) -> dict: - """Return the experimental source of a signal read from a scan.""" +def _source( + spec: SpecFile, scan: SpecScan, selection: dict, conversions: Sequence[str] +) -> dict: + """Return the experimental source of an object read from a scan.""" details = {"command": scan.info.command, "date": scan.info.date} if scan.motors: details["motors"] = dict(scan.motors) if scan.problems: details["skipped_lines"] = len(scan.problems) + if scan.ignored: + details["ignored_lines"] = dict(scan.ignored) return make_experimental_source( "spec", spec.path, entry=scan.info.key, selection=selection, - conversions=conversions or None, + conversions=list(conversions) or None, details=details, ) +def _value_and_note(text: str) -> tuple[float, str]: + """Return the number and the parenthesized note of a header line, e.g. + ``60 (Clock)``.""" + match = re.match(r"(\S+)\s*(?:\((.*)\))?", text) + if match is None: + raise ValueError(f"No value in '{text}'") + return float(match.group(1)), (match.group(2) or "").strip() + + +def scan_metadata(scan: SpecScan) -> dict[str, object]: + """Return the scan header information kept as object metadata. + + Keys: ``SPEC count time`` and ``SPEC count time unit`` (``#T``), ``SPEC monitor + count`` and ``SPEC monitor counter`` (``#M``), ``SPEC hkl`` (``#Q``), ``SPEC + geometry G`` (``#G``), ``SPEC comments`` (``#C`` lines, one per line), + and ``SPEC MD `` (``#MD = `` lines written by Bluesky). + Header lines that cannot be read are left out. + """ + metadata: dict[str, object] = {} + comments: list[str] = [] + for line in scan.header[1:]: + tag, _sep, text = line.strip().partition(" ") + text = text.strip() + try: + if tag in ("#T", "#M") and text: + value, note = _value_and_note(text) + name, note_name = ( + ("count time", "count time unit") + if tag == "#T" + else ("monitor count", "monitor counter") + ) + metadata[f"SPEC {name}"] = value + if note: + metadata[f"SPEC {note_name}"] = note + elif tag == "#Q" and text: + metadata["SPEC hkl"] = np.array([float(word) for word in text.split()]) + elif re.fullmatch(r"#G\d+", tag) and text: + metadata[f"SPEC geometry {tag[1:]}"] = np.array( + [float(word) for word in text.split()] + ) + elif tag == "#C" and text: + comments.append(text) + elif tag == "#MD" and "=" in text: + name, value = text.split("=", 1) + if name.strip(): + metadata[f"SPEC MD {name.strip()}"] = value.strip() + except ValueError: + continue + if comments: + metadata["SPEC comments"] = "\n".join(comments) + return metadata + + +def _finish(obj: SignalObj | ImageObj, source: dict, metadata: dict) -> None: + """Record the header metadata and the origin of an object.""" + obj.metadata.update(metadata) + set_experimental_source(obj, source) + + def scan_signals( spec: SpecFile, scan: SpecScan, x: str, counters: Sequence[str] ) -> list[SignalObj]: @@ -61,6 +145,7 @@ def scan_signals( ValueError: if a column does not exist """ xdata = scan.column(x) + metadata = scan_metadata(scan) signals = [] for counter in counters: signal = create_signal( @@ -69,51 +154,208 @@ def scan_signals( scan.column(counter).copy(), labels=(x, counter), ) - selection = {"x": x, "y": counter} - set_experimental_source(signal, _source(spec, scan, selection, None)) + source = _source(spec, scan, {"x": x, "y": counter}, ()) + _finish(signal, source, metadata) signals.append(signal) return signals -def mca_signals( - spec: SpecFile, scan: SpecScan, indices: Sequence[int] | None = None -) -> list[SignalObj]: - """Return MCA spectra of a scan, on calibrated X if the scan declares it. +def _is_strictly_monotonic(values: np.ndarray) -> bool: + """Return True if values are finite and strictly increasing or decreasing.""" + if not np.all(np.isfinite(values)): + return False + steps = np.diff(values) + return bool(np.all(steps > 0) or np.all(steps < 0)) - The X axis is the channel number, or ``a + b * channel + c * channel**2`` - with the ``#@CALIB`` coefficients of the scan. + +def _image( + title: str, + data: np.ndarray, + coords: tuple[np.ndarray, np.ndarray], + labels: tuple[str, str, str], + conversions: list[str], +) -> ImageObj: + """Return an image with increasing coordinates, uniform if evenly spaced. + + Args: + title: image title + data: image values, one row per Y coordinate + coords: strictly monotonic X and Y coordinates + labels: X, Y and Z labels + conversions: conversions applied to the values, completed with the axes + reversed to increasing order """ - signals = [] - for index in range(len(scan.mca)) if indices is None else indices: - channels = scan.mca_channels_axis(index) - conversions = [] - xlabel = "Channel" - if scan.mca_calibration is not None: - a, b, c = scan.mca_calibration - x = a + b * channels + c * channels**2 - conversions.append(f"channels -> a + b*ch + c*ch^2 with {a}, {b}, {c}") - xlabel = "Calibrated channel" - else: - x = channels - signal = create_signal( - f"MCA {index} (scan {scan.info.key})", - np.asarray(x, dtype=float), - scan.mca[index].astype(float), - labels=(xlabel, "Counts"), + xvals, yvals = (np.asarray(values, dtype=float) for values in coords) + for axis, (values, label) in enumerate(((yvals, labels[1]), (xvals, labels[0]))): + if values.size > 1 and values[-1] < values[0]: + data = np.flip(data, axis=axis) + conversions.append(f"axis '{label}': reversed to increasing order") + xvals, yvals = np.sort(xvals), np.sort(yvals) + image = create_image(title, np.ascontiguousarray(data), labels=labels) + steps = [np.diff(values) for values in (xvals, yvals)] + if all( + step.size < 2 or np.allclose(step, step[0], rtol=_UNIFORM_RTOL, atol=0.0) + for step in steps + ): + dx, dy = (float(step[0]) if step.size else 1.0 for step in steps) + image.set_uniform_coords(dx, dy, float(xvals[0]), float(yvals[0])) + else: + image.set_coords(xvals, yvals) + return image + + +def mca_images( + spec: SpecFile, scan: SpecScan, x: str = "", devices: Sequence[str] | None = None +) -> list[ImageObj]: + """Return the MCA spectra of a scan as one image per device. + + Each row holds the spectrum of a point. The X axis is the channel number, or + ``a + b * channel + c * channel**2`` with the ``#@CALIB`` coefficients of the + device, if they give strictly monotonic values. The Y axis is the X column + of the scan, if it holds one strictly monotonic value per spectrum, else the + point number. + + Args: + spec: SPEC file + scan: scan read from the file + x: X column of the scan (empty: point numbers) + devices: MCA devices (default: all devices of the scan) + + Raises: + ValueError: if a device has no spectrum, or spectra of several sizes + """ + metadata = scan_metadata(scan) + images = [] + for device in scan.mca if devices is None else devices: + spectra = scan.mca.get(device, ()) + channels = scan.channel_numbers(device) + conversions = [f"spectra -> image rows ({len(spectra)} spectra)"] + xvals, xlabel = channels, "Channel" + calibration = scan.calibration(device) + if calibration is not None: + a, b, c = calibration + calibrated = a + b * channels + c * channels**2 + if _is_strictly_monotonic(calibrated): + xvals, xlabel = calibrated, "Calibrated channel" + conversions.append(f"channels -> a + b*ch + c*ch^2 with {a}, {b}, {c}") + else: + conversions.append("calibration not monotonic: channel numbers kept") + yvals, ylabel = np.arange(len(spectra), dtype=float), "Point" + if x in scan.info.labels: + column = scan.column(x) + if column.size == len(spectra) and _is_strictly_monotonic(column): + yvals, ylabel = column, x + image = _image( + f"{device} (scan {scan.info.key})", + np.vstack(spectra), + (xvals, yvals), + (xlabel, ylabel, "Counts"), + conversions, ) - set_experimental_source( - signal, _source(spec, scan, {"mca": index}, conversions) + selection = {"mca": device, "y": ylabel if ylabel == x else "point"} + _finish(image, _source(spec, scan, selection, conversions), metadata) + images.append(image) + return images + + +@dataclasses.dataclass(frozen=True) +class MeshShape: + """Grid of a mesh scan, as given by its command. + + Args: + columns: number of points along the first motor (fastest) + rows: number of points along the second motor + steps: steps of the first and second motors + """ + + columns: int + rows: int + steps: tuple[float, float] + + +def mesh_shape(info: SpecScanInfo) -> MeshShape | None: + """Return the grid of a mesh scan, or None if the scan is not a mesh scan. + + Mesh scans are written by ``mesh`` and ``dmesh`` commands: + ``mesh +