Keep user curve styles; convert legacy automatic styles when loading DataLab <= 1.3 files

datalab/gui/h5io.py

@@ -111,14 +111,6 @@ def __read_provenance(
                 _("The provenance data of this workspace are invalid: %s") % exc
             ) from exc
 
-    @staticmethod
-    def __saved_version(reader: NativeH5Reader) -> str | None:
-        """Return the DataLab version that saved the file, as stored."""
-        value = reader.version[()]
-        if isinstance(value, bytes):
-            value = value.decode("utf-8", "replace")
-        return value if isinstance(value, str) else None
-
     def __data_objects(self) -> list[SignalObj | ImageObj]:
         """Return the objects of the signal and image panels."""
         return [
@@ -176,7 +168,7 @@ def open_file_headless(self, filename: str, reset_all: bool) -> bool:
         try:
             reader = NativeH5Reader(filename)
             block = self.__read_provenance(reader)
-            saved_version = self.__saved_version(reader)
+            saved_version = reader.saved_version
             if reset_all:
                 self.mainwindow.reset_all()
             before = {id(obj) for obj in self.__data_objects()}
@@ -233,7 +225,7 @@ def __open_native(self, filename: str, reset_all: bool) -> bool:
         try:
             reader = NativeH5Reader(filename)
             block = self.__read_provenance(reader)
-            saved_version = self.__saved_version(reader)
+            saved_version = reader.saved_version
             if block is not None and not reset_all and not self.__confirm_append():
                 self.cancelled_files.append(filename)
                 return True

datalab/gui/panel/signal.py

@@ -38,6 +38,7 @@
     from qtpy import QtWidgets as QW
 
     from datalab.gui.docks import DockablePlotWidget
+    from datalab.h5.native import NativeH5Reader
 
 
 class SignalPanel(BaseDataPanel[SignalObj, SignalROI, roieditor.SignalROIEditor]):
@@ -88,6 +89,32 @@ def __init__(
         view_toolbar = dockableplotwidget.toolbar
         self.acthandler = SignalActionHandler(self, panel_toolbar, view_toolbar)
 
+    def deserialize_object_from_hdf5(
+        self, reader: NativeH5Reader, name: str, reset_all: bool = False
+    ) -> SignalObj:
+        """Deserialize object from a HDF5 file
+
+        DataLab 1.3 and earlier stored automatic curve styles in signal metadata:
+        when loading a file saved by these versions, the colors and line styles of
+        their automatic style cycle are removed, so that curves are given the
+        current automatic styles. Other stored styles are kept.
+
+        Args:
+            reader: HDF5 reader
+            name: Object name in HDF5 file
+            reset_all: If True, preserve original UUIDs (workspace reload).
+             If False, regenerate only UUIDs that conflict with existing
+             objects (object import).
+        """
+        obj = super().deserialize_object_from_hdf5(reader, name, reset_all)
+        metadata = obj.metadata
+        if reader.has_legacy_curve_styles and CURVESTYLES.is_legacy_automatic(
+            *(metadata.get(key) for key in LINE_STYLE_KEYS)
+        ):
+            for key in LINE_STYLE_KEYS:
+                metadata.pop(key, None)
+        return obj
+
     # ------Creating, adding, removing objects------------------------------------------
     def get_newparam_from_current(
         self, newparam: NewSignalParam | None = None, title: str | None = None

datalab/h5/native.py

@@ -19,6 +19,7 @@
 
 from guidata.io import HDF5Reader, HDF5Writer
 from guidata.io.h5fmt import NoDefault
+from packaging.version import InvalidVersion, Version
 
 import datalab
 
@@ -27,6 +28,30 @@
 
 H5_CALLABLE_PREFIX = "#callable#"
 
+#: First DataLab version which no longer stores automatic curve styles in signal
+#: metadata (earlier versions stored them, see :func:`is_legacy_style_version`)
+NON_PERSISTENT_CURVE_STYLES_VERSION = (1, 4)
+
+
+def is_legacy_style_version(version: str | None) -> bool:
+    """Return True if a file saved by the given DataLab version may store automatic
+    curve styles in signal metadata (DataLab 1.3 or earlier)
+
+    Args:
+        version: DataLab version that saved the file, as stored in the file
+
+    Returns:
+        True if the version is earlier than 1.4 (pre-releases of 1.4 included in
+        1.4); False if it is 1.4 or later, missing or invalid
+    """
+    if not isinstance(version, str):
+        return False
+    try:
+        release = Version(version).release
+    except InvalidVersion:
+        return False
+    return release < NON_PERSISTENT_CURVE_STYLES_VERSION
+
 
 class NativeH5Writer(HDF5Writer):
     """DataLab signal/image objects HDF5 guidata Dataset Writer class
@@ -140,6 +165,25 @@ class NativeH5Reader(HDF5Reader):
     def __init__(self, filename: str) -> None:
         super().__init__(filename)
         self.version = self.h5[DATALAB_VERSION_NAME]
+        self.__saved_version: str | None = None
+        self.__saved_version_read = False
+
+    @property
+    def saved_version(self) -> str | None:
+        """DataLab version that saved the file, or None if it is not readable"""
+        if not self.__saved_version_read:
+            value = self.version[()]
+            if isinstance(value, bytes):
+                value = value.decode("utf-8", "replace")
+            self.__saved_version = value if isinstance(value, str) else None
+            self.__saved_version_read = True
+        return self.__saved_version
+
+    @property
+    def has_legacy_curve_styles(self) -> bool:
+        """True if the file was saved by a DataLab version which stored automatic
+        curve styles in signal metadata (DataLab 1.3 or earlier)"""
+        return is_legacy_style_version(self.saved_version)
 
     @staticmethod
     def deserialize_func_or_class(obj: str) -> Callable | type:

datalab/tests/features/hdf5/h5workspace_unit_test.py

@@ -89,7 +89,7 @@ def test_open_file_headless_refreshes_reference_displays(monkeypatch) -> None:
         provenance=SimpleNamespace(load=Mock()),
     )
     # A workspace file without provenance block, saved by DataLab 1.4
-    reader = SimpleNamespace(close=Mock(), h5={}, version={(): "1.4.0"})
+    reader = SimpleNamespace(close=Mock(), h5={}, saved_version="1.4.0")
     monkeypatch.setattr("datalab.gui.h5io.NativeH5Reader", lambda _filename: reader)
 
     assert H5InputOutput(mainwindow).open_file_headless("workspace.h5", False)

datalab/tests/features/hdf5/native_reader_unit_test.py

@@ -0,0 +1,42 @@
+# Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file.
+
+"""
+Native HDF5 reader unit tests
+-----------------------------
+
+The DataLab version that saved a native HDF5 file is read once, whatever the way
+it is stored, and tells whether the file stores automatic curve styles.
+"""
+
+# guitest: skip
+
+from __future__ import annotations
+
+import h5py
+import numpy as np
+import pytest
+
+from datalab.h5.native import DATALAB_VERSION_NAME, NativeH5Reader
+
+
+@pytest.mark.parametrize(
+    "stored, saved_version, legacy",
+    [
+        ("1.4.0", "1.4.0", False),
+        ("1.3.2", "1.3.2", True),
+        (np.bytes_(b"1.2.1"), "1.2.1", True),  # Stored as bytes
+        ("not a version", "not a version", False),
+        (42, None, False),  # Not a string
+    ],
+)
+def test_saved_version(tmp_path, stored, saved_version, legacy: bool) -> None:
+    """The saved version is decoded, and legacy curve styles are detected"""
+    filename = str(tmp_path / "workspace.h5")
+    with h5py.File(filename, "w") as h5file:
+        h5file[DATALAB_VERSION_NAME] = stored
+    reader = NativeH5Reader(filename)
+    try:
+        assert reader.saved_version == saved_version
+        assert reader.has_legacy_curve_styles is legacy
+    finally:
+        reader.close()

datalab/tests/features/signal/curve_styles_unit_test.py

@@ -6,7 +6,8 @@
 
 Automatic curve styles depend on the position of the curves on the plot, are not
 stored in signal metadata, and are not shifted by dialogs showing signals. Curve
-styles chosen by the user are kept.
+styles chosen by the user are kept. Automatic styles stored by DataLab 1.3 or
+earlier are removed when loading files saved by these versions.
 """
 
 # guitest: show
@@ -15,7 +16,9 @@
 
 import os.path as osp
 
+import h5py
 import numpy as np
+import pytest
 from plotpy.items import CurveItem
 from qtpy import QtGui as QG
 from sigima.objects import SignalObj, create_signal
@@ -26,10 +29,15 @@
     CurveStyles,
     create_adapter_from_object,
 )
+from datalab.gui.main import DLMainWindow
 from datalab.gui.panel.signal import SignalPanel
+from datalab.h5.native import DATALAB_VERSION_NAME, is_legacy_style_version
 from datalab.objectmodel import get_uuid
 from datalab.tests import datalab_test_app_context, helpers
 
+#: Style of the automatic style cycle of DataLab 1.3 or earlier
+LEGACY_STYLE = (CurveStyles.LEGACY_COLORS[2], "DashDotDotLine")
+
 
 def _create_signals(count: int) -> list[SignalObj]:
     """Create test signals"""
@@ -47,13 +55,60 @@ def _style(item: CurveItem) -> tuple[str, str]:
     return QG.QColor(item.param.line.color).name(), item.param.line.style
 
 
-def _set_user_style(panel: SignalPanel, item: CurveItem, color: str) -> None:
-    """Set a curve color as the user does with the curve parameters dialog"""
+def _stored_style(obj: SignalObj) -> tuple[str | None, str | None]:
+    """Return the color and the line style stored in signal metadata"""
+    return obj.metadata.get("color"), obj.metadata.get("style")
+
+
+def _set_user_style(
+    panel: SignalPanel, item: CurveItem, color: str, linestyle: str | None = None
+) -> None:
+    """Set a curve style as the user does with the curve parameters dialog"""
     item.param.line.color = color
+    if linestyle is not None:
+        item.param.line.style = linestyle
     item.param.update_item(item)
     panel.plot_item_parameters_changed(item)
 
 
+def _set_saved_version(filename: str, version: str) -> None:
+    """Set the DataLab version that saved a native HDF5 file"""
+    with h5py.File(filename, "r+") as h5file:
+        del h5file[DATALAB_VERSION_NAME]
+        h5file[DATALAB_VERSION_NAME] = version
+
+
+def _reload_workspace(win: DLMainWindow, filename: str) -> list[SignalObj]:
+    """Reload a workspace and show all its signals"""
+    panel = win.signalpanel
+    panel.remove_all_objects()
+    win.load_h5_workspace([filename], reset_all=True)
+    loaded = panel.objmodel.get_all_objects()
+    panel.objview.select_objects(loaded)
+    return loaded
+
+
+@pytest.mark.parametrize(
+    "version, expected",
+    [
+        ("1.3.0", True),
+        ("1.3.10", True),
+        ("0.20.1", True),
+        ("1.4.0", False),
+        ("1.4.0rc1", False),  # Pre-releases of 1.4 do not store automatic styles
+        ("1.4.0.dev0", False),
+        ("1.10.0", False),
+        ("2.0.0", False),
+        ("invalid", False),
+        ("", False),
+        (None, False),
+    ],
+)
+def test_legacy_style_version(version: str | None, expected: bool) -> None:
+    """Versions which stored automatic curve styles are DataLab 1.3 or earlier"""
+    assert is_legacy_style_version(version) is expected
+
+
 def test_styles_follow_curve_positions() -> None:
     """Visible curves are given the automatic styles of their positions"""
     with datalab_test_app_context(console=False, exec_loop=False) as win:
@@ -123,23 +178,80 @@ def test_user_styles_are_kept() -> None:
             assert _style(_item(panel, loaded[2])) == CurveStyles.style_at(2)
 
 
-def test_legacy_styles_are_automatic() -> None:
-    """Styles stored by earlier versions from their style cycle are automatic"""
+def test_user_style_from_legacy_cycle_is_kept() -> None:
+    """A style chosen by the user is kept over several workspace round trips, even
+    if it matches a style of the automatic style cycle of DataLab 1.3 or earlier"""
     with helpers.WorkdirRestoringTempDir() as tmpdir:
         with datalab_test_app_context(console=False, exec_loop=False) as win:
             panel = win.signalpanel
             signals = _create_signals(2)
-            for obj in signals:
-                obj.metadata["color"] = CurveStyles.LEGACY_COLORS[2]
-                obj.metadata["style"] = "DashDotDotLine"
             panel.add_objects(signals, set_current=False)
             panel.objview.select_objects(signals)
-            for index, obj in enumerate(signals):
-                assert _style(_item(panel, obj)) == CurveStyles.style_at(index)
-            win.save_h5_workspace(osp.join(tmpdir, "legacy_styles.h5"))
-            for obj in signals:
-                for key in LINE_STYLE_KEYS:
-                    assert key not in obj.metadata
+            _set_user_style(panel, _item(panel, signals[1]), "#1f77b4", "SolidLine")
+            assert CurveStyles.is_legacy_automatic("#1f77b4", "SolidLine")
+            fname = osp.join(tmpdir, "user_styles.h5")
+            for _cycle in range(2):
+                win.save_h5_workspace(fname)
+                loaded = _reload_workspace(win, fname)
+                assert _stored_style(loaded[1]) == ("#1f77b4", "SolidLine")
+                assert _style(_item(panel, loaded[1])) == ("#1f77b4", "SolidLine")
+                assert _stored_style(loaded[0]) == (None, None)
+                assert _style(_item(panel, loaded[0])) == CurveStyles.style_at(0)
+
+
+def test_legacy_styles_converted_on_load() -> None:
+    """Automatic styles stored by DataLab 1.3 or earlier are removed on load"""
+    with helpers.WorkdirRestoringTempDir() as tmpdir:
+        with datalab_test_app_context(console=False, exec_loop=False) as win:
+            panel = win.signalpanel
+            signals = _create_signals(3)
+            signals[0].metadata["color"], signals[0].metadata["style"] = LEGACY_STYLE
+            signals[1].metadata["color"], signals[1].metadata["style"] = LEGACY_STYLE
+            # A legacy color with a line style which is not a legacy automatic one:
+            signals[2].metadata["color"] = LEGACY_STYLE[0]
+            signals[2].metadata["style"] = "DotLine"
+            panel.add_objects(signals, set_current=False)
+            panel.objview.select_objects(signals)
+            fname = osp.join(tmpdir, "legacy_styles.h5")
+            win.save_h5_workspace(fname)
+
+            # Saved by the current version: stored styles are kept
+            loaded = _reload_workspace(win, fname)
+            assert _stored_style(loaded[0]) == LEGACY_STYLE
+            assert _style(_item(panel, loaded[0])) == LEGACY_STYLE
+
+            # Saved by DataLab 1.3: legacy automatic styles are removed
+            _set_saved_version(fname, "1.3.0")
+            loaded = _reload_workspace(win, fname)
+            for index in (0, 1):
+                assert _stored_style(loaded[index]) == (None, None)
+                assert _style(_item(panel, loaded[index])) == CurveStyles.style_at(
+                    index
+                )
+            assert _stored_style(loaded[2]) == (LEGACY_STYLE[0], "DotLine")
+            assert _style(_item(panel, loaded[2])) == (LEGACY_STYLE[0], "DotLine")
+
+
+def test_legacy_styles_converted_on_dlhist_import() -> None:
+    """Automatic styles stored by DataLab 1.3 or earlier are removed when importing
+    a history file in a workspace in use"""
+    with helpers.WorkdirRestoringTempDir() as tmpdir:
+        with datalab_test_app_context(console=False, exec_loop=False) as win:
+            panel = win.signalpanel
+            signals = _create_signals(1)
+            signals[0].metadata["color"], signals[0].metadata["style"] = LEGACY_STYLE
+            panel.add_objects(signals, set_current=False)
+            fname = osp.join(tmpdir, "legacy_styles.dlhist")
+            assert win.historypanel.save_to_dlhist_file(fname)
+            _set_saved_version(fname, "1.3.0")
+            existing = panel.objmodel.get_all_objects()
+            assert win.historypanel.open_dlhist_file(fname)
+            imported = [
+                obj for obj in panel.objmodel.get_all_objects() if obj not in existing
+            ]
+            assert len(imported) == 1
+            assert _stored_style(imported[0]) == (None, None)
+            assert _stored_style(existing[0]) == LEGACY_STYLE
 
 
 def test_reset_curve_styles() -> None:
@@ -168,5 +280,7 @@ def test_reset_curve_styles() -> None:
     test_styles_follow_curve_positions()
     test_style_cycle_not_used_by_panel_and_dialogs()
     test_user_styles_are_kept()
-    test_legacy_styles_are_automatic()
+    test_user_style_from_legacy_cycle_is_kept()
+    test_legacy_styles_converted_on_load()
+    test_legacy_styles_converted_on_dlhist_import()
     test_reset_curve_styles()