diff --git a/datalab/config/options.py b/datalab/config/options.py index 94e0695e6..79b93f424 100644 --- a/datalab/config/options.py +++ b/datalab/config/options.py @@ -44,6 +44,7 @@ from datalab import __docurl__, __homeurl__, __supporturl__, __version__ from datalab.config.appinfo import APP_NAME, DATAPATH +from datalab.plugins.catalog import DEFAULT_CATALOG_URL if TYPE_CHECKING: from datalab.config.persistence import OptionStore @@ -147,6 +148,14 @@ def __init__(self) -> None: expected_type=list, description="List of extra plugin directories.", ) + self.plugins_catalog_url = TypedOptionField( + self, + "plugins_catalog_url", + category="main", + default=DEFAULT_CATALOG_URL, + expected_type=str, + description="URL of the plugin catalog (empty to disable it).", + ) self.welcome_on_startup = TypedOptionField( self, "welcome_on_startup", diff --git a/datalab/gui/plugins/available.py b/datalab/gui/plugins/available.py new file mode 100644 index 000000000..db66b5a0a --- /dev/null +++ b/datalab/gui/plugins/available.py @@ -0,0 +1,338 @@ +# Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file. + +""" +Available plugins +----------------- + +Browse the DataLab plugin catalog and install or update its plugins. +""" + +from __future__ import annotations + +import shutil +import tempfile +from collections.abc import Callable +from html import escape +from importlib import metadata as importlib_metadata + +from guidata.configtools import get_icon +from packaging.utils import canonicalize_name +from packaging.version import InvalidVersion, Version +from qtpy import QtCore as QC +from qtpy import QtGui as QG +from qtpy import QtWidgets as QW + +from datalab.config import PLUGIN_ERROR_COLOR, Conf, _ +from datalab.gui.plugins.install import InstalledPluginsWidget +from datalab.plugins.catalog import ( + CatalogPlugin, + CatalogRelease, + download_release, + fetch_catalog, +) +from datalab.plugins.store import WHEEL_KIND +from datalab.widgets.expandabletext import apply_palette_color, apply_subdued_color + +CATALOG_PAGE_URL = "https://datalab-platform.com/plugins/" + +#: Running tasks, kept alive until their thread finishes +_RUNNING_TASKS: set[_Task] = set() + + +class _Task(QC.QThread): + """Run a function in a background thread.""" + + succeeded = QC.Signal(object) + failed = QC.Signal(str) + + def __init__(self, function: Callable[[], object]) -> None: + super().__init__() + self._function = function + + def run(self) -> None: # pragma: no cover - thread entry point + """Call the function and emit its result or error message.""" + try: + result = self._function() + except (OSError, ValueError) as exc: + self.failed.emit(str(exc)) + return + self.succeeded.emit(result) + + +def _is_newer(version: str, installed: str) -> bool: + try: + return Version(version) > Version(installed) + except InvalidVersion: + return version != installed + + +def _tier_label(plugin: CatalogPlugin) -> str: + return _("Official") if plugin.tier == "official" else _("Community") + + +class AvailablePluginItemWidget(QW.QWidget): + """Row describing a catalog plugin and the action it allows. + + Args: + plugin: Catalog plugin + installed: How the plugin is installed (``""``, ``"file"`` or + ``"environment"``) and its installed version + parent: Parent widget + """ + + def __init__( + self, + plugin: CatalogPlugin, + installed: tuple[str, str], + parent: QW.QWidget | None = None, + ) -> None: + super().__init__(parent) + self.plugin = plugin + self.action_button: QW.QPushButton | None = None + release = plugin.get_installable_release() + layout = QW.QHBoxLayout() + layout.setContentsMargins(0, 4, 0, 4) + self.setLayout(layout) + + version = f" {escape(release.version)}" if release else "" + details = " · ".join( + part + for part in ( + escape(plugin.license), + f'' + f"{escape(_('Source code'))}", + ) + if part + ) + label = QW.QLabel( + f"{escape(plugin.name)}{version} " + f"({escape(_tier_label(plugin))})
" + f"{escape(plugin.summary)}
{details}" + ) + label.setWordWrap(True) + label.setOpenExternalLinks(True) + label.setTextInteractionFlags(QC.Qt.TextBrowserInteraction) + layout.addWidget(label, 1) + + status, action = self._describe(release, installed) + self.status_label = QW.QLabel(status) + self.status_label.setWordWrap(True) + self.status_label.setAlignment(QC.Qt.AlignRight | QC.Qt.AlignVCenter) + if plugin.status == "revoked": + apply_palette_color(self.status_label, QG.QColor(PLUGIN_ERROR_COLOR)) + else: + apply_subdued_color(self.status_label) + layout.addWidget(self.status_label) + if action: + self.action_button = QW.QPushButton(action) + layout.addWidget(self.action_button) + + def _describe( + self, release: CatalogRelease | None, installed: tuple[str, str] + ) -> tuple[str, str | None]: + """Return the status text and the action button text, if any.""" + plugin = self.plugin + if plugin.status == "revoked": + return _("Withdrawn: %s") % plugin.status_reason, None + if release is None: + return _("Not available for DataLab desktop"), None + how, version = installed + if how == "environment": + return _("Installed in the Python environment (%s)") % version, None + if how == "file" and not _is_newer(release.version, version): + return _("Installed"), None + status = "" + if plugin.status == "deprecated": + status = _("Deprecated: %s") % plugin.status_reason + if how == "file": + status = status or _("Installed: %s") % version + return status, _("Update to %s") % release.version + return status, _("Install") + + +class AvailablePluginsWidget(QW.QWidget): + """Tab listing the catalog plugins, with install and update actions. + + Args: + installer: Tab installing plugin files, used once a wheel is downloaded + parent: Parent widget + """ + + def __init__( + self, installer: InstalledPluginsWidget, parent: QW.QWidget | None = None + ) -> None: + super().__init__(parent) + self._installer = installer + self.plugins: list[CatalogPlugin] | None = None + self.item_widgets: list[AvailablePluginItemWidget] = [] + self._download: tuple[CatalogPlugin, str] | None = None + self._busy = False + layout = QW.QVBoxLayout() + self.setLayout(layout) + + intro = QW.QLabel( + _( + 'Plugins listed in the DataLab plugin catalog. ' + "Official plugins are maintained by the DataLab team, community " + "plugins by their authors: install only plugins from authors you " + "trust." + ) + % CATALOG_PAGE_URL + ) + intro.setWordWrap(True) + intro.setOpenExternalLinks(True) + layout.addWidget(intro) + + controls = QW.QHBoxLayout() + self.search_edit = QW.QLineEdit() + self.search_edit.setPlaceholderText(_("Search plugins")) + self.search_edit.setClearButtonEnabled(True) + self.search_edit.textChanged.connect(self._populate) + controls.addWidget(self.search_edit, 1) + self.refresh_button = QW.QPushButton(get_icon("refresh-auto.svg"), _("Refresh")) + self.refresh_button.clicked.connect(self.refresh) + controls.addWidget(self.refresh_button) + layout.addLayout(controls) + + self.status_label = QW.QLabel() + self.status_label.setWordWrap(True) + apply_subdued_color(self.status_label) + layout.addWidget(self.status_label) + + scroll = QW.QScrollArea() + scroll.setWidgetResizable(True) + scroll.setHorizontalScrollBarPolicy(QC.Qt.ScrollBarAlwaysOff) + container = QW.QWidget() + self.items_layout = QW.QVBoxLayout() + self.items_layout.setContentsMargins(6, 0, 6, 0) + self.items_layout.addStretch() + container.setLayout(self.items_layout) + scroll.setWidget(container) + layout.addWidget(scroll, 1) + + def showEvent(self, event: QG.QShowEvent) -> None: # pylint: disable=C0103 + """Load the catalog the first time the tab is shown.""" + super().showEvent(event) + if self.plugins is None and not self._busy: + self.refresh() + + def run_task( + self, + function: Callable[[], object], + on_success: Callable[[object], None], + on_failure: Callable[[str], None], + ) -> None: + """Run a function in a background thread and report its outcome.""" + task = _Task(function) + _RUNNING_TASKS.add(task) + task.succeeded.connect(on_success) + task.failed.connect(on_failure) + task.finished.connect(lambda: _RUNNING_TASKS.discard(task)) + task.start() + + def refresh(self) -> None: + """Download the catalog again.""" + url = Conf.plugins_catalog_url.get() + if not url: + self.status_label.setText(_("No plugin catalog is configured.")) + return + self._set_busy(_("Loading the plugin catalog...")) + self.run_task(lambda: fetch_catalog(url), self._on_catalog, self._on_error) + + def install(self, plugin: CatalogPlugin) -> None: + """Download the newest release of a plugin, then install it.""" + release = plugin.get_installable_release() + if release is None or self._busy: + return + directory = tempfile.mkdtemp(prefix="datalab-plugin-") + self._download = (plugin, directory) + self._set_busy(_("Downloading %s...") % release.filename) + self.run_task( + lambda: download_release(release, directory), + self._on_downloaded, + self._on_download_failed, + ) + + def _set_busy(self, message: str | None) -> None: + self._busy = message is not None + self.refresh_button.setEnabled(not self._busy) + for widget in self.item_widgets: + if widget.action_button is not None: + widget.action_button.setEnabled(not self._busy) + if message is not None: + apply_subdued_color(self.status_label) + self.status_label.setText(message) + + def _on_catalog(self, plugins: list[CatalogPlugin]) -> None: + self.plugins = plugins + self._set_busy(None) + self.status_label.setText(_("Plugins in the catalog: %d") % len(plugins)) + self._populate() + + def _on_error(self, message: str) -> None: + self._set_busy(None) + apply_palette_color(self.status_label, QG.QColor(PLUGIN_ERROR_COLOR)) + self.status_label.setText( + _("The plugin catalog cannot be loaded:") + f" {message}" + ) + + def _on_downloaded(self, path: str) -> None: + plugin, directory = self._download + self._download = None + self._set_busy(None) + self.status_label.setText("") + try: + self._installer.install_path( + path, origin=_("DataLab plugin catalog (%s)") % _tier_label(plugin) + ) + finally: + shutil.rmtree(directory, ignore_errors=True) + self._populate() + + def _on_download_failed(self, message: str) -> None: + _plugin, directory = self._download + self._download = None + shutil.rmtree(directory, ignore_errors=True) + self._set_busy(None) + self.status_label.setText("") + QW.QMessageBox.warning(self, _("Install plugin"), message) + + def _get_installed(self, plugin: CatalogPlugin) -> tuple[str, str]: + """Return how a catalog plugin is installed, and its version.""" + if not plugin.distribution: + return "", "" + key = canonicalize_name(plugin.distribution) + try: + installed_plugins = self._installer.store.get_plugins() + except ValueError: + installed_plugins = [] + for installed in installed_plugins: + if ( + installed.kind == WHEEL_KIND + and canonicalize_name(installed.name) == key + ): + return "file", installed.version + try: + return "environment", importlib_metadata.version(plugin.distribution) + except importlib_metadata.PackageNotFoundError: + return "", "" + + def _populate(self) -> None: + """Rebuild the plugin rows matching the search text.""" + for widget in self.item_widgets: + self.items_layout.removeWidget(widget) + widget.deleteLater() + self.item_widgets.clear() + if self.plugins is None: + return + text = self.search_edit.text() + for plugin in self.plugins: + if not plugin.matches(text): + continue + widget = AvailablePluginItemWidget(plugin, self._get_installed(plugin)) + if widget.action_button is not None: + widget.action_button.clicked.connect( + lambda _checked=False, item=plugin: self.install(item) + ) + self.item_widgets.append(widget) + self.items_layout.insertWidget(self.items_layout.count() - 1, widget) diff --git a/datalab/gui/plugins/config.py b/datalab/gui/plugins/config.py index 47194721b..ba755654e 100644 --- a/datalab/gui/plugins/config.py +++ b/datalab/gui/plugins/config.py @@ -36,6 +36,7 @@ set_user_plugin_paths, ) from datalab.gui.plugins.applications import get_plugin_icon +from datalab.gui.plugins.available import AvailablePluginsWidget from datalab.gui.plugins.install import InstalledPluginsWidget from datalab.plugins import ( PLUGINS_DEFAULT_PATH, @@ -619,6 +620,9 @@ def __init__(self, parent: DLMainWindow): self.installed_plugins_widget = InstalledPluginsWidget( self._apply_and_reload_plugins ) + self.available_plugins_widget = AvailablePluginsWidget( + self.installed_plugins_widget + ) self.setWindowTitle(_("Plugin Configuration")) self.setMinimumWidth(DIALOG_MIN_WIDTH) @@ -631,6 +635,7 @@ def __init__(self, parent: DLMainWindow): self.tabs = QW.QTabWidget() self.tabs.addTab(self._create_plugins_tab(), _("Enable/disable plugins")) self.tabs.addTab(self._create_search_paths_tab(), _("Plugin settings")) + self.tabs.addTab(self.available_plugins_widget, _("Available plugins")) self.tabs.addTab(self.installed_plugins_widget, _("Install plugins")) self.tabs.setCornerWidget(self._create_corner_widget(), QC.Qt.TopRightCorner) layout.addWidget(self.tabs, 1) @@ -1003,6 +1008,7 @@ def _update_global_plugins_ui(self) -> None: if self.settings_disabled_label is not None: self.settings_disabled_label.setVisible(not enabled) self.installed_plugins_widget.setEnabled(enabled) + self.available_plugins_widget.setEnabled(enabled) for widget in self.fixed_path_widgets + self.extra_path_widgets: widget.set_links_enabled(enabled) diff --git a/datalab/gui/plugins/install.py b/datalab/gui/plugins/install.py index 4a16a477e..d2809b94b 100644 --- a/datalab/gui/plugins/install.py +++ b/datalab/gui/plugins/install.py @@ -61,9 +61,15 @@ class PluginConsentDialog(QW.QDialog): Args: manifest: Wheel or module manifest returned by the installed plugin store parent: Parent widget + origin: Where the file comes from, when not chosen by the user """ - def __init__(self, manifest: dict, parent: QW.QWidget | None = None) -> None: + def __init__( + self, + manifest: dict, + parent: QW.QWidget | None = None, + origin: str | None = None, + ) -> None: super().__init__(parent) win32_fix_title_bar_background(self) self.setWindowTitle(_("Install plugin")) @@ -71,6 +77,8 @@ def __init__(self, manifest: dict, parent: QW.QWidget | None = None) -> None: self.setLayout(layout) form = QW.QFormLayout() + if origin: + form.addRow(_("Source:"), QW.QLabel(origin)) if "distribution" in manifest: form.addRow(_("Plugin:"), QW.QLabel(manifest["distribution"])) form.addRow(_("Version:"), QW.QLabel(manifest["version"])) @@ -249,9 +257,9 @@ def select_file(self) -> str | None: ) return filename or None - def confirm_installation(self, manifest: dict) -> bool: + def confirm_installation(self, manifest: dict, origin: str | None = None) -> bool: """Return True if the user agrees to install the inspected file.""" - return bool(PluginConsentDialog(manifest, self).exec()) + return bool(PluginConsentDialog(manifest, self, origin).exec()) def install_from_file(self) -> InstallResult | None: """Install a plugin file chosen by the user. @@ -262,13 +270,27 @@ def install_from_file(self) -> InstallResult | None: path = self.select_file() if not path: return None + return self.install_path(path) + + def install_path( + self, path: str, origin: str | None = None + ) -> InstallResult | None: + """Install a plugin file after the user's consent. + + Args: + path: Wheel or module file path + origin: Where the file comes from, shown in the consent dialog + + Returns: + Installation result, or None if refused + """ is_wheel = path.lower().endswith(".whl") try: if is_wheel: manifest = self.store.inspect_wheel_file(path) else: manifest = self.store.inspect_module_file(path) - if not self.confirm_installation(manifest): + if not self.confirm_installation(manifest, origin=origin): return None install = ( self.store.install_wheel if is_wheel else self.store.install_module diff --git a/datalab/locale/fr/LC_MESSAGES/datalab.po b/datalab/locale/fr/LC_MESSAGES/datalab.po index c9b49549e..2292fb053 100644 --- a/datalab/locale/fr/LC_MESSAGES/datalab.po +++ b/datalab/locale/fr/LC_MESSAGES/datalab.po @@ -2276,6 +2276,75 @@ msgstr "%s a été désinstallé." msgid "Do you want to reload plugins now?" msgstr "Voulez-vous recharger les plugins maintenant ?" +msgid "Source:" +msgstr "Source :" + +msgid "Available plugins" +msgstr "Plugins disponibles" + +msgid "Official" +msgstr "Officiel" + +msgid "Community" +msgstr "Communauté" + +msgid "Source code" +msgstr "Code source" + +#, python-format +msgid "Withdrawn: %s" +msgstr "Retiré : %s" + +msgid "Not available for DataLab desktop" +msgstr "Non disponible pour DataLab desktop" + +#, python-format +msgid "Installed in the Python environment (%s)" +msgstr "Installé dans l'environnement Python (%s)" + +msgid "Installed" +msgstr "Installé" + +#, python-format +msgid "Deprecated: %s" +msgstr "Obsolète : %s" + +#, python-format +msgid "Installed: %s" +msgstr "Installé : %s" + +#, python-format +msgid "Update to %s" +msgstr "Mettre à jour vers %s" + +#, python-format +msgid "Plugins listed in the DataLab plugin catalog. Official plugins are maintained by the DataLab team, community plugins by their authors: install only plugins from authors you trust." +msgstr "Plugins répertoriés dans le catalogue de plugins DataLab. Les plugins officiels sont maintenus par l'équipe DataLab, les plugins de la communauté par leurs auteurs : n'installez que des plugins provenant d'auteurs de confiance." + +msgid "Search plugins" +msgstr "Rechercher des plugins" + +msgid "No plugin catalog is configured." +msgstr "Aucun catalogue de plugins n'est configuré." + +msgid "Loading the plugin catalog..." +msgstr "Chargement du catalogue de plugins..." + +#, python-format +msgid "Downloading %s..." +msgstr "Téléchargement de %s..." + +#, python-format +msgid "Plugins in the catalog: %d" +msgstr "Plugins dans le catalogue : %d" + +msgid "The plugin catalog cannot be loaded:" +msgstr "Le catalogue de plugins ne peut pas être chargé :" + +#, python-format +msgid "DataLab plugin catalog (%s)" +msgstr "Catalogue de plugins DataLab (%s)" + msgid "Failed to deserialize processing parameters from JSON." msgstr "Échec de la désérialisation des paramètres de traitement depuis le format JSON." diff --git a/datalab/plugins/catalog.py b/datalab/plugins/catalog.py new file mode 100644 index 000000000..d7625571e --- /dev/null +++ b/datalab/plugins/catalog.py @@ -0,0 +1,204 @@ +# Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file. + +""" +Plugin catalog client +--------------------- + +Read the DataLab plugin catalog (``catalog.json``) and download release wheels +whose size and SHA-256 digest match the catalog. This module is Qt-free: the +plugin dialog calls it from a worker thread. +""" + +from __future__ import annotations + +import dataclasses +import hashlib +import json +import os.path as osp +import re +import urllib.error +import urllib.parse +import urllib.request + +from datalab import __version__ +from datalab.plugins.wheels import MAX_WHEEL_BYTES + +DEFAULT_CATALOG_URL = "https://datalab-platform.com/plugins/catalog.json" +CATALOG_SCHEMA_VERSION = 1 +MAX_CATALOG_BYTES = 8 * 1024 * 1024 +DOWNLOAD_TIMEOUT = 60 + +# file: URLs serve catalogs mirrored on a local or network folder +_ALLOWED_SCHEMES = ("https", "file") +_SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") + + +class CatalogError(ValueError): + """Raised when the catalog or a release cannot be read or verified.""" + + +@dataclasses.dataclass(frozen=True) +class CatalogRelease: + """Release of a catalog plugin. + + Args: + version: Plugin version + sha256: SHA-256 digest of the wheel + filename: Wheel file name (empty for revoked plugins) + url: Absolute wheel URL (empty for revoked plugins) + size: Wheel size in bytes + targets: Hosts supported by the wheel (``desktop``, ``web``) + yanked: Reason why the release should not be installed, if any + """ + + version: str + sha256: str + filename: str = "" + url: str = "" + size: int = 0 + targets: tuple[str, ...] = () + yanked: str = "" + + +@dataclasses.dataclass(frozen=True) +class CatalogPlugin: + """Plugin listed in the catalog, with its releases from the newest.""" + + id: str + name: str + tier: str + status: str + repository: str + releases: tuple[CatalogRelease, ...] + status_reason: str = "" + distribution: str = "" + summary: str = "" + license: str = "" + keywords: tuple[str, ...] = () + + def get_installable_release(self, target: str = "desktop") -> CatalogRelease | None: + """Return the newest release that may be installed on a host.""" + if self.status == "revoked": + return None + return next( + ( + release + for release in self.releases + if release.url and not release.yanked and target in release.targets + ), + None, + ) + + def matches(self, text: str) -> bool: + """Return True if the plugin matches a search text.""" + words = text.casefold().split() + haystack = " ".join( + (self.name, self.id, self.summary, " ".join(self.keywords)) + ).casefold() + return all(word in haystack for word in words) + + +def _read(url: str, limit: int) -> bytes: + """Return the content of an HTTPS or file URL, up to a size limit.""" + if urllib.parse.urlparse(url).scheme not in _ALLOWED_SCHEMES: + raise CatalogError(f"Unsupported URL scheme: {url}") + request = urllib.request.Request( + url, headers={"User-Agent": f"DataLab/{__version__}"} + ) + try: + with urllib.request.urlopen(request, timeout=DOWNLOAD_TIMEOUT) as response: + data = response.read(limit + 1) + except (urllib.error.URLError, OSError) as exc: + raise CatalogError(f"Cannot download {url}: {exc}") from exc + if len(data) > limit: + raise CatalogError(f"{url} exceeds the {limit} byte size limit") + return data + + +def _parse_release(item: dict, catalog_url: str) -> CatalogRelease: + if not _SHA256_PATTERN.fullmatch(item["sha256"]): + raise ValueError(f"Invalid SHA-256 digest: {item['sha256']!r}") + url = item.get("url", "") + return CatalogRelease( + version=str(item["version"]), + sha256=item["sha256"], + filename=osp.basename(item.get("filename", "")), + url=urllib.parse.urljoin(catalog_url, url) if url else "", + size=int(item.get("size", 0)), + targets=tuple(item.get("targets", ())), + yanked=item.get("yanked", ""), + ) + + +def _parse_plugin(item: dict, catalog_url: str) -> CatalogPlugin: + return CatalogPlugin( + id=item["id"], + name=item["name"], + tier=item["tier"], + status=item["status"], + repository=item["repository"], + releases=tuple( + _parse_release(release, catalog_url) for release in item["releases"] + ), + status_reason=item.get("status_reason", ""), + distribution=item.get("distribution", ""), + summary=item.get("summary", ""), + license=item.get("license", ""), + keywords=tuple(item.get("keywords", ())), + ) + + +def fetch_catalog(url: str = DEFAULT_CATALOG_URL) -> list[CatalogPlugin]: + """Download and parse the plugin catalog. + + Raises: + CatalogError: The catalog cannot be downloaded or is invalid + """ + data = _read(url, MAX_CATALOG_BYTES) + try: + payload = json.loads(data.decode("utf-8")) + except (UnicodeDecodeError, ValueError) as exc: + raise CatalogError(f"Invalid plugin catalog: {url}") from exc + if ( + not isinstance(payload, dict) + or payload.get("schema_version") != CATALOG_SCHEMA_VERSION + ): + raise CatalogError(f"Unsupported plugin catalog format: {url}") + try: + return [_parse_plugin(item, url) for item in payload["plugins"]] + except (AttributeError, KeyError, TypeError, ValueError) as exc: + raise CatalogError(f"Invalid plugin catalog: {url}") from exc + + +def download_release(release: CatalogRelease, directory: str) -> str: + """Download a release wheel into a directory after checking it. + + Returns: + Path of the downloaded wheel, named as in the catalog + + Raises: + CatalogError: The download failed or does not match the catalog + """ + if not release.url or not release.filename.endswith(".whl"): + raise CatalogError("This release cannot be downloaded") + if not 0 < release.size <= MAX_WHEEL_BYTES: + raise CatalogError(f"Invalid wheel size in the catalog: {release.size}") + data = _read(release.url, release.size) + if len(data) != release.size or hashlib.sha256(data).hexdigest() != release.sha256: + raise CatalogError( + f"{release.filename} does not match the size and digest of the catalog" + ) + path = osp.join(directory, release.filename) + with open(path, "wb") as file: + file.write(data) + return path + + +__all__ = [ + "DEFAULT_CATALOG_URL", + "CatalogError", + "CatalogPlugin", + "CatalogRelease", + "download_release", + "fetch_catalog", +] diff --git a/datalab/tests/backbone/plugins/plugin_catalog_unit_test.py b/datalab/tests/backbone/plugins/plugin_catalog_unit_test.py new file mode 100644 index 000000000..3a2f19d55 --- /dev/null +++ b/datalab/tests/backbone/plugins/plugin_catalog_unit_test.py @@ -0,0 +1,151 @@ +# Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file. + +"""Unit tests for the plugin catalog client, with catalogs served from files.""" + +from __future__ import annotations + +import hashlib +import json +from pathlib import Path + +import pytest + +from datalab.plugins.catalog import CatalogError, download_release, fetch_catalog + +FILENAME = "example_plugin-1.0.0-py3-none-any.whl" + + +def write_catalog(directory: Path, plugins: list[dict], schema_version=1) -> str: + """Write a catalog file and return its URL.""" + path = directory / "catalog.json" + path.write_text( + json.dumps({"schema_version": schema_version, "plugins": plugins}), + encoding="utf-8", + ) + return path.as_uri() + + +def publish(directory: Path, data: bytes, version: str = "1.0.0", **kwargs) -> dict: + """Write a wheel in the catalog folder and return its release record.""" + digest = hashlib.sha256(data).hexdigest() + filename = FILENAME.replace("1.0.0", version) + path = directory / "wheels" / digest / filename + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + release = { + "version": version, + "filename": filename, + "url": f"wheels/{digest}/{filename}", + "sha256": digest, + "size": len(data), + "targets": ["desktop", "web"], + } + release.update(kwargs) + return release + + +def plugin_record(releases: list[dict], **kwargs) -> dict: + """Return a catalog plugin record.""" + record = { + "id": "io.github.someone.example", + "name": "Example Plugin", + "tier": "community", + "status": "active", + "repository": "https://github.com/someone/example-plugin", + "distribution": "example-plugin", + "summary": "Spectral baseline tools", + "license": "MIT", + "keywords": ["spectroscopy"], + "releases": releases, + } + record.update(kwargs) + return record + + +def test_release_is_downloaded_from_a_relative_url(tmp_path: Path) -> None: + """Release URLs are relative to the catalog and verified once downloaded.""" + release = publish(tmp_path, b"wheel content") + url = write_catalog(tmp_path, [plugin_record([release])]) + + (plugin,) = fetch_catalog(url) + latest = plugin.get_installable_release() + destination = tmp_path / "download" + destination.mkdir() + path = download_release(latest, str(destination)) + + assert plugin.name == "Example Plugin" and plugin.tier == "community" + assert latest.url == (tmp_path / release["url"]).as_uri() + assert Path(path) == destination / FILENAME + assert Path(path).read_bytes() == b"wheel content" + + +def test_installable_release_skips_yanked_web_only_and_revoked(tmp_path: Path) -> None: + """Only the newest desktop release that is not withdrawn is offered.""" + releases = [ + publish(tmp_path, b"3", "3.0.0", yanked="Corrupted results"), + publish(tmp_path, b"2", "2.0.0", targets=["web"]), + publish(tmp_path, b"1", "1.0.0"), + ] + url = write_catalog( + tmp_path, + [ + plugin_record(releases), + plugin_record( + [{"version": "1.0.0", "sha256": "a" * 64}], + id="io.github.someone.revoked", + status="revoked", + status_reason="Malicious code", + ), + ], + ) + + plugin, revoked = fetch_catalog(url) + + assert plugin.get_installable_release().version == "1.0.0" + assert plugin.get_installable_release("web").version == "2.0.0" + assert revoked.get_installable_release() is None + assert plugin.matches("SPECTRO baseline") and not plugin.matches("camera") + + +def test_downloads_must_match_the_catalog(tmp_path: Path) -> None: + """A wheel changed after publication is refused.""" + release = publish(tmp_path, b"original") + (tmp_path / release["url"]).write_bytes(b"tampered") + url = write_catalog(tmp_path, [plugin_record([release])]) + (plugin,) = fetch_catalog(url) + + with pytest.raises(CatalogError, match="does not match"): + download_release(plugin.get_installable_release(), str(tmp_path)) + + (tmp_path / release["url"]).write_bytes(b"original and more") + with pytest.raises(CatalogError, match="size limit"): + download_release(plugin.get_installable_release(), str(tmp_path)) + + +@pytest.mark.parametrize( + ("content", "message"), + [ + ('{"schema_version": 2, "plugins": []}', "Unsupported plugin catalog"), + ("not json", "Invalid plugin catalog"), + ('{"schema_version": 1, "plugins": [{"id": "x"}]}', "Invalid plugin catalog"), + ], +) +def test_invalid_catalogs_are_refused( + tmp_path: Path, content: str, message: str +) -> None: + """Unknown formats and incomplete records are reported.""" + path = tmp_path / "catalog.json" + path.write_text(content, encoding="utf-8") + + with pytest.raises(CatalogError, match=message): + fetch_catalog(path.as_uri()) + + +def test_only_https_and_file_urls_are_used() -> None: + """Plain HTTP could be tampered with on the way.""" + with pytest.raises(CatalogError, match="Unsupported URL scheme"): + fetch_catalog("http://example.org/catalog.json") + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/datalab/tests/backbone/plugins/plugins_contracts_imports_unit_test.py b/datalab/tests/backbone/plugins/plugins_contracts_imports_unit_test.py index 4095d2428..4f70cd9b3 100644 --- a/datalab/tests/backbone/plugins/plugins_contracts_imports_unit_test.py +++ b/datalab/tests/backbone/plugins/plugins_contracts_imports_unit_test.py @@ -17,7 +17,7 @@ "tools", ) # Installer modules, reused outside the Qt host (DataLab-Web, catalog tooling) -HEADLESS_HOST_MODULES = ("store", "wheels") +HEADLESS_HOST_MODULES = ("catalog", "store", "wheels") def test_plugin_contracts_do_not_import_plugin_host() -> None: diff --git a/datalab/tests/features/plugins/plugin_available_dialog_test.py b/datalab/tests/features/plugins/plugin_available_dialog_test.py new file mode 100644 index 000000000..d6cda94df --- /dev/null +++ b/datalab/tests/features/plugins/plugin_available_dialog_test.py @@ -0,0 +1,160 @@ +# Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file. + +"""Installing plugins from the catalog in the configuration dialog.""" + +from __future__ import annotations + +import hashlib +import json +import sys +from pathlib import Path + +import pytest +from qtpy import QtWidgets as QW + +from datalab import __version__ +from datalab.config import Conf +from datalab.config.persistence import has_persisted_option, remove_persisted_option +from datalab.gui.plugins.config import PluginConfigDialog +from datalab.plugins import PluginRegistry +from datalab.plugins import base as plugin_base +from datalab.plugins.store import InstalledPluginStore +from datalab.tests import datalab_test_app_context +from datalab.tests.backbone.plugins.wheel_factory import ( + make_plugin_wheel, + wheel_filename, +) + +PACKAGE = "datalab_catalog_ui_probe" +DISTRIBUTION = "datalab-catalog-ui-probe" +PLUGIN_ID = "org.example.catalog-ui-probe" + + +@pytest.fixture(name="store") +def fixture_store(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): + """Use an empty installed plugin store and restore the import state.""" + store = InstalledPluginStore( + str(tmp_path / "installed_plugins"), + host_distributions={"datalab-platform": __version__}, + ) + monkeypatch.setattr(plugin_base, "_INSTALLED_PLUGIN_STORE", store) + monkeypatch.setattr(sys, "path", list(sys.path)) + had_url = has_persisted_option(Conf, "plugins_catalog_url") + original_url = Conf.plugins_catalog_url.get() + yield store + if had_url: + Conf.plugins_catalog_url.set(original_url) + else: + remove_persisted_option(Conf, "plugins_catalog_url") + for name in [name for name in sys.modules if name.startswith(PACKAGE)]: + del sys.modules[name] + + +def publish_catalog(site: Path, versions: tuple[str, ...]) -> str: + """Publish probe plugin releases in a catalog folder and return its URL.""" + releases = [] + for version in sorted(versions, reverse=True): + data = make_plugin_wheel( + distribution=DISTRIBUTION, + version=version, + package=PACKAGE, + plugin_id=PLUGIN_ID, + requires_dist=("datalab-platform>=1.0",), + ) + digest = hashlib.sha256(data).hexdigest() + filename = wheel_filename(DISTRIBUTION, version) + path = site / "wheels" / digest / filename + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + releases.append( + { + "version": version, + "filename": filename, + "url": f"wheels/{digest}/{filename}", + "sha256": digest, + "size": len(data), + "targets": ["desktop"], + } + ) + catalog = { + "schema_version": 1, + "plugins": [ + { + "id": PLUGIN_ID, + "name": "Catalog probe", + "tier": "community", + "status": "active", + "repository": "https://github.com/someone/catalog-probe", + "distribution": DISTRIBUTION, + "summary": "Probe plugin listed in a test catalog", + "license": "MIT", + "releases": releases, + } + ], + } + (site / "catalog.json").write_text(json.dumps(catalog), encoding="utf-8") + return (site / "catalog.json").as_uri() + + +def run_now(function, on_success, on_failure) -> None: + """Run a catalog task synchronously.""" + try: + result = function() + except (OSError, ValueError) as exc: + on_failure(str(exc)) + return + on_success(result) + + +def test_catalog_plugin_is_installed_then_offered_as_update( + store: InstalledPluginStore, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A catalog plugin is downloaded, verified, installed and later updated.""" + site = tmp_path / "site" + Conf.plugins_catalog_url.set(publish_catalog(site, ("1.0.0",))) + origins: list[str] = [] + monkeypatch.setattr( + QW.QMessageBox, "question", lambda *_args, **_kwargs: QW.QMessageBox.Yes + ) + with datalab_test_app_context(console=False) as win: + dialog = PluginConfigDialog(win) + available = dialog.available_plugins_widget + assert dialog.tabs.indexOf(available) == 2 + monkeypatch.setattr(available, "run_task", run_now) + monkeypatch.setattr( + dialog.installed_plugins_widget, + "confirm_installation", + lambda _manifest, origin=None: origins.append(origin) or True, + ) + + available.refresh() + (row,) = available.item_widgets + assert row.action_button.text() == "Install" + row.action_button.click() + QW.QApplication.processEvents() + + assert origins == ["DataLab plugin catalog (Community)"] + assert PluginRegistry.get_plugin(PLUGIN_ID) is not None + assert [plugin.version for plugin in store.get_plugins()] == ["1.0.0"] + (row,) = available.item_widgets + assert row.action_button is None + assert row.status_label.text() == "Installed" + + available.search_edit.setText("camera") + assert not available.item_widgets + available.search_edit.clear() + + Conf.plugins_catalog_url.set(publish_catalog(site, ("1.0.0", "1.1.0"))) + available.refresh() + (row,) = available.item_widgets + assert row.action_button.text() == "Update to 1.1.0" + + Conf.plugins_catalog_url.set((tmp_path / "missing.json").as_uri()) + available.refresh() + assert "cannot be loaded" in available.status_label.text() + dialog.close() + dialog.deleteLater() + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/datalab/tests/features/plugins/plugin_install_dialog_test.py b/datalab/tests/features/plugins/plugin_install_dialog_test.py index 1c6eab963..8bb1b779f 100644 --- a/datalab/tests/features/plugins/plugin_install_dialog_test.py +++ b/datalab/tests/features/plugins/plugin_install_dialog_test.py @@ -78,12 +78,12 @@ def test_installed_wheel_is_enabled_loaded_and_uninstalled( ) dialog = PluginConfigDialog(win) widget = dialog.installed_plugins_widget - assert dialog.tabs.indexOf(widget) == 2 + assert dialog.tabs.indexOf(widget) == 3 monkeypatch.setattr(widget, "select_file", lambda: path) monkeypatch.setattr( widget, "confirm_installation", - lambda manifest: consented.append(manifest) or True, + lambda manifest, origin=None: consented.append(manifest) or True, ) widget.install_button.click() @@ -136,7 +136,9 @@ def test_refused_or_invalid_files_are_not_installed( ) path = write_wheel(tmp_path) monkeypatch.setattr(widget, "select_file", lambda: path) - monkeypatch.setattr(widget, "confirm_installation", lambda _manifest: False) + monkeypatch.setattr( + widget, "confirm_installation", lambda _manifest, origin=None: False + ) assert widget.install_from_file() is None diff --git a/doc/features/advanced/plugins.rst b/doc/features/advanced/plugins.rst index f1cbffcb3..96bc807a4 100644 --- a/doc/features/advanced/plugins.rst +++ b/doc/features/advanced/plugins.rst @@ -148,8 +148,7 @@ Managing plugins in DataLab The **Plugins** menu provides two dedicated actions: - **Configure plugins...** - Opens the plugin configuration dialog, organized in three tabs: - **Enable/disable plugins**, **Plugin settings** and **Install plugins**. + Opens the plugin configuration dialog, organized in four tabs: **Enable/disable plugins**, **Plugin settings**, **Available plugins** and **Install plugins**. - **Reload plugins** Reloads plugin modules from disk without restarting DataLab. @@ -190,6 +189,17 @@ This tab lists every directory scanned at startup, in two groups: The tab also holds a **Compatibility warnings** option to hide warnings for incompatible DataLab v0.20 plugins. +.. _plugin_catalog: + +Available plugins +~~~~~~~~~~~~~~~~~ + +This tab lists the plugins of the `DataLab plugin catalog `_, with their description, license and source code link. Plugins maintained by the DataLab team are marked **Official**; the others are **Community** plugins, maintained by their authors. A search field filters the list. + +**Install** downloads the newest release available for DataLab desktop, checks that its size and SHA-256 digest match the catalog, then installs it as described in :ref:`plugin_install_from_file`, after the same confirmation. When the catalog offers a newer version of a plugin installed this way, the button becomes **Update to...**. Plugins installed with ``pip`` in the Python environment are shown as such and are not modified. + +The catalog is downloaded only when the tab is shown or refreshed. Its address is the ``plugins_catalog_url`` option of the DataLab configuration file: it may point to a private copy of the catalog, including a ``file:`` URL for a folder shared on the network, or be emptied to disable the catalog. + .. _plugin_install_from_file: Install plugins diff --git a/doc/locale/fr/LC_MESSAGES/features/advanced/plugins.po b/doc/locale/fr/LC_MESSAGES/features/advanced/plugins.po index c31c4ad89..5bb06d980 100644 --- a/doc/locale/fr/LC_MESSAGES/features/advanced/plugins.po +++ b/doc/locale/fr/LC_MESSAGES/features/advanced/plugins.po @@ -115,8 +115,8 @@ msgstr "Gestion des plugins dans DataLab" msgid "The **Plugins** menu provides two dedicated actions:" msgstr "Le menu **Plugins** propose deux actions dédiées :" -msgid "**Configure plugins...** Opens the plugin configuration dialog, organized in three tabs: **Enable/disable plugins**, **Plugin settings** and **Install plugins**." -msgstr "**Configurer les plugins...** Ouvre la boîte de dialogue de configuration des plugins, organisée en trois onglets : **Activer/désactiver les plugins**, **Paramètres des plugins** et **Installer des plugins**." +msgid "**Configure plugins...** Opens the plugin configuration dialog, organized in four tabs: **Enable/disable plugins**, **Plugin settings**, **Available plugins** and **Install plugins**." +msgstr "**Configurer les plugins...** Ouvre la boîte de dialogue de configuration des plugins, organisée en quatre onglets : **Activer/désactiver les plugins**, **Paramètres des plugins**, **Plugins disponibles** et **Installer des plugins**." msgid "**Reload plugins** Reloads plugin modules from disk without restarting DataLab." msgstr "**Recharger les plugins** Recharge les modules de plugins depuis le disque sans redémarrer DataLab." @@ -154,6 +154,18 @@ msgstr "**Répertoires de plugins supplémentaires** : vos propres répertoires, msgid "The tab also holds a **Compatibility warnings** option to hide warnings for incompatible DataLab v0.20 plugins." msgstr "L'onglet contient également une option **Avertissements de compatibilité** pour masquer les avertissements concernant les plugins DataLab v0.20 incompatibles." +msgid "Available plugins" +msgstr "Plugins disponibles" + +msgid "This tab lists the plugins of the `DataLab plugin catalog `_, with their description, license and source code link. Plugins maintained by the DataLab team are marked **Official**; the others are **Community** plugins, maintained by their authors. A search field filters the list." +msgstr "Cet onglet liste les plugins du `catalogue de plugins DataLab `_, avec leur description, leur licence et un lien vers leur code source. Les plugins maintenus par l'équipe DataLab sont marqués **Officiel** ; les autres sont des plugins de la **Communauté**, maintenus par leurs auteurs. Un champ de recherche filtre la liste." + +msgid "**Install** downloads the newest release available for DataLab desktop, checks that its size and SHA-256 digest match the catalog, then installs it as described in :ref:`plugin_install_from_file`, after the same confirmation. When the catalog offers a newer version of a plugin installed this way, the button becomes **Update to...**. Plugins installed with ``pip`` in the Python environment are shown as such and are not modified." +msgstr "**Installer** télécharge la version la plus récente disponible pour DataLab desktop, vérifie que sa taille et son empreinte SHA-256 correspondent au catalogue, puis l'installe comme décrit dans :ref:`plugin_install_from_file`, après la même confirmation. Lorsque le catalogue propose une version plus récente d'un plugin installé ainsi, le bouton devient **Mettre à jour vers...**. Les plugins installés avec ``pip`` dans l'environnement Python sont signalés comme tels et ne sont pas modifiés." + +msgid "The catalog is downloaded only when the tab is shown or refreshed. Its address is the ``plugins_catalog_url`` option of the DataLab configuration file: it may point to a private copy of the catalog, including a ``file:`` URL for a folder shared on the network, or be emptied to disable the catalog." +msgstr "Le catalogue n'est téléchargé que lorsque l'onglet est affiché ou actualisé. Son adresse est l'option ``plugins_catalog_url`` du fichier de configuration de DataLab : elle peut désigner une copie privée du catalogue, y compris une URL ``file:`` pour un dossier partagé sur le réseau, ou être vidée pour désactiver le catalogue." + msgid "Install plugins" msgstr "Installer des plugins" diff --git a/doc/locale/fr/LC_MESSAGES/release_notes/release_1.04.po b/doc/locale/fr/LC_MESSAGES/release_notes/release_1.04.po index b8b37eae1..5d8b622e9 100644 --- a/doc/locale/fr/LC_MESSAGES/release_notes/release_1.04.po +++ b/doc/locale/fr/LC_MESSAGES/release_notes/release_1.04.po @@ -120,6 +120,9 @@ msgstr "Avant l'installation, DataLab vérifie le fichier sans l'exécuter et af msgid "Installed plugins are listed in the same tab, where they can be uninstalled; a newly installed plugin is enabled and loaded after a plugin reload, and a new version of an already loaded plugin is used at the next start" msgstr "Les plugins installés sont listés dans le même onglet, où ils peuvent être désinstallés ; un plugin nouvellement installé est activé et chargé après un rechargement des plugins, et une nouvelle version d'un plugin déjà chargé est utilisée au prochain démarrage" +msgid "The new **Available plugins** tab lists the plugins of the [DataLab plugin catalog](https://datalab-platform.com/plugins/), official or from the community, with a search field: **Install** and **Update** download the wheel, check it against the catalog and install it after confirmation. The catalog address may be changed to use a private copy" +msgstr "Le nouvel onglet **Plugins disponibles** liste les plugins du [catalogue de plugins DataLab](https://datalab-platform.com/plugins/), officiels ou issus de la communauté, avec un champ de recherche : **Installer** et **Mettre à jour** téléchargent le wheel, le vérifient par rapport au catalogue et l'installent après confirmation. L'adresse du catalogue peut être modifiée pour utiliser une copie privée" + msgid "**Plugin project generator:**" msgstr "**Générateur de projets de plugins :**" diff --git a/doc/release_notes/release_1.04.md b/doc/release_notes/release_1.04.md index 3065a9b3c..befc69b90 100644 --- a/doc/release_notes/release_1.04.md +++ b/doc/release_notes/release_1.04.md @@ -46,6 +46,7 @@ * Plugins can now be installed from a file, without any Python tool, including in the standalone version of DataLab: the new **Install plugins** tab of **Plugins > Configure plugins...** accepts a pure-Python wheel (`.whl`) or a single `datalab_.py` module * Before installing, DataLab checks the file without running it and shows its name, version, plugin classes, dependencies and SHA-256 digest; a wheel requiring a package that DataLab does not provide is refused * Installed plugins are listed in the same tab, where they can be uninstalled; a newly installed plugin is enabled and loaded after a plugin reload, and a new version of an already loaded plugin is used at the next start +* The new **Available plugins** tab lists the plugins of the [DataLab plugin catalog](https://datalab-platform.com/plugins/), official or from the community, with a search field: **Install** and **Update** download the wheel, check it against the catalog and install it after confirmation. The catalog address may be changed to use a private copy **Plugin project generator:**