diff --git a/datalab/config/__init__.py b/datalab/config/__init__.py index 969144931..a9fb9d96a 100644 --- a/datalab/config/__init__.py +++ b/datalab/config/__init__.py @@ -11,6 +11,7 @@ from .appinfo import ( APP_DESC, APP_NAME, + DATALAB_PLUGINS_CATALOG_ENV_VAR, DATALAB_PLUGINS_ENV_PATHS, DATALAB_PLUGINS_ENV_VAR, DATAPATH, @@ -24,6 +25,7 @@ _, get_config_filename, get_config_path, + get_env_plugin_catalog_paths, normalize_plugin_paths, ) from .core import ( @@ -31,10 +33,12 @@ Conf, DataLabShapeParam, ensure_initialized, + get_plugin_catalog_paths, get_user_plugin_paths, initialize, reset, reset_to_defaults, + set_plugin_catalog_paths, set_user_plugin_paths, ) from .persistence import save_runtime_option @@ -42,6 +46,7 @@ __all__ = [ "APP_DESC", "APP_NAME", + "DATALAB_PLUGINS_CATALOG_ENV_VAR", "DATALAB_PLUGINS_ENV_PATHS", "DATALAB_PLUGINS_ENV_VAR", "DATAPATH", @@ -60,11 +65,14 @@ "ensure_initialized", "get_config_filename", "get_config_path", + "get_env_plugin_catalog_paths", + "get_plugin_catalog_paths", "get_user_plugin_paths", "initialize", "normalize_plugin_paths", "reset", "reset_to_defaults", "save_runtime_option", + "set_plugin_catalog_paths", "set_user_plugin_paths", ] diff --git a/datalab/config/appinfo.py b/datalab/config/appinfo.py index f5cc633f1..4471c362d 100644 --- a/datalab/config/appinfo.py +++ b/datalab/config/appinfo.py @@ -189,6 +189,22 @@ def normalize_plugin_paths(paths: list[str] | tuple[str, ...] | None) -> list[st return normalized +# Local plugin catalog folders provided via the `DATALAB_PLUGINS_CATALOG` +# environment variable, separated by `os.pathsep` as for `DATALAB_PLUGINS`. +DATALAB_PLUGINS_CATALOG_ENV_VAR = "DATALAB_PLUGINS_CATALOG" + + +def get_env_plugin_catalog_paths() -> list[str]: + """Return the local plugin catalog folders of ``DATALAB_PLUGINS_CATALOG``. + + Unlike plugin directories, missing folders are kept: a network share may be + unavailable when DataLab starts, and the plugin dialog reports the folders + it cannot read. + """ + value = os.environ.get(DATALAB_PLUGINS_CATALOG_ENV_VAR, "") + return normalize_plugin_paths([path.strip() for path in value.split(os.pathsep)]) + + def migrate_legacy_plugin_paths( options: DataLabOptions, ) -> list[str]: diff --git a/datalab/config/core.py b/datalab/config/core.py index 5cff6b5bd..32eb4f23b 100644 --- a/datalab/config/core.py +++ b/datalab/config/core.py @@ -65,6 +65,16 @@ def set_user_plugin_paths(paths: list[str] | tuple[str, ...]) -> None: Conf.plugins_path_list.set(normalized) +def get_plugin_catalog_paths() -> list[str]: + """Return the user-configured local plugin catalog folders.""" + return normalize_plugin_paths(Conf.plugins_catalog_path_list.get([]) or []) + + +def set_plugin_catalog_paths(paths: list[str] | tuple[str, ...]) -> None: + """Persist the user-configured local plugin catalog folders.""" + Conf.plugins_catalog_path_list.set(normalize_plugin_paths(list(paths))) + + #: Active initialization mode: ``None`` (not yet initialized), ``"user"`` #: (user configuration loaded/migrated, INI persistence enabled) or #: ``"defaults"`` (production defaults only, no persistence). diff --git a/datalab/config/options.py b/datalab/config/options.py index a937bd9c9..022545bd4 100644 --- a/datalab/config/options.py +++ b/datalab/config/options.py @@ -164,6 +164,14 @@ def __init__(self) -> None: expected_type=str, description="URL of the plugin catalog (empty to disable it).", ) + self.plugins_catalog_path_list = TypedOptionField( + self, + "plugins_catalog_path_list", + category="main", + default=[], + expected_type=list, + description="Local plugin catalog folders, by priority.", + ) self.welcome_on_startup = TypedOptionField( self, "welcome_on_startup", diff --git a/datalab/gui/plugins/available.py b/datalab/gui/plugins/available.py index afafe05eb..8b5553eb1 100644 --- a/datalab/gui/plugins/available.py +++ b/datalab/gui/plugins/available.py @@ -4,7 +4,8 @@ Available plugins ----------------- -Browse the DataLab plugin catalog and install or update its plugins. +Browse the DataLab plugin catalog and the local plugin catalogs, and install or +update their plugins. """ from __future__ import annotations @@ -12,7 +13,7 @@ import shutil import sys import tempfile -from collections.abc import Callable +from collections.abc import Callable, Sequence from html import escape from importlib import metadata as importlib_metadata @@ -23,13 +24,21 @@ from qtpy import QtGui as QG from qtpy import QtWidgets as QW -from datalab.config import PLUGIN_ERROR_COLOR, Conf, _ +from datalab.config import ( + PLUGIN_ERROR_COLOR, + Conf, + _, + get_env_plugin_catalog_paths, + get_plugin_catalog_paths, + normalize_plugin_paths, +) from datalab.gui.plugins.install import InstalledPluginsWidget from datalab.plugins.catalog import ( CatalogPlugin, CatalogRelease, + CatalogsContent, download_release, - fetch_catalog, + load_catalogs, ) from datalab.plugins.store import WHEEL_KIND from datalab.widgets.expandabletext import apply_palette_color, apply_subdued_color @@ -71,6 +80,22 @@ def _tier_label(plugin: CatalogPlugin) -> str: return _("Official") if plugin.tier == "official" else _("Community") +def _source_label(plugin: CatalogPlugin) -> str: + return _("Local catalog") if plugin.source else _tier_label(plugin) + + +def merge_catalog_directories(configured: Sequence[str]) -> list[str]: + """Return the local catalog folders, by priority. + + Folders of the ``DATALAB_PLUGINS_CATALOG`` environment variable come first, + then the folders configured in DataLab. + + Args: + configured: Folders configured in DataLab + """ + return normalize_plugin_paths([*get_env_plugin_catalog_paths(), *configured]) + + class AvailablePluginItemWidget(QW.QWidget): """Row describing a catalog plugin and the action it allows. @@ -96,20 +121,22 @@ def __init__( self.setLayout(layout) version = f" {escape(release.version)}" if release else "" + source_link = ( + f'' + f"{escape(_('Source code'))}" + if plugin.repository + else "" + ) details = " · ".join( - part - for part in ( - escape(plugin.license), - f'' - f"{escape(_('Source code'))}", - ) - if part + part for part in (escape(plugin.license), source_link) if part ) label = QW.QLabel( f"{escape(plugin.name)}{version} " - f"({escape(_tier_label(plugin))})
" + f"({escape(_source_label(plugin))})
" f"{escape(plugin.summary)}
{details}" ) + if plugin.source: + label.setToolTip(_("Local catalog: %s") % plugin.source) label.setWordWrap(True) label.setOpenExternalLinks(True) label.setTextInteractionFlags(QC.Qt.TextBrowserInteraction) @@ -157,6 +184,9 @@ def _describe( class AvailablePluginsWidget(QW.QWidget): """Tab listing the catalog plugins, with install and update actions. + The local catalogs are listed first: a plugin they provide hides the same + plugin in the online catalog. + Args: installer: Tab installing plugin files, used once a wheel is downloaded parent: Parent widget @@ -171,15 +201,20 @@ def __init__( self.item_widgets: list[AvailablePluginItemWidget] = [] self._download: tuple[CatalogPlugin, str] | None = None self._busy = False + self._directories = merge_catalog_directories(get_plugin_catalog_paths()) + #: Catalog URL and local folders of the last loading + self._loaded_sources: tuple[str, tuple[str, ...]] | None = None layout = QW.QVBoxLayout() self.setLayout(layout) intro = QW.QLabel( _( - 'Plugins listed in the DataLab plugin catalog. ' - "Official plugins are maintained by the DataLab team, community " - "plugins by their authors: install only plugins from authors you " - "trust." + 'Plugins listed in the DataLab plugin catalog ' + "and in the local catalogs of the Plugin settings tab, " + "which take precedence. Official plugins are maintained by the " + "DataLab team, community plugins by their authors, and local " + "catalogs by your organization: install only plugins from " + "authors you trust." ) % CATALOG_PAGE_URL ) @@ -203,6 +238,13 @@ def __init__( apply_subdued_color(self.status_label) layout.addWidget(self.status_label) + self.errors_label = QW.QLabel() + self.errors_label.setWordWrap(True) + self.errors_label.setTextFormat(QC.Qt.PlainText) + apply_palette_color(self.errors_label, QG.QColor(PLUGIN_ERROR_COLOR)) + self.errors_label.hide() + layout.addWidget(self.errors_label) + scroll = QW.QScrollArea() scroll.setWidgetResizable(True) scroll.setHorizontalScrollBarPolicy(QC.Qt.ScrollBarAlwaysOff) @@ -215,11 +257,28 @@ def __init__( layout.addWidget(scroll, 1) def showEvent(self, event: QG.QShowEvent) -> None: # pylint: disable=C0103 - """Load the catalog the first time the tab is shown.""" + """Load the catalogs when the tab is shown, unless already loaded.""" super().showEvent(event) - if self.plugins is None and not self._busy: + if not self._busy and self._loaded_sources != self._get_sources(): self.refresh() + def get_catalog_directories(self) -> list[str]: + """Return the local catalog folders, by priority.""" + return list(self._directories) + + def set_catalog_directories(self, directories: Sequence[str]) -> None: + """Set the local catalog folders, by priority. + + The catalogs are loaded again when the tab is visible, otherwise when it + is shown. + """ + self._directories = list(directories) + if self.isVisible() and not self._busy: + self.refresh() + + def _get_sources(self) -> tuple[str, tuple[str, ...]]: + return Conf.plugins_catalog_url.get(), tuple(self._directories) + def run_task( self, function: Callable[[], object], @@ -235,13 +294,21 @@ def run_task( task.start() def refresh(self) -> None: - """Download the catalog again.""" - url = Conf.plugins_catalog_url.get() - if not url: + """Load the catalogs again.""" + sources = self._get_sources() + self._loaded_sources = sources + url, directories = sources + self.errors_label.hide() + if not url and not directories: + self.plugins = None + self._populate() + apply_subdued_color(self.status_label) self.status_label.setText(_("No plugin catalog is configured.")) return - self._set_busy(_("Loading the plugin catalog...")) - self.run_task(lambda: fetch_catalog(url), self._on_catalog, self._on_error) + self._set_busy(_("Loading the plugin catalogs...")) + self.run_task( + lambda: load_catalogs(url, directories), self._on_catalog, self._on_error + ) def install(self, plugin: CatalogPlugin) -> None: """Download the newest release of a plugin, then install it.""" @@ -267,11 +334,23 @@ def _set_busy(self, message: str | None) -> None: apply_subdued_color(self.status_label) self.status_label.setText(message) - def _on_catalog(self, plugins: list[CatalogPlugin]) -> None: - self.plugins = plugins + def _on_catalog(self, content: CatalogsContent) -> None: + self.plugins = list(content.plugins) self._set_busy(None) - self.status_label.setText(_("Plugins in the catalog: %d") % len(plugins)) + url, directories = self._loaded_sources + if len(directories) + bool(url) > 1: + text = _("Plugins in the catalogs: %d") % len(self.plugins) + else: + text = _("Plugins in the catalog: %d") % len(self.plugins) + self.status_label.setText(text) + self.errors_label.setText( + "\n".join(f"\u26a0 {message}" for message in content.errors) + ) + self.errors_label.setVisible(bool(content.errors)) self._populate() + if self.isVisible() and self._loaded_sources != self._get_sources(): + # The sources changed during the loading + self.refresh() def _on_error(self, message: str) -> None: self._set_busy(None) @@ -286,9 +365,11 @@ def _on_downloaded(self, path: str) -> None: self._set_busy(None) self.status_label.setText("") try: - self._installer.install_path( - path, origin=_("DataLab plugin catalog (%s)") % _tier_label(plugin) - ) + if plugin.source: + origin = _("Local plugin catalog (%s)") % plugin.source + else: + origin = _("DataLab plugin catalog (%s)") % _tier_label(plugin) + self._installer.install_path(path, origin=origin) finally: shutil.rmtree(directory, ignore_errors=True) self._populate() diff --git a/datalab/gui/plugins/config.py b/datalab/gui/plugins/config.py index ba755654e..45204fe86 100644 --- a/datalab/gui/plugins/config.py +++ b/datalab/gui/plugins/config.py @@ -24,6 +24,7 @@ from qtpy.compat import getexistingdirectory from datalab.config import ( + DATALAB_PLUGINS_CATALOG_ENV_VAR, DATALAB_PLUGINS_ENV_PATHS, DATALAB_PLUGINS_ENV_VAR, OTHER_PLUGINS_PATHLIST, @@ -31,12 +32,18 @@ PLUGIN_OK_COLOR, Conf, _, + get_env_plugin_catalog_paths, + get_plugin_catalog_paths, get_user_plugin_paths, normalize_plugin_paths, + set_plugin_catalog_paths, set_user_plugin_paths, ) from datalab.gui.plugins.applications import get_plugin_icon -from datalab.gui.plugins.available import AvailablePluginsWidget +from datalab.gui.plugins.available import ( + AvailablePluginsWidget, + merge_catalog_directories, +) from datalab.gui.plugins.install import InstalledPluginsWidget from datalab.plugins import ( PLUGINS_DEFAULT_PATH, @@ -498,19 +505,26 @@ def matches_filter(filter_mode: str) -> bool: class SearchPathItemWidget(QW.QWidget): - """Widget representing one plugin search path.""" + """Widget representing one plugin search path or local catalog folder. + + Args: + path: Directory path + editable: Whether to show the edit and remove buttons + env_var: Environment variable providing the directory, if any + parent: Parent widget + """ def __init__( self, path: str, *, editable: bool, - from_env: bool = False, + env_var: str = "", parent: QW.QWidget | None = None, ) -> None: super().__init__(parent) self.path = path - self.from_env = from_env + self.env_var = env_var self.path_label: QW.QLabel | None = None self.edit_button: QW.QToolButton | None = None self.delete_button: QW.QToolButton | None = None @@ -572,8 +586,8 @@ def _build_path_label_text(self, enabled: bool) -> str: .name() ) link_text = f'{path}' - if self.from_env: - env_var = escape(DATALAB_PLUGINS_ENV_VAR) + if self.env_var: + env_var = escape(self.env_var) link_text += f" ({_('from')} {env_var})" return link_text @@ -594,6 +608,8 @@ def __init__(self, parent: DLMainWindow): self.failed_plugin_widgets: list[FailedPluginInfoWidget] = [] self.fixed_path_widgets: list[SearchPathItemWidget] = [] self.extra_path_widgets: list[SearchPathItemWidget] = [] + self.env_catalog_path_widgets: list[SearchPathItemWidget] = [] + self.catalog_path_widgets: list[SearchPathItemWidget] = [] self.original_plugins_enabled = Conf.plugins_enabled.get(True) self.plugins_enabled = self.original_plugins_enabled self.original_v020_plugins_warning_ignore = ( @@ -602,6 +618,8 @@ def __init__(self, parent: DLMainWindow): self.v020_plugins_warning_ignore = self.original_v020_plugins_warning_ignore self.original_extra_plugin_paths = get_user_plugin_paths() self.extra_plugin_paths = list(self.original_extra_plugin_paths) + self.original_catalog_paths = get_plugin_catalog_paths() + self.catalog_paths = list(self.original_catalog_paths) self.tabs: QW.QTabWidget | None = None self.toggle_all_checkbox: QW.QCheckBox | None = None self.filter_combo: QW.QComboBox | None = None @@ -614,6 +632,9 @@ def __init__(self, parent: DLMainWindow): self.settings_scroll: QW.QScrollArea | None = None self.settings_disabled_label: QW.QLabel | None = None self.add_path_button: QW.QPushButton | None = None + self.catalog_paths_layout: QW.QVBoxLayout | None = None + self.catalog_paths_placeholder: QW.QLabel | None = None + self.add_catalog_button: QW.QPushButton | None = None self.reload_button: QW.QPushButton | None = None self.global_toggle_button: QW.QPushButton | None = None self.v020_warning_checkbox: QW.QCheckBox | None = None @@ -726,6 +747,8 @@ def _create_search_paths_tab(self) -> QW.QWidget: container.setLayout(container_layout) container_layout.addLayout(self._create_search_paths_layout()) container_layout.addSpacing(18) + container_layout.addLayout(self._create_catalog_paths_layout()) + container_layout.addSpacing(18) container_layout.addLayout(self._create_warning_settings_layout()) container_layout.addStretch() self.settings_scroll.setWidget(container) @@ -784,7 +807,11 @@ def _create_search_paths_layout(self) -> QW.QVBoxLayout: fixed_layout.setContentsMargins(18, 0, 0, 0) fixed_layout.setSpacing(4) for path, from_env in self._collect_fixed_search_paths(): - widget = SearchPathItemWidget(path, editable=False, from_env=from_env) + widget = SearchPathItemWidget( + path, + editable=False, + env_var=DATALAB_PLUGINS_ENV_VAR if from_env else "", + ) self.fixed_path_widgets.append(widget) fixed_layout.addWidget(widget) paths_layout.addLayout(fixed_layout) @@ -838,6 +865,66 @@ def _create_search_paths_layout(self) -> QW.QVBoxLayout: paths_layout.addWidget(hint) return paths_layout + def _create_catalog_paths_layout(self) -> QW.QVBoxLayout: + """Create the layout listing local plugin catalog folders.""" + catalogs_layout = QW.QVBoxLayout() + + header_layout = QW.QHBoxLayout() + title = QW.QLabel(_("Local plugin catalogs")) + title_font = title.font() + title_font.setBold(True) + title.setFont(title_font) + header_layout.addWidget(title) + header_layout.addStretch() + self.add_catalog_button = QW.QPushButton(get_icon("metadata_add.svg"), _("Add")) + self.add_catalog_button.clicked.connect(self._add_catalog_path) + header_layout.addWidget(self.add_catalog_button) + catalogs_layout.addLayout(header_layout) + + intro = QW.QLabel( + _( + "Folders, possibly shared on the network, whose plugins are " + "listed in the Available plugins tab before those of the " + "DataLab plugin catalog. A folder provides the plugins of its " + "catalog.json file if any, otherwise its plugin " + "wheels." + ) + ) + intro.setWordWrap(True) + apply_subdued_color(intro) + catalogs_layout.addWidget(intro) + + self.catalog_paths_layout = QW.QVBoxLayout() + self.catalog_paths_layout.setContentsMargins(18, 0, 0, 0) + self.catalog_paths_layout.setSpacing(4) + for path in get_env_plugin_catalog_paths(): + widget = SearchPathItemWidget( + path, editable=False, env_var=DATALAB_PLUGINS_CATALOG_ENV_VAR + ) + self.env_catalog_path_widgets.append(widget) + self.catalog_paths_layout.addWidget(widget) + self.catalog_paths_placeholder = QW.QLabel( + _("No local plugin catalog is configured.") + ) + self.catalog_paths_placeholder.setWordWrap(True) + apply_subdued_color(self.catalog_paths_placeholder) + self.catalog_paths_layout.addWidget(self.catalog_paths_placeholder) + catalogs_layout.addLayout(self.catalog_paths_layout) + self._refresh_catalog_path_widgets() + + hint = QW.QLabel( + _( + "Folders provided via the %s environment variable " + "(multiple paths separated by '%s') come first and " + "appear above as read-only entries." + ) + % (DATALAB_PLUGINS_CATALOG_ENV_VAR, os.pathsep) + ) + hint.setWordWrap(True) + apply_subdued_color(hint) + catalogs_layout.addWidget(hint) + return catalogs_layout + def _create_warning_settings_layout(self) -> QW.QVBoxLayout: """Create the layout for plugin compatibility warning options.""" warnings_layout = QW.QVBoxLayout() @@ -934,6 +1021,64 @@ def _remove_search_path(self, item: SearchPathItemWidget) -> None: ] self._refresh_extra_path_widgets() + def _refresh_catalog_path_widgets(self) -> None: + """Rebuild the editable local catalog widgets and update the catalogs.""" + for widget in self.catalog_path_widgets: + self.catalog_paths_layout.removeWidget(widget) + widget.deleteLater() + self.catalog_path_widgets.clear() + self.catalog_paths_placeholder.setVisible( + not self.catalog_paths and not self.env_catalog_path_widgets + ) + for path in self.catalog_paths: + widget = SearchPathItemWidget(path, editable=True) + widget.edit_button.clicked.connect( + lambda _checked=False, item=widget: self._edit_catalog_path(item) + ) + widget.delete_button.clicked.connect( + lambda _checked=False, item=widget: self._remove_catalog_path(item) + ) + self.catalog_path_widgets.append(widget) + self.catalog_paths_layout.addWidget(widget) + self.available_plugins_widget.set_catalog_directories( + merge_catalog_directories(self.catalog_paths) + ) + + def _browse_catalog_directory(self, initial_path: str | None = None) -> str | None: + """Open a directory chooser for a local plugin catalog folder.""" + basedir = initial_path or Conf.base_dir.get(osp.expanduser("~")) + directory = getexistingdirectory( + self, _("Select local plugin catalog folder"), basedir + ) + normalized = normalize_plugin_paths([directory]) + return normalized[0] if normalized else None + + def _is_known_catalog_path(self, path: str, ignored: str | None = None) -> bool: + """Return whether a folder is already a local catalog.""" + others = [other for other in self.catalog_paths if other != ignored] + return path in get_env_plugin_catalog_paths() or path in others + + def _add_catalog_path(self) -> None: + """Append a local plugin catalog folder.""" + path = self._browse_catalog_directory() + if not path or self._is_known_catalog_path(path): + return + self.catalog_paths.append(path) + self._refresh_catalog_path_widgets() + + def _edit_catalog_path(self, item: SearchPathItemWidget) -> None: + """Replace a local plugin catalog folder.""" + path = self._browse_catalog_directory(item.path) + if not path or self._is_known_catalog_path(path, ignored=item.path): + return + self.catalog_paths[self.catalog_paths.index(item.path)] = path + self._refresh_catalog_path_widgets() + + def _remove_catalog_path(self, item: SearchPathItemWidget) -> None: + """Remove a local plugin catalog folder.""" + self.catalog_paths.remove(item.path) + self._refresh_catalog_path_widgets() + def _create_controls_layout(self) -> QW.QHBoxLayout: """Create the row with master controls.""" controls_layout = QW.QHBoxLayout() @@ -1009,7 +1154,12 @@ def _update_global_plugins_ui(self) -> None: self.settings_disabled_label.setVisible(not enabled) self.installed_plugins_widget.setEnabled(enabled) self.available_plugins_widget.setEnabled(enabled) - for widget in self.fixed_path_widgets + self.extra_path_widgets: + for widget in ( + self.fixed_path_widgets + + self.extra_path_widgets + + self.env_catalog_path_widgets + + self.catalog_path_widgets + ): widget.set_links_enabled(enabled) def _toggle_global_plugins_enabled(self) -> None: @@ -1027,6 +1177,7 @@ def _has_changes(self) -> bool: widget.has_changed() for widget in self.plugin_widgets ) path_changes_made = self.extra_plugin_paths != self.original_extra_plugin_paths + catalog_changes_made = self.catalog_paths != self.original_catalog_paths plugins_enabled_changed = self.plugins_enabled != self.original_plugins_enabled warning_changed = ( self.v020_plugins_warning_ignore @@ -1035,6 +1186,7 @@ def _has_changes(self) -> bool: return ( plugin_changes_made or path_changes_made + or catalog_changes_made or plugins_enabled_changed or warning_changed ) @@ -1074,12 +1226,14 @@ def _save_configuration(self) -> None: ) Conf.plugins_enabled_list.set(enabled_plugins) set_user_plugin_paths(self.extra_plugin_paths) + set_plugin_catalog_paths(self.catalog_paths) def _mark_current_state_as_saved(self) -> None: """Synchronize original values with the current dialog state.""" self.original_plugins_enabled = self.plugins_enabled self.original_v020_plugins_warning_ignore = self.v020_plugins_warning_ignore self.original_extra_plugin_paths = list(self.extra_plugin_paths) + self.original_catalog_paths = list(self.catalog_paths) def _refresh_plugin_list(self) -> None: """Rebuild the plugin list from the current registry state.""" diff --git a/datalab/locale/fr/LC_MESSAGES/datalab.po b/datalab/locale/fr/LC_MESSAGES/datalab.po index 02de595b7..71ecc9e9d 100644 --- a/datalab/locale/fr/LC_MESSAGES/datalab.po +++ b/datalab/locale/fr/LC_MESSAGES/datalab.po @@ -2198,6 +2198,13 @@ msgstr "Officiel" msgid "Community" msgstr "Communauté" +msgid "Local catalog" +msgstr "Catalogue local" + +#, python-format +msgid "Local catalog: %s" +msgstr "Catalogue local : %s" + #, python-format msgid "Withdrawn: %s" msgstr "Retiré : %s" @@ -2232,8 +2239,8 @@ msgid "Install" msgstr "Installer" #, python-format -msgid "Plugins listed in the DataLab plugin catalog. Official plugins are maintained by the DataLab team, community plugins by their authors: install only plugins from authors you trust." -msgstr "Plugins répertoriés dans le catalogue de plugins DataLab. Les plugins officiels sont maintenus par l'équipe DataLab, les plugins de la communauté par leurs auteurs : n'installez que des plugins provenant d'auteurs de confiance." +msgid "Plugins listed in the DataLab plugin catalog and in the local catalogs of the Plugin settings tab, which take precedence. Official plugins are maintained by the DataLab team, community plugins by their authors, and local catalogs by your organization: install only plugins from authors you trust." +msgstr "Plugins répertoriés dans le catalogue de plugins DataLab et dans les catalogues locaux de l'onglet Paramètres des plugins, qui sont prioritaires. Les plugins officiels sont maintenus par l'équipe DataLab, les plugins de la communauté par leurs auteurs et les catalogues locaux par votre organisation : n'installez que des plugins provenant d'auteurs de confiance." msgid "Search plugins" msgstr "Rechercher des plugins" @@ -2241,13 +2248,17 @@ msgstr "Rechercher des plugins" msgid "No plugin catalog is configured." msgstr "Aucun catalogue de plugins n'est configuré." -msgid "Loading the plugin catalog..." -msgstr "Chargement du catalogue de plugins..." +msgid "Loading the plugin catalogs..." +msgstr "Chargement des catalogues de plugins..." #, python-format msgid "Downloading %s..." msgstr "Téléchargement de %s..." +#, python-format +msgid "Plugins in the catalogs: %d" +msgstr "Plugins dans les catalogues : %d" + #, python-format msgid "Plugins in the catalog: %d" msgstr "Plugins dans le catalogue : %d" @@ -2255,6 +2266,10 @@ msgstr "Plugins dans le catalogue : %d" msgid "The plugin catalog cannot be loaded:" msgstr "Le catalogue de plugins ne peut pas être chargé :" +#, python-format +msgid "Local plugin catalog (%s)" +msgstr "Catalogue local de plugins (%s)" + #, python-format msgid "DataLab plugin catalog (%s)" msgstr "Catalogue de plugins DataLab (%s)" @@ -2348,6 +2363,19 @@ msgstr "Aucun répertoire de plugins supplémentaire n'est configuré." msgid "Directories provided via the %s environment variable (multiple paths separated by '%s') also appear above as read-only entries. Changes take effect at DataLab startup." msgstr "Les répertoires fournis via la variable d'environnement %s (plusieurs chemins séparés par '%s') apparaissent également ci-dessus en lecture seule. Les modifications prennent effet au démarrage de DataLab." +msgid "Local plugin catalogs" +msgstr "Catalogues locaux de plugins" + +msgid "Folders, possibly shared on the network, whose plugins are listed in the Available plugins tab before those of the DataLab plugin catalog. A folder provides the plugins of its catalog.json file if any, otherwise its plugin wheels." +msgstr "Dossiers, éventuellement partagés sur le réseau, dont les plugins sont proposés dans l'onglet Plugins disponibles avant ceux du catalogue de plugins DataLab. Un dossier fournit les plugins de son fichier catalog.json s'il existe, sinon ses wheels de plugins." + +msgid "No local plugin catalog is configured." +msgstr "Aucun catalogue local de plugins n'est configuré." + +#, python-format +msgid "Folders provided via the %s environment variable (multiple paths separated by '%s') come first and appear above as read-only entries." +msgstr "Les dossiers fournis via la variable d'environnement %s (plusieurs chemins séparés par '%s') sont prioritaires et apparaissent ci-dessus en lecture seule." + msgid "Compatibility warnings" msgstr "Avertissements de compatibilité" @@ -2360,6 +2388,9 @@ msgstr "Si activé, DataLab ne vous avertira pas des plugins v0.20 qui ne sont p msgid "Select plugin directory" msgstr "Sélectionner un répertoire de plugins" +msgid "Select local plugin catalog folder" +msgstr "Sélectionner un dossier de catalogue local de plugins" + msgid "Enable all plugins" msgstr "Activer tous les plugins" @@ -4389,6 +4420,34 @@ msgstr "Type d'objet utilisé par le traitement d'exemple" msgid "Project license" msgstr "Licence du projet" +msgid "Build or check a local plugin catalog folder." +msgstr "Construire ou vérifier un dossier de catalogue local de plugins." + +msgid "Write the catalog.json file listing the plugin wheels." +msgstr "Écrire le fichier catalog.json répertoriant les wheels de plugins." + +msgid "Check the catalog.json file against the plugin wheels." +msgstr "Vérifier le fichier catalog.json par rapport aux wheels de plugins." + +msgid "Local plugin catalog folder" +msgstr "Dossier du catalogue local de plugins" + +#, python-format +msgid "Warning: %s" +msgstr "Avertissement : %s" + +#, python-format +msgid "%s was not written." +msgstr "%s n'a pas été écrit." + +#, python-format +msgid "Wrote %s, listing %d plugins." +msgstr "%s écrit, répertoriant %d plugins." + +#, python-format +msgid "The local plugin catalog lists %d plugins." +msgstr "Le catalogue local de plugins répertorie %d plugins." + #, python-format msgid "Created DataLab plugin project at %s" msgstr "Projet de plugin DataLab créé dans %s" diff --git a/datalab/plugins/catalog.py b/datalab/plugins/catalog.py index 9f3fb9c27..2689e3f63 100644 --- a/datalab/plugins/catalog.py +++ b/datalab/plugins/catalog.py @@ -7,30 +7,54 @@ Read the DataLab plugin catalog (``catalog.json``) and download release wheels whose size and SHA-256 digest match the catalog. This module is Qt-free: the plugin dialog calls it from a worker thread. + +Besides the online catalog, local catalogs are folders, typically shared on the +network, read as files: their ``catalog.json`` if any, otherwise the plugin +wheels they hold. """ from __future__ import annotations +import configparser import dataclasses import hashlib +import io import json +import os import os.path as osp import re import urllib.error import urllib.parse import urllib.request +import zipfile +from collections.abc import Callable, Sequence +from email.message import Message +from email.parser import BytesParser +from email.policy import default as email_policy + +from packaging.utils import canonicalize_name, parse_wheel_filename +from packaging.version import InvalidVersion, Version from datalab import __version__ -from datalab.plugins.wheels import MAX_WHEEL_BYTES, is_compatible_wheel +from datalab.plugins.wheels import ( + DESKTOP_ENTRY_POINT_GROUP, + MAX_WHEEL_BYTES, + is_compatible_wheel, +) DEFAULT_CATALOG_URL = "https://datalab-platform.com/plugins/catalog.json" +CATALOG_FILENAME = "catalog.json" CATALOG_SCHEMA_VERSION = 1 +LOCAL_TIER = "local" MAX_CATALOG_BYTES = 8 * 1024 * 1024 +MAX_FOLDER_WHEELS = 1000 DOWNLOAD_TIMEOUT = 60 -# file: URLs serve catalogs mirrored on a local or network folder +# file: URLs serve catalogs mirrored on a local folder _ALLOWED_SCHEMES = ("https", "file") _SHA256_PATTERN = re.compile(r"^[0-9a-f]{64}$") +# Normalized Project-URL labels (PEP 753) of the source repository, by priority +_REPOSITORY_LABELS = ("source", "sourcecode", "repository", "code", "homepage") class CatalogError(ValueError): @@ -45,7 +69,8 @@ class CatalogRelease: version: Plugin version sha256: SHA-256 digest of the wheel filename: Wheel file name (empty for revoked plugins) - url: Absolute wheel URL (empty for revoked plugins) + url: Absolute wheel URL, or file path for local catalogs (empty for + revoked plugins) size: Wheel size in bytes targets: Hosts supported by the wheel (``desktop``, ``web``) yanked: Reason why the release should not be installed, if any @@ -64,7 +89,11 @@ class CatalogRelease: @dataclasses.dataclass(frozen=True) class CatalogPlugin: - """Plugin listed in the catalog, with its releases from the newest.""" + """Plugin listed in the catalog, with its releases from the newest. + + ``source`` is the folder of the local catalog listing the plugin, empty for + the online catalog. + """ id: str name: str @@ -77,6 +106,7 @@ class CatalogPlugin: summary: str = "" license: str = "" keywords: tuple[str, ...] = () + source: str = "" def get_installable_release( self, target: str = "desktop", python_version: str | None = None @@ -116,24 +146,61 @@ def matches(self, text: str) -> bool: return all(word in haystack for word in words) -def _read(url: str, limit: int) -> bytes: - """Return the content of an HTTPS or file URL, up to a size limit.""" - if urllib.parse.urlparse(url).scheme not in _ALLOWED_SCHEMES: - raise CatalogError(f"Unsupported URL scheme: {url}") - request = urllib.request.Request( - url, headers={"User-Agent": f"DataLab/{__version__}"} - ) - try: - with urllib.request.urlopen(request, timeout=DOWNLOAD_TIMEOUT) as response: - data = response.read(limit + 1) - except (urllib.error.URLError, OSError) as exc: - raise CatalogError(f"Cannot download {url}: {exc}") from exc +def _read(location: str, limit: int) -> bytes: + """Return the content of a file path, or of an HTTPS or file URL. + + Args: + location: Absolute file path (local catalogs) or URL + limit: Maximum size in bytes + """ + # Testing the path first: urlparse("C:\\x").scheme is "c" + if osp.isabs(location): + try: + with open(location, "rb") as file: + data = file.read(limit + 1) + except OSError as exc: + raise CatalogError(f"Cannot read {location}: {exc}") from exc + else: + if urllib.parse.urlparse(location).scheme not in _ALLOWED_SCHEMES: + raise CatalogError(f"Unsupported URL scheme: {location}") + request = urllib.request.Request( + location, headers={"User-Agent": f"DataLab/{__version__}"} + ) + try: + with urllib.request.urlopen(request, timeout=DOWNLOAD_TIMEOUT) as response: + data = response.read(limit + 1) + except (urllib.error.URLError, OSError) as exc: + raise CatalogError(f"Cannot download {location}: {exc}") from exc if len(data) > limit: - raise CatalogError(f"{url} exceeds the {limit} byte size limit") + raise CatalogError(f"{location} exceeds the {limit} byte size limit") return data -def _parse_release(item: dict, catalog_url: str) -> CatalogRelease: +def _folder_resolver(root: str) -> Callable[[str], str]: + """Return a function resolving release URLs relative to a catalog folder. + + Relative URLs become file paths, which must stay inside the folder; absolute + URLs are kept. + """ + + def resolve(url: str) -> str: + parsed = urllib.parse.urlparse(url) + if parsed.scheme and parsed.netloc: + return url + relative = urllib.parse.unquote(parsed.path).split("/") + path = osp.normpath(osp.join(root, *relative)) + try: + inside = osp.commonpath([root, path]) == root and path != root + except ValueError: # Paths on different drives + inside = False + if not inside: + raise ValueError(f"Release outside the catalog folder: {url!r}") + return path + + return resolve + + +def _parse_release(item: dict, resolve: Callable[[str], str]) -> CatalogRelease: if not _SHA256_PATTERN.fullmatch(item["sha256"]): raise ValueError(f"Invalid SHA-256 digest: {item['sha256']!r}") url = item.get("url", "") @@ -141,7 +208,7 @@ def _parse_release(item: dict, catalog_url: str) -> CatalogRelease: version=str(item["version"]), sha256=item["sha256"], filename=osp.basename(item.get("filename", "")), - url=urllib.parse.urljoin(catalog_url, url) if url else "", + url=resolve(url) if url else "", size=int(item.get("size", 0)), targets=tuple(item.get("targets", ())), yanked=item.get("yanked", ""), @@ -149,7 +216,9 @@ def _parse_release(item: dict, catalog_url: str) -> CatalogRelease: ) -def _parse_plugin(item: dict, catalog_url: str) -> CatalogPlugin: +def _parse_plugin( + item: dict, resolve: Callable[[str], str], source: str +) -> CatalogPlugin: return CatalogPlugin( id=item["id"], name=item["name"], @@ -157,36 +226,329 @@ def _parse_plugin(item: dict, catalog_url: str) -> CatalogPlugin: status=item["status"], repository=item["repository"], releases=tuple( - _parse_release(release, catalog_url) for release in item["releases"] + _parse_release(release, resolve) for release in item["releases"] ), status_reason=item.get("status_reason", ""), distribution=item.get("distribution", ""), summary=item.get("summary", ""), license=item.get("license", ""), keywords=tuple(item.get("keywords", ())), + source=source, ) -def fetch_catalog(url: str = DEFAULT_CATALOG_URL) -> list[CatalogPlugin]: - """Download and parse the plugin catalog. +def load_catalog_json(data: bytes, location: str) -> dict: + """Return the payload of a ``catalog.json`` file, checking its format. Raises: - CatalogError: The catalog cannot be downloaded or is invalid + CatalogError: The content is not a supported catalog """ - data = _read(url, MAX_CATALOG_BYTES) try: payload = json.loads(data.decode("utf-8")) except (UnicodeDecodeError, ValueError) as exc: - raise CatalogError(f"Invalid plugin catalog: {url}") from exc + raise CatalogError(f"Invalid plugin catalog: {location}") from exc if ( not isinstance(payload, dict) or payload.get("schema_version") != CATALOG_SCHEMA_VERSION ): - raise CatalogError(f"Unsupported plugin catalog format: {url}") + raise CatalogError(f"Unsupported plugin catalog format: {location}") + return payload + + +def _parse_catalog( + payload: dict, location: str, resolve: Callable[[str], str], source: str = "" +) -> list[CatalogPlugin]: try: - return [_parse_plugin(item, url) for item in payload["plugins"]] + return [_parse_plugin(item, resolve, source) for item in payload["plugins"]] except (AttributeError, KeyError, TypeError, ValueError) as exc: - raise CatalogError(f"Invalid plugin catalog: {url}") from exc + raise CatalogError(f"Invalid plugin catalog: {location}") from exc + + +def fetch_catalog(url: str = DEFAULT_CATALOG_URL) -> list[CatalogPlugin]: + """Download and parse the plugin catalog. + + Raises: + CatalogError: The catalog cannot be downloaded or is invalid + """ + payload = load_catalog_json(_read(url, MAX_CATALOG_BYTES), url) + return _parse_catalog( + payload, url, lambda release_url: urllib.parse.urljoin(url, release_url) + ) + + +def _dist_info_file(archive: zipfile.ZipFile, name: str) -> bytes | None: + paths = [ + path + for path in archive.namelist() + if path.count("/") == 1 and path.endswith(f".dist-info/{name}") + ] + return archive.read(paths[0]) if len(paths) == 1 else None + + +def _get_repository(metadata: Message) -> str: + """Return the source repository URL declared in wheel metadata, if any.""" + urls: dict[str, str] = {} + for value in metadata.get_all("Project-URL", []): + label, _sep, url = str(value).partition(",") + urls.setdefault(re.sub(r"[^a-z]", "", label.lower()), url.strip()) + candidates = [urls.get(label, "") for label in _REPOSITORY_LABELS] + candidates.append(str(metadata.get("Home-page") or "").strip()) + return next( + (url for url in candidates if url.startswith(("https://", "http://"))), "" + ) + + +def _get_license(metadata: Message) -> str: + """Return a short license description from wheel metadata.""" + expression = str(metadata.get("License-Expression") or "").strip() + if expression: + return expression + # The License field may hold the full license text + text = str(metadata.get("License") or "").strip() + if text and "\n" not in text and len(text) <= 80: + return text + for classifier in metadata.get_all("Classifier", []): + if str(classifier).startswith("License :: "): + return str(classifier).rsplit(" :: ", 1)[-1] + return "" + + +def _get_keywords(metadata: Message) -> list[str]: + text = str(metadata.get("Keywords") or "") + words = text.split(",") if "," in text else text.split() + return [word.strip() for word in words if word.strip()] + + +def read_wheel_entry(path: str) -> tuple[dict, dict]: + """Return the catalog records describing a plugin wheel file. + + The wheel is only read, not validated as thoroughly as when installing it. + + Args: + path: Wheel file path + + Returns: + Plugin record without releases, and release record whose URL is the + file name, both as written in ``catalog.json`` + + Raises: + CatalogError: The file is not a pure-Python DataLab desktop plugin wheel + """ + filename = osp.basename(path) + try: + name, version, _build, tags = parse_wheel_filename(filename) + except (InvalidVersion, ValueError) as exc: + raise CatalogError("invalid wheel file name") from exc + if any(tag.abi != "none" or tag.platform != "any" for tag in tags): + raise CatalogError("not a pure-Python wheel") + try: + if not 0 < osp.getsize(path) <= MAX_WHEEL_BYTES: + raise CatalogError(f"size not between 1 and {MAX_WHEEL_BYTES} bytes") + with open(path, "rb") as file: + data = file.read(MAX_WHEEL_BYTES + 1) + except OSError as exc: + raise CatalogError(f"cannot be read: {exc}") from exc + if not 0 < len(data) <= MAX_WHEEL_BYTES: + raise CatalogError(f"size not between 1 and {MAX_WHEEL_BYTES} bytes") + try: + with zipfile.ZipFile(io.BytesIO(data)) as archive: + metadata_data = _dist_info_file(archive, "METADATA") + entry_points = _dist_info_file(archive, "entry_points.txt") or b"" + except zipfile.BadZipFile as exc: + raise CatalogError("not a valid ZIP archive") from exc + if metadata_data is None: + raise CatalogError("no METADATA file") + parser = configparser.ConfigParser(interpolation=None) + parser.optionxform = str + try: + parser.read_string(entry_points.decode("utf-8")) + except (UnicodeDecodeError, configparser.Error) as exc: + raise CatalogError("invalid entry_points.txt") from exc + if not parser.has_section(DESKTOP_ENTRY_POINT_GROUP): + raise CatalogError( + f"not a DataLab desktop plugin (no {DESKTOP_ENTRY_POINT_GROUP} entry point)" + ) + metadata = BytesParser(policy=email_policy).parsebytes(metadata_data) + distribution = str(metadata.get("Name") or "") + try: + same_version = Version(str(metadata.get("Version") or "")) == version + except InvalidVersion: + same_version = False + if canonicalize_name(distribution) != name or not same_version: + raise CatalogError("file name and METADATA differ") + plugin = { + "id": name, + "name": distribution, + "tier": LOCAL_TIER, + "status": "active", + "repository": _get_repository(metadata), + "distribution": distribution, + "summary": str(metadata.get("Summary") or ""), + "license": _get_license(metadata), + "keywords": _get_keywords(metadata), + } + release = { + "version": str(version), + "filename": filename, + "url": urllib.parse.quote(filename), + "sha256": hashlib.sha256(data).hexdigest(), + "size": len(data), + "targets": ["desktop"], + } + requires_python = str(metadata.get("Requires-Python") or "") + if requires_python: + release["requires_python"] = requires_python + return plugin, release + + +def list_folder_wheels(directory: str) -> list[str]: + """Return the paths of the wheel files of a folder, sorted by name. + + Raises: + CatalogError: The folder cannot be read or holds too many wheels + """ + try: + names = sorted( + name + for name in os.listdir(directory) + if name.lower().endswith(".whl") and osp.isfile(osp.join(directory, name)) + ) + except OSError as exc: + raise CatalogError(f"Cannot read the plugin catalog folder: {exc}") from exc + if len(names) > MAX_FOLDER_WHEELS: + raise CatalogError( + f"{directory} holds more than {MAX_FOLDER_WHEELS} wheels: " + f"list the plugins in a {CATALOG_FILENAME} file" + ) + return [osp.join(directory, name) for name in names] + + +def scan_wheel_folder(directory: str) -> tuple[dict, list[str]]: + """Return the catalog describing the plugin wheels of a folder. + + Releases of a plugin are grouped by distribution, from the newest. + + Returns: + Catalog payload, as written in ``catalog.json``, and messages about the + ignored files + + Raises: + CatalogError: The folder cannot be read + """ + warnings: list[str] = [] + plugins: dict[str, dict] = {} + releases: dict[str, dict[Version, dict]] = {} + for path in list_folder_wheels(directory): + try: + plugin, release = read_wheel_entry(path) + except CatalogError as exc: + warnings.append(f"{path} ignored: {exc}") + continue + versions = releases.setdefault(plugin["id"], {}) + version = Version(release["version"]) + if version in versions: + duplicate = versions[version]["filename"] + warnings.append(f"{path} ignored: same release as {duplicate}") + continue + versions[version] = release + if version == max(versions): + plugins[plugin["id"]] = plugin + records = [] + for key, plugin in sorted( + plugins.items(), key=lambda item: item[1]["name"].casefold() + ): + versions = releases[key] + plugin["releases"] = [versions[version] for version in sorted(versions)[::-1]] + records.append(plugin) + return {"schema_version": CATALOG_SCHEMA_VERSION, "plugins": records}, warnings + + +def read_folder_catalog(directory: str) -> tuple[list[CatalogPlugin], list[str]]: + """Read a local catalog folder. + + The folder ``catalog.json`` lists the plugins if it exists, with release URLs + relative to the folder; otherwise, the plugin wheels of the folder are listed. + + Args: + directory: Catalog folder, possibly shared on the network + + Returns: + Plugins, and messages about the ignored files + + Raises: + CatalogError: The folder or its catalog cannot be read + """ + root = osp.normpath(osp.abspath(directory)) + if not osp.isdir(root): + raise CatalogError(f"Plugin catalog folder not found: {root}") + path = osp.join(root, CATALOG_FILENAME) + if osp.isfile(path): + payload = load_catalog_json(_read(path, MAX_CATALOG_BYTES), path) + warnings: list[str] = [] + location = path + else: + payload, warnings = scan_wheel_folder(root) + location = root + return _parse_catalog(payload, location, _folder_resolver(root), root), warnings + + +@dataclasses.dataclass(frozen=True) +class CatalogsContent: + """Plugins of the configured catalogs. + + Args: + plugins: Plugins, a plugin listed by several catalogs appearing once + errors: Messages about the catalogs that cannot be read, the ignored + files and the plugins withdrawn from a masked catalog + """ + + plugins: tuple[CatalogPlugin, ...] + errors: tuple[str, ...] + + +def load_catalogs(url: str, directories: Sequence[str] = ()) -> CatalogsContent: + """Read the local catalogs, then download the online catalog. + + A plugin listed in several catalogs is taken from the first one: local + catalogs take precedence over the online catalog, so that an organization + may provide its own releases. A catalog that cannot be read does not prevent + the others from being used. + + Args: + url: Online catalog URL (empty to ignore it) + directories: Local catalog folders, by priority + """ + plugins: list[CatalogPlugin] = [] + errors: list[str] = [] + ids: set[str] = set() + distributions: set[str] = set() + readers: list[Callable[[], tuple[list[CatalogPlugin], list[str]]]] = [ + lambda directory=directory: read_folder_catalog(directory) + for directory in directories + ] + if url: + readers.append(lambda: (fetch_catalog(url), [])) + for read in readers: + try: + items, warnings = read() + except CatalogError as exc: + errors.append(str(exc)) + continue + errors.extend(warnings) + for plugin in items: + distribution = canonicalize_name(plugin.distribution) + if plugin.id in ids or (distribution and distribution in distributions): + if plugin.status == "revoked": + errors.append( + f"{plugin.name} ({plugin.id}) is withdrawn from " + f"{plugin.source or url}: {plugin.status_reason}" + ) + continue + ids.add(plugin.id) + if distribution: + distributions.add(distribution) + plugins.append(plugin) + return CatalogsContent(tuple(plugins), tuple(errors)) def download_release(release: CatalogRelease, directory: str) -> str: @@ -214,10 +576,19 @@ def download_release(release: CatalogRelease, directory: str) -> str: __all__ = [ + "CATALOG_FILENAME", "DEFAULT_CATALOG_URL", + "LOCAL_TIER", "CatalogError", "CatalogPlugin", "CatalogRelease", + "CatalogsContent", "download_release", "fetch_catalog", + "list_folder_wheels", + "load_catalog_json", + "load_catalogs", + "read_folder_catalog", + "read_wheel_entry", + "scan_wheel_folder", ] diff --git a/datalab/plugins/catalogtool.py b/datalab/plugins/catalogtool.py new file mode 100644 index 000000000..6417b3b05 --- /dev/null +++ b/datalab/plugins/catalogtool.py @@ -0,0 +1,309 @@ +# Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file. + +""" +Local plugin catalog tool +------------------------- + +Build and check the ``catalog.json`` file of a local plugin catalog: a folder +holding plugin wheels, typically shared on the network. This module is Qt-free +and backs the ``datalab-plugin catalog`` command. +""" + +from __future__ import annotations + +import dataclasses +import json +import os +import os.path as osp +import tempfile +from collections.abc import Mapping + +from packaging.utils import canonicalize_name + +from datalab import __version__ +from datalab.plugins.catalog import ( + CATALOG_FILENAME, + MAX_CATALOG_BYTES, + CatalogError, + list_folder_wheels, + load_catalog_json, + read_folder_catalog, + scan_wheel_folder, +) +from datalab.plugins.store import get_available_distributions +from datalab.plugins.wheels import ( + DESKTOP_ENTRY_POINT_GROUP, + MAX_WHEEL_BYTES, + WheelInspectionError, + inspect_wheel, +) + +#: Python versions supported by DataLab: a plugin wheel must install on one of +#: them (update when DataLab supports a new Python version) +DESKTOP_PYTHONS = ("3.9", "3.10", "3.11", "3.12", "3.13", "3.14") +#: Plugin fields edited in ``catalog.json`` and kept when building it again +CURATED_PLUGIN_FIELDS = ( + "id", + "name", + "tier", + "status", + "status_reason", + "repository", + "documentation", + "capabilities", + "keywords", +) + + +@dataclasses.dataclass +class CatalogReport: + """Outcome of building or checking a local plugin catalog. + + Args: + path: Path of the ``catalog.json`` file + catalog: Catalog payload, built or read + errors: Problems preventing DataLab from using the catalog as expected + warnings: Other remarks + written: Whether ``catalog.json`` was written + """ + + path: str + catalog: dict = dataclasses.field(default_factory=dict) + errors: list[str] = dataclasses.field(default_factory=list) + warnings: list[str] = dataclasses.field(default_factory=list) + written: bool = False + + +def _get_root(directory: str) -> str: + root = osp.normpath(osp.abspath(directory)) + if not osp.isdir(root): + raise CatalogError(f"Plugin catalog folder not found: {root}") + return root + + +def inspect_plugin_wheel(path: str, available: Mapping[str, str]) -> dict: + """Check a plugin wheel as DataLab does before installing it. + + The wheel must install on one of the Python versions supported by DataLab. + + Args: + path: Wheel file path + available: Distribution versions a plugin wheel may depend on + + Returns: + Wheel manifest + + Raises: + WheelInspectionError: DataLab would refuse to install the wheel + """ + with open(path, "rb") as file: + data = file.read(MAX_WHEEL_BYTES + 1) + error = WheelInspectionError("No supported Python version") + for python_version in DESKTOP_PYTHONS: + try: + return inspect_wheel( + data, + filename=osp.basename(path), + entry_point_group=DESKTOP_ENTRY_POINT_GROUP, + available_distributions=available, + python_version=python_version, + marker_environment={ + "python_version": python_version, + "python_full_version": f"{python_version}.0", + }, + ) + except WheelInspectionError as exc: + error = exc + raise error + + +def _read_existing_plugins(path: str) -> list[dict]: + """Return the plugins of an existing ``catalog.json``, if any.""" + if not osp.isfile(path): + return [] + try: + with open(path, "rb") as file: + data = file.read(MAX_CATALOG_BYTES + 1) + except OSError as exc: + raise CatalogError(f"Cannot read {path}: {exc}") from exc + plugins = load_catalog_json(data, path).get("plugins") + if not isinstance(plugins, list) or not all( + isinstance(plugin, dict) for plugin in plugins + ): + raise CatalogError(f"Invalid plugin catalog: {path}") + return plugins + + +def _keep_curated_fields(plugin: dict, previous: dict) -> None: + """Copy into a built plugin record the fields edited in the previous one.""" + for field in CURATED_PLUGIN_FIELDS: + if field in previous: + plugin[field] = previous[field] + yanked = { + str(release.get("version")): release["yanked"] + for release in previous.get("releases", ()) + if isinstance(release, dict) and "yanked" in release + } + for release in plugin["releases"]: + if release["version"] in yanked: + release["yanked"] = yanked[release["version"]] + if plugin.get("status") == "revoked": + # As in the online catalog, revoked plugins cannot be installed + plugin["releases"] = [ + {"version": release["version"], "sha256": release["sha256"]} + for release in plugin["releases"] + ] + + +def _write_atomically(path: str, catalog: dict) -> None: + """Write a catalog file, which DataLab may read meanwhile.""" + directory = osp.dirname(path) + descriptor, temporary = tempfile.mkstemp( + prefix=".catalog-", suffix=".json", dir=directory + ) + try: + with os.fdopen(descriptor, "w", encoding="utf-8", newline="\n") as file: + json.dump(catalog, file, indent=2, ensure_ascii=False) + file.write("\n") + os.replace(temporary, path) + except BaseException: + if osp.exists(temporary): + os.remove(temporary) + raise + + +def build_local_catalog(directory: str, *, write: bool = True) -> CatalogReport: + """Build the ``catalog.json`` file listing the plugin wheels of a folder. + + Every wheel is checked as DataLab does before installing it. Fields edited + in an existing ``catalog.json`` are kept: plugin ID, name, tier, status and + its reason, repository, documentation, capabilities and keywords, and the + reason why a release is yanked. Revoked plugins remain listed. + + The catalog is not written if a wheel is refused. + + Args: + directory: Catalog folder + write: Whether to write ``catalog.json`` + + Raises: + CatalogError: The folder or its existing catalog cannot be read + """ + root = _get_root(directory) + report = CatalogReport(osp.join(root, CATALOG_FILENAME)) + previous_plugins = _read_existing_plugins(report.path) + payload, ignored = scan_wheel_folder(root) + report.errors.extend(ignored) + available = get_available_distributions({"datalab-platform": __version__}) + plugins: list[dict] = [] + for plugin in payload["plugins"]: + releases = [] + for release in plugin["releases"]: + path = osp.join(root, release["filename"]) + try: + manifest = inspect_plugin_wheel(path, available) + except (OSError, WheelInspectionError) as exc: + report.errors.append(f"{path} refused: {exc}") + continue + release["requires_dist"] = [ + item["requirement"] for item in manifest["dependencies"] + ] + releases.append(release) + if releases: + plugin["releases"] = releases + plugins.append(plugin) + + for previous in previous_plugins: + distribution = canonicalize_name(str(previous.get("distribution", ""))) + plugin = next( + (item for item in plugins if distribution and item["id"] == distribution), + None, + ) + if plugin is not None: + _keep_curated_fields(plugin, previous) + elif previous.get("status") == "revoked": + plugins.append(previous) + else: + report.warnings.append( + f"{previous.get('name')}: no wheel left, removed from the catalog" + ) + plugins.sort(key=lambda item: str(item.get("name", "")).casefold()) + report.catalog = {"schema_version": payload["schema_version"], "plugins": plugins} + if write and not report.errors: + try: + _write_atomically(report.path, report.catalog) + except OSError as exc: + raise CatalogError(f"Cannot write {report.path}: {exc}") from exc + report.written = True + return report + + +def check_local_catalog(directory: str) -> CatalogReport: + """Check that a local catalog folder may be used by DataLab. + + The wheels of the ``catalog.json`` file must exist, match their size and + digest, and be accepted by DataLab; wheels of the folder missing from the + catalog are reported. Without ``catalog.json``, DataLab lists the wheels of + the folder, which are checked. + + Args: + directory: Catalog folder + + Raises: + CatalogError: The folder cannot be read + """ + root = _get_root(directory) + path = osp.join(root, CATALOG_FILENAME) + if not osp.isfile(path): + report = build_local_catalog(root, write=False) + report.warnings = [ + f"No {CATALOG_FILENAME}: DataLab lists the plugin wheels of the folder" + ] + return report + report = CatalogReport(path) + try: + plugins, _ignored = read_folder_catalog(root) + except CatalogError as exc: + report.errors.append(str(exc)) + return report + with open(path, "rb") as file: + report.catalog = load_catalog_json(file.read(MAX_CATALOG_BYTES + 1), path) + available = get_available_distributions({"datalab-platform": __version__}) + listed: set[str] = set() + for plugin in plugins: + for release in plugin.releases: + if not osp.isabs(release.url): + continue + label = f"{plugin.name} {release.version}" + listed.add(osp.normcase(release.url)) + try: + manifest = inspect_plugin_wheel(release.url, available) + except OSError as exc: + report.errors.append(f"{label}: cannot read {release.url}: {exc}") + continue + except WheelInspectionError as exc: + report.errors.append(f"{label}: {release.url} refused: {exc}") + continue + if ( + manifest["size_bytes"] != release.size + or manifest["sha256"] != release.sha256 + ): + report.errors.append( + f"{label}: {release.url} does not match the size and digest " + "of the catalog" + ) + report.warnings.extend( + f"{wheel} is not listed in {CATALOG_FILENAME}" + for wheel in list_folder_wheels(root) + if osp.normcase(wheel) not in listed + ) + return report + + +__all__ = [ + "DESKTOP_PYTHONS", + "CatalogReport", + "build_local_catalog", + "check_local_catalog", + "inspect_plugin_wheel", +] diff --git a/datalab/plugins/generator.py b/datalab/plugins/generator.py index f6ee2f1c0..910b2fc65 100644 --- a/datalab/plugins/generator.py +++ b/datalab/plugins/generator.py @@ -1,6 +1,9 @@ # Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file. -"""Generate a layered installable DataLab plugin project.""" +"""Generate a layered installable DataLab plugin project. + +The ``datalab-plugin`` command also builds and checks local plugin catalogs. +""" from __future__ import annotations @@ -9,11 +12,14 @@ import json import keyword import re +import sys from collections.abc import Sequence from pathlib import Path from datalab import __version__ from datalab.config import _ +from datalab.plugins.catalog import CatalogError +from datalab.plugins.catalogtool import build_local_catalog, check_local_catalog _PACKAGE_PATTERN = re.compile(r"^[a-z][a-z0-9_]*$") _PLUGIN_ID_PATTERN = re.compile(r"^[a-z0-9]+(?:[.-][a-z0-9]+)+$") @@ -722,13 +728,59 @@ def build_parser() -> argparse.ArgumentParser: default="BSD-3-Clause", help=_("Project license"), ) + catalog_parser = subparsers.add_parser( + "catalog", help=_("Build or check a local plugin catalog folder.") + ) + catalog_subparsers = catalog_parser.add_subparsers( + dest="catalog_command", required=True + ) + for name, text in ( + ("build", _("Write the catalog.json file listing the plugin wheels.")), + ("check", _("Check the catalog.json file against the plugin wheels.")), + ): + subparser = catalog_subparsers.add_parser(name, help=text) + subparser.add_argument( + "directory", type=Path, help=_("Local plugin catalog folder") + ) return parser +def run_catalog_command(args: argparse.Namespace) -> int: + """Build or check a local plugin catalog and print the outcome. + + Returns: + Exit code: 0 on success, 1 if the catalog has problems + """ + try: + if args.catalog_command == "build": + report = build_local_catalog(str(args.directory)) + else: + report = check_local_catalog(str(args.directory)) + except CatalogError as exc: + print(_("Error: %s") % exc, file=sys.stderr) + return 1 + for message in report.warnings: + print(_("Warning: %s") % message) + for message in report.errors: + print(_("Error: %s") % message, file=sys.stderr) + if report.errors: + if args.catalog_command == "build": + print(_("%s was not written.") % report.path, file=sys.stderr) + return 1 + count = len(report.catalog["plugins"]) + if report.written: + print(_("Wrote %s, listing %d plugins.") % (report.path, count)) + else: + print(_("The local plugin catalog lists %d plugins.") % count) + return 0 + + def main(argv: Sequence[str] | None = None) -> int: - """Run the plugin project generator command.""" + """Run the ``datalab-plugin`` command.""" parser = build_parser() args = parser.parse_args(argv) + if args.command == "catalog": + return run_catalog_command(args) try: project = _project_from_args(args) destination = create_plugin_project(project) diff --git a/datalab/plugins/store.py b/datalab/plugins/store.py index 364c16b66..4db775277 100644 --- a/datalab/plugins/store.py +++ b/datalab/plugins/store.py @@ -149,6 +149,24 @@ def _release_archives() -> None: gc.collect() +def get_available_distributions( + host_distributions: Mapping[str, str] | None = None, +) -> dict[str, str]: + """Return distribution versions a plugin wheel may depend on. + + Args: + host_distributions: Distribution versions provided by the host that the + environment metadata may lack (e.g. when running from sources) + """ + distributions: dict[str, str] = {} + for distribution in importlib_metadata.distributions(): + name = distribution.metadata["Name"] + if name: + distributions.setdefault(name, distribution.version) + distributions.update(host_distributions or {}) + return distributions + + class InstalledPluginStore: """Store of the plugins installed from a file. @@ -214,13 +232,7 @@ def find_plugin(self, filepath: str) -> InstalledPlugin | None: def get_available_distributions(self) -> dict[str, str]: """Return distribution versions a plugin wheel may depend on.""" - distributions: dict[str, str] = {} - for distribution in importlib_metadata.distributions(): - name = distribution.metadata["Name"] - if name: - distributions.setdefault(name, distribution.version) - distributions.update(self._host_distributions) - return distributions + return get_available_distributions(self._host_distributions) def _get_environment_distributions(self) -> set[str]: """Return canonical names of distributions installed outside the store.""" @@ -485,4 +497,5 @@ def _collect_garbage(self, plugins: list[InstalledPlugin]) -> None: "InstalledPlugin", "InstalledPluginStore", "PluginInstallError", + "get_available_distributions", ] diff --git a/datalab/tests/backbone/plugins/plugin_catalogtool_unit_test.py b/datalab/tests/backbone/plugins/plugin_catalogtool_unit_test.py new file mode 100644 index 000000000..827d616e6 --- /dev/null +++ b/datalab/tests/backbone/plugins/plugin_catalogtool_unit_test.py @@ -0,0 +1,132 @@ +# Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file. + +"""Unit tests for the ``datalab-plugin catalog`` command.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from datalab.plugins.catalog import CATALOG_FILENAME, read_folder_catalog +from datalab.plugins.catalogtool import build_local_catalog +from datalab.plugins.generator import main +from datalab.tests.backbone.plugins.plugin_local_catalog_unit_test import ( + DISTRIBUTION, + add_wheel, +) + + +def read_catalog(folder: Path) -> dict: + """Return the content of the catalog.json file of a folder.""" + return json.loads((folder / CATALOG_FILENAME).read_text(encoding="utf-8")) + + +def write_catalog(folder: Path, catalog: dict) -> None: + """Write the catalog.json file of a folder.""" + (folder / CATALOG_FILENAME).write_text(json.dumps(catalog), encoding="utf-8") + + +def test_build_lists_the_wheels_and_keeps_curated_fields( + tmp_path: Path, capsys: pytest.CaptureFixture +) -> None: + """A rebuilt catalog keeps what the administrator edited.""" + add_wheel(tmp_path, "1.0.0") + assert main(["catalog", "build", str(tmp_path)]) == 0 + assert "listing 1 plugins" in capsys.readouterr().out + + catalog = read_catalog(tmp_path) + (plugin,) = catalog["plugins"] + assert plugin["id"] == DISTRIBUTION and plugin["tier"] == "local" + (release,) = plugin["releases"] + assert release["url"] == release["filename"] + assert release["requires_dist"] == ["datalab-platform>=1.0"] + (listed,), warnings = read_folder_catalog(str(tmp_path)) + assert listed.get_installable_release().version == "1.0.0" and not warnings + + plugin.update( + id="org.example.spectro", + name="Spectro tools", + status="deprecated", + status_reason="Use Spectro tools 2", + ) + release["yanked"] = "Wrong baseline" + catalog["plugins"].append( + { + "id": "org.example.gone", + "name": "Gone", + "tier": "local", + "status": "active", + "repository": "", + "releases": [], + } + ) + catalog["plugins"].append( + { + "id": "org.example.revoked", + "name": "Revoked", + "tier": "local", + "status": "revoked", + "status_reason": "Malicious code", + "repository": "", + "releases": [{"version": "1.0.0", "sha256": "c" * 64}], + } + ) + write_catalog(tmp_path, catalog) + add_wheel(tmp_path, "1.1.0") + + report = build_local_catalog(str(tmp_path)) + + assert report.written and not report.errors + assert report.warnings == ["Gone: no wheel left, removed from the catalog"] + revoked, plugin = read_catalog(tmp_path)["plugins"] + assert revoked["status_reason"] == "Malicious code" + assert plugin["id"] == "org.example.spectro" and plugin["status"] == "deprecated" + assert [item["version"] for item in plugin["releases"]] == ["1.1.0", "1.0.0"] + assert plugin["releases"][1]["yanked"] == "Wrong baseline" + assert "yanked" not in plugin["releases"][0] + + +def test_build_refuses_wheels_that_datalab_would_not_install( + tmp_path: Path, capsys: pytest.CaptureFixture +) -> None: + """A wheel requiring a package that DataLab lacks prevents the build.""" + add_wheel(tmp_path, "1.0.0", requires_dist=("unknown-package-xyz>=1",)) + + assert main(["catalog", "build", str(tmp_path)]) == 1 + + error = capsys.readouterr().err + assert "unknown-package-xyz" in error and "was not written" in error + assert not (tmp_path / CATALOG_FILENAME).exists() + assert main(["catalog", "build", str(tmp_path / "missing")]) == 1 + + +def test_check_compares_the_catalog_with_the_wheels( + tmp_path: Path, capsys: pytest.CaptureFixture +) -> None: + """The check reports changed wheels and wheels missing from the catalog.""" + wheel = add_wheel(tmp_path, "1.0.0") + assert main(["catalog", "check", str(tmp_path)]) == 0 + assert f"No {CATALOG_FILENAME}" in capsys.readouterr().out + + assert main(["catalog", "build", str(tmp_path)]) == 0 + assert main(["catalog", "check", str(tmp_path)]) == 0 + unlisted = add_wheel(tmp_path, "1.1.0") + capsys.readouterr() + assert main(["catalog", "check", str(tmp_path)]) == 0 + assert f"{unlisted} is not listed" in capsys.readouterr().out + + other = tmp_path / "other" + other.mkdir() + wheel.write_bytes(add_wheel(other, "1.0.0", distribution="other-tool").read_bytes()) + assert main(["catalog", "check", str(tmp_path)]) == 1 + assert "refused" in capsys.readouterr().err + + wheel.unlink() + assert main(["catalog", "check", str(tmp_path)]) == 1 + assert "cannot read" in capsys.readouterr().err + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/datalab/tests/backbone/plugins/plugin_local_catalog_unit_test.py b/datalab/tests/backbone/plugins/plugin_local_catalog_unit_test.py new file mode 100644 index 000000000..0defe9aa9 --- /dev/null +++ b/datalab/tests/backbone/plugins/plugin_local_catalog_unit_test.py @@ -0,0 +1,210 @@ +# Copyright (c) DataLab Platform Developers, BSD 3-Clause license, see LICENSE file. + +"""Unit tests for local plugin catalogs: folders read as files.""" + +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path + +import pytest + +from datalab.config.appinfo import ( + DATALAB_PLUGINS_CATALOG_ENV_VAR, + get_env_plugin_catalog_paths, +) +from datalab.plugins.catalog import ( + CatalogError, + download_release, + load_catalogs, + read_folder_catalog, +) +from datalab.tests.backbone.plugins.wheel_factory import ( + make_plugin_wheel, + wheel_filename, +) + +DISTRIBUTION = "datalab-spectro-tools" + + +def add_wheel( + folder: Path, version: str, distribution: str = DISTRIBUTION, **kwargs +) -> Path: + """Write a plugin wheel in a folder and return its path.""" + kwargs.setdefault("requires_dist", ("datalab-platform>=1.0",)) + path = folder / wheel_filename(distribution, version) + path.write_bytes( + make_plugin_wheel( + distribution=distribution, + version=version, + package=distribution.replace("-", "_"), + **kwargs, + ) + ) + return path + + +def online_catalog(folder: Path, plugins: list[dict]) -> str: + """Write an online-like catalog and return its URL.""" + path = folder / "catalog.json" + path.write_text( + json.dumps({"schema_version": 1, "plugins": plugins}), encoding="utf-8" + ) + return path.as_uri() + + +def online_record(plugin_id: str, distribution: str, **kwargs) -> dict: + """Return an online catalog plugin record with one fake release.""" + record = { + "id": plugin_id, + "name": f"Online {distribution}", + "tier": "community", + "status": "active", + "repository": "https://github.com/someone/plugin", + "distribution": distribution, + "releases": [ + { + "version": "9.0.0", + "filename": wheel_filename(distribution, "9.0.0"), + "url": f"wheels/{wheel_filename(distribution, '9.0.0')}", + "sha256": "b" * 64, + "size": 10, + "targets": ["desktop"], + } + ], + } + record.update(kwargs) + return record + + +def test_folder_wheels_are_listed_from_the_newest(tmp_path: Path) -> None: + """Without catalog.json, the plugin wheels of a folder are the catalog.""" + metadata = ( + "License-Expression: MIT", + "Keywords: spectroscopy,baseline", + "Project-URL: Source Code, https://git.example.org/spectro", + ) + add_wheel(tmp_path, "1.0.0") + newest = add_wheel(tmp_path, "1.1.0", extra_metadata=metadata) + add_wheel( + tmp_path, "1.0.0", distribution="other-tool", entry_points="[console]\nx=a:b\n" + ) + (tmp_path / "broken-1.0-py3-none-any.whl").write_bytes(b"not a zip") + (tmp_path / "native-1.0-cp311-cp311-win_amd64.whl").write_bytes(b"") + (tmp_path / "notes.txt").write_text("not a wheel", encoding="utf-8") + + (plugin,), warnings = read_folder_catalog(str(tmp_path)) + + assert plugin.id == DISTRIBUTION and plugin.name == DISTRIBUTION + assert plugin.tier == "local" and plugin.source == str(tmp_path) + assert plugin.license == "MIT" and plugin.keywords == ("spectroscopy", "baseline") + assert plugin.repository == "https://git.example.org/spectro" + assert plugin.summary == "Example DataLab plugin" + assert [release.version for release in plugin.releases] == ["1.1.0", "1.0.0"] + release = plugin.get_installable_release() + assert release.url == str(newest) and release.requires_python == ">=3.9" + assert release.sha256 == hashlib.sha256(newest.read_bytes()).hexdigest() + assert len(warnings) == 3 + assert any("broken" in message and "ZIP" in message for message in warnings) + assert any("pure-Python" in message for message in warnings) + assert any("no datalab.plugins entry point" in message for message in warnings) + + destination = tmp_path / "download" + destination.mkdir() + path = download_release(release, str(destination)) + assert Path(path).read_bytes() == newest.read_bytes() + + newest.write_bytes(newest.read_bytes()[:-1] + b"!") + with pytest.raises(CatalogError, match="does not match"): + download_release(release, str(destination)) + + +def test_folder_catalog_json_lists_the_plugins(tmp_path: Path) -> None: + """A catalog.json file lists the plugins, with paths relative to the folder.""" + (tmp_path / "wheels").mkdir() + listed = add_wheel(tmp_path / "wheels", "2.0.0") + add_wheel(tmp_path, "3.0.0") + data = listed.read_bytes() + release = { + "version": "2.0.0", + "filename": listed.name, + "url": f"wheels/{listed.name}", + "sha256": hashlib.sha256(data).hexdigest(), + "size": len(data), + "targets": ["desktop"], + } + record = online_record("org.example.spectro", DISTRIBUTION, releases=[release]) + online_catalog(tmp_path, [record]) + + (plugin,), warnings = read_folder_catalog(str(tmp_path)) + + assert not warnings + assert plugin.id == "org.example.spectro" and plugin.source == str(tmp_path) + assert [release.version for release in plugin.releases] == ["2.0.0"] + assert plugin.releases[0].url == str(listed) + + release["url"] = "../outside.whl" + online_catalog(tmp_path, [record]) + with pytest.raises(CatalogError, match="Invalid plugin catalog"): + read_folder_catalog(str(tmp_path)) + + +def test_local_catalogs_take_precedence(tmp_path: Path) -> None: + """Local catalogs hide the same plugins in the next catalogs.""" + first, second, site = tmp_path / "first", tmp_path / "second", tmp_path / "site" + for folder in (first, second, site): + folder.mkdir() + add_wheel(first, "1.0.0") + add_wheel(second, "2.0.0") + add_wheel(second, "1.0.0", distribution="datalab-camera-tools") + url = online_catalog( + site, + [ + online_record("io.github.someone.spectro", DISTRIBUTION), + online_record( + "io.github.someone.camera", + "datalab-camera-tools", + status="revoked", + status_reason="Malicious code", + ), + online_record("io.github.someone.pulse", "datalab-pulse-tools"), + ], + ) + + content = load_catalogs(url, [str(tmp_path / "missing"), str(first), str(second)]) + + assert [(plugin.distribution, plugin.source) for plugin in content.plugins] == [ + (DISTRIBUTION, str(first)), + ("datalab-camera-tools", str(second)), + ("datalab-pulse-tools", ""), + ] + assert content.plugins[0].releases[0].version == "1.0.0" + missing, withdrawn = content.errors + assert missing.startswith("Plugin catalog folder not found") + assert "withdrawn" in withdrawn and "Malicious code" in withdrawn + + content = load_catalogs((site / "missing.json").as_uri(), [str(first)]) + assert [plugin.source for plugin in content.plugins] == [str(first)] + assert content.errors[0].startswith("Cannot download") + + +def test_catalog_folders_from_the_environment( + tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Folders of the environment variable are kept even when unavailable.""" + existing = tmp_path / "existing" + existing.mkdir() + missing = tmp_path / "missing" + value = os.pathsep.join((f" {existing} ", "", str(missing), str(existing))) + monkeypatch.setenv(DATALAB_PLUGINS_CATALOG_ENV_VAR, value) + + assert get_env_plugin_catalog_paths() == [str(existing), str(missing)] + + monkeypatch.delenv(DATALAB_PLUGINS_CATALOG_ENV_VAR) + assert not get_env_plugin_catalog_paths() + + +if __name__ == "__main__": + pytest.main([__file__]) diff --git a/datalab/tests/backbone/plugins/plugins_contracts_imports_unit_test.py b/datalab/tests/backbone/plugins/plugins_contracts_imports_unit_test.py index 4f70cd9b3..a012c3a35 100644 --- a/datalab/tests/backbone/plugins/plugins_contracts_imports_unit_test.py +++ b/datalab/tests/backbone/plugins/plugins_contracts_imports_unit_test.py @@ -17,7 +17,7 @@ "tools", ) # Installer modules, reused outside the Qt host (DataLab-Web, catalog tooling) -HEADLESS_HOST_MODULES = ("catalog", "store", "wheels") +HEADLESS_HOST_MODULES = ("catalog", "catalogtool", "store", "wheels") def test_plugin_contracts_do_not_import_plugin_host() -> None: diff --git a/datalab/tests/backbone/plugins/wheel_factory.py b/datalab/tests/backbone/plugins/wheel_factory.py index 7d6c8ba33..8e22c1dd3 100644 --- a/datalab/tests/backbone/plugins/wheel_factory.py +++ b/datalab/tests/backbone/plugins/wheel_factory.py @@ -38,11 +38,13 @@ def make_plugin_wheel( top_level: str | None | object = _DEFAULT, files: dict[str, str | bytes] | None = None, extra_files: dict[str, str | bytes] | None = None, + extra_metadata: tuple[str, ...] = (), ) -> bytes: """Return the bytes of a DataLab plugin wheel. By default, the wheel provides a real desktop plugin class declared in the - ``datalab.plugins`` entry-point group. + ``datalab.plugins`` entry-point group. ``extra_metadata`` lists additional + ``METADATA`` lines, such as ``"Keywords: spectroscopy"``. """ dist_info = f"{distribution.replace('-', '_')}-{version}.dist-info" if entry_points is None: @@ -66,6 +68,7 @@ def make_plugin_wheel( f"Requires-Python: {requires_python}", ] metadata.extend(f"Requires-Dist: {requirement}" for requirement in requires_dist) + metadata.extend(extra_metadata) buffer = io.BytesIO() with zipfile.ZipFile(buffer, "w", zipfile.ZIP_DEFLATED) as archive: for name, data in {**files, **(extra_files or {})}.items(): diff --git a/datalab/tests/features/plugins/plugin_available_dialog_test.py b/datalab/tests/features/plugins/plugin_available_dialog_test.py index fee536421..75030eaf0 100644 --- a/datalab/tests/features/plugins/plugin_available_dialog_test.py +++ b/datalab/tests/features/plugins/plugin_available_dialog_test.py @@ -13,7 +13,12 @@ from qtpy import QtWidgets as QW from datalab import __version__ -from datalab.config import Conf +from datalab.config import ( + DATALAB_PLUGINS_CATALOG_ENV_VAR, + Conf, + get_plugin_catalog_paths, + set_plugin_catalog_paths, +) from datalab.config.persistence import has_persisted_option, remove_persisted_option from datalab.gui.plugins.config import PluginConfigDialog from datalab.plugins import PluginRegistry @@ -28,6 +33,7 @@ PACKAGE = "datalab_catalog_ui_probe" DISTRIBUTION = "datalab-catalog-ui-probe" PLUGIN_ID = "org.example.catalog-ui-probe" +OPTIONS = ("plugins_catalog_url", "plugins_catalog_path_list") @pytest.fixture(name="store") @@ -39,13 +45,18 @@ def fixture_store(tmp_path: Path, monkeypatch: pytest.MonkeyPatch): ) monkeypatch.setattr(plugin_base, "_INSTALLED_PLUGIN_STORE", store) monkeypatch.setattr(sys, "path", list(sys.path)) - had_url = has_persisted_option(Conf, "plugins_catalog_url") - original_url = Conf.plugins_catalog_url.get() + monkeypatch.delenv(DATALAB_PLUGINS_CATALOG_ENV_VAR, raising=False) + saved = { + name: (has_persisted_option(Conf, name), getattr(Conf, name).get()) + for name in OPTIONS + } + set_plugin_catalog_paths([]) yield store - if had_url: - Conf.plugins_catalog_url.set(original_url) - else: - remove_persisted_option(Conf, "plugins_catalog_url") + for name, (persisted, value) in saved.items(): + if persisted: + getattr(Conf, name).set(value) + else: + remove_persisted_option(Conf, name) for name in [name for name in sys.modules if name.startswith(PACKAGE)]: del sys.modules[name] @@ -167,10 +178,116 @@ def test_catalog_plugin_is_installed_then_offered_as_update( Conf.plugins_catalog_url.set((tmp_path / "missing.json").as_uri()) available.refresh() - assert "cannot be loaded" in available.status_label.text() + assert "Cannot download" in available.errors_label.text() + assert not available.item_widgets + dialog.close() + dialog.deleteLater() + + +def publish_wheel(folder: Path, version: str) -> None: + """Drop a probe plugin wheel in a local catalog folder.""" + folder.mkdir(parents=True, exist_ok=True) + (folder / wheel_filename(DISTRIBUTION, version)).write_bytes( + make_plugin_wheel( + distribution=DISTRIBUTION, + version=version, + package=PACKAGE, + plugin_id=PLUGIN_ID, + requires_dist=("datalab-platform>=1.0",), + ) + ) + + +def test_local_catalog_plugin_takes_precedence_and_is_installed( + store: InstalledPluginStore, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """A plugin of a local catalog hides the online one and is installed from it.""" + local = tmp_path / "shared" / "plugins" + publish_wheel(local, "1.1.0") + Conf.plugins_catalog_url.set(publish_catalog(tmp_path / "site", ("1.0.0",))) + set_plugin_catalog_paths([str(local)]) + origins: list[str] = [] + monkeypatch.setattr( + QW.QMessageBox, "question", lambda *_args, **_kwargs: QW.QMessageBox.Yes + ) + with datalab_test_app_context(console=False) as win: + dialog = PluginConfigDialog(win) + available = dialog.available_plugins_widget + assert available.get_catalog_directories() == [str(local)] + monkeypatch.setattr(available, "run_task", run_now) + monkeypatch.setattr( + dialog.installed_plugins_widget, + "confirm_installation", + lambda _manifest, origin=None: origins.append(origin) or True, + ) + + available.refresh() + (row,) = available.item_widgets + assert row.plugin.source == str(local) + assert row.action_button.text() == "Install" + assert available.status_label.text() == "Plugins in the catalogs: 1" + assert available.errors_label.isHidden() + row.action_button.click() + QW.QApplication.processEvents() + + assert origins == [f"Local plugin catalog ({local})"] + assert [plugin.version for plugin in store.get_plugins()] == ["1.1.0"] + + Conf.plugins_catalog_url.set((tmp_path / "missing.json").as_uri()) + available.refresh() + (row,) = available.item_widgets + assert row.status_label.text() == "Installed" + assert "Cannot download" in available.errors_label.text() dialog.close() dialog.deleteLater() +def test_local_catalog_folders_are_set_in_plugin_settings( + store: InstalledPluginStore, tmp_path: Path, monkeypatch: pytest.MonkeyPatch +) -> None: + """Local catalog folders are edited without reloading plugins.""" + del store + env_folder, added, edited = (tmp_path / name for name in ("env", "a", "b")) + monkeypatch.setenv(DATALAB_PLUGINS_CATALOG_ENV_VAR, str(env_folder)) + Conf.plugins_catalog_url.set("") + + def no_reload_question(*_args, **_kwargs): + raise AssertionError("Changing catalogs must not reload plugins") + + monkeypatch.setattr(QW.QMessageBox, "question", no_reload_question) + with datalab_test_app_context(console=False) as win: + dialog = PluginConfigDialog(win) + available = dialog.available_plugins_widget + (env_widget,) = dialog.env_catalog_path_widgets + assert env_widget.path == str(env_folder) and env_widget.edit_button is None + assert DATALAB_PLUGINS_CATALOG_ENV_VAR in env_widget.path_label.text() + assert dialog.catalog_paths_placeholder.isHidden() + monkeypatch.setattr(available, "run_task", run_now) + selected = iter([str(added), str(env_folder), str(edited)]) + monkeypatch.setattr( + dialog, "_browse_catalog_directory", lambda _path=None: next(selected) + ) + + dialog.add_catalog_button.click() + dialog.add_catalog_button.click() + assert dialog.catalog_paths == [str(added)] + assert available.get_catalog_directories() == [str(env_folder), str(added)] + dialog.catalog_path_widgets[0].edit_button.click() + assert available.get_catalog_directories() == [str(env_folder), str(edited)] + + available.refresh() + assert available.errors_label.text().count("not found") == 2 + + dialog.accept() + assert get_plugin_catalog_paths() == [str(edited)] + + with datalab_test_app_context(console=False) as win: + dialog = PluginConfigDialog(win) + dialog.catalog_path_widgets[0].delete_button.click() + assert not dialog.catalog_paths + dialog.reject() + assert get_plugin_catalog_paths() == [str(edited)] + + if __name__ == "__main__": pytest.main([__file__]) diff --git a/doc/features/advanced/plugins.rst b/doc/features/advanced/plugins.rst index 110ebbaae..cb02fdc0c 100644 --- a/doc/features/advanced/plugins.rst +++ b/doc/features/advanced/plugins.rst @@ -118,7 +118,7 @@ Plugins are automatically discovered at startup from multiple locations: If using a frozen (standalone) build, the `plugins` folder located next to the executable is scanned. - Installed plugins: - Wheels and modules installed from **Plugins > Configure plugins...**, either from a file in the **Install plugins** tab (see :ref:`plugin_install_from_file`) or from the plugin catalog in the **Available plugins** tab (see :ref:`plugin_catalog`). + Wheels and modules installed from **Plugins > Configure plugins...**, either from a file in the **Install plugins** tab (see :ref:`plugin_install_from_file`) or from a plugin catalog in the **Available plugins** tab (see :ref:`plugin_catalog`). - The internal `datalab/plugins/builtin` folder (not recommended for user plugins): This location is reserved for built-in or bundled plugins and should not be modified manually. @@ -186,6 +186,8 @@ This tab lists every directory scanned at startup, in two groups: buttons. Unlike the environment variable, these directories are saved in the DataLab configuration and therefore persist across sessions. +It also lists the **Local plugin catalogs**: folders, possibly shared on the network, whose plugins are offered in the **Available plugins** tab (see :ref:`plugin_local_catalogs`). They are managed with the same buttons, and the folders declared through the ``DATALAB_PLUGINS_CATALOG`` environment variable come first, as read-only entries. Changing these folders does not require reloading plugins. + The tab also holds a **Compatibility warnings** option to hide warnings for incompatible DataLab v0.20 plugins. @@ -194,11 +196,47 @@ incompatible DataLab v0.20 plugins. Available plugins ~~~~~~~~~~~~~~~~~ -This tab lists the plugins of the `DataLab plugin catalog `_, with their description, license and source code link. Plugins maintained by the DataLab team are marked **Official**; the others are **Community** plugins, maintained by their authors. A search field filters the list. +This tab lists the plugins of the `DataLab plugin catalog `_ and of the local plugin catalogs (see :ref:`plugin_local_catalogs`), with their description, license and source code link. Plugins maintained by the DataLab team are marked **Official**; the others are **Community** plugins, maintained by their authors, or **Local catalog** plugins, provided by your organization (the tooltip of such a plugin shows the folder of its catalog). A search field filters the list. **Install** downloads the newest release available for DataLab desktop and for the running Python version, checks that its size and SHA-256 digest match the catalog, then installs it as described in :ref:`plugin_install_from_file`, after the same confirmation. When the catalog offers a newer version of a plugin installed this way, the button becomes **Update to...**. Plugins installed with ``pip`` in the Python environment are shown as such and are not modified. -The catalog is downloaded only when the tab is shown or refreshed. Its address is the ``plugins_catalog_url`` option of the DataLab configuration file: it may point to a private copy of the catalog, including a ``file:`` URL for a folder shared on the network, or be emptied to disable the catalog. +The catalogs are read only when the tab is shown or refreshed, and again when the local catalog folders change. A catalog that cannot be read, for example a network share that is not connected, is reported below the plugin count without hiding the plugins of the other catalogs. The address of the DataLab plugin catalog is the ``plugins_catalog_url`` option of the DataLab configuration file: it may point to a private copy of the catalog, or be emptied to disable it, for example on computers without Internet access. For a catalog stored in a folder, including a folder shared on the network, use a local plugin catalog instead. + +.. _plugin_local_catalogs: + +Local plugin catalogs +^^^^^^^^^^^^^^^^^^^^^ + +A local plugin catalog is a folder, typically shared on the network (for example ``\\fileserver\datalab\plugins``), from which an organization distributes its own plugins, or the versions of public plugins it has approved, without publishing them. Local catalogs complement the DataLab plugin catalog and take precedence over it: when several catalogs list the same plugin (same distribution name or plugin ID), only the first one is shown, so that users install the version chosen by the organization. If the DataLab plugin catalog withdraws a plugin hidden this way, the tab reports it. + +Local catalogs are declared: + +- in the **Plugin settings** tab (see :ref:`plugin_search_paths`): these folders are saved in the DataLab configuration (``plugins_catalog_path_list`` option); +- through the ``DATALAB_PLUGINS_CATALOG`` environment variable, for example to configure all the computers of an organization: folders are separated by the OS path separator (``;`` on Windows, ``:`` on Linux/macOS) and take precedence over the folders of the configuration. Unlike ``DATALAB_PLUGINS``, folders unavailable when DataLab starts are not ignored: they are reported in the **Available plugins** tab, and used as soon as they become available and the catalogs are refreshed. + +.. code-block:: bat + + :: Windows + set DATALAB_PLUGINS_CATALOG=\\fileserver\datalab\plugins;D:\approved_plugins + +A local catalog folder holds either: + +- **plugin wheels only**: DataLab lists every pure-Python wheel of the folder (subfolders are not scanned) declaring a ``datalab.plugins`` entry point, and groups the versions of each plugin. Other files are ignored and reported. The plugin name is its distribution name; its description, license, keywords and source code link come from the wheel metadata. Copying a new version to the folder is enough to offer it as an update; +- **or a** ``catalog.json`` **file**, in the format of the DataLab plugin catalog, whose release URLs are relative to the folder: only the plugins it lists are shown. It may be a copy of the DataLab plugin catalog with its wheels, or a catalog built from the wheels of the folder with ``datalab-plugin catalog build``, then edited to give a display name to the plugins, deprecate or revoke them (``status`` and ``status_reason`` fields), or withdraw a release (``yanked`` field). + +Whatever the catalog, DataLab checks the size and SHA-256 digest of the wheel before installing it, then inspects it without running it and asks for confirmation; the confirmation dialog shows the catalog folder as the source of the plugin. Being listed in a local catalog does not mean that a plugin was audited: the organization managing the folder is responsible for its content, and write access to the folder should be restricted accordingly. + +The ``datalab-plugin`` command maintains the ``catalog.json`` file of a folder: + +.. code-block:: bat + + :: Write catalog.json, listing the plugin wheels of the folder + datalab-plugin catalog build \\fileserver\datalab\plugins + + :: Check catalog.json against the plugin wheels of the folder + datalab-plugin catalog check \\fileserver\datalab\plugins + +``build`` checks every wheel as DataLab does before installing it (pure Python, plugin entry point, dependencies provided by DataLab, Python version supported by DataLab) and does not write the file if a wheel is refused. It keeps the fields edited in an existing ``catalog.json`` file (plugin ID, name, tier, status and its reason, repository, documentation, capabilities and keywords, and the reasons why releases are withdrawn), and revoked plugins remain listed. The file is replaced atomically, so that users may read the catalog meanwhile. ``check`` reports the wheels of the catalog that are missing, changed or refused, and the wheels of the folder that the catalog does not list: once a folder holds a ``catalog.json`` file, run ``build`` again after adding wheels. Both commands exit with a non-zero status when they find a problem. .. _plugin_install_from_file: diff --git a/doc/locale/fr/LC_MESSAGES/features/advanced/plugins.po b/doc/locale/fr/LC_MESSAGES/features/advanced/plugins.po index 5921691d2..d0a80f388 100644 --- a/doc/locale/fr/LC_MESSAGES/features/advanced/plugins.po +++ b/doc/locale/fr/LC_MESSAGES/features/advanced/plugins.po @@ -97,8 +97,8 @@ msgstr "Répertoires de plugins supplémentaires : Tout nombre de répertoires s msgid "The standalone distribution directory: If using a frozen (standalone) build, the `plugins` folder located next to the executable is scanned." msgstr "Le répertoire de distribution autonome : Si vous utilisez une version gelée (autonome), le dossier `plugins` situé à côté de l'exécutable est analysé." -msgid "Installed plugins: Wheels and modules installed from **Plugins > Configure plugins...**, either from a file in the **Install plugins** tab (see :ref:`plugin_install_from_file`) or from the plugin catalog in the **Available plugins** tab (see :ref:`plugin_catalog`)." -msgstr "Les plugins installés : wheels et modules installés depuis **Plugins > Configurer les plugins...**, soit depuis un fichier dans l'onglet **Installer des plugins** (voir :ref:`plugin_install_from_file`), soit depuis le catalogue de plugins dans l'onglet **Plugins disponibles** (voir :ref:`plugin_catalog`)." +msgid "Installed plugins: Wheels and modules installed from **Plugins > Configure plugins...**, either from a file in the **Install plugins** tab (see :ref:`plugin_install_from_file`) or from a plugin catalog in the **Available plugins** tab (see :ref:`plugin_catalog`)." +msgstr "Les plugins installés : wheels et modules installés depuis **Plugins > Configurer les plugins...**, soit depuis un fichier dans l'onglet **Installer des plugins** (voir :ref:`plugin_install_from_file`), soit depuis un catalogue de plugins dans l'onglet **Plugins disponibles** (voir :ref:`plugin_catalog`)." msgid "The internal `datalab/plugins/builtin` folder (not recommended for user plugins): This location is reserved for built-in or bundled plugins and should not be modified manually." msgstr "Le dossier interne `datalab/plugins/builtin` (non recommandé pour les plugins utilisateur) : Cet emplacement est réservé aux plugins intégrés ou fournis et ne doit pas être modifié manuellement." @@ -151,20 +151,56 @@ msgstr "**Répertoires de plugins par défaut** (lecture seule) : le répertoire msgid "**Additional plugin directories**: your own directories, added with the **Add** button and managed with the *Edit directory* and *Remove directory* buttons. Unlike the environment variable, these directories are saved in the DataLab configuration and therefore persist across sessions." msgstr "**Répertoires de plugins supplémentaires** : vos propres répertoires, ajoutés avec le bouton **Ajouter** et gérés avec les boutons *Modifier le répertoire* et *Supprimer le répertoire*. Contrairement à la variable d'environnement, ces répertoires sont enregistrés dans la configuration de DataLab et persistent donc entre les sessions." +msgid "It also lists the **Local plugin catalogs**: folders, possibly shared on the network, whose plugins are offered in the **Available plugins** tab (see :ref:`plugin_local_catalogs`). They are managed with the same buttons, and the folders declared through the ``DATALAB_PLUGINS_CATALOG`` environment variable come first, as read-only entries. Changing these folders does not require reloading plugins." +msgstr "Il liste aussi les **Catalogues locaux de plugins** : des dossiers, éventuellement partagés sur le réseau, dont les plugins sont proposés dans l'onglet **Plugins disponibles** (voir :ref:`plugin_local_catalogs`). Ils se gèrent avec les mêmes boutons, et les dossiers déclarés par la variable d'environnement ``DATALAB_PLUGINS_CATALOG`` apparaissent en premier, en lecture seule. Modifier ces dossiers ne nécessite pas de recharger les plugins." + msgid "The tab also holds a **Compatibility warnings** option to hide warnings for incompatible DataLab v0.20 plugins." msgstr "L'onglet contient également une option **Avertissements de compatibilité** pour masquer les avertissements concernant les plugins DataLab v0.20 incompatibles." msgid "Available plugins" msgstr "Plugins disponibles" -msgid "This tab lists the plugins of the `DataLab plugin catalog `_, with their description, license and source code link. Plugins maintained by the DataLab team are marked **Official**; the others are **Community** plugins, maintained by their authors. A search field filters the list." -msgstr "Cet onglet liste les plugins du `catalogue de plugins DataLab `_, avec leur description, leur licence et un lien vers leur code source. Les plugins maintenus par l'équipe DataLab sont marqués **Officiel** ; les autres sont des plugins de la **Communauté**, maintenus par leurs auteurs. Un champ de recherche filtre la liste." +msgid "This tab lists the plugins of the `DataLab plugin catalog `_ and of the local plugin catalogs (see :ref:`plugin_local_catalogs`), with their description, license and source code link. Plugins maintained by the DataLab team are marked **Official**; the others are **Community** plugins, maintained by their authors, or **Local catalog** plugins, provided by your organization (the tooltip of such a plugin shows the folder of its catalog). A search field filters the list." +msgstr "Cet onglet liste les plugins du `catalogue de plugins DataLab `_ et des catalogues locaux de plugins (voir :ref:`plugin_local_catalogs`), avec leur description, leur licence et un lien vers leur code source. Les plugins maintenus par l'équipe DataLab sont marqués **Officiel** ; les autres sont des plugins de la **Communauté**, maintenus par leurs auteurs, ou des plugins d'un **Catalogue local**, fournis par votre organisation (l'infobulle d'un tel plugin indique le dossier de son catalogue). Un champ de recherche filtre la liste." msgid "**Install** downloads the newest release available for DataLab desktop and for the running Python version, checks that its size and SHA-256 digest match the catalog, then installs it as described in :ref:`plugin_install_from_file`, after the same confirmation. When the catalog offers a newer version of a plugin installed this way, the button becomes **Update to...**. Plugins installed with ``pip`` in the Python environment are shown as such and are not modified." msgstr "**Installer** télécharge la version la plus récente disponible pour DataLab desktop et pour la version de Python utilisée, vérifie que sa taille et son empreinte SHA-256 correspondent au catalogue, puis l'installe comme décrit dans :ref:`plugin_install_from_file`, après la même confirmation. Lorsque le catalogue propose une version plus récente d'un plugin installé ainsi, le bouton devient **Mettre à jour vers...**. Les plugins installés avec ``pip`` dans l'environnement Python sont signalés comme tels et ne sont pas modifiés." -msgid "The catalog is downloaded only when the tab is shown or refreshed. Its address is the ``plugins_catalog_url`` option of the DataLab configuration file: it may point to a private copy of the catalog, including a ``file:`` URL for a folder shared on the network, or be emptied to disable the catalog." -msgstr "Le catalogue n'est téléchargé que lorsque l'onglet est affiché ou actualisé. Son adresse est l'option ``plugins_catalog_url`` du fichier de configuration de DataLab : elle peut désigner une copie privée du catalogue, y compris une URL ``file:`` pour un dossier partagé sur le réseau, ou être vidée pour désactiver le catalogue." +msgid "The catalogs are read only when the tab is shown or refreshed, and again when the local catalog folders change. A catalog that cannot be read, for example a network share that is not connected, is reported below the plugin count without hiding the plugins of the other catalogs. The address of the DataLab plugin catalog is the ``plugins_catalog_url`` option of the DataLab configuration file: it may point to a private copy of the catalog, or be emptied to disable it, for example on computers without Internet access. For a catalog stored in a folder, including a folder shared on the network, use a local plugin catalog instead." +msgstr "Les catalogues ne sont lus que lorsque l'onglet est affiché ou actualisé, ainsi que lorsque les dossiers des catalogues locaux changent. Un catalogue illisible, par exemple un partage réseau non connecté, est signalé sous le nombre de plugins sans masquer les plugins des autres catalogues. L'adresse du catalogue de plugins DataLab est l'option ``plugins_catalog_url`` du fichier de configuration de DataLab : elle peut désigner une copie privée du catalogue, ou être vidée pour le désactiver, par exemple sur des ordinateurs sans accès à Internet. Pour un catalogue stocké dans un dossier, y compris un dossier partagé sur le réseau, utilisez plutôt un catalogue local de plugins." + +msgid "Local plugin catalogs" +msgstr "Catalogues locaux de plugins" + +msgid "A local plugin catalog is a folder, typically shared on the network (for example ``\\\\fileserver\\datalab\\plugins``), from which an organization distributes its own plugins, or the versions of public plugins it has approved, without publishing them. Local catalogs complement the DataLab plugin catalog and take precedence over it: when several catalogs list the same plugin (same distribution name or plugin ID), only the first one is shown, so that users install the version chosen by the organization. If the DataLab plugin catalog withdraws a plugin hidden this way, the tab reports it." +msgstr "Un catalogue local de plugins est un dossier, généralement partagé sur le réseau (par exemple ``\\\\fileserver\\datalab\\plugins``), à partir duquel une organisation diffuse ses propres plugins, ou les versions de plugins publics qu'elle a validées, sans les publier. Les catalogues locaux complètent le catalogue de plugins DataLab et sont prioritaires sur lui : lorsque plusieurs catalogues proposent le même plugin (même nom de distribution ou même identifiant de plugin), seul le premier est affiché, afin que les utilisateurs installent la version choisie par l'organisation. Si le catalogue de plugins DataLab retire un plugin ainsi masqué, l'onglet le signale." + +msgid "Local catalogs are declared:" +msgstr "Les catalogues locaux sont déclarés :" + +msgid "in the **Plugin settings** tab (see :ref:`plugin_search_paths`): these folders are saved in the DataLab configuration (``plugins_catalog_path_list`` option);" +msgstr "dans l'onglet **Paramètres des plugins** (voir :ref:`plugin_search_paths`) : ces dossiers sont enregistrés dans la configuration de DataLab (option ``plugins_catalog_path_list``) ;" + +msgid "through the ``DATALAB_PLUGINS_CATALOG`` environment variable, for example to configure all the computers of an organization: folders are separated by the OS path separator (``;`` on Windows, ``:`` on Linux/macOS) and take precedence over the folders of the configuration. Unlike ``DATALAB_PLUGINS``, folders unavailable when DataLab starts are not ignored: they are reported in the **Available plugins** tab, and used as soon as they become available and the catalogs are refreshed." +msgstr "par la variable d'environnement ``DATALAB_PLUGINS_CATALOG``, par exemple pour configurer tous les ordinateurs d'une organisation : les dossiers sont séparés par le séparateur de chemins du système (``;`` sous Windows, ``:`` sous Linux/macOS) et sont prioritaires sur les dossiers de la configuration. Contrairement à ``DATALAB_PLUGINS``, les dossiers indisponibles au démarrage de DataLab ne sont pas ignorés : ils sont signalés dans l'onglet **Plugins disponibles**, et utilisés dès qu'ils deviennent disponibles et que les catalogues sont actualisés." + +msgid "A local catalog folder holds either:" +msgstr "Un dossier de catalogue local contient :" + +msgid "**plugin wheels only**: DataLab lists every pure-Python wheel of the folder (subfolders are not scanned) declaring a ``datalab.plugins`` entry point, and groups the versions of each plugin. Other files are ignored and reported. The plugin name is its distribution name; its description, license, keywords and source code link come from the wheel metadata. Copying a new version to the folder is enough to offer it as an update;" +msgstr "**soit uniquement des wheels de plugins** : DataLab liste chaque wheel en Python pur du dossier (les sous-dossiers ne sont pas parcourus) déclarant un point d'entrée ``datalab.plugins``, et regroupe les versions de chaque plugin. Les autres fichiers sont ignorés et signalés. Le nom du plugin est son nom de distribution ; sa description, sa licence, ses mots-clés et le lien vers son code source proviennent des métadonnées de la wheel. Copier une nouvelle version dans le dossier suffit pour la proposer en mise à jour ;" + +msgid "**or a** ``catalog.json`` **file**, in the format of the DataLab plugin catalog, whose release URLs are relative to the folder: only the plugins it lists are shown. It may be a copy of the DataLab plugin catalog with its wheels, or a catalog built from the wheels of the folder with ``datalab-plugin catalog build``, then edited to give a display name to the plugins, deprecate or revoke them (``status`` and ``status_reason`` fields), or withdraw a release (``yanked`` field)." +msgstr "**soit un fichier** ``catalog.json``, au format du catalogue de plugins DataLab, dont les URL des versions sont relatives au dossier : seuls les plugins qu'il liste sont affichés. Il peut s'agir d'une copie du catalogue de plugins DataLab avec ses wheels, ou d'un catalogue construit à partir des wheels du dossier avec ``datalab-plugin catalog build``, puis modifié pour donner un nom d'affichage aux plugins, les déprécier ou les révoquer (champs ``status`` et ``status_reason``), ou retirer une version (champ ``yanked``)." + +msgid "Whatever the catalog, DataLab checks the size and SHA-256 digest of the wheel before installing it, then inspects it without running it and asks for confirmation; the confirmation dialog shows the catalog folder as the source of the plugin. Being listed in a local catalog does not mean that a plugin was audited: the organization managing the folder is responsible for its content, and write access to the folder should be restricted accordingly." +msgstr "Quel que soit le catalogue, DataLab vérifie la taille et l'empreinte SHA-256 de la wheel avant de l'installer, puis l'inspecte sans l'exécuter et demande une confirmation ; la boîte de dialogue de confirmation indique le dossier du catalogue comme source du plugin. Figurer dans un catalogue local ne signifie pas qu'un plugin a été audité : l'organisation qui gère le dossier est responsable de son contenu, et l'accès en écriture au dossier doit être restreint en conséquence." + +msgid "The ``datalab-plugin`` command maintains the ``catalog.json`` file of a folder:" +msgstr "La commande ``datalab-plugin`` gère le fichier ``catalog.json`` d'un dossier :" + +msgid "``build`` checks every wheel as DataLab does before installing it (pure Python, plugin entry point, dependencies provided by DataLab, Python version supported by DataLab) and does not write the file if a wheel is refused. It keeps the fields edited in an existing ``catalog.json`` file (plugin ID, name, tier, status and its reason, repository, documentation, capabilities and keywords, and the reasons why releases are withdrawn), and revoked plugins remain listed. The file is replaced atomically, so that users may read the catalog meanwhile. ``check`` reports the wheels of the catalog that are missing, changed or refused, and the wheels of the folder that the catalog does not list: once a folder holds a ``catalog.json`` file, run ``build`` again after adding wheels. Both commands exit with a non-zero status when they find a problem." +msgstr "``build`` vérifie chaque wheel comme DataLab le fait avant de l'installer (Python pur, point d'entrée de plugin, dépendances fournies par DataLab, version de Python prise en charge par DataLab) et n'écrit pas le fichier si une wheel est refusée. Il conserve les champs modifiés d'un fichier ``catalog.json`` existant (identifiant, nom, niveau, statut et sa raison, dépôt, documentation, capacités et mots-clés des plugins, ainsi que les raisons du retrait des versions), et les plugins révoqués restent listés. Le fichier est remplacé de manière atomique, afin que les utilisateurs puissent lire le catalogue pendant ce temps. ``check`` signale les wheels du catalogue manquantes, modifiées ou refusées, ainsi que les wheels du dossier que le catalogue ne liste pas : dès qu'un dossier contient un fichier ``catalog.json``, relancez ``build`` après avoir ajouté des wheels. Les deux commandes se terminent avec un code de sortie non nul lorsqu'elles détectent un problème." msgid "Install plugins" msgstr "Installer des plugins" diff --git a/doc/locale/fr/LC_MESSAGES/release_notes/release_1.04.po b/doc/locale/fr/LC_MESSAGES/release_notes/release_1.04.po index 27d1d1017..f4c8efeef 100644 --- a/doc/locale/fr/LC_MESSAGES/release_notes/release_1.04.po +++ b/doc/locale/fr/LC_MESSAGES/release_notes/release_1.04.po @@ -123,6 +123,15 @@ msgstr "Les plugins installés sont listés dans le même onglet, où ils peuven msgid "The new **Available plugins** tab lists the plugins of the [DataLab plugin catalog](https://datalab-platform.com/plugins/), official or from the community, with a search field: **Install** and **Update** download the wheel, check it against the catalog and install it after confirmation. The catalog address may be changed to use a private copy" msgstr "Le nouvel onglet **Plugins disponibles** liste les plugins du [catalogue de plugins DataLab](https://datalab-platform.com/plugins/), officiels ou issus de la communauté, avec un champ de recherche : **Installer** et **Mettre à jour** téléchargent le wheel, le vérifient par rapport au catalogue et l'installent après confirmation. L'adresse du catalogue peut être modifiée pour utiliser une copie privée" +msgid "Organizations can distribute their own plugins, or the versions of public plugins they have approved, from local plugin catalogs: folders, typically shared on the network, declared in the **Plugin settings** tab or with the `DATALAB_PLUGINS_CATALOG` environment variable. Their plugins are listed in the **Available plugins** tab before those of the DataLab plugin catalog, and hide them when both offer the same plugin" +msgstr "Les organisations peuvent diffuser leurs propres plugins, ou les versions de plugins publics qu'elles ont validées, depuis des catalogues locaux de plugins : des dossiers, généralement partagés sur le réseau, déclarés dans l'onglet **Paramètres des plugins** ou par la variable d'environnement `DATALAB_PLUGINS_CATALOG`. Leurs plugins sont listés dans l'onglet **Plugins disponibles** avant ceux du catalogue de plugins DataLab, et les masquent lorsque les deux proposent le même plugin" + +msgid "A local catalog folder may simply hold plugin wheels, listed as they are copied to it, or a `catalog.json` file giving display names, deprecations and withdrawn releases; `datalab-plugin catalog build` writes this file from the wheels of the folder, keeping the edited fields, and `datalab-plugin catalog check` reports wheels that are missing, changed, refused by DataLab or not listed" +msgstr "Un dossier de catalogue local peut simplement contenir des wheels de plugins, listées dès qu'elles y sont copiées, ou un fichier `catalog.json` indiquant les noms d'affichage, les dépréciations et les versions retirées ; `datalab-plugin catalog build` écrit ce fichier à partir des wheels du dossier en conservant les champs modifiés, et `datalab-plugin catalog check` signale les wheels manquantes, modifiées, refusées par DataLab ou non listées" + +msgid "A catalog that cannot be read, such as a network share that is not connected or the online catalog on a computer without Internet access, is reported in the **Available plugins** tab without hiding the plugins of the other catalogs" +msgstr "Un catalogue illisible, comme un partage réseau non connecté ou le catalogue en ligne sur un ordinateur sans accès à Internet, est signalé dans l'onglet **Plugins disponibles** sans masquer les plugins des autres catalogues" + msgid "**Plugin project generator:**" msgstr "**Générateur de projets de plugins :**" diff --git a/doc/release_notes/release_1.04.md b/doc/release_notes/release_1.04.md index 0c2e639ab..6546eee18 100644 --- a/doc/release_notes/release_1.04.md +++ b/doc/release_notes/release_1.04.md @@ -47,6 +47,9 @@ * Before installing, DataLab checks the file without running it and shows its name, version, plugin classes, dependencies and SHA-256 digest; a wheel requiring a package that DataLab does not provide is refused * Installed plugins are listed in the same tab, where they can be uninstalled; a newly installed plugin is enabled and loaded after a plugin reload, and a new version of an already loaded plugin is used at the next start * The new **Available plugins** tab lists the plugins of the [DataLab plugin catalog](https://datalab-platform.com/plugins/), official or from the community, with a search field: **Install** and **Update** download the wheel, check it against the catalog and install it after confirmation. The catalog address may be changed to use a private copy +* Organizations can distribute their own plugins, or the versions of public plugins they have approved, from local plugin catalogs: folders, typically shared on the network, declared in the **Plugin settings** tab or with the `DATALAB_PLUGINS_CATALOG` environment variable. Their plugins are listed in the **Available plugins** tab before those of the DataLab plugin catalog, and hide them when both offer the same plugin +* A local catalog folder may simply hold plugin wheels, listed as they are copied to it, or a `catalog.json` file giving display names, deprecations and withdrawn releases; `datalab-plugin catalog build` writes this file from the wheels of the folder, keeping the edited fields, and `datalab-plugin catalog check` reports wheels that are missing, changed, refused by DataLab or not listed +* A catalog that cannot be read, such as a network share that is not connected or the online catalog on a computer without Internet access, is reported in the **Available plugins** tab without hiding the plugins of the other catalogs **Plugin project generator:**