Skip to content

Commit ee4c6a0

Browse files
Merge pull request #412 from DataLab-Platform/fix/411-plugin-folders
Load plugin packages from their folder without installing them
2 parents 2d15b01 + 2398f9b commit ee4c6a0

8 files changed

Lines changed: 611 additions & 80 deletions

File tree

‎datalab/plugins/base.py‎

Lines changed: 133 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,11 @@
7373
PLUGIN_ENTRY_POINT_GROUP = DESKTOP_ENTRY_POINT_GROUP
7474
#: Plugins installed from a file (kept out of the plugin search path)
7575
INSTALLED_PLUGINS_PATH = get_config_path("installed_plugins")
76+
#: Module of a ``datalab_*`` package defining its plugin class, as laid out by
77+
#: ``datalab-plugin create``: the package root stays free of Qt
78+
PACKAGE_DESKTOP_MODULE = "adapters.desktop"
79+
#: Subfolder of a plugin project holding its ``datalab_*`` packages
80+
PROJECT_SOURCE_FOLDER = "src"
7681

7782
if not osp.isdir(PLUGINS_DEFAULT_PATH):
7883
os.makedirs(PLUGINS_DEFAULT_PATH)
@@ -1092,6 +1097,86 @@ def _add_plugin_discovery_source(plugin_class: type[PluginBase], source: str) ->
10921097
plugin_class.__plugin_discovery_sources__ = tuple(dict.fromkeys((*sources, source)))
10931098

10941099

1100+
def _holds_plugin_modules(directory: str) -> bool:
1101+
"""Return True if a directory holds ``datalab_*`` modules or packages."""
1102+
prefix = f"{MOD_NAME}_"
1103+
try:
1104+
with os.scandir(directory) as entries:
1105+
for entry in entries:
1106+
if not entry.name.startswith(prefix):
1107+
continue
1108+
if entry.is_file() and entry.name.endswith(".py"):
1109+
return True
1110+
if entry.is_dir() and osp.isfile(osp.join(entry.path, "__init__.py")):
1111+
return True
1112+
except OSError:
1113+
return False
1114+
return False
1115+
1116+
1117+
def _get_plugin_search_paths() -> list[str]:
1118+
"""Return the directories searched for plugins, in discovery order.
1119+
1120+
The ``src`` subfolder of a plugin project is searched too when it holds
1121+
``datalab_*`` modules or packages, so that users may simply add the project
1122+
folder.
1123+
"""
1124+
paths: list[str] = []
1125+
for path in (
1126+
get_user_plugin_paths() + [PLUGINS_DEFAULT_PATH] + OTHER_PLUGINS_PATHLIST
1127+
):
1128+
rpath = osp.realpath(path)
1129+
source_path = osp.join(rpath, PROJECT_SOURCE_FOLDER)
1130+
candidates = [rpath]
1131+
if _holds_plugin_modules(source_path):
1132+
candidates.append(source_path)
1133+
paths.extend(candidate for candidate in candidates if candidate not in paths)
1134+
return paths
1135+
1136+
1137+
def _extend_sys_path_with_plugin_paths() -> None:
1138+
"""Append the plugin search paths missing from ``sys.path``."""
1139+
for path in _get_plugin_search_paths():
1140+
if path not in sys.path:
1141+
sys.path.append(path)
1142+
1143+
1144+
def _reload_once(
1145+
name: str, reloadable_modules: Collection[str], reloaded_modules: set[str]
1146+
) -> ModuleType | None:
1147+
"""Reload a module imported before the discovery pass, at most once per pass.
1148+
1149+
Reloading takes code changes into account (hot reload). A module shared by
1150+
several entry points, or found both through an entry point and by naming
1151+
convention, is reloaded only once.
1152+
1153+
Args:
1154+
name: Module name
1155+
reloadable_modules: Modules imported before the discovery pass
1156+
reloaded_modules: Modules already reloaded during the pass (updated)
1157+
1158+
Returns:
1159+
The reloaded module, or None if the module was not imported before the
1160+
pass or was already reloaded
1161+
"""
1162+
module = sys.modules.get(name)
1163+
if module is None or name not in reloadable_modules or name in reloaded_modules:
1164+
return None
1165+
reloaded_modules.add(name)
1166+
return importlib.reload(module)
1167+
1168+
1169+
def _find_package_desktop_module(package_name: str) -> str | None:
1170+
"""Return the full name of the Desktop module of a package, if it has one."""
1171+
module_name = package_name
1172+
for part in PACKAGE_DESKTOP_MODULE.split("."):
1173+
spec = importlib.util.find_spec(module_name)
1174+
if spec is None or spec.submodule_search_locations is None:
1175+
return None
1176+
module_name = f"{module_name}.{part}"
1177+
return module_name if importlib.util.find_spec(module_name) else None
1178+
1179+
10951180
def _get_plugin_entry_points() -> list[importlib_metadata.EntryPoint]:
10961181
"""Return installed DataLab plugin entry points in deterministic order."""
10971182
entry_points = importlib_metadata.entry_points()
@@ -1121,8 +1206,15 @@ def _record_plugin_discovery_failure(
11211206
PluginRegistry.add_failed_plugin(name, filepath, tb_text, source)
11221207

11231208

1124-
def _discover_entry_point_plugins() -> list[ModuleType]:
1125-
"""Load and register plugin classes declared through package entry points."""
1209+
def _discover_entry_point_plugins(
1210+
reloadable_modules: Collection[str], reloaded_modules: set[str]
1211+
) -> list[ModuleType]:
1212+
"""Load and register plugin classes declared through package entry points.
1213+
1214+
Args:
1215+
reloadable_modules: Modules imported before the discovery pass
1216+
reloaded_modules: Modules already reloaded during the pass (updated)
1217+
"""
11261218
try:
11271219
entry_points = _get_plugin_entry_points()
11281220
# Installed distribution metadata is external input. A malformed package
@@ -1135,21 +1227,15 @@ def _discover_entry_point_plugins() -> list[ModuleType]:
11351227
return []
11361228

11371229
discovered_modules: list[ModuleType] = []
1138-
reloadable_modules = set(sys.modules)
1139-
reloaded_modules: set[str] = set()
11401230
for entry_point in entry_points:
11411231
source = f"entry point {entry_point.name!r} ({entry_point.value})"
11421232
plugin_classes = PluginRegistry.get_plugin_classes()
11431233
previous_classes = list(plugin_classes)
11441234
try:
11451235
try:
11461236
module_name = getattr(entry_point, "module", None)
1147-
if (
1148-
module_name in reloadable_modules
1149-
and module_name not in reloaded_modules
1150-
):
1151-
importlib.reload(sys.modules[module_name])
1152-
reloaded_modules.add(module_name)
1237+
if module_name:
1238+
_reload_once(module_name, reloadable_modules, reloaded_modules)
11531239
plugin_class = entry_point.load()
11541240
finally:
11551241
# Importing the target module may invoke PluginBaseMeta. The entry
@@ -1262,7 +1348,9 @@ def discover_plugins() -> list[ModuleType]:
12621348
12631349
Installed packages may expose a :class:`PluginBase` subclass through the
12641350
``datalab.plugins`` entry-point group. This function also reloads or imports
1265-
modules matching the historical ``"{MOD_NAME}_*"`` naming scheme.
1351+
modules and packages matching the historical ``"{MOD_NAME}_*"`` naming scheme,
1352+
as well as the ``adapters.desktop`` module of such packages, which defines
1353+
the plugin class of packages laid out by ``datalab-plugin create``.
12661354
12671355
Import errors for individual plugins are captured and logged so that
12681356
one broken plugin does not prevent the others from loading. Error
@@ -1282,13 +1370,7 @@ def discover_plugins() -> list[ModuleType]:
12821370
if not Conf.plugins_enabled.get():
12831371
return []
12841372

1285-
# Ensure plugin search paths are present in sys.path
1286-
for path in (
1287-
get_user_plugin_paths() + [PLUGINS_DEFAULT_PATH] + OTHER_PLUGINS_PATHLIST
1288-
):
1289-
rpath = osp.realpath(path)
1290-
if rpath not in sys.path:
1291-
sys.path.append(rpath)
1373+
_extend_sys_path_with_plugin_paths()
12921374
try:
12931375
get_installed_plugin_store().activate()
12941376
except (OSError, ValueError):
@@ -1299,52 +1381,67 @@ def discover_plugins() -> list[ModuleType]:
12991381
INSTALLED_PLUGINS_PATH,
13001382
)
13011383

1302-
modules = _discover_entry_point_plugins()
1303-
for finder, name, _ispkg in pkgutil.iter_modules():
1384+
reloadable_modules = set(sys.modules)
1385+
reloaded_modules: set[str] = set()
1386+
modules = _discover_entry_point_plugins(reloadable_modules, reloaded_modules)
1387+
for finder, name, ispkg in pkgutil.iter_modules():
13041388
if not name.startswith(f"{MOD_NAME}_"):
13051389
continue
1390+
source = f"module convention {name!r}"
1391+
module_name = name
13061392
try:
13071393
previous_classes = list(PluginRegistry.get_plugin_classes())
13081394
# If module is already loaded, reload it so that code changes
13091395
# are taken into account (useful for hot-reload in dev).
1310-
if name in sys.modules:
1311-
module = importlib.reload(sys.modules[name])
1312-
else:
1313-
module = importlib.import_module(name)
1314-
source = f"module convention {name!r}"
1396+
loaded_modules = [
1397+
_reload_once(name, reloadable_modules, reloaded_modules)
1398+
or importlib.import_module(name)
1399+
]
1400+
desktop_module_name = _find_package_desktop_module(name) if ispkg else None
1401+
if desktop_module_name is not None:
1402+
module_name = desktop_module_name
1403+
loaded_modules.append(
1404+
_reload_once(module_name, reloadable_modules, reloaded_modules)
1405+
or importlib.import_module(module_name)
1406+
)
1407+
loaded_names = {module.__name__ for module in loaded_modules}
13151408
for plugin_class in PluginRegistry.get_plugin_classes():
1316-
if plugin_class not in previous_classes:
1409+
# Classes already registered by an entry point in this pass are
1410+
# also found by convention, without executing their module again
1411+
if (
1412+
plugin_class not in previous_classes
1413+
or plugin_class.__module__ in loaded_names
1414+
):
13171415
_add_plugin_discovery_source(plugin_class, source)
1318-
_set_plugin_class_filepaths(module)
1319-
if module not in modules:
1320-
modules.append(module)
1416+
for module in loaded_modules:
1417+
_set_plugin_class_filepaths(module)
1418+
if module not in modules:
1419+
modules.append(module)
13211420
# Plugin discovery imports arbitrary third-party modules. We must catch
13221421
# every failure here so discovery can continue and the error is exposed
13231422
# through the console, log files, and plugin configuration dialog.
13241423
except Exception as e: # pylint: disable=broad-except
13251424
tb_text = traceback.format_exc()
1326-
print(f"Error loading plugin '{name}': {e}")
1425+
print(f"Error loading plugin '{module_name}': {e}")
13271426
traceback.print_exc()
13281427
# Log to file so it appears in Log Files viewer
13291428
logger = logging.getLogger(__name__)
1330-
logger.error("Error loading plugin '%s'", name, exc_info=True)
1429+
logger.error("Error loading plugin '%s'", module_name, exc_info=True)
13311430
Conf.traceback_log_available.set(True)
13321431
# Accumulate for replay in internal console
13331432
PluginRegistry.add_discovery_error(tb_text)
13341433
# Record structured info about the failed plugin
13351434
filepath = ""
13361435
try:
1337-
spec = importlib.util.find_spec(name)
1436+
spec = importlib.util.find_spec(module_name)
13381437
if spec and spec.origin:
13391438
filepath = spec.origin
13401439
# Best effort only: failing to resolve the file path must never mask
13411440
# the original plugin import error already captured above.
13421441
except Exception: # pylint: disable=broad-except
13431442
if hasattr(finder, "path"):
13441443
filepath = osp.join(finder.path, name)
1345-
PluginRegistry.add_failed_plugin(
1346-
name, filepath, tb_text, f"module convention {name!r}"
1347-
)
1444+
PluginRegistry.add_failed_plugin(module_name, filepath, tb_text, source)
13481445
_normalize_discovered_plugin_classes()
13491446
return modules
13501447

@@ -1381,12 +1478,7 @@ def discover_v020_plugins() -> list[tuple[str, str]]:
13811478
"""
13821479
v020_plugins = []
13831480
if Conf.plugins_enabled.get():
1384-
for path in (
1385-
get_user_plugin_paths() + [PLUGINS_DEFAULT_PATH] + OTHER_PLUGINS_PATHLIST
1386-
):
1387-
rpath = osp.realpath(path)
1388-
if rpath not in sys.path:
1389-
sys.path.append(rpath)
1481+
_extend_sys_path_with_plugin_paths()
13901482
for finder, name, _ispkg in pkgutil.iter_modules():
13911483
if name.startswith("cdl_"):
13921484
# Get the directory path from the module finder

‎datalab/plugins/generator.py‎

Lines changed: 17 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -408,6 +408,12 @@ def _render_readme(project: PluginProject) -> str:
408408
409409
{project.description}
410410
411+
## Trying the plugin in DataLab
412+
413+
Add this project folder in **Plugins > Configure plugins... > Plugin settings**,
414+
then reload the plugins: DataLab loads the plugin from
415+
`src/{project.package}/adapters/desktop.py`, without installing the project.
416+
411417
## Development
412418
413419
```bash
@@ -416,11 +422,11 @@ def _render_readme(project: PluginProject) -> str:
416422
python -m ruff check .
417423
```
418424
419-
Installing the project registers `{project.plugin_id}` through the
420-
`datalab.plugins` entry-point group. Put host-independent algorithms in `core`,
421-
compose them into headless recipes in `workflow`, and keep DataLab or browser
422-
integration in `adapters`. The generated architecture test preserves these
423-
dependency boundaries as the plugin grows.
425+
Installing the project also registers `{project.plugin_id}` through the
426+
`datalab.plugins` entry-point group, as the published wheel does. Put
427+
host-independent algorithms in `core`, compose them into headless recipes in
428+
`workflow`, and keep DataLab or browser integration in `adapters`. The generated
429+
architecture test preserves these dependency boundaries as the plugin grows.
424430
425431
## Publishing
426432
@@ -514,7 +520,8 @@ def _render_architecture_doc(project: PluginProject) -> str:
514520
515521
- `core` owns host-independent domain behavior and does not import DataLab.
516522
- `workflow` may use DataLab's headless recipe contracts, but not GUI modules.
517-
- `adapters/desktop.py` is the installed DataLab plugin entry point.
523+
- `adapters/desktop.py` defines the DataLab plugin class: DataLab loads it from
524+
the project folder, and the installed entry point targets it.
518525
- `adapters/web.py` records Web support explicitly and starts as unsupported.
519526
520527
The package root exposes identity metadata without importing a host adapter.
@@ -526,6 +533,10 @@ def _render_contributing() -> str:
526533
"""Render concise development and architecture rules."""
527534
return """# Contributing
528535
536+
To try a change in DataLab, add the project folder in **Plugins > Configure
537+
plugins... > Plugin settings** and reload the plugins: no installation is
538+
needed.
539+
529540
Install the project in editable mode and run its checks before submitting a
530541
change:
531542

0 commit comments

Comments
 (0)