diff --git a/README.md b/README.md index 74c6f2a..dee4a5d 100644 --- a/README.md +++ b/README.md @@ -20,7 +20,7 @@ Any author may list a plugin by opening a pull request that adds one small YAML ## How the catalog works - Each plugin is described by `plugins/.yaml`, validated against [schema/plugin-entry.schema.json](schema/plugin-entry.schema.json). -- For each release, the CI downloads the wheel from the GitHub release (or from PyPI), checks its SHA-256 digest against the entry, and inspects it without running its code, with the same rules as DataLab: pure-Python wheel, `datalab.plugins` (desktop) and/or `datalab.web_plugins` (web) entry points, dependencies provided by the latest DataLab release, OSI-approved license. +- For each release, the CI downloads the wheel from the GitHub release (or from PyPI), checks its SHA-256 digest against the entry, and inspects it without running its code, with the same rules as DataLab: pure-Python wheel, `datalab.plugins` (desktop) and/or `datalab.web_plugins` (web) entry points, dependencies provided by the latest DataLab release, OSI-approved license. Python compatibility is checked per target: a desktop wheel must install on at least one Python version supported by DataLab (3.9 to 3.14), a web wheel on the Python of the Pyodide release used by DataLab-Web (3.12). - When a pull request is merged, the catalog is rebuilt and published: `catalog.json`, a copy of every wheel in `wheels//`, and a human-readable `index.html`. Wheels are downloaded again at each build and must still match the reviewed digests. - Plugins from DataLab-Platform repositories are marked **official**, others **community**. IDs starting with `org.datalab.` are reserved for official plugins. @@ -37,4 +37,4 @@ python tools/catalog.py check python tools/catalog.py build --output site ``` -[tools/wheels.py](tools/wheels.py) is a verbatim copy of `datalab/plugins/wheels.py` from DataLab: update both together. +[tools/wheels.py](tools/wheels.py) is a verbatim copy of `datalab/plugins/wheels.py` from DataLab: update both together. A test compares them when a sibling DataLab checkout is available. Update `TARGET_PYTHONS` in [tools/catalog.py](tools/catalog.py) when DataLab supports a new Python version or DataLab-Web moves to another Pyodide release. diff --git a/tests/test_catalog.py b/tests/test_catalog.py index c5efe59..6ca6c53 100644 --- a/tests/test_catalog.py +++ b/tests/test_catalog.py @@ -30,6 +30,7 @@ def make_wheel( entry_points: str = DESKTOP_AND_WEB, license_expression: str = "BSD-3-Clause", requires: tuple[str, ...] = ("datalab-platform>=1.4", "numpy>=1.22"), + requires_python: str = ">=3.9", ) -> tuple[str, bytes]: """Return the file name and content of a plugin wheel.""" stem = f"{distribution.replace('-', '_')}-{version}" @@ -39,7 +40,7 @@ def make_wheel( f"Version: {version}", "Summary: Example plugin", f"License-Expression: {license_expression}", - "Requires-Python: >=3.9", + f"Requires-Python: {requires_python}", *(f"Requires-Dist: {requirement}" for requirement in requires), ] buffer = io.BytesIO() @@ -331,3 +332,48 @@ def test_wheels_must_install_on_the_latest_datalab( ) with pytest.raises(catalog.CatalogError, match="already listed"): catalog.build_catalog(entries, HOST) + + +@pytest.mark.parametrize( + ("requires_python", "error"), + [ + (">=3.12", None), + (">=3.13", r"\(web, Python 3.12\)"), + ("<3.9", r"\(desktop, Python 3.9, 3.10, .*3.14\)"), + ], +) +def test_targets_are_checked_with_their_python_versions( + served: dict, tmp_path: Path, requires_python: str, error: str | None +) -> None: + """Wheels are judged by the Python of each target, not by the CI interpreter.""" + repository = "https://github.com/someone/example-plugin" + wheel = make_wheel(requires_python=requires_python) + digest = publish_on_github(served, repository, "v1.0.0", wheel) + entries = tmp_path / "plugins" + write_entry( + entries, + "io.github.someone.example", + f"id: io.github.someone.example\nname: Example\nrepository: {repository}\n" + f"releases:\n - version: 1.0.0\n sha256: {digest}\n", + ) + + if error is None: + (plugin,) = catalog.build_catalog(entries, HOST)["plugins"] + assert plugin["releases"][0]["targets"] == ["desktop", "web"] + assert plugin["releases"][0]["requires_python"] == requires_python + else: + with pytest.raises(catalog.CatalogError, match=error): + catalog.build_catalog(entries, HOST) + + +def test_wheels_module_is_a_verbatim_copy_of_datalab() -> None: + """The catalog must judge a wheel exactly as DataLab and DataLab-Web do.""" + root = Path(__file__).resolve().parents[1] + upstream = root.parent / "DataLab" / "datalab" / "plugins" / "wheels.py" + if not upstream.is_file(): + pytest.skip("no sibling DataLab checkout") + + def read(path: Path) -> str: + return path.read_text(encoding="utf-8").replace("\r\n", "\n") + + assert read(root / "tools" / "wheels.py") == read(upstream) diff --git a/tools/catalog.py b/tools/catalog.py index 12d6901..bf4d557 100644 --- a/tools/catalog.py +++ b/tools/catalog.py @@ -69,6 +69,12 @@ ("desktop", wheels.DESKTOP_ENTRY_POINT_GROUP), ("web", wheels.WEB_ENTRY_POINT_GROUP), ) +#: Python versions of each target: a wheel must install on at least one of them +TARGET_PYTHONS = { + "desktop": ("3.9", "3.10", "3.11", "3.12", "3.13", "3.14"), + # Python of the Pyodide release used by DataLab-Web (Pyodide 0.26) + "web": ("3.12",), +} #: OSI-approved licenses accepted in the catalog (SPDX identifiers) ALLOWED_LICENSES = frozenset( { @@ -235,6 +241,29 @@ def _check_license(metadata: Message, label: str) -> str: raise CatalogError(f"{label}: the wheel declares no License-Expression metadata") +def _inspect_for_target( + data: bytes, filename: str, target: str, group: str, available: Mapping[str, str] +) -> dict: + """Inspect a wheel with the first Python version of a target that accepts it.""" + pythons = TARGET_PYTHONS[target] + for python_version in pythons: + try: + return wheels.inspect_wheel( + data, + filename=filename, + entry_point_group=group, + available_distributions=available, + python_version=python_version, + marker_environment={ + "python_version": python_version, + "python_full_version": f"{python_version}.0", + }, + ) + except wheels.WheelInspectionError as exc: + error = exc + raise CatalogError(f"{error} ({target}, Python {', '.join(pythons)})") + + def check_release( entry: dict, release: dict, available: Mapping[str, str] ) -> tuple[dict, dict, bytes]: @@ -257,13 +286,10 @@ def check_release( try: for target, group in TARGET_GROUPS: if group in groups: - manifests[target] = wheels.inspect_wheel( - data, - filename=filename, - entry_point_group=group, - available_distributions=available, + manifests[target] = _inspect_for_target( + data, filename, target, group, available ) - except wheels.WheelInspectionError as exc: + except CatalogError as exc: raise CatalogError(f"{label}: {exc}") from exc if not manifests: raise CatalogError( diff --git a/tools/wheels.py b/tools/wheels.py index 90421eb..621206c 100644 --- a/tools/wheels.py +++ b/tools/wheels.py @@ -27,6 +27,7 @@ from packaging.markers import default_environment from packaging.requirements import InvalidRequirement, Requirement from packaging.specifiers import InvalidSpecifier, SpecifierSet +from packaging.tags import Tag, compatible_tags, interpreter_name from packaging.utils import canonicalize_name, parse_wheel_filename from packaging.version import InvalidVersion, Version @@ -72,6 +73,40 @@ class WheelInspectionError(ValueError): """Raised when an archive cannot be accepted as a plugin wheel.""" +def _host_python(python_version: str | None) -> Version: + return Version( + python_version or f"{sys.version_info.major}.{sys.version_info.minor}" + ) + + +def _pure_python_tags(python: Version) -> frozenset[Tag]: + """Return the pure-Python wheel tags a Python version installs, as pip does.""" + interpreter = f"{interpreter_name()}{python.major}{python.minor}" + return frozenset( + compatible_tags((python.major, python.minor), interpreter, ["any"]) + ) + + +def is_compatible_wheel( + filename: str, requires_python: str | None, python_version: str | None = None +) -> bool: + """Return True if a Python version can install a pure-Python wheel. + + Args: + filename: Wheel file name, which carries its tags + requires_python: ``Requires-Python`` metadata of the wheel, if any + python_version: Host Python version (default: running interpreter) + """ + try: + wheel_tags = parse_wheel_filename(filename)[3] + python = _host_python(python_version) + if requires_python and python not in SpecifierSet(requires_python): + return False + except (InvalidSpecifier, InvalidVersion, ValueError): + return False + return bool(wheel_tags & _pure_python_tags(python)) + + def _is_stdlib_module(name: str) -> bool: stdlib_names = getattr(sys, "stdlib_module_names", None) if stdlib_names is not None: @@ -310,6 +345,13 @@ def inspect_wheel( raise WheelInspectionError(f"Invalid wheel filename: {filename!r}") from exc if any(tag.abi != "none" or tag.platform != "any" for tag in wheel_tags): raise WheelInspectionError("Only pure-Python *-none-any wheels are supported") + current_python = _host_python(python_version) + if not wheel_tags & _pure_python_tags(current_python): + interpreters = ".".join(sorted({tag.interpreter for tag in wheel_tags})) + raise WheelInspectionError( + f"Plugin wheel is built for {interpreters}, " + f"host has Python {current_python}" + ) try: archive = zipfile.ZipFile(io.BytesIO(data)) @@ -350,9 +392,6 @@ def inspect_wheel( raise WheelInspectionError("Wheel filename and METADATA version differ") required_python = package_metadata.get("Requires-Python") - current_python = Version( - python_version or f"{sys.version_info.major}.{sys.version_info.minor}" - ) if required_python: try: specifier = SpecifierSet(required_python) @@ -401,4 +440,5 @@ def inspect_wheel( "WEB_ENTRY_POINT_GROUP", "WheelInspectionError", "inspect_wheel", + "is_compatible_wheel", ]