diff --git a/guide/src/type-stub.md b/guide/src/type-stub.md index 0a1ed38723f..69b3c43ac9c 100644 --- a/guide/src/type-stub.md +++ b/guide/src/type-stub.md @@ -55,6 +55,8 @@ will generate the following stub file: ```python import typing +__all__ = ["CONSTANT", "Class", "list_of_int_identity"] + CONSTANT: typing.Final = "FOO" @@ -92,5 +94,6 @@ PyO3 also provides the smaller `pyo3-introspection` binary that allows to genera If you commit generated stubs to your repository, generate them with the oldest Python version you support (or with the matching `abi3-pyXY` feature) so that they are valid for every version. - PyO3 is not able to introspect the content of `#[pymodule]` and `#[pymodule_init]` functions. If they are present, the module is tagged as incomplete using a fake `def __getattr__(name: str) -> Incomplete: ...` function [following best practices](https://typing.python.org/en/latest/guides/writing_stubs.html#incomplete-stubs). - A `#[pymodule_init]` function [declared without the module argument](module.md#procedural-initialization) is exempt: it is not handed the module, so the module is taken to be complete. + Such a module also gets no `__all__` declaration, because the set of names it exports at runtime is not known. + A `#[pymodule_init]` function [declared without the module argument](module.md#procedural-initialization) is exempt: it is not handed the module, so the module is taken to be complete and keeps its `__all__`. A `Python<'_>` marker on its own does not count as the module argument. diff --git a/newsfragments/6242.fixed.md b/newsfragments/6242.fixed.md new file mode 100644 index 00000000000..d2d2c8cf3a8 --- /dev/null +++ b/newsfragments/6242.fixed.md @@ -0,0 +1,2 @@ +`experimental-inspect`: generate the `__all__` declaration of complete modules in the type stubs, matching the `__all__` that `#[pymodule]` builds at runtime. +It is placed directly after the imports, following the conventions used by `typeshed`, and lists the members in the order the stub declares them. diff --git a/noxfile.py b/noxfile.py index a0aaf4505c2..20982c3a462 100644 --- a/noxfile.py +++ b/noxfile.py @@ -1766,6 +1766,8 @@ def test_introspection(session: nox.Session): with tempfile.TemporaryDirectory() as stub_dir: session.install("maturin") session.install("ruff") + # `pytest-benchmark` backs the `--benchmark-disable` in `pytests/pyproject.toml` + session.install("pytest", "pytest-benchmark") options = [] target = os.environ.get("CARGO_BUILD_TARGET") if target is not None: @@ -1801,6 +1803,9 @@ def test_introspection(session: nox.Session): ) _run(session, "ruff", "format", stub_dir) _ensure_directory_equals(Path(stub_dir), Path("pytests/stubs")) + # The stubs only match a module built with the features used above, so this test + # is skipped by the regular `pytests` session and runs here instead. + _run(session, "pytest", "pytests/tests/test_stubs.py") def _ensure_directory_equals(expected_dir: Path, actual_dir: Path): diff --git a/pyo3-introspection/src/stubs.rs b/pyo3-introspection/src/stubs.rs index 35491a17375..a0dac641baa 100644 --- a/pyo3-introspection/src/stubs.rs +++ b/pyo3-introspection/src/stubs.rs @@ -95,6 +95,8 @@ fn module_stubs(module: &Module, parents: &[&str]) -> String { )); } + let dunder_all = dunder_all_stubs(module, &imports); + let mut final_elements = Vec::new(); if let Some(docstring) = &module.docstring { let mut buffer = String::new(); @@ -102,19 +104,21 @@ fn module_stubs(module: &Module, parents: &[&str]) -> String { final_elements.push(buffer); } final_elements.extend(imports.imports); + final_elements.extend(dunder_all); final_elements.extend(elements); let mut output = String::new(); - // We insert two line jumps (i.e. empty strings) only above and below multiple line elements (classes with methods, functions with decorators) + // We insert two line jumps (i.e. empty strings) only above and below multiple line elements + // (classes with methods, functions with decorators) and the `__all__` declaration for element in final_elements { - let is_multiline = element.contains('\n'); - if is_multiline && !output.is_empty() && !output.ends_with("\n\n") { + let needs_empty_lines = element.contains('\n') || element.starts_with("__all__"); + if needs_empty_lines && !output.is_empty() && !output.ends_with("\n\n") { output.push('\n'); } output.push_str(&element); output.push('\n'); - if is_multiline { + if needs_empty_lines { output.push('\n'); } } @@ -126,6 +130,49 @@ fn module_stubs(module: &Module, parents: &[&str]) -> String { output } +/// Generates the `__all__` declaration of a module, if we are able to write an accurate one. +/// +/// `PyModuleMethods::add` appends every name it adds to the module `__all__`, so any `#[pymodule]` +/// with at least one member has one at runtime. Incomplete modules are skipped: we do not know all +/// of their members, and a partial `__all__` would hide names that do exist at runtime. +fn dunder_all_stubs(module: &Module, imports: &Imports) -> Option { + if module.incomplete { + return None; + } + if module.attributes.iter().any(|a| a.name == "__all__") { + // The introspection data carries an explicit `__all__`, it is more accurate than ours + return None; + } + let member_count = module.attributes.len() + + module.classes.len() + + module.functions.len() + + module.modules.len(); + if member_count == 0 { + // Nothing was ever added to the module, so it has no `__all__` at runtime either + return None; + } + + // Each of these lists is already sorted by name, so listing them in the order the stub declares + // them keeps the output stable. + let mut elts = Vec::with_capacity(member_count); + elts.extend( + module + .attributes + .iter() + .map(|a| &a.name) + .chain(module.classes.iter().map(|c| &c.name)) + .chain(module.functions.iter().map(|f| &f.name)) + .chain(module.modules.iter().map(|m| &m.name)) + .map(|name| Expr::Constant { + value: Constant::Str(name.clone()), + }), + ); + + let mut buffer = "__all__ = ".to_string(); + imports.serialize_expr(&Expr::List { elts }, &mut buffer); + Some(buffer) +} + fn class_stubs(class: &Class, imports: &Imports) -> String { let mut buffer = String::new(); for decorator in &class.decorators { @@ -992,10 +1039,8 @@ mod tests { assert_eq!(output, "dict[A, (A3.C, A3.D, B, A2, int, int2, float)]"); } - /// A submodule is an attribute of its parent at runtime, so the parent stub must re-export it. - #[test] - fn submodules_are_re_exported_by_their_parent() { - let submodule = |name: &str| Module { + fn empty_module(name: &str) -> Module { + Module { name: name.into(), modules: Vec::new(), classes: Vec::new(), @@ -1003,10 +1048,15 @@ mod tests { attributes: Vec::new(), incomplete: false, docstring: None, - }; + } + } + + /// A submodule is an attribute of its parent at runtime, so the parent stub must re-export it. + #[test] + fn submodules_are_re_exported_by_their_parent() { let module = Module { name: "foo".into(), - modules: vec![submodule("zulu"), submodule("alpha")], + modules: vec![empty_module("zulu"), empty_module("alpha")], classes: Vec::new(), functions: Vec::new(), attributes: Vec::new(), @@ -1015,10 +1065,81 @@ mod tests { }; assert_eq!( module_stubs(&module, &[]), - "from . import alpha as alpha, zulu as zulu\n" + "from . import alpha as alpha, zulu as zulu\n\n__all__ = [\"zulu\", \"alpha\"]\n" ); // A module without submodules gets no such line. - assert_eq!(module_stubs(&submodule("alpha"), &["foo"]), ""); + assert_eq!(module_stubs(&empty_module("alpha"), &["foo"]), ""); + } + + /// A module with one member of every kind that lands in `__all__` at runtime. + fn populated_module() -> Module { + Module { + modules: vec![empty_module("sub")], + classes: vec![Class { + name: "Zulu".into(), + bases: Vec::new(), + methods: Vec::new(), + attributes: Vec::new(), + decorators: Vec::new(), + inner_classes: Vec::new(), + docstring: None, + }], + functions: vec![Function { + name: "func".into(), + decorators: Vec::new(), + arguments: Arguments { + positional_only_arguments: Vec::new(), + arguments: Vec::new(), + vararg: None, + keyword_only_arguments: Vec::new(), + kwarg: None, + }, + returns: None, + is_async: false, + docstring: None, + }], + attributes: vec![Attribute { + name: "CONST".into(), + value: Some(Expr::Constant { + value: Constant::Int("1".into()), + }), + annotation: None, + docstring: None, + }], + ..empty_module("bar") + } + } + + /// The `pytests` stubs cover the common cases, we only test the ones they don't have: + /// a module with a submodule, and an empty module. + #[test] + fn test_dunder_all() { + // The names are the ones `PyModuleMethods::add` puts in `__all__` at runtime, including + // the submodule which is declared in its own stub file. + assert_eq!( + module_stubs(&populated_module(), &["foo"]), + "from . import sub as sub\n\n__all__ = [\"CONST\", \"Zulu\", \"func\", \"sub\"]\n\nCONST = 1\nclass Zulu: ...\ndef func(): ...\n" + ); + // Nothing was added to an empty module, so it has no `__all__` at runtime either + assert_eq!(module_stubs(&empty_module("bar"), &["foo"]), ""); + } + + /// A `#[pymodule_init]` that is handed the module can add anything to it, so the module is + /// tagged incomplete and the members we know about are not the whole of `__all__`. An + /// initialiser declared without the module argument leaves the module complete and keeps its + /// `__all__`, which `test_dunder_all` and the `othermod` stubs of `pytests` cover. + #[test] + fn test_dunder_all_omitted_for_incomplete_module() { + let module = Module { + incomplete: true, + ..populated_module() + }; + let stubs = module_stubs(&module, &["foo"]); + assert!( + !stubs.contains("__all__"), + "an incomplete module must not declare `__all__`:\n{stubs}" + ); + assert!(stubs.contains("def __getattr__(name: str) -> Incomplete: ...\n")); } #[test] @@ -1269,7 +1390,7 @@ mod tests { }; assert_eq!( module_stubs(&module, &[]), - "from math import inf, nan\nX = inf\ndef func(a=nan): ...\n" + "from math import inf, nan\n\n__all__ = [\"X\", \"func\"]\n\nX = inf\ndef func(a=nan): ...\n" ); } @@ -1315,7 +1436,7 @@ mod tests { }; assert_eq!( module_stubs(&module, &[]), - "from builtins import int as int2\nclass int: ...\ndef make(a: int2) -> int: ...\n" + "from builtins import int as int2\n\n__all__ = [\"int\", \"make\"]\n\nclass int: ...\ndef make(a: int2) -> int: ...\n" ); } } diff --git a/pytests/noxfile.py b/pytests/noxfile.py index b518bf6b7ea..f528841b4fe 100644 --- a/pytests/noxfile.py +++ b/pytests/noxfile.py @@ -33,7 +33,10 @@ def try_install_binary(package: str, constraint: str): # hypothesis itself depends on PyO3 so newer Python versions may fail # to build try_install_binary("hypothesis", ">=6.156.1") - ignored_paths = [] + # The stubs are generated from a build with `experimental-async,experimental-inspect`, + # so only such a build exports the members they declare. The `test-introspection` + # session runs this test against one. + ignored_paths = ["tests/test_stubs.py"] if sys.version_info < (3, 10): # Match syntax is only available in Python >= 3.10 ignored_paths.append("tests/test_enums_match.py") diff --git a/pytests/src/othermod.rs b/pytests/src/othermod.rs index 797afa24cb0..6e28fd9ca17 100644 --- a/pytests/src/othermod.rs +++ b/pytests/src/othermod.rs @@ -38,7 +38,7 @@ pub mod othermod { pub const USIZE_MAX: usize = usize::MAX; // An initialiser without the module argument leaves the module complete for introspection: - // `stubs/othermod.pyi` has no `__getattr__` catch-all. + // `stubs/othermod.pyi` has no `__getattr__` catch-all and keeps its `__all__`. #[pymodule_init] fn init() {} } diff --git a/pytests/stubs/annotations.pyi b/pytests/stubs/annotations.pyi index b9059252602..4f2ef34e9b4 100644 --- a/pytests/stubs/annotations.pyi +++ b/pytests/stubs/annotations.pyi @@ -1,5 +1,7 @@ from .pyclasses import EmptyClass +__all__ = ["cross_module_imports", "with_custom_type_annotations"] + def cross_module_imports(_a: EmptyClass) -> None: ... def with_custom_type_annotations( a: "list[int]", *_args: "str", _b: "int | None" = None, **_kwargs: "bool" diff --git a/pytests/stubs/awaitable.pyi b/pytests/stubs/awaitable.pyi index fb5bc19d6c5..38e3ec7d612 100644 --- a/pytests/stubs/awaitable.pyi +++ b/pytests/stubs/awaitable.pyi @@ -1,5 +1,7 @@ from typing import Any, final +__all__ = ["FutureAwaitable", "IterAwaitable"] + @final class FutureAwaitable: def __await__(self, /) -> FutureAwaitable: ... diff --git a/pytests/stubs/buf_and_str.pyi b/pytests/stubs/buf_and_str.pyi index 393df0ceab4..3e507cf2c88 100644 --- a/pytests/stubs/buf_and_str.pyi +++ b/pytests/stubs/buf_and_str.pyi @@ -6,6 +6,14 @@ from _typeshed import SupportsGetItem from collections.abc import Buffer from typing import SupportsIndex, final +__all__ = [ + "BytesExtractor", + "map_byte_cow", + "map_byte_slice", + "map_byte_vec", + "return_memoryview", +] + @final class BytesExtractor: """ diff --git a/pytests/stubs/comparisons.pyi b/pytests/stubs/comparisons.pyi index ba34eb78387..fea7553a479 100644 --- a/pytests/stubs/comparisons.pyi +++ b/pytests/stubs/comparisons.pyi @@ -1,5 +1,15 @@ from typing import SupportsIndex, final +__all__ = [ + "Eq", + "EqDefaultNe", + "EqDerived", + "Ordered", + "OrderedDefaultNe", + "OrderedDerived", + "OrderedRichCmp", +] + @final class Eq: def __eq__(self, other: object, /) -> bool: ... diff --git a/pytests/stubs/consts.pyi b/pytests/stubs/consts.pyi index 695dda43165..26ff92d4a70 100644 --- a/pytests/stubs/consts.pyi +++ b/pytests/stubs/consts.pyi @@ -1,5 +1,7 @@ from typing import Final, final +__all__ = ["ESCAPING", "PI", "ClassWithConst"] + ESCAPING: Final = "S\0\x01\t\n\r\"'\\" """ We experiment with "escaping" diff --git a/pytests/stubs/datetime.pyi b/pytests/stubs/datetime.pyi index 0615d18afd0..dec3edcc8df 100644 --- a/pytests/stubs/datetime.pyi +++ b/pytests/stubs/datetime.pyi @@ -1,6 +1,25 @@ from datetime import date, datetime, time, timedelta, tzinfo from typing import SupportsFloat, SupportsIndex, final +__all__ = [ + "TzClass", + "date_from_timestamp", + "datetime_from_timestamp", + "get_date_tuple", + "get_datetime_tuple", + "get_datetime_tuple_fold", + "get_datetime_tzinfo", + "get_delta_tuple", + "get_time_tuple", + "get_time_tuple_fold", + "get_time_tzinfo", + "make_date", + "make_datetime", + "make_delta", + "make_time", + "time_with_fold", +] + @final class TzClass(tzinfo): def __new__(cls, /) -> TzClass: ... diff --git a/pytests/stubs/dict_iter.pyi b/pytests/stubs/dict_iter.pyi index ccf11b56119..9539308e376 100644 --- a/pytests/stubs/dict_iter.pyi +++ b/pytests/stubs/dict_iter.pyi @@ -1,5 +1,7 @@ from typing import SupportsIndex, final +__all__ = ["DictSize"] + @final class DictSize: def __new__(cls, /, expected: SupportsIndex) -> DictSize: ... diff --git a/pytests/stubs/enums.pyi b/pytests/stubs/enums.pyi index c4af59c23cc..237d534b2a5 100644 --- a/pytests/stubs/enums.pyi +++ b/pytests/stubs/enums.pyi @@ -1,6 +1,19 @@ from typing import Any, Final, SupportsFloat, SupportsIndex, final from typing_extensions import disjoint_base +__all__ = [ + "ComplexEnum", + "MixedComplexEnum", + "SimpleEnum", + "SimpleEnumWithoutDerive", + "SimpleTupleEnum", + "TupleEnum", + "do_complex_stuff", + "do_mixed_complex_stuff", + "do_simple_stuff", + "do_tuple_stuff", +] + @disjoint_base class ComplexEnum: @final diff --git a/pytests/stubs/misc.pyi b/pytests/stubs/misc.pyi index 797c6a74520..b984297ab40 100644 --- a/pytests/stubs/misc.pyi +++ b/pytests/stubs/misc.pyi @@ -1,5 +1,14 @@ from typing import Any +__all__ = [ + "accepts_bool", + "detach_during_finalization", + "get_item_and_run_callback", + "get_type_fully_qualified_name", + "hammer_attaching_in_thread", + "issue_219", +] + def accepts_bool(val: bool) -> bool: ... def detach_during_finalization() -> Any: ... def get_item_and_run_callback(dict: dict, callback: Any) -> None: ... diff --git a/pytests/stubs/objstore.pyi b/pytests/stubs/objstore.pyi index c8ee5a638cf..5b6af241b5c 100644 --- a/pytests/stubs/objstore.pyi +++ b/pytests/stubs/objstore.pyi @@ -1,5 +1,7 @@ from typing import Any, final +__all__ = ["ObjStore"] + @final class ObjStore: def __new__(cls, /) -> ObjStore: ... diff --git a/pytests/stubs/othermod.pyi b/pytests/stubs/othermod.pyi index be871bc3701..270b92e8664 100644 --- a/pytests/stubs/othermod.pyi +++ b/pytests/stubs/othermod.pyi @@ -1,5 +1,7 @@ from typing import Final, SupportsIndex, final +__all__ = ["USIZE_MAX", "USIZE_MIN", "ModClass", "double"] + USIZE_MAX: Final[int] USIZE_MIN: Final[int] diff --git a/pytests/stubs/path.pyi b/pytests/stubs/path.pyi index c73a08c842f..09117d41bcb 100644 --- a/pytests/stubs/path.pyi +++ b/pytests/stubs/path.pyi @@ -1,6 +1,8 @@ from os import PathLike from pathlib import Path +__all__ = ["make_path", "take_name_or_path", "take_pathbuf"] + def make_path() -> Path: ... def take_name_or_path(value: str | PathLike[str]) -> Path: ... def take_pathbuf(path: str | PathLike[str]) -> Path: ... diff --git a/pytests/stubs/pyclasses.pyi b/pytests/stubs/pyclasses.pyi index f34f7cd3bee..715582ea86a 100644 --- a/pytests/stubs/pyclasses.pyi +++ b/pytests/stubs/pyclasses.pyi @@ -3,6 +3,23 @@ from _typeshed import Incomplete from typing import Final, SupportsIndex, final from typing_extensions import disjoint_base +__all__ = [ + "AssertingBaseClass", + "ClassWithDecorators", + "ClassWithDict", + "ClassWithoutConstructor", + "EmptyClass", + "Number", + "PlainObject", + "PyClassIter", + "PyClassOptionAsyncIter", + "PyClassOptionIter", + "PyClassResultOptionIter", + "PyClassThreadIter", + "SubClassWithInit", + "map_a_class", +] + @disjoint_base class AssertingBaseClass: """ diff --git a/pytests/stubs/pyfunctions.pyi b/pytests/stubs/pyfunctions.pyi index f3eadc491fd..c699a50d885 100644 --- a/pytests/stubs/pyfunctions.pyi +++ b/pytests/stubs/pyfunctions.pyi @@ -1,5 +1,20 @@ from typing import Any, SupportsFloat, SupportsIndex +__all__ = [ + "args_kwargs", + "many_keyword_arguments", + "nested_empty_tuples", + "none", + "positional_only", + "renamed", + "simple", + "simple_args", + "simple_args_kwargs", + "simple_kwargs", + "with_async", + "with_typed_args", +] + def args_kwargs(*args, **kwargs) -> tuple[tuple, dict | None]: ... def many_keyword_arguments( *, diff --git a/pytests/stubs/sequence.pyi b/pytests/stubs/sequence.pyi index 5a52c88e286..32b9d44762d 100644 --- a/pytests/stubs/sequence.pyi +++ b/pytests/stubs/sequence.pyi @@ -1,6 +1,8 @@ from _typeshed import SupportsGetItem, SupportsLenAndGetItem from typing import SupportsIndex +__all__ = ["array_to_array_i32", "vec_to_vec_i32", "vec_to_vec_pystring"] + def array_to_array_i32(arr: SupportsLenAndGetItem[SupportsIndex]) -> list[int]: ... def vec_to_vec_i32(vec: SupportsGetItem[int, SupportsIndex]) -> list[int]: ... def vec_to_vec_pystring(vec: SupportsGetItem[int, str]) -> list[str]: ... diff --git a/pytests/stubs/subclassing.pyi b/pytests/stubs/subclassing.pyi index 5c17a929a41..389eb5b3705 100644 --- a/pytests/stubs/subclassing.pyi +++ b/pytests/stubs/subclassing.pyi @@ -1,6 +1,8 @@ from typing import final from typing_extensions import disjoint_base +__all__ = ["SubDict", "Subclass", "Subclassable"] + @final class SubDict(dict): def __new__(cls, /) -> SubDict: ... diff --git a/pytests/tests/test_stubs.py b/pytests/tests/test_stubs.py new file mode 100644 index 00000000000..4002d07e06b --- /dev/null +++ b/pytests/tests/test_stubs.py @@ -0,0 +1,37 @@ +import ast +from pathlib import Path + +import pyo3_pytests +import pytest + +STUBS_DIR = Path(__file__).parent.parent / "stubs" + + +def _stub_dunder_all(path: Path): + """The `__all__` a stub file declares, or `None` if it declares none.""" + for node in ast.parse(path.read_text()).body: + if isinstance(node, ast.Assign) and any( + isinstance(target, ast.Name) and target.id == "__all__" + for target in node.targets + ): + return ast.literal_eval(node.value) + return None + + +@pytest.mark.parametrize( + "stub_file", sorted(STUBS_DIR.glob("*.pyi")), ids=lambda path: path.name +) +def test_stub_dunder_all_matches_runtime(stub_file: Path): + """The `__all__` in the stubs must name what the module exports at import time. + + Sorted, because the stubs declare the members in a different order than + `PyModuleMethods::add` appends them in, and `__all__` is only consumed as a set of names. + """ + stub_all = _stub_dunder_all(stub_file) + if stub_all is None: + pytest.skip("incomplete modules get no `__all__`, see `guide/src/type-stub.md`") + root = pyo3_pytests.pyo3_pytests + module = root if stub_file.name == "__init__.pyi" else getattr(root, stub_file.stem) + runtime_all = module.__all__ + assert sorted(stub_all) == sorted(runtime_all) + assert len(stub_all) == len(set(stub_all)), "`__all__` has duplicates"