diff --git a/Include/internal/pycore_import.h b/Include/internal/pycore_import.h index a1078828afa572e..cec63dba2ee30e4 100644 --- a/Include/internal/pycore_import.h +++ b/Include/internal/pycore_import.h @@ -39,8 +39,10 @@ extern PyObject * _PyImport_GetAbsName( // Symbol is exported for the JIT on Windows builds. PyAPI_FUNC(PyObject *) _PyImport_LoadLazyImportTstate( PyThreadState *tstate, PyObject *lazy_import); -extern PyObject * _PyImport_TryLoadLazySubmodule( - PyObject *mod_name, PyObject *attr_name); +// Returns 1 with a new reference in result, 0 if not pending, or -1 on error. +extern int _PyImport_TryLoadLazySubmodule( + PyObject *mod_name, PyObject *attr_name, PyObject *mod_dict, + PyObject **result); extern PyObject * _PyImport_LazyImportModuleLevelObject( PyThreadState *tstate, PyObject *name, PyObject *builtins, PyObject *globals, PyObject *locals, PyObject *fromlist, int level); diff --git a/Include/internal/pycore_lazyimportobject.h b/Include/internal/pycore_lazyimportobject.h index b81e4211b08ff39..c74b03d3e02af82 100644 --- a/Include/internal/pycore_lazyimportobject.h +++ b/Include/internal/pycore_lazyimportobject.h @@ -19,6 +19,10 @@ typedef struct { PyObject *lz_builtins; PyObject *lz_from; PyObject *lz_attr; + PyObject *lz_globals; + PyObject *lz_key; + PyObject *lz_resolved; // Protected by the lazy object lock. + Py_hash_t lz_key_hash; // Frame information for the original import location. PyCodeObject *lz_code; // Code object where the lazy import was created. int lz_instr_offset; // Instruction offset where the lazy import was created. @@ -26,6 +30,14 @@ typedef struct { PyAPI_FUNC(PyObject *) _PyLazyImport_GetName(PyObject *lazy_import); +PyAPI_FUNC(PyObject *) _PyLazyImport_GetResolved(PyObject *lazy_import); +PyAPI_FUNC(int) _PyLazyImport_FinishResolve( + PyObject *lazy_import, PyObject *resolved); +PyAPI_FUNC(int) _PyLazyImport_ReplaceDictItemIfCurrent( + PyObject *lazy_import, PyObject *dict, PyObject *name, + PyObject *resolved); +PyAPI_FUNC(int) _PyLazyImport_SetGlobalBindingAndDictItem( + PyObject *lazy_import, PyObject *globals, PyObject *name); PyAPI_FUNC(PyObject *) _PyLazyImport_New( struct _PyInterpreterFrame *frame, PyObject *import_func, PyObject *from, PyObject *attr); diff --git a/Lib/test/test_lazy_import/__init__.py b/Lib/test/test_lazy_import/__init__.py index cf1d3eb793a8b94..5b604681d573a55 100644 --- a/Lib/test/test_lazy_import/__init__.py +++ b/Lib/test/test_lazy_import/__init__.py @@ -457,6 +457,105 @@ def test_lazy_submodule_stored_in_parent_dict(self): self.assertIs(pkg.__dict__["bar"], sys.modules["test.test_lazy_import.data.pkg.bar"]) self.assertIn("BAR_MODULE_LOADED", out.getvalue()) + @support.requires_subprocess() + def test_lazy_submodule_missing_from_sys_modules_raises_key_error(self): + """A sys.modules race after import should surface the hard error.""" + code = textwrap.dedent(""" + import os + import sys + import tempfile + + class HidingModules(dict): + hide = False + hide_name = None + + def __getitem__(self, name): + if self.hide and name == self.hide_name: + raise KeyError(name) + return super().__getitem__(name) + + with tempfile.TemporaryDirectory() as tmpdir: + pkg_dir = os.path.join(tmpdir, "lazy_sysmodules_pkg") + os.mkdir(pkg_dir) + with open(os.path.join(pkg_dir, "__init__.py"), "w", encoding="utf-8") as f: + f.write("") + with open(os.path.join(pkg_dir, "sub.py"), "w", encoding="utf-8") as f: + f.write("VALUE = 42\\n") + + original_modules = sys.modules + modules = HidingModules(original_modules) + sys.modules = modules + sys.path.insert(0, tmpdir) + lazy import lazy_sysmodules_pkg.sub + try: + modules.hide_name = "lazy_sysmodules_pkg.sub" + modules.hide = True + + try: + lazy_sysmodules_pkg.sub + except KeyError as exc: + assert exc.args == ( + "'lazy_sysmodules_pkg.sub' not in sys.modules as expected", + ), exc + else: + raise AssertionError("KeyError was not raised") + finally: + sys.path.remove(tmpdir) + sys.modules = original_modules + for name in ("lazy_sysmodules_pkg", "lazy_sysmodules_pkg.sub"): + sys.modules.pop(name, None) + + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + @support.requires_subprocess() + def test_lazy_submodule_sys_modules_none_sentinel_raises(self): + """The sys.modules None sentinel should preserve import failure.""" + code = textwrap.dedent(""" + import os + import sys + import tempfile + + with tempfile.TemporaryDirectory() as tmpdir: + pkg_dir = os.path.join(tmpdir, "lazy_sentinel_pkg") + os.mkdir(pkg_dir) + with open(os.path.join(pkg_dir, "__init__.py"), "w", encoding="utf-8") as f: + f.write("") + with open(os.path.join(pkg_dir, "sub.py"), "w", encoding="utf-8") as f: + f.write("VALUE = 42\\n") + + sys.path.insert(0, tmpdir) + lazy import lazy_sentinel_pkg.sub + try: + sys.modules["lazy_sentinel_pkg.sub"] = None + try: + lazy_sentinel_pkg.sub + except ModuleNotFoundError as exc: + assert exc.name == "lazy_sentinel_pkg.sub", exc + else: + raise AssertionError("ModuleNotFoundError was not raised") + finally: + sys.path.remove(tmpdir) + for name in ("lazy_sentinel_pkg", "lazy_sentinel_pkg.sub"): + sys.modules.pop(name, None) + + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + def test_lazy_import_pkg_cross_import(self): """Cross-imports within package should preserve lazy imports.""" import test.test_lazy_import.data.pkg.c @@ -731,25 +830,27 @@ def test_reification_retries_on_failure(self): """ code = textwrap.dedent(""" import sys - import types lazy import test.test_lazy_import.data.broken_module # First access - should fail try: x = test.test_lazy_import.data.broken_module - except AttributeError: - pass + except ValueError as exc: + assert str(exc) == "This module always fails to import", exc + else: + raise AssertionError("ValueError was not raised") + + assert "test.test_lazy_import.data.broken_module" not in sys.modules - # The lazy object should still be a lazy proxy (not reified) - g = globals() - lazy_obj = g['test'] - # The root 'test' binding should still allow retry # Second access - should also fail (retry the import) try: x = test.test_lazy_import.data.broken_module - except AttributeError: + except ValueError as exc: + assert str(exc) == "This module always fails to import", exc print("OK - retry worked") + else: + raise AssertionError("ValueError was not raised") """) result = subprocess.run( [sys.executable, "-c", code], @@ -767,9 +868,11 @@ def test_error_during_module_execution_propagates(self): try: _ = test.test_lazy_import.data.broken_module - print("FAIL - should have raised") - except AttributeError: + except ValueError as exc: + assert str(exc) == "This module always fails to import", exc print("OK") + else: + raise AssertionError("ValueError was not raised") """) result = subprocess.run( [sys.executable, "-c", code], @@ -905,33 +1008,456 @@ def test_direct_access_triggers_reification(self): result = test.test_lazy_import.data.globals_access.get_direct() self.assertIn("test.test_lazy_import.data.basic2", sys.modules) - def test_resolve_method_forces_reification(self): - """Calling resolve() on lazy proxy should force reification. + def test_resolve_method_updates_original_global(self): + """resolve() should update the original global binding.""" + code = textwrap.dedent(""" + import builtins + import types - Note: Must access lazy proxy from within a function to avoid automatic - reification by LOAD_NAME at module level. - """ + real_import = builtins.__import__ + calls = [] + + lazy import target_module as target + + def custom_import(name, globals=None, locals=None, fromlist=None, level=0): + if name == "target_module": + index = len(calls) + 1 + calls.append((index, name, globals.get("__name__"), fromlist)) + module = types.ModuleType(name) + module.VALUE = f"value-{index}" + return module + return real_import(name, globals, locals, fromlist, level) + + builtins.__import__ = custom_import + try: + def test_resolve(): + g = globals() + lazy_obj = g["target"] + assert type(lazy_obj) is types.LazyImportType, ( + f"Expected lazy proxy, got {type(lazy_obj)}" + ) + g["alias"] = lazy_obj + + resolved = lazy_obj.resolve() + + assert resolved.VALUE == "value-1" + assert g["target"] is resolved + assert g["alias"] is lazy_obj + return True + + assert test_resolve() + assert target.VALUE == "value-1" + assert calls == [(1, "target_module", "__main__", None)], calls + finally: + builtins.__import__ = real_import + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_resolve_method_respects_rebound_global(self): + """resolve() should not overwrite a rebound original global.""" code = textwrap.dedent(""" - import sys + import builtins import types - lazy from test.test_lazy_import.data.basic2 import x + real_import = builtins.__import__ + calls = [] + sentinel = object() - assert 'test.test_lazy_import.data.basic2' not in sys.modules + lazy import target_module as target - def test_resolve(): - g = globals() - lazy_obj = g['x'] - assert type(lazy_obj) is types.LazyImportType, f"Expected lazy proxy, got {type(lazy_obj)}" + def custom_import(name, globals=None, locals=None, fromlist=None, level=0): + if name == "target_module": + calls.append(name) + module = types.ModuleType(name) + module.VALUE = "resolved" + return module + return real_import(name, globals, locals, fromlist, level) - resolved = lazy_obj.resolve() + builtins.__import__ = custom_import + try: + def test_resolve(): + g = globals() + lazy_obj = g["target"] + g["target"] = sentinel + resolved = lazy_obj.resolve() + assert resolved.VALUE == "resolved" + assert g["target"] is sentinel + return True - # Now module should be loaded - assert 'test.test_lazy_import.data.basic2' in sys.modules - assert resolved == 42 # x is 42 in basic2.py - return True + assert test_resolve() + assert calls == ["target_module"], calls + finally: + builtins.__import__ = real_import + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_resolve_method_respects_deleted_global(self): + """resolve() should not recreate a deleted original global.""" + code = textwrap.dedent(""" + import builtins + import types + + real_import = builtins.__import__ + + lazy import target_module as target + + def custom_import(name, globals=None, locals=None, fromlist=None, level=0): + if name == "target_module": + module = types.ModuleType(name) + module.VALUE = "resolved" + return module + return real_import(name, globals, locals, fromlist, level) + + builtins.__import__ = custom_import + try: + def test_resolve(): + g = globals() + lazy_obj = g["target"] + del g["target"] + resolved = lazy_obj.resolve() + assert resolved.VALUE == "resolved" + assert "target" not in g + return True + + assert test_resolve() + finally: + builtins.__import__ = real_import + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_resolve_method_failure_preserves_global_proxy(self): + """Failed resolve() should keep the global proxy for retry.""" + code = textwrap.dedent(""" + import builtins + + real_import = builtins.__import__ + calls = [] + + lazy import broken_target as target + + def custom_import(name, globals=None, locals=None, fromlist=None, level=0): + if name == "broken_target": + calls.append(name) + raise ImportError("boom") + return real_import(name, globals, locals, fromlist, level) - assert test_resolve() + builtins.__import__ = custom_import + try: + def test_resolve(): + g = globals() + lazy_obj = g["target"] + try: + lazy_obj.resolve() + except ImportError: + pass + else: + raise AssertionError("resolve() unexpectedly succeeded") + assert g["target"] is lazy_obj + + try: + lazy_obj.resolve() + except ImportError: + pass + else: + raise AssertionError("resolve() unexpectedly succeeded") + assert g["target"] is lazy_obj + return True + + assert test_resolve() + assert calls == ["broken_target", "broken_target"], calls + finally: + builtins.__import__ = real_import + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_resolve_method_updates_from_import_binding(self): + """resolve() should update the stored from-import binding key.""" + code = textwrap.dedent(""" + import builtins + import types + + real_import = builtins.__import__ + calls = [] + + lazy from target_module import VALUE as value + + def custom_import(name, globals=None, locals=None, fromlist=None, level=0): + if name == "target_module": + calls.append((name, fromlist)) + module = types.ModuleType(name) + module.VALUE = "value-1" + return module + return real_import(name, globals, locals, fromlist, level) + + builtins.__import__ = custom_import + try: + def test_resolve(): + g = globals() + lazy_obj = g["value"] + assert type(lazy_obj) is types.LazyImportType + g["alias"] = lazy_obj + + resolved = lazy_obj.resolve() + + assert resolved == "value-1" + assert g["value"] == "value-1" + assert g["alias"] is lazy_obj + return True + + assert test_resolve() + assert value == "value-1" + assert calls == [("target_module", ("VALUE",))], calls + finally: + builtins.__import__ = real_import + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_resolve_method_caches_copied_proxy(self): + """Repeated resolve() on a copied proxy should not import again.""" + code = textwrap.dedent(""" + import builtins + import types + + real_import = builtins.__import__ + calls = [] + + lazy import target_module as target + + def custom_import(name, globals=None, locals=None, fromlist=None, level=0): + if name == "target_module": + index = len(calls) + 1 + calls.append(name) + module = types.ModuleType(name) + module.VALUE = f"value-{index}" + return module + return real_import(name, globals, locals, fromlist, level) + + builtins.__import__ = custom_import + try: + def test_resolve(): + g = globals() + lazy_obj = g["target"] + g["alias"] = lazy_obj + + resolved = lazy_obj.resolve() + again = g["alias"].resolve() + + assert again is resolved + assert resolved.VALUE == "value-1" + assert g["target"] is resolved + assert g["alias"] is lazy_obj + return True + + assert test_resolve() + assert calls == ["target_module"], calls + finally: + builtins.__import__ = real_import + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_resolve_method_does_not_reclaim_stale_binding(self): + """A stale owner binding should not be retargeted by a later resolve.""" + code = textwrap.dedent(""" + import builtins + import types + + real_import = builtins.__import__ + calls = [] + sentinel = object() + + lazy import target_module as target + + def custom_import(name, globals=None, locals=None, fromlist=None, level=0): + if name == "target_module": + calls.append(name) + module = types.ModuleType(name) + module.VALUE = "resolved" + return module + return real_import(name, globals, locals, fromlist, level) + + builtins.__import__ = custom_import + try: + def test_resolve(): + g = globals() + lazy_obj = g["target"] + g["target"] = sentinel + + resolved = lazy_obj.resolve() + assert g["target"] is sentinel + + g["target"] = lazy_obj + again = lazy_obj.resolve() + assert again is resolved + assert g["target"] is lazy_obj + return True + + assert test_resolve() + assert calls == ["target_module"], calls + finally: + builtins.__import__ = real_import + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_load_global_respects_rebound_global_during_reification(self): + """LOAD_GLOBAL should not overwrite a global rebound by the import hook.""" + code = textwrap.dedent(""" + import builtins + import types + + real_import = builtins.__import__ + calls = [] + sentinel = object() + + lazy import target_module as target + + def custom_import(name, globals=None, locals=None, fromlist=None, level=0): + if name == "target_module": + calls.append(name) + globals["target"] = sentinel + module = types.ModuleType(name) + module.VALUE = "resolved" + return module + return real_import(name, globals, locals, fromlist, level) + + builtins.__import__ = custom_import + try: + def trigger_load_global(): + return target + + resolved = trigger_load_global() + assert resolved.VALUE == "resolved" + assert globals()["target"] is sentinel + assert calls == ["target_module"], calls + finally: + builtins.__import__ = real_import + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_load_name_respects_deleted_global_during_reification(self): + """LOAD_NAME should not recreate a global deleted by the import hook.""" + code = textwrap.dedent(""" + import builtins + import types + + real_import = builtins.__import__ + calls = [] + + lazy import target_module as target + + def custom_import(name, globals=None, locals=None, fromlist=None, level=0): + if name == "target_module": + calls.append(name) + del globals["target"] + module = types.ModuleType(name) + module.VALUE = "resolved" + return module + return real_import(name, globals, locals, fromlist, level) + + builtins.__import__ = custom_import + try: + resolved = target + assert resolved.VALUE == "resolved" + assert "target" not in globals() + assert calls == ["target_module"], calls + finally: + builtins.__import__ = real_import + print("OK") + """) + result = subprocess.run( + [sys.executable, "-c", code], + capture_output=True, + text=True + ) + self.assertEqual(result.returncode, 0, f"stdout: {result.stdout}, stderr: {result.stderr}") + self.assertIn("OK", result.stdout) + + def test_module_attr_respects_rebound_global_during_reification(self): + """Module attribute access should not overwrite a rebound lazy binding.""" + code = textwrap.dedent(""" + import builtins + import types + + real_import = builtins.__import__ + calls = [] + sentinel = object() + holder = types.ModuleType("holder") + holder.__dict__["__builtins__"] = builtins + + exec("lazy import target_module as target", holder.__dict__) + + def custom_import(name, globals=None, locals=None, fromlist=None, level=0): + if name == "target_module": + calls.append(name) + holder.__dict__["target"] = sentinel + module = types.ModuleType(name) + module.VALUE = "resolved" + return module + return real_import(name, globals, locals, fromlist, level) + + builtins.__import__ = custom_import + try: + resolved = holder.target + assert resolved.VALUE == "resolved" + assert holder.__dict__["target"] is sentinel + assert calls == ["target_module"], calls + finally: + builtins.__import__ = real_import print("OK") """) result = subprocess.run( diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2026-06-29-03-18-22.gh-issue-152298.X3zQpA.rst b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-29-03-18-22.gh-issue-152298.X3zQpA.rst new file mode 100644 index 000000000000000..aa32f64025c3543 --- /dev/null +++ b/Misc/NEWS.d/next/Core_and_Builtins/2026-06-29-03-18-22.gh-issue-152298.X3zQpA.rst @@ -0,0 +1,5 @@ +Fix :meth:`!types.LazyImportType.resolve` so resolving a module-level lazy +import replaces the original lazy proxy in the module globals. +Lazy submodule attribute access now also preserves importlib's +``sys.modules[name] is None`` sentinel behavior while treating absent requested +lazy submodules as normal attribute misses. diff --git a/Modules/_testinternalcapi/test_cases.c.h b/Modules/_testinternalcapi/test_cases.c.h index 708aa4d966e5357..5017b4a76f45509 100644 --- a/Modules/_testinternalcapi/test_cases.c.h +++ b/Modules/_testinternalcapi/test_cases.c.h @@ -10336,10 +10336,20 @@ _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - int err = PyDict_SetItem(GLOBALS(), name, l_v); - _PyFrame_StackPointerInvalidate(frame); + int err; + if (PyDict_CheckExact(GLOBALS())) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + err = _PyLazyImport_ReplaceDictItemIfCurrent( + v_o, GLOBALS(), name, l_v); + _PyFrame_StackPointerInvalidate(frame); + } + else { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + err = PyDict_SetItem(GLOBALS(), name, l_v); + _PyFrame_StackPointerInvalidate(frame); + } if (err < 0) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); @@ -12525,10 +12535,20 @@ } } else { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); - _PyFrame_StackPointerInvalidate(frame); + PyObject *value = PyStackRef_AsPyObjectBorrow(v); + if (PyLazyImport_CheckExact(value)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + err = _PyLazyImport_SetGlobalBindingAndDictItem( + value, GLOBALS(), name); + _PyFrame_StackPointerInvalidate(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + err = PyDict_SetItem(GLOBALS(), name, value); + _PyFrame_StackPointerInvalidate(frame); + } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); diff --git a/Objects/lazyimportobject.c b/Objects/lazyimportobject.c index fa1eb25047d9617..f4149a25aeef1de 100644 --- a/Objects/lazyimportobject.c +++ b/Objects/lazyimportobject.c @@ -2,6 +2,8 @@ #include "Python.h" #include "pycore_ceval.h" +#include "pycore_critical_section.h" +#include "pycore_dict.h" #include "pycore_frame.h" #include "pycore_import.h" #include "pycore_interpframe.h" @@ -33,6 +35,10 @@ _PyLazyImport_New(_PyInterpreterFrame *frame, PyObject *builtins, PyObject *name m->lz_builtins = Py_XNewRef(builtins); m->lz_from = Py_NewRef(name); m->lz_attr = Py_XNewRef(fromlist); + m->lz_globals = NULL; + m->lz_key = NULL; + m->lz_resolved = NULL; + m->lz_key_hash = -1; // Capture frame information for the original import location. m->lz_code = NULL; @@ -58,6 +64,9 @@ lazy_import_traverse(PyObject *op, visitproc visit, void *arg) Py_VISIT(m->lz_builtins); Py_VISIT(m->lz_from); Py_VISIT(m->lz_attr); + Py_VISIT(m->lz_globals); + Py_VISIT(m->lz_key); + Py_VISIT(m->lz_resolved); Py_VISIT(m->lz_code); return 0; } @@ -69,6 +78,9 @@ lazy_import_clear(PyObject *op) Py_CLEAR(m->lz_builtins); Py_CLEAR(m->lz_from); Py_CLEAR(m->lz_attr); + Py_CLEAR(m->lz_globals); + Py_CLEAR(m->lz_key); + Py_CLEAR(m->lz_resolved); Py_CLEAR(m->lz_code); return 0; } @@ -116,6 +128,172 @@ _PyLazyImport_GetName(PyObject *op) return lazy_import_name(lazy_import); } +PyObject * +_PyLazyImport_GetResolved(PyObject *op) +{ + PyLazyImportObject *m = PyLazyImportObject_CAST(op); + assert(PyLazyImport_CheckExact(op)); + + PyObject *resolved = NULL; + Py_BEGIN_CRITICAL_SECTION(op); + resolved = Py_XNewRef(m->lz_resolved); + Py_END_CRITICAL_SECTION(); + return resolved; +} + +int +_PyLazyImport_SetGlobalBindingAndDictItem(PyObject *op, PyObject *globals, + PyObject *name) +{ + assert(PyLazyImport_CheckExact(op)); + assert(PyDict_CheckExact(globals)); + + PyLazyImportObject *m = PyLazyImportObject_CAST(op); + Py_hash_t hash = PyObject_Hash(name); + if (hash == -1) { + return -1; + } + + PyObject *discard_globals = NULL; + PyObject *discard_key = NULL; + int err; + int recorded = 0; + + Py_BEGIN_CRITICAL_SECTION2(op, globals); + if (m->lz_key == NULL && m->lz_resolved == NULL) { + // Record the owner binding before publishing the proxy. resolve() + // may update only this key; aliases must not retarget it. + m->lz_globals = Py_NewRef(globals); + m->lz_key = Py_NewRef(name); + m->lz_key_hash = hash; + recorded = 1; + } + err = _PyDict_SetItem_KnownHash_LockHeld( + (PyDictObject *)globals, name, op, hash); + if (err < 0 && recorded) { + discard_globals = m->lz_globals; + discard_key = m->lz_key; + m->lz_globals = NULL; + m->lz_key = NULL; + m->lz_key_hash = -1; + } + Py_END_CRITICAL_SECTION2(); + + Py_XDECREF(discard_globals); + Py_XDECREF(discard_key); + return err; +} + +static int +lazy_import_replace_dict_item_if_current(PyObject *op, PyObject *globals, + PyObject *key, Py_hash_t key_hash, + PyObject *resolved) +{ + assert(PyLazyImport_CheckExact(op)); + assert(PyDict_CheckExact(globals)); + assert(!PyLazyImport_CheckExact(resolved)); + + PyObject *current = NULL; + int err = 0; + + Py_BEGIN_CRITICAL_SECTION(globals); + int found = _PyDict_GetItemRef_KnownHash_LockHeld( + (PyDictObject *)globals, key, key_hash, ¤t); + if (found < 0) { + err = -1; + } + else if (found && current == op) { + err = _PyDict_SetItem_KnownHash_LockHeld( + (PyDictObject *)globals, key, resolved, key_hash); + } + Py_END_CRITICAL_SECTION(); + + Py_XDECREF(current); + return err; +} + +int +_PyLazyImport_ReplaceDictItemIfCurrent(PyObject *op, PyObject *globals, + PyObject *key, PyObject *resolved) +{ + assert(PyLazyImport_CheckExact(op)); + assert(PyDict_CheckExact(globals)); + + Py_hash_t key_hash = PyObject_Hash(key); + if (key_hash == -1) { + return -1; + } + return lazy_import_replace_dict_item_if_current( + op, globals, key, key_hash, resolved); +} + +int +_PyLazyImport_FinishResolve(PyObject *op, PyObject *resolved) +{ + PyLazyImportObject *m = PyLazyImportObject_CAST(op); + PyObject *globals = NULL; + PyObject *key = NULL; + Py_hash_t key_hash = -1; + int err = 0; + int already_resolved = 0; + + assert(PyLazyImport_CheckExact(op)); + assert(!PyLazyImport_CheckExact(resolved)); + + Py_BEGIN_CRITICAL_SECTION(op); + if (m->lz_resolved != NULL) { + already_resolved = 1; + } + else if (m->lz_globals != NULL && m->lz_key != NULL) { + globals = Py_NewRef(m->lz_globals); + key = Py_NewRef(m->lz_key); + key_hash = m->lz_key_hash; + } + Py_END_CRITICAL_SECTION(); + + if (already_resolved) { + return 0; + } + + if (globals != NULL) { + assert(key != NULL); + assert(PyDict_CheckExact(globals)); + + err = lazy_import_replace_dict_item_if_current( + op, globals, key, key_hash, resolved); + if (err < 0) { + Py_DECREF(globals); + Py_DECREF(key); + return err; + } + } + + PyObject *discard_globals = NULL; + PyObject *discard_key = NULL; + Py_BEGIN_CRITICAL_SECTION(op); + if (m->lz_resolved == NULL) { + m->lz_resolved = Py_NewRef(resolved); + } + if (globals != NULL && + m->lz_globals == globals && + m->lz_key == key && + m->lz_key_hash == key_hash) + { + discard_globals = m->lz_globals; + discard_key = m->lz_key; + m->lz_globals = NULL; + m->lz_key = NULL; + m->lz_key_hash = -1; + } + Py_END_CRITICAL_SECTION(); + + Py_XDECREF(discard_globals); + Py_XDECREF(discard_key); + Py_XDECREF(globals); + Py_XDECREF(key); + return 0; +} + static PyObject * lazy_import_resolve(PyObject *self, PyObject *args) { @@ -137,9 +315,10 @@ PyDoc_STRVAR(lazy_import_doc, "\n" "Represents a lazy import that will be resolved on first use.\n" "\n" -"Instances of this object accessed from the global scope will be\n" -"automatically imported based upon their name and then replaced with\n" -"the imported value."); +"A successful resolution is cached. Instances of this object accessed\n" +"from the global scope will be automatically imported based upon their\n" +"name. The original global is replaced only if it still refers to this\n" +"lazy import object."); PyTypeObject PyLazyImport_Type = { PyVarObject_HEAD_INIT(&PyType_Type, 0) diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index f447403ef31b43a..c57d7ce19aaa549 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -1300,7 +1300,7 @@ _PyModule_IsPossiblyShadowing(PyObject *origin) } // Check if `name` is a lazily pending submodule of module `m`. -// Returns a new reference on success, or NULL with no error set. +// Returns a new reference on success, or NULL on miss or error. static PyObject * try_load_lazy_submodule(PyModuleObject *m, PyObject *name) { @@ -1313,14 +1313,11 @@ try_load_lazy_submodule(PyModuleObject *m, PyObject *name) Py_DECREF(mod_name); return NULL; } - PyObject *result = _PyImport_TryLoadLazySubmodule(mod_name, name); + PyObject *result = NULL; + int lazy_rc = _PyImport_TryLoadLazySubmodule( + mod_name, name, m->md_dict, &result); Py_DECREF(mod_name); - if (result == NULL) { - PyErr_Clear(); - return NULL; - } - if (PyDict_SetItem(m->md_dict, name, result) < 0) { - Py_DECREF(result); + if (lazy_rc <= 0) { return NULL; } return result; @@ -1368,7 +1365,9 @@ _Py_module_getattro_impl(PyModuleObject *m, PyObject *name, int suppress) return NULL; } - if (PyDict_SetItem(m->md_dict, name, new_value) < 0) { + if (_PyLazyImport_ReplaceDictItemIfCurrent( + attr, m->md_dict, name, new_value) < 0) + { Py_CLEAR(new_value); } Py_DECREF(attr); diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 6d5ae1b074db5b7..f2d35a679eb7eba 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -19,7 +19,7 @@ #include "pycore_instruments.h" #include "pycore_interpolation.h" // _PyInterpolation_Build() #include "pycore_intrinsics.h" -#include "pycore_lazyimportobject.h" // PyLazyImport_CheckExact() +#include "pycore_lazyimportobject.h" // PyLazyImport_CheckExact(), _PyLazyImport_SetGlobalBindingAndDictItem() #include "pycore_long.h" // _PyLong_ExactDealloc(), _PyLong_GetZero() #include "pycore_moduleobject.h" // PyModuleObject #include "pycore_object.h" // _PyObject_GC_TRACK() @@ -2174,7 +2174,14 @@ dummy_func( } } else { - err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); + PyObject *value = PyStackRef_AsPyObjectBorrow(v); + if (PyLazyImport_CheckExact(value)) { + err = _PyLazyImport_SetGlobalBindingAndDictItem( + value, GLOBALS(), name); + } + else { + err = PyDict_SetItem(GLOBALS(), name, value); + } PyStackRef_CLOSE(v); } ERROR_IF(err < 0); @@ -2257,7 +2264,14 @@ dummy_func( Py_DECREF(v_o); ERROR_IF(true); } - int err = PyDict_SetItem(GLOBALS(), name, l_v); + int err; + if (PyDict_CheckExact(GLOBALS())) { + err = _PyLazyImport_ReplaceDictItemIfCurrent( + v_o, GLOBALS(), name, l_v); + } + else { + err = PyDict_SetItem(GLOBALS(), name, l_v); + } if (err < 0) { Py_DECREF(v_o); Py_DECREF(l_v); diff --git a/Python/ceval.c b/Python/ceval.c index f3f03b28112137a..5f69f30b9b181b9 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1,6 +1,7 @@ /* Execute compiled code */ #include "ceval.h" +#include "pycore_lazyimportobject.h" #include "pycore_long.h" int @@ -1171,6 +1172,11 @@ int _PyEval_StoreName(PyThreadState *tstate, _PyStackRef v, PyObject *name, PyOb PyObject *v_o = PyStackRef_AsPyObjectBorrow(v); if (PyDict_CheckExact(ns)) { + if (ns == PyEval_GetGlobals() && + PyLazyImport_CheckExact(v_o)) + { + return _PyLazyImport_SetGlobalBindingAndDictItem(v_o, ns, name); + } return PyDict_SetItem(ns, name, v_o); } @@ -3682,13 +3688,21 @@ _PyEval_LoadGlobalStackRef(PyObject *globals, PyObject *builtins, PyObject *name PyObject *res_o = PyStackRef_AsPyObjectBorrow(*writeto); if (res_o != NULL && PyLazyImport_CheckExact(res_o)) { PyObject *l_v = _PyImport_LoadLazyImportTstate(PyThreadState_GET(), res_o); - PyStackRef_CLOSE(writeto[0]); if (l_v == NULL) { assert(PyErr_Occurred()); + PyStackRef_CLOSE(writeto[0]); *writeto = PyStackRef_NULL; return; } - int err = PyDict_SetItem(globals, name, l_v); + int err; + if (PyDict_CheckExact(globals)) { + err = _PyLazyImport_ReplaceDictItemIfCurrent( + res_o, globals, name, l_v); + } + else { + err = PyDict_SetItem(globals, name, l_v); + } + PyStackRef_CLOSE(writeto[0]); if (err < 0) { Py_DECREF(l_v); *writeto = PyStackRef_NULL; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 6be53213966678e..678687e59646369 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -10033,13 +10033,26 @@ } } else { - stack_pointer[0] = v; - stack_pointer += 1; - ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); - _PyFrame_StackPointerInvalidate(frame); + PyObject *value = PyStackRef_AsPyObjectBorrow(v); + if (PyLazyImport_CheckExact(value)) { + stack_pointer[0] = v; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + err = _PyLazyImport_SetGlobalBindingAndDictItem( + value, GLOBALS(), name); + _PyFrame_StackPointerInvalidate(frame); + } + else { + stack_pointer[0] = v; + stack_pointer += 1; + ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + err = PyDict_SetItem(GLOBALS(), name, value); + _PyFrame_StackPointerInvalidate(frame); + } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); @@ -10164,10 +10177,20 @@ SET_CURRENT_CACHED_VALUES(0); JUMP_TO_ERROR(); } - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - int err = PyDict_SetItem(GLOBALS(), name, l_v); - _PyFrame_StackPointerInvalidate(frame); + int err; + if (PyDict_CheckExact(GLOBALS())) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + err = _PyLazyImport_ReplaceDictItemIfCurrent( + v_o, GLOBALS(), name, l_v); + _PyFrame_StackPointerInvalidate(frame); + } + else { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + err = PyDict_SetItem(GLOBALS(), name, l_v); + _PyFrame_StackPointerInvalidate(frame); + } if (err < 0) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index d336b9fbcd57124..42561fdeb4d5246 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -10334,10 +10334,20 @@ _PyFrame_StackPointerInvalidate(frame); JUMP_TO_LABEL(error); } - assert(stack_pointer == _PyFrame_GetStackPointer(frame)); - _PyFrame_StackPointerValidate(frame); - int err = PyDict_SetItem(GLOBALS(), name, l_v); - _PyFrame_StackPointerInvalidate(frame); + int err; + if (PyDict_CheckExact(GLOBALS())) { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + err = _PyLazyImport_ReplaceDictItemIfCurrent( + v_o, GLOBALS(), name, l_v); + _PyFrame_StackPointerInvalidate(frame); + } + else { + assert(stack_pointer == _PyFrame_GetStackPointer(frame)); + _PyFrame_StackPointerValidate(frame); + err = PyDict_SetItem(GLOBALS(), name, l_v); + _PyFrame_StackPointerInvalidate(frame); + } if (err < 0) { assert(stack_pointer == _PyFrame_GetStackPointer(frame)); _PyFrame_StackPointerValidate(frame); @@ -12522,10 +12532,20 @@ } } else { - _PyFrame_SetStackPointer(frame, stack_pointer); - _PyFrame_StackPointerValidate(frame); - err = PyDict_SetItem(GLOBALS(), name, PyStackRef_AsPyObjectBorrow(v)); - _PyFrame_StackPointerInvalidate(frame); + PyObject *value = PyStackRef_AsPyObjectBorrow(v); + if (PyLazyImport_CheckExact(value)) { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + err = _PyLazyImport_SetGlobalBindingAndDictItem( + value, GLOBALS(), name); + _PyFrame_StackPointerInvalidate(frame); + } + else { + _PyFrame_SetStackPointer(frame, stack_pointer); + _PyFrame_StackPointerValidate(frame); + err = PyDict_SetItem(GLOBALS(), name, value); + _PyFrame_StackPointerInvalidate(frame); + } stack_pointer += -1; ASSERT_WITHIN_STACK_BOUNDS(__FILE__, __LINE__); _PyFrame_SetStackPointer(frame, stack_pointer); diff --git a/Python/import.c b/Python/import.c index 63e23e21beb1266..1ea1d3e64e7c934 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3897,6 +3897,12 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) // Acquire the global import lock to serialize reification _PyImport_AcquireLock(interp); + obj = _PyLazyImport_GetResolved(lazy_import); + if (obj != NULL) { + _PyImport_ReleaseLock(interp); + return obj; + } + // Check if we are already importing this module, if so, then we want to // return an error that indicates we've hit a cycle which will indicate // the value isn't yet available. @@ -4093,6 +4099,13 @@ _PyImport_LoadLazyImportTstate(PyThreadState *tstate, PyObject *lazy_import) if (PySet_Discard(importing, lazy_import) < 0) { Py_CLEAR(obj); } + else if (obj != NULL) { + // Cache the result and update the original global before releasing + // the import lock, so later reification through this proxy reuses it. + if (_PyLazyImport_FinishResolve(lazy_import, obj) < 0) { + Py_CLEAR(obj); + } + } // Release the global import lock. _PyImport_ReleaseLock(interp); @@ -4439,52 +4452,135 @@ register_from_lazy_on_parent(PyThreadState *tstate, PyObject *abs_name, return res; } -PyObject * -_PyImport_TryLoadLazySubmodule(PyObject *mod_name, PyObject *attr_name) +static int +is_module_not_found_for_name(PyThreadState *tstate, PyObject *name) { - PyInterpreterState *interp = _PyInterpreterState_GET(); + // Return true only for the importlib._handle_fromlist() compatibility + // case where "name itself was not found" and sys.modules[name] is not the + // None sentinel. Consume the current exception so callers can continue + // normal attribute fallback. + if (!_PyErr_ExceptionMatches(tstate, PyExc_ModuleNotFoundError)) { + return 0; + } + + PyObject *exc = _PyErr_GetRaisedException(tstate); + PyObject *missing_name = PyObject_GetAttr(exc, &_Py_ID(name)); + if (missing_name == NULL) { + PyErr_Clear(); + _PyErr_SetRaisedException(tstate, exc); + return 0; + } + + int is_same = PyObject_RichCompareBool(missing_name, name, Py_EQ); + Py_DECREF(missing_name); + if (is_same <= 0) { + if (is_same < 0) { + PyErr_Clear(); + } + _PyErr_SetRaisedException(tstate, exc); + return 0; + } + + PyObject *mod = import_get_module(tstate, name); + if (mod == Py_None) { + Py_DECREF(mod); + _PyErr_SetRaisedException(tstate, exc); + return 0; + } + if (mod == NULL && _PyErr_Occurred(tstate)) { + PyErr_Clear(); + _PyErr_SetRaisedException(tstate, exc); + return 0; + } + Py_XDECREF(mod); + Py_DECREF(exc); + return 1; +} + +int +_PyImport_TryLoadLazySubmodule(PyObject *mod_name, PyObject *attr_name, + PyObject *mod_dict, PyObject **result) +{ + *result = NULL; + + PyThreadState *tstate = _PyThreadState_GET(); + PyInterpreterState *interp = tstate->interp; PyObject *lazy_pending = LAZY_PENDING_SUBMODULES(interp); if (lazy_pending == NULL) { - return NULL; + return 0; } PyObject *pending_set; int rc = PyDict_GetItemRef(lazy_pending, mod_name, &pending_set); - if (rc <= 0) { - return NULL; + if (rc < 0) { + return -1; + } + if (rc == 0) { + return 0; } int contains = PySet_Contains(pending_set, attr_name); - if (contains <= 0) { + if (contains < 0) { Py_DECREF(pending_set); - return NULL; + return -1; + } + if (contains == 0) { + Py_DECREF(pending_set); + return 0; } PyObject *full_name = PyUnicode_FromFormat("%U.%U", mod_name, attr_name); if (full_name == NULL) { Py_DECREF(pending_set); - return NULL; + return -1; } PyObject *mod = PyImport_ImportModuleLevelObject( full_name, NULL, NULL, NULL, 0); if (mod == NULL) { + if (is_module_not_found_for_name(tstate, full_name)) { + Py_DECREF(pending_set); + Py_DECREF(full_name); + return 0; + } Py_DECREF(pending_set); Py_DECREF(full_name); - return NULL; + return -1; } Py_DECREF(mod); - if (PySet_Discard(pending_set, attr_name) < 0) { - Py_DECREF(pending_set); + PyObject *submod = PyImport_GetModule(full_name); + if (submod == NULL) { + if (!_PyErr_Occurred(tstate)) { + _PyErr_Format(tstate, PyExc_KeyError, + "%R not in sys.modules as expected", + full_name); + } Py_DECREF(full_name); - return NULL; + Py_DECREF(pending_set); + return -1; } - Py_DECREF(pending_set); - - PyObject *submod = PyImport_GetModule(full_name); Py_DECREF(full_name); - return submod; + + if (PyDict_SetItem(mod_dict, attr_name, submod) < 0) { + Py_DECREF(pending_set); + Py_DECREF(submod); + return -1; + } + + // This completes the successful lazy-submodule publish. pending_set is an + // internal set and attr_name is a Unicode attribute name, so discard is not + // expected to fail after the parent dict has been updated. + int discard_rc = PySet_Discard(pending_set, attr_name); + assert(discard_rc >= 0); + if (discard_rc < 0) { + Py_DECREF(pending_set); + Py_DECREF(submod); + return -1; + } + Py_DECREF(pending_set); + *result = submod; + return 1; } PyObject *