diff --git a/Include/internal/pycore_interp_structs.h b/Include/internal/pycore_interp_structs.h index 0623adce693d465..fe748c8fcdde2c7 100644 --- a/Include/internal/pycore_interp_structs.h +++ b/Include/internal/pycore_interp_structs.h @@ -319,6 +319,8 @@ struct _import_runtime_state { struct _import_state { /* cached sys.modules dictionary */ PyObject *modules; + /* Incremented when sys.modules or module lookup metadata changes. */ + uint64_t module_registry_version; /* This is the list of module objects for all legacy (single-phase init) extension modules ever loaded in this process (i.e. imported in this interpreter or in any other). Py_None stands in for diff --git a/Lib/inspect.py b/Lib/inspect.py index 3f8991c79652d3f..73916e3d72ee6f4 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -143,6 +143,7 @@ import abc +import _imp from annotationlib import Format, ForwardRef from annotationlib import get_annotations # re-exported import ast @@ -925,6 +926,104 @@ def getabsfile(object, _filename=None): modulesbyfile = {} _filesbymodname = {} +# (registry version, filename map, per-module metadata, volatile metadata) +_modulesbyfile_cache = (None, {}, {}, ()) +_module_cache_missing = object() + + +def _get_volatile_module_metadata(module): + """Return unwatchable module metadata used to validate the cache.""" + try: + module_file = getattr(module, '__file__') + except AttributeError: + return _module_cache_missing, _module_cache_missing + if module_file is None: + return None, _module_cache_missing + return module_file, getattr(module, '__name__') + + +def _volatile_modules_unchanged(modules, volatile_modules): + for modname, module_file, module_name in volatile_modules: + module = modules.get(modname, _module_cache_missing) + if not ismodule(module): + return False + current_file, current_name = _get_volatile_module_metadata(module) + if current_file != module_file or current_name != module_name: + return False + return True + + +def _module_cache_is_current(modules, version, cache): + if (modules is not sys.modules or + version is None or version != cache[0]): + return False + volatile_modules = cache[3] + if not volatile_modules: + return True + if not _volatile_modules_unchanged(modules, volatile_modules): + return False + # Dynamic metadata may execute arbitrary Python code. + return (modules is sys.modules and + version == _imp._get_module_registry_version(modules)) + + +def _publish_module_cache(version, modules_by_file, files_by_module, + volatile_modules): + """Publish a consistent filename cache and update its legacy mirrors.""" + global _modulesbyfile_cache + + # Keep these longstanding, externally visible dictionaries alive, but do + # not use them as the authoritative cache: updating two dictionaries and a + # version cannot be made atomic for concurrent getmodule() calls. + modulesbyfile.clear() + modulesbyfile.update(modules_by_file) + _filesbymodname.clear() + _filesbymodname.update( + (modname, metadata[0]) + for modname, metadata in files_by_module.items() + ) + # This single assignment is the authoritative publication point. + _modulesbyfile_cache = ( + version, modules_by_file, files_by_module, volatile_modules) + + +def _getframemodule(frame, _filename=None): + """Return a module owned by a frame's globals, or None if not found.""" + frame_globals = frame.f_globals + module_name = frame_globals.get('__name__') + if not isinstance(module_name, str): + return None + module = sys.modules.get(module_name) + if not (ismodule(module) and module.__dict__ is frame_globals): + return None + try: + if type(module) is types.ModuleType: + module_file = module.__file__ + else: + # A module subclass can supply dynamic metadata through + # descriptors. getattr() avoids a module-attribute specialization + # that may read the subclass's underlying module dict. + module_file = getattr(module, '__file__') + except AttributeError: + return None + if module_file is None: + return None + frame_filename = ( + frame.f_code.co_filename if _filename is None else _filename + ) + try: + file = getabsfile(frame, frame_filename) + except (TypeError, FileNotFoundError): + return None + if frame_filename == module_file: + return module + try: + module_file = getabsfile(module, module_file) + except (TypeError, FileNotFoundError): + return None + if file == module_file or file == os.path.realpath(module_file): + return module + def getmodule(object, _filename=None): """Return the module an object was defined in, or None if not found.""" @@ -932,32 +1031,182 @@ def getmodule(object, _filename=None): return object if hasattr(object, '__module__'): return sys.modules.get(object.__module__) + if istraceback(object): + object = object.tb_frame + if isframe(object): + # Frame globals identify the execution namespace directly. Preserve + # the private filename override when it names a different file. + if _filename is None or _filename == object.f_code.co_filename: + module = _getframemodule(object, _filename) + if module is not None: + return module + + modules = sys.modules + version = _imp._get_module_registry_version(modules) + cache = _modulesbyfile_cache + cache_is_current = _module_cache_is_current(modules, version, cache) + cached_modulesbyfile = cache[1] + + # Try the filename to modulename cache. Recheck the version after reading + # the module so a concurrent registry mutation cannot make the result + # stale. + if (cache_is_current and _filename is not None and + _filename in cached_modulesbyfile): + module = modules.get(cached_modulesbyfile[_filename]) + if _module_cache_is_current( + modules, _imp._get_module_registry_version(modules), cache): + return module + cache_is_current = False - # Try the filename to modulename cache - if _filename is not None and _filename in modulesbyfile: - return sys.modules.get(modulesbyfile[_filename]) # Try the cache again with the absolute file name try: file = getabsfile(object, _filename) except (TypeError, FileNotFoundError): return None - if file in modulesbyfile: - return sys.modules.get(modulesbyfile[file]) - # Update the filename to module name cache and check yet again - # Copy sys.modules in order to cope with changes while iterating - for modname, module in sys.modules.copy().items(): - if ismodule(module) and hasattr(module, '__file__'): - f = module.__file__ - if f == _filesbymodname.get(modname, None): - # Have already mapped this module, so skip it - continue - _filesbymodname[modname] = f - f = getabsfile(module) - # Always map to the name the module knows itself by - modulesbyfile[f] = modulesbyfile[ - os.path.realpath(f)] = module.__name__ - if file in modulesbyfile: - return sys.modules.get(modulesbyfile[file]) + if modules is not sys.modules: + # getabsfile() may execute user code through dynamic module metadata. + modules = sys.modules + cache_is_current = False + if not cache_is_current: + # getabsfile() can recurse through getsourcefile() and populate this + # cache. Avoid rebuilding it a second time in the outer call. + version = _imp._get_module_registry_version(modules) + cache = _modulesbyfile_cache + cache_is_current = _module_cache_is_current(modules, version, cache) + cached_modulesbyfile = cache[1] + if cache_is_current and file in cached_modulesbyfile: + module = modules.get(cached_modulesbyfile[file]) + if _module_cache_is_current( + modules, _imp._get_module_registry_version(modules), cache): + return module + cache_is_current = False + + if cache_is_current: + # A stable cache miss is a negative-cache hit. Validate it after the + # lookup just as for a positive hit. + cache_is_current = _module_cache_is_current( + modules, _imp._get_module_registry_version(modules), cache) + + # Update the filename to module name cache when the canonical module + # registry or relevant module metadata has changed. Replacement + # sys.modules mappings cannot be watched, so scan those on every miss. + if not cache_is_current: + # Retry once if a mutation overlaps the scan. This handles imports + # triggered by the first scan without spinning under sustained churn. + for _ in range(2): + # Copy sys.modules in order to cope with changes while iterating. + modules = sys.modules + version = _imp._get_module_registry_version(modules) + modules_copy = modules.copy() + new_modulesbyfile = {} + new_filesbymodname = {} + volatile_modules = [] + cached_filesbymodname = _modulesbyfile_cache[2] + for modname, module in modules_copy.items(): + if ismodule(module): + if type(module) is types.ModuleType: + module_dict = module.__dict__ + try: + f = module.__file__ + except AttributeError: + if '__getattr__' in module_dict: + volatile_modules.append(( + modname, + _module_cache_missing, + _module_cache_missing, + )) + continue + volatile = ( + module_dict.get( + '__file__', _module_cache_missing) is not f + ) + else: + # Module subclasses may resolve __file__ or __name__ + # dynamically. Validate just these entries on cache + # hits instead of making every fallback rescan all of + # sys.modules. + module_dict = None + volatile = True + try: + f = getattr(module, '__file__') + except AttributeError: + volatile_modules.append(( + modname, + _module_cache_missing, + _module_cache_missing, + )) + continue + if f is None: + if volatile: + volatile_modules.append(( + modname, None, _module_cache_missing)) + continue + if module_dict is None: + module_name = getattr(module, '__name__') + else: + module_name = module.__name__ + if (module_dict is not None and + module_dict.get( + '__name__', _module_cache_missing) + is not module_name): + volatile = True + if volatile: + volatile_modules.append( + (modname, f, module_name)) + cached = cached_filesbymodname.get(modname) + if (isinstance(cached, tuple) and len(cached) == 4 and + f == cached[0] and module_name == cached[1]): + _, _, absfile, realpath = cached + else: + # Canonicalize the same metadata value that was + # validated above. Re-reading __file__ here can + # observe a different dynamic value. + absfile = getabsfile(module, f) + realpath = os.path.realpath(absfile) + new_filesbymodname[modname] = ( + f, module_name, absfile, realpath) + # Always map to the name the module knows itself by. + new_modulesbyfile[absfile] = new_modulesbyfile[ + realpath] = module_name + module = None + found = file in new_modulesbyfile + if found: + module = modules.get(new_modulesbyfile[file]) + # Watcher callbacks run before their mutations. Validate both the + # watched generation and the small set of dynamic metadata before + # retaining the rebuilt caches. + current_version = _imp._get_module_registry_version(modules) + if version is not None: + if (modules is not sys.modules or + version != current_version): + continue + volatile_current = _volatile_modules_unchanged( + modules, volatile_modules) + current_version = _imp._get_module_registry_version(modules) + if (modules is not sys.modules or + version != current_version or + not volatile_current): + continue + _publish_module_cache( + version, + new_modulesbyfile, + new_filesbymodname, + tuple(volatile_modules), + ) + if found: + return module + break + if modules is sys.modules: + # A replacement sys.modules mapping cannot be watched, but its + # just-scanned result is still useful for this call. + _publish_module_cache( + None, new_modulesbyfile, new_filesbymodname, + tuple(volatile_modules)) + if found: + return module + break + # If both attempts overlap mutations, fall through rather than using + # a snapshot that could not be validated. # Check the main module main = sys.modules['__main__'] if not hasattr(object, '__name__'): diff --git a/Lib/test/libregrtest/utils.py b/Lib/test/libregrtest/utils.py index f6c13227dbbcdf7..dd0b82041518329 100644 --- a/Lib/test/libregrtest/utils.py +++ b/Lib/test/libregrtest/utils.py @@ -296,6 +296,7 @@ def clear_caches(): inspect._shadowed_dict_from_weakref_mro_tuple.cache_clear() inspect._filesbymodname.clear() inspect.modulesbyfile.clear() + inspect._modulesbyfile_cache = (None, {}, {}, ()) try: importlib_metadata = sys.modules['importlib.metadata'] diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 7e92028cb955cb2..2f6b7ecaccb26bb 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -373,6 +373,109 @@ def test_import_null_byte_in_name_raises_ModuleNotFoundError(self): self.assertEqual(set(sys.modules), before) + def test_module_registry_version(self): + self.assertIsNone(_imp._get_module_registry_version({})) + + name = f"{TESTFN}_module_registry_version" + module = ModuleType(name) + sys.modules[name] = module + try: + version = _imp._get_module_registry_version(sys.modules) + self.assertIsInstance(version, int) + + # Use a non-constant key to verify that the watcher compares the + # key by value rather than relying on interned-string identity. + file_key = ''.join(('__fi', 'le__')) + module.__dict__[file_key] = "first.py" + changed = _imp._get_module_registry_version(sys.modules) + self.assertNotEqual(changed, version) + + module.unrelated = None + self.assertEqual( + _imp._get_module_registry_version(sys.modules), changed) + module.__dict__["12345678"] = None + self.assertEqual( + _imp._get_module_registry_version(sys.modules), changed) + + module.__file__ = "second.py" + changed_again = _imp._get_module_registry_version(sys.modules) + self.assertNotEqual(changed_again, changed) + + del module.__file__ + self.assertNotEqual( + _imp._get_module_registry_version(sys.modules), + changed_again) + + before_name_change = _imp._get_module_registry_version( + sys.modules) + module.__name__ = name + "_renamed" + after_name_change = _imp._get_module_registry_version( + sys.modules) + self.assertNotEqual(after_name_change, before_name_change) + + getattr_key = ''.join(('__get', 'attr__')) + module.__dict__[getattr_key] = lambda name: None + after_getattr_change = _imp._get_module_registry_version( + sys.modules) + self.assertNotEqual( + after_getattr_change, after_name_change) + + class ModuleSubclass(ModuleType): + pass + + module.__class__ = ModuleSubclass + after_class_change = _imp._get_module_registry_version( + sys.modules) + self.assertNotEqual( + after_class_change, after_getattr_change) + + replacement = ModuleType(name) + before_replacement = _imp._get_module_registry_version( + sys.modules) + sys.modules[name] = replacement + self.assertNotEqual( + _imp._get_module_registry_version(sys.modules), + before_replacement) + finally: + sys.modules.pop(name, None) + + def test_module_registry_version_clear(self): + name = f"{TESTFN}_module_registry_clear" + module = ModuleType(name) + module.__file__ = "module.py" + sys.modules[name] = module + try: + version = _imp._get_module_registry_version(sys.modules) + module.__dict__.clear() + self.assertNotEqual( + _imp._get_module_registry_version(sys.modules), version) + finally: + sys.modules.pop(name, None) + + @requires_subinterpreters + def test_module_registry_version_is_per_interpreter(self): + version = _imp._get_module_registry_version(sys.modules) + code = textwrap.dedent(''' + import _imp + import sys + import types + + before = _imp._get_module_registry_version(sys.modules) + module = types.ModuleType("_registry_version_subinterp") + sys.modules[module.__name__] = module + after_add = _imp._get_module_registry_version(sys.modules) + assert after_add != before + module.__file__ = "subinterp.py" + assert _imp._get_module_registry_version(sys.modules) != after_add + ''') + interpid = _interpreters.create() + try: + self.assertIsNone(_interpreters.run_string(interpid, code)) + finally: + _interpreters.destroy(interpid) + self.assertEqual( + _imp._get_module_registry_version(sys.modules), version) + def test_from_import_missing_module_raises_ModuleNotFoundError(self): with self.assertRaises(ModuleNotFoundError): from something_that_should_not_exist_anywhere import blah diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index ff7475447e95a03..95288f08bdd6241 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -20,6 +20,7 @@ import stat import sys import subprocess +import threading import time import types import tempfile @@ -36,7 +37,7 @@ except ImportError: ThreadPoolExecutor = None -from test.support import cpython_only, import_helper +from test.support import cpython_only, import_helper, threading_helper from test.support import MISSING_C_DOCSTRINGS, ALWAYS_EQ from test.support import run_no_yield_async_fn, EqualToForwardRef from test.support.import_helper import DirsOnSysPath, ready_to_import @@ -831,6 +832,576 @@ def test_getmodule(self): self.assertEqual(inspect.getmodule(str), sys.modules["builtins"]) # Check filename override self.assertEqual(inspect.getmodule(None, modfile), mod) + self.assertEqual( + inspect.getmodule(inspect.currentframe(), modfile), mod) + # Check code, frame, and traceback objects + self.assertIs(inspect.getmodule(mod.eggs.__code__), mod) + self.assertIs(inspect.getmodule(inspect.currentframe()), + sys.modules[__name__]) + try: + 1 / 0 + except ZeroDivisionError as error: + self.assertIs(inspect.getmodule(error.__traceback__), + sys.modules[__name__]) + + def test_getmodule_unregistered_exec_frame(self): + def exec_namespace(namespace, filename, expected): + exec(compile(textwrap.dedent(""" + frame = inspect.currentframe() + try: + 1 / 0 + except ZeroDivisionError as error: + traceback = error.__traceback__ + """), filename, "exec"), namespace) + self.assertIs(inspect.getmodule(namespace["frame"]), expected) + self.assertIs( + inspect.getmodule(namespace["frame"].f_code), expected) + self.assertIs(inspect.getmodule(namespace["traceback"]), expected) + + with (unittest.mock.patch.object(inspect, "modulesbyfile", {}), + unittest.mock.patch.object(inspect, "_filesbymodname", {}), + unittest.mock.patch.object( + inspect, "_modulesbyfile_cache", (None, {}, {}, ()))): + # Preserve filename-based resolution when the execution namespace + # does not identify the module that supplied the source. + exec_namespace({"inspect": inspect}, modfile, mod) + exec_namespace({"inspect": inspect, "__name__": []}, + modfile, mod) + + module_name = f"{__name__}.not_registered" + for module in (None, object(), types.ModuleType(module_name)): + with self.subTest(module=module): + sys.modules[module_name] = module + try: + exec_namespace({ + "inspect": inspect, + "__name__": module_name, + }, modfile, mod) + finally: + del sys.modules[module_name] + + fileless_name = f"{__name__}.fileless" + fileless = types.ModuleType(fileless_name) + fileless.__file__ = None + sys.modules[fileless_name] = fileless + try: + # Namespace packages and other fileless modules must not + # prevent the filename fallback from scanning later entries. + exec_namespace({"inspect": inspect}, modfile, mod) + finally: + del sys.modules[fileless_name] + + # A namespace and filename with no registered module still + # resolves to None after the filename fallback. + with temp_cwd() as cwd: + filename = os.path.join(cwd, "not_registered.py") + exec_namespace({"inspect": inspect}, filename, None) + + def test_getmodule_skips_unchanged_module_rescan(self): + with temp_cwd() as cwd: + filename = os.path.join(cwd, "not_registered.py") + namespace = {"inspect": inspect} + exec(compile("frame = inspect.currentframe()", filename, "exec"), + namespace) + frame = namespace["frame"] + module_name = f"{__name__}.late_registered" + marker_name = f"{__name__}.scan_marker" + marker = types.ModuleType(marker_name) + marker.__file__ = os.path.join(cwd, "scan_marker.py") + with open(marker.__file__, "w"): + pass + sys.modules[marker_name] = marker + + original_ismodule = inspect.ismodule + scan_count = 0 + + def ismodule(object): + nonlocal scan_count + if object is marker: + scan_count += 1 + return original_ismodule(object) + + try: + with (unittest.mock.patch.object( + inspect, "modulesbyfile", {}), + unittest.mock.patch.object( + inspect, "_filesbymodname", {}), + unittest.mock.patch.object( + inspect, "ismodule", ismodule), + unittest.mock.patch.object( + inspect, "_modulesbyfile_cache", (None, {}, {}, ()))): + self.assertIsNone(inspect.getmodule(frame)) + first_scan_count = scan_count + self.assertGreater(first_scan_count, 0) + self.assertIsNone(inspect.getmodule(frame)) + self.assertEqual(scan_count, first_scan_count) + + module = types.ModuleType(module_name) + module.__file__ = filename + sys.modules[module_name] = module + self.assertIs(inspect.getmodule(frame.f_code), module) + self.assertGreater(scan_count, first_scan_count) + second_scan_count = scan_count + + # Replacement mappings cannot be watched, so every miss + # scans even when the entries are unchanged. + replacement = sys.modules.copy() + unknown = os.path.join(cwd, "unknown.py") + with unittest.mock.patch.object( + sys, "modules", replacement): + self.assertIsNone( + inspect.getmodule(None, unknown)) + third_scan_count = scan_count + self.assertGreater( + third_scan_count, second_scan_count) + self.assertIsNone( + inspect.getmodule(None, unknown)) + self.assertGreater(scan_count, third_scan_count) + finally: + sys.modules.pop(module_name, None) + sys.modules.pop(marker_name, None) + + def test_getmodule_rescans_replaced_module_with_same_name(self): + module_name = f"{__name__}.replaced" + with (temp_cwd() as cwd, + unittest.mock.patch.object(inspect, "modulesbyfile", {}), + unittest.mock.patch.object(inspect, "_filesbymodname", {}), + unittest.mock.patch.object( + inspect, "_modulesbyfile_cache", (None, {}, {}, ()))): + old_module = types.ModuleType(module_name) + old_module.__file__ = os.path.join(cwd, "old.py") + new_module = types.ModuleType(module_name) + new_module.__file__ = os.path.join(cwd, "new.py") + unknown = os.path.join(cwd, "unknown.py") + sys.modules[module_name] = old_module + try: + self.assertIsNone(inspect.getmodule(None, unknown)) + self.assertIs( + inspect.getmodule(None, old_module.__file__), old_module) + del sys.modules[module_name] + sys.modules[module_name] = new_module + self.assertIsNone( + inspect.getmodule(None, old_module.__file__)) + self.assertIs( + inspect.getmodule(None, new_module.__file__), new_module) + finally: + sys.modules.pop(module_name, None) + + def test_getmodule_sys_modules_replaced_during_lookup(self): + module_name = f"{__name__}.registry_replaced_during_lookup" + with (temp_cwd() as cwd, + unittest.mock.patch.object(inspect, "modulesbyfile", {}), + unittest.mock.patch.object(inspect, "_filesbymodname", {}), + unittest.mock.patch.object( + inspect, "_modulesbyfile_cache", (None, {}, {}, ()))): + old_registry = sys.modules + old_module = types.ModuleType(module_name) + old_module.__file__ = os.path.join(cwd, "old.py") + new_module = types.ModuleType(module_name) + new_module.__file__ = os.path.join(cwd, "new.py") + old_registry[module_name] = old_module + replacement = old_registry.copy() + replacement[module_name] = new_module + original_getabsfile = inspect.getabsfile + replace_registry = True + + def getabsfile(object, _filename=None): + nonlocal replace_registry + if replace_registry: + replace_registry = False + sys.modules = replacement + return original_getabsfile(object, _filename) + + try: + with unittest.mock.patch.object( + inspect, "getabsfile", getabsfile): + self.assertIs( + inspect.getmodule(None, new_module.__file__), + new_module) + finally: + sys.modules = old_registry + old_registry.pop(module_name, None) + + def test_getmodule_rescans_changed_module_file(self): + module_name = f"{__name__}.changed_file" + with (temp_cwd() as cwd, + unittest.mock.patch.object(inspect, "modulesbyfile", {}), + unittest.mock.patch.object(inspect, "_filesbymodname", {}), + unittest.mock.patch.object( + inspect, "_modulesbyfile_cache", (None, {}, {}, ()))): + module = types.ModuleType(module_name) + module.__file__ = os.path.join(cwd, "old.py") + new_file = os.path.join(cwd, "new.py") + renamed = module_name + ".renamed" + unknown = os.path.join(cwd, "unknown.py") + sys.modules[module_name] = module + try: + self.assertIsNone(inspect.getmodule(None, unknown)) + self.assertIs( + inspect.getmodule(None, module.__file__), module) + old_file = module.__file__ + module.__file__ = new_file + self.assertIsNone(inspect.getmodule(None, old_file)) + self.assertIs(inspect.getmodule(None, new_file), module) + module.__name__ = renamed + self.assertIsNone(inspect.getmodule(None, new_file)) + sys.modules[renamed] = module + self.assertIs(inspect.getmodule(None, new_file), module) + finally: + sys.modules.pop(renamed, None) + sys.modules.pop(module_name, None) + + def test_getmodule_retries_overlapping_scan(self): + marker_name = f"{__name__}.scan_marker" + module_name = f"{__name__}.added_during_scan" + with (temp_cwd() as cwd, + unittest.mock.patch.object(inspect, "modulesbyfile", {}), + unittest.mock.patch.object(inspect, "_filesbymodname", {}), + unittest.mock.patch.object( + inspect, "_modulesbyfile_cache", (None, {}, {}, ()))): + marker = types.ModuleType(marker_name) + marker.__file__ = os.path.join(cwd, "marker.py") + module = types.ModuleType(module_name) + module.__file__ = os.path.join(cwd, "added.py") + sys.modules[marker_name] = marker + original_ismodule = inspect.ismodule + mutate = True + + def ismodule(object): + nonlocal mutate + if object is marker and mutate: + mutate = False + sys.modules[module_name] = module + return original_ismodule(object) + + try: + with unittest.mock.patch.object( + inspect, "ismodule", ismodule): + self.assertIs( + inspect.getmodule(None, module.__file__), module) + self.assertIsNotNone(inspect._modulesbyfile_cache[0]) + finally: + sys.modules.pop(module_name, None) + sys.modules.pop(marker_name, None) + + @threading_helper.requires_working_threading() + def test_getmodule_cache_publish_is_atomic(self): + module_name = f"{__name__}.concurrent_publish" + with (temp_cwd() as cwd, + unittest.mock.patch.object(inspect, "modulesbyfile", {}), + unittest.mock.patch.object(inspect, "_filesbymodname", {}), + unittest.mock.patch.object( + inspect, "_modulesbyfile_cache", (None, {}, {}, ()))): + old_module = types.ModuleType(module_name) + old_module.__file__ = os.path.join(cwd, "old.py") + new_module = types.ModuleType(module_name) + new_module.__file__ = os.path.join(cwd, "new.py") + unknown = os.path.join(cwd, "unknown.py") + sys.modules[module_name] = old_module + + original_publish = inspect._publish_module_cache + publish_ready = threading.Event() + release_publish = threading.Event() + errors = [] + + def publish(version, modules_by_file, files_by_module, + volatile_modules): + if threading.current_thread().name == "inspect-old-cache": + publish_ready.set() + release_publish.wait() + original_publish( + version, modules_by_file, files_by_module, + volatile_modules) + + def build_old_cache(): + try: + inspect.getmodule(None, unknown) + except BaseException as error: + errors.append(error) + + thread = threading.Thread( + target=build_old_cache, name="inspect-old-cache") + try: + with unittest.mock.patch.object( + inspect, "_publish_module_cache", publish): + thread.start() + self.assertTrue( + publish_ready.wait(support.SHORT_TIMEOUT)) + + # Let a newer cache publish while the old publisher is + # paused after validating its registry version. + sys.modules[module_name] = new_module + self.assertIsNone(inspect.getmodule(None, unknown)) + release_publish.set() + thread.join(support.SHORT_TIMEOUT) + + self.assertFalse(thread.is_alive()) + if errors: + raise errors[0] + # The delayed old snapshot may replace the newer one, but its + # version must travel with its maps so it is never accepted as + # current. + self.assertIsNone( + inspect.getmodule(None, old_module.__file__)) + self.assertIs( + inspect.getmodule(None, new_module.__file__), new_module) + finally: + release_publish.set() + thread.join() + sys.modules.pop(module_name, None) + + def test_getmodule_dynamic_module_class(self): + class DynamicFileModule(types.ModuleType): + @property + def __file__(self): + return self._origin + + @property + def __name__(self): + return self._dynamic_name + + module_name = f"{__name__}.dynamic_class" + with (temp_cwd() as cwd, + unittest.mock.patch.object(inspect, "modulesbyfile", {}), + unittest.mock.patch.object(inspect, "_filesbymodname", {}), + unittest.mock.patch.object( + inspect, "_modulesbyfile_cache", (None, {}, {}, ()))): + module = types.ModuleType(module_name) + module.__file__ = os.path.join(cwd, "static.py") + module._origin = os.path.join(cwd, "dynamic.py") + module._dynamic_name = module_name + # Exercise a warmed getfile() call site before changing the + # module's class. getmodule() must canonicalize the dynamic value + # it reads rather than re-reading stale module-dict metadata. + for _ in range(100): + inspect.getfile(module) + unknown = os.path.join(cwd, "unknown.py") + sys.modules[module_name] = module + try: + self.assertIsNone(inspect.getmodule(None, unknown)) + self.assertIsNotNone(inspect._modulesbyfile_cache[0]) + + module.__class__ = DynamicFileModule + self.assertIs( + inspect.getmodule(None, module._origin), module) + self.assertIsNotNone(inspect._modulesbyfile_cache[0]) + self.assertTrue(inspect._modulesbyfile_cache[3]) + + module.inspect = inspect + frame_origin = os.path.join(cwd, ".", "dynamic.py") + exec(compile("frame = inspect.currentframe()", + frame_origin, "exec"), module.__dict__) + self.assertIs(inspect.getmodule(module.frame), module) + + module._origin = os.path.join(cwd, "changed.py") + self.assertIs( + inspect.getmodule(None, module._origin), module) + + renamed = module_name + ".renamed" + module._dynamic_name = renamed + self.assertIsNone( + inspect.getmodule(None, module._origin)) + sys.modules[renamed] = module + self.assertIs( + inspect.getmodule(None, module._origin), module) + finally: + sys.modules.pop(module_name + ".renamed", None) + sys.modules.pop(module_name, None) + + def test_getmodule_dynamic_module_getattr(self): + module_name = f"{__name__}.dynamic_getattr" + with (temp_cwd() as cwd, + unittest.mock.patch.object(inspect, "modulesbyfile", {}), + unittest.mock.patch.object(inspect, "_filesbymodname", {}), + unittest.mock.patch.object( + inspect, "_modulesbyfile_cache", (None, {}, {}, ()))): + module = types.ModuleType(module_name) + module._origin = None + unknown = os.path.join(cwd, "unknown.py") + sys.modules[module_name] = module + try: + self.assertIsNone(inspect.getmodule(None, unknown)) + self.assertIsNotNone(inspect._modulesbyfile_cache[0]) + + def __getattr__(name): + if name == "__file__" and module._origin is not None: + return module._origin + raise AttributeError(name) + + module.__getattr__ = __getattr__ + self.assertIsNone(inspect.getmodule(None, unknown)) + self.assertTrue(inspect._modulesbyfile_cache[3]) + + module._origin = os.path.join(cwd, "dynamic.py") + self.assertIs( + inspect.getmodule(None, module._origin), module) + self.assertIsNotNone(inspect._modulesbyfile_cache[0]) + self.assertTrue(inspect._modulesbyfile_cache[3]) + + module._origin = os.path.join(cwd, "changed.py") + self.assertIs( + inspect.getmodule(None, module._origin), module) + finally: + sys.modules.pop(module_name, None) + + def test_getmodule_dynamic_module_does_not_disable_cache(self): + class DynamicFileModule(types.ModuleType): + @property + def __file__(self): + return self._origin + + module_name = f"{__name__}.dynamic_cached" + marker_name = f"{__name__}.dynamic_scan_marker" + with (temp_cwd() as cwd, + unittest.mock.patch.object(inspect, "modulesbyfile", {}), + unittest.mock.patch.object(inspect, "_filesbymodname", {}), + unittest.mock.patch.object( + inspect, "_modulesbyfile_cache", (None, {}, {}, ()))): + module = DynamicFileModule(module_name) + module._origin = os.path.join(cwd, "dynamic.py") + marker = types.ModuleType(marker_name) + marker.__file__ = os.path.join(cwd, "marker.py") + unknown = os.path.join(cwd, "unknown.py") + original_ismodule = inspect.ismodule + scan_count = 0 + + def ismodule(object): + nonlocal scan_count + if object is marker: + scan_count += 1 + return original_ismodule(object) + + sys.modules[module_name] = module + sys.modules[marker_name] = marker + try: + with unittest.mock.patch.object( + inspect, "ismodule", ismodule): + self.assertIs( + inspect.getmodule(None, module._origin), module) + first_scan_count = scan_count + self.assertGreater(first_scan_count, 0) + + # Only the dynamic entry is revalidated; the stable + # negative lookup does not rescan the whole registry. + self.assertIsNone(inspect.getmodule(None, unknown)) + self.assertEqual(scan_count, first_scan_count) + + # An unwatchable change to the dynamic metadata is found + # by that targeted validation and rebuilds the cache. + module._origin = os.path.join(cwd, "changed.py") + self.assertIs( + inspect.getmodule(None, module._origin), module) + self.assertGreater(scan_count, first_scan_count) + finally: + sys.modules.pop(marker_name, None) + sys.modules.pop(module_name, None) + + def test_getmodule_preserves_legacy_cache_objects(self): + with (temp_cwd() as cwd, + unittest.mock.patch.object(inspect, "modulesbyfile", {}), + unittest.mock.patch.object(inspect, "_filesbymodname", {}), + unittest.mock.patch.object( + inspect, "_modulesbyfile_cache", (None, {}, {}, ()))): + modulesbyfile = inspect.modulesbyfile + filesbymodname = inspect._filesbymodname + unknown = os.path.join(cwd, "unknown.py") + self.assertIsNone(inspect.getmodule(None, unknown)) + self.assertIs(inspect.modulesbyfile, modulesbyfile) + self.assertIs(inspect._filesbymodname, filesbymodname) + self.assertTrue(all( + not isinstance(value, tuple) + for value in filesbymodname.values() + )) + + def test_getmodule_version_does_not_retain_modules(self): + with temp_cwd() as cwd: + module_name = f"{__name__}.snapshot_module" + module = types.ModuleType(module_name) + module.__file__ = os.path.join(cwd, "snapshot_module.py") + with open(module.__file__, "w"): + pass + module_ref = weakref.ref(module) + modules = sys.modules.copy() + modules[module_name] = module + + with (unittest.mock.patch.object(sys, "modules", modules), + unittest.mock.patch.object(inspect, "modulesbyfile", {}), + unittest.mock.patch.object(inspect, "_filesbymodname", {}), + unittest.mock.patch.object( + inspect, "_modulesbyfile_cache", (None, {}, {}, ()))): + self.assertIs(inspect.getmodule(None, module.__file__), module) + del modules[module_name] + del module + support.gc_collect() + self.assertIsNone(module_ref()) + + def test_getmodule_registered_exec_frame(self): + def exec_module(module, filename): + module.inspect = inspect + exec(compile(textwrap.dedent(""" + frame = inspect.currentframe() + try: + 1 / 0 + except ZeroDivisionError as error: + traceback = error.__traceback__ + """), filename, "exec"), module.__dict__) + + module_name = f"{__name__}.registered" + module = types.ModuleType(module_name) + sys.modules[module_name] = module + try: + with temp_cwd() as cwd: + filename = os.path.join(cwd, "registered.py") + with open(filename, "w"): + pass + module.__file__ = filename + + exec_module(module, filename) + self.assertIs(inspect.getmodule(module.frame), module) + self.assertIs(inspect.getmodule(module.traceback), module) + + # The execution namespace identifies the owner even if a + # later registry entry happens to use the same filename. + collision_name = f"{module_name}.collision" + collision = types.ModuleType(collision_name) + collision.__file__ = filename + sys.modules[collision_name] = collision + try: + exec_module(module, filename) + self.assertIs(inspect.getmodule(module.frame), module) + finally: + del sys.modules[collision_name] + + # Globals identity is insufficient when the code came from a + # different origin than the registered module. + exec_module(module, filename + ".other") + self.assertIsNone(inspect.getmodule(module.frame)) + self.assertIsNone(inspect.getmodule(module.traceback)) + + # If another module supplied the source, fall back to the + # filename-based lookup instead of trusting frame globals. + source_name = f"{module_name}.source" + source_module = types.ModuleType(source_name) + source_module.__file__ = filename + ".source" + with open(source_module.__file__, "w"): + pass + sys.modules[source_name] = source_module + try: + exec_module(module, source_module.__file__) + self.assertIs(inspect.getmodule(module.frame), + source_module) + self.assertIs(inspect.getmodule(module.traceback), + source_module) + finally: + del sys.modules[source_name] + + # Preserve the existing result for fileless modules. + del module.__file__ + exec_module(module, "") + self.assertIsNone(inspect.getmodule(module.frame)) + self.assertIsNone(inspect.getmodule(module.traceback)) + finally: + del sys.modules[module_name] def test_getmodule_file_not_found(self): # See bpo-45406 diff --git a/Lib/test/test_zipimport_support.py b/Lib/test/test_zipimport_support.py index 2b28f46149b4ff3..6aa1950c2f628dd 100644 --- a/Lib/test/test_zipimport_support.py +++ b/Lib/test/test_zipimport_support.py @@ -10,6 +10,7 @@ import zipfile import zipimport import doctest +import importlib import inspect import linecache import unittest @@ -96,6 +97,36 @@ def test_inspect_getsource_issue4223(self): finally: del sys.modules["zip_pkg"] + def test_inspect_fresh_namespace_uses_module_loader(self): + test_src = textwrap.dedent("""\ + import inspect + + def capture(): + return inspect.currentframe() + + frame = capture() + """) + expected_source = textwrap.dedent("""\ + def capture(): + return inspect.currentframe() + """) + module_name = "test_zipped_inspect" + with os_helper.temp_dir() as d: + script_name = make_script(d, module_name, test_src) + zip_name, _ = make_zip_script(d, "test_zip", script_name) + os.remove(script_name) + sys.path.insert(0, zip_name) + module = importlib.import_module(module_name) + try: + namespace = {} + exec(compile(test_src, module.__file__, "exec"), namespace) + frame = namespace["frame"] + + self.assertIs(inspect.getmodule(frame), module) + self.assertEqual(inspect.getsource(frame), expected_source) + finally: + del sys.modules[module_name] + def test_doctest_issue4197(self): # To avoid having to keep two copies of the doctest module's # unit tests in sync, this test works by taking the source of diff --git a/Misc/NEWS.d/next/Library/2026-08-14-16-30-00.gh-issue-92041.kD7pQm.rst b/Misc/NEWS.d/next/Library/2026-08-14-16-30-00.gh-issue-92041.kD7pQm.rst new file mode 100644 index 000000000000000..2e9e1cacc153bdd --- /dev/null +++ b/Misc/NEWS.d/next/Library/2026-08-14-16-30-00.gh-issue-92041.kD7pQm.rst @@ -0,0 +1,4 @@ +Improve :func:`inspect.getmodule` performance for frame, traceback, and code +objects by resolving registered modules directly from frame globals and +avoiding repeated full scans while :data:`sys.modules` and relevant module +metadata are unchanged. diff --git a/Objects/moduleobject.c b/Objects/moduleobject.c index b8cd6025c20ba56..d4813b433bb36c6 100644 --- a/Objects/moduleobject.c +++ b/Objects/moduleobject.c @@ -12,10 +12,11 @@ #include "pycore_modsupport.h" // _PyModule_CreateInitialized() #include "pycore_moduleobject.h" // _PyModule_GetDefOrNull() #include "pycore_object.h" // _PyType_AllocNoTrack +#include "pycore_pyatomic_ft_wrappers.h" // FT_ATOMIC_ADD_UINT64() #include "pycore_pyerrors.h" // _PyErr_FormatFromCause() #include "pycore_pystate.h" // _PyInterpreterState_GET() #include "pycore_slots.h" // _PySlotIterator_Init -#include "pycore_unicodeobject.h" // _PyUnicode_EqualToASCIIString() +#include "pycore_unicodeobject.h" // _PyUnicode_Equal() #include "pycore_weakref.h" // FT_CLEAR_WEAKREFS() #include "osdefs.h" // MAXPATHLEN @@ -196,15 +197,56 @@ new_module_notrack(PyTypeObject *mt) return m; } -/* Module dict watcher callback. - * When a module dictionary is modified, we need to clear the keys version - * to invalidate any cached lookups that depend on the dictionary structure. +/* Module and module-registry dict watcher callback. + * When a module dictionary is modified, we may need to clear the keys version + * or invalidate filename-based module lookups. */ +static bool +is_module_lookup_attr(PyObject *key) +{ + if (key == &_Py_ID(__file__) || key == &_Py_ID(__name__) || + key == &_Py_ID(__getattr__)) { + return true; + } + if (!PyUnicode_Check(key)) { + return false; + } + // Interned strings compare by identity, so the common module-global key + // cannot match any of the identifiers checked above. + if (PyUnicode_CheckExact(key) && PyUnicode_CHECK_INTERNED(key)) { + return false; + } + Py_ssize_t length = PyUnicode_GET_LENGTH(key); + return ((length == 8 && + (_PyUnicode_Equal(key, &_Py_ID(__file__)) || + _PyUnicode_Equal(key, &_Py_ID(__name__)))) || + (length == 11 && + _PyUnicode_Equal(key, &_Py_ID(__getattr__)))); +} + static int module_dict_watcher(PyDict_WatchEvent event, PyObject *dict, PyObject *key, PyObject *new_value) { assert(PyDict_Check(dict)); + PyInterpreterState *interp = _PyInterpreterState_GET(); + bool registry_changed = dict == interp->imports.modules; + if (!registry_changed) { + if ((event == PyDict_EVENT_ADDED || + event == PyDict_EVENT_MODIFIED || + event == PyDict_EVENT_DELETED) && + is_module_lookup_attr(key)) { + registry_changed = true; + } + else if (event == PyDict_EVENT_CLONED || + event == PyDict_EVENT_CLEARED) { + // These events may change lookup metadata without a key event. + registry_changed = true; + } + } + if (registry_changed) { + FT_ATOMIC_ADD_UINT64(interp->imports.module_registry_version, 1); + } // Only if a new lazy object shows up do we need to clear the dictionary. If // this is adding a new key then the version will be reset anyway. if (event == PyDict_EVENT_MODIFIED && @@ -215,6 +257,21 @@ module_dict_watcher(PyDict_WatchEvent event, PyObject *dict, return 0; } +static int +module_setattro(PyObject *self, PyObject *name, PyObject *value) +{ + PyTypeObject *old_type = Py_TYPE(self); + int result = PyObject_GenericSetAttr(self, name, value); + if (result == 0 && Py_TYPE(self) != old_type) { + // Module dict changes are handled by the watcher above. A class + // change is not a dict mutation, but may make lookup metadata + // dynamic. + PyInterpreterState *interp = _PyInterpreterState_GET(); + FT_ATOMIC_ADD_UINT64(interp->imports.module_registry_version, 1); + } + return result; +} + int _PyModule_InitModuleDictWatcher(PyInterpreterState *interp) { @@ -1812,7 +1869,7 @@ PyTypeObject PyModule_Type = { 0, /* tp_call */ 0, /* tp_str */ _Py_module_getattro, /* tp_getattro */ - PyObject_GenericSetAttr, /* tp_setattro */ + module_setattro, /* tp_setattro */ 0, /* tp_as_buffer */ Py_TPFLAGS_DEFAULT | Py_TPFLAGS_HAVE_GC | Py_TPFLAGS_BASETYPE, /* tp_flags */ diff --git a/Python/clinic/import.c.h b/Python/clinic/import.c.h index 2e4e178b3084063..0c62be9ce08e72e 100644 --- a/Python/clinic/import.c.h +++ b/Python/clinic/import.c.h @@ -70,6 +70,15 @@ _imp_release_lock(PyObject *module, PyObject *Py_UNUSED(ignored)) return _imp_release_lock_impl(module); } +PyDoc_STRVAR(_imp__get_module_registry_version__doc__, +"_get_module_registry_version($module, modules, /)\n" +"--\n" +"\n" +"Return the canonical module registry\'s filename-lookup version, or None."); + +#define _IMP__GET_MODULE_REGISTRY_VERSION_METHODDEF \ + {"_get_module_registry_version", (PyCFunction)_imp__get_module_registry_version, METH_O, _imp__get_module_registry_version__doc__}, + PyDoc_STRVAR(_imp__fix_co_filename__doc__, "_fix_co_filename($module, code, path, /)\n" "--\n" @@ -665,4 +674,4 @@ _imp__set_lazy_attributes(PyObject *module, PyObject *const *args, Py_ssize_t na #ifndef _IMP_EXEC_DYNAMIC_METHODDEF #define _IMP_EXEC_DYNAMIC_METHODDEF #endif /* !defined(_IMP_EXEC_DYNAMIC_METHODDEF) */ -/*[clinic end generated code: output=0974db098d601372 input=a9049054013a1b77]*/ +/*[clinic end generated code: output=e96d82cd2522bb44 input=a9049054013a1b77]*/ diff --git a/Python/import.c b/Python/import.c index 5ca78a971fa54c6..19be9f3cb165337 100644 --- a/Python/import.c +++ b/Python/import.c @@ -181,6 +181,11 @@ _PyImport_InitModules(PyInterpreterState *interp) if (MODULES(interp) == NULL) { return NULL; } + interp->imports.module_registry_version = 1; + if (PyDict_Watch(MODULE_WATCHER_ID, MODULES(interp)) < 0) { + Py_CLEAR(MODULES(interp)); + return NULL; + } return MODULES(interp); } @@ -5104,6 +5109,29 @@ _imp_release_lock_impl(PyObject *module) } +/*[clinic input] +_imp._get_module_registry_version + + modules: object + / + +Return the canonical module registry's filename-lookup version, or None. +[clinic start generated code]*/ + +static PyObject * +_imp__get_module_registry_version(PyObject *module, PyObject *modules) +/*[clinic end generated code: output=f4aa8c97595da312 input=a223282dad9ff65d]*/ +{ + PyInterpreterState *interp = _PyInterpreterState_GET(); + if (modules != MODULES(interp)) { + Py_RETURN_NONE; + } + uint64_t version = FT_ATOMIC_LOAD_UINT64_ACQUIRE( + interp->imports.module_registry_version); + return PyLong_FromUnsignedLongLong(version); +} + + /*[clinic input] _imp._fix_co_filename @@ -5678,6 +5706,7 @@ static PyMethodDef imp_methods[] = { _IMP_IS_BUILTIN_METHODDEF _IMP_IS_FROZEN_METHODDEF _IMP__FROZEN_MODULE_NAMES_METHODDEF + _IMP__GET_MODULE_REGISTRY_VERSION_METHODDEF _IMP__OVERRIDE_FROZEN_MODULES_FOR_TESTS_METHODDEF _IMP__OVERRIDE_MULTI_INTERP_EXTENSIONS_CHECK_METHODDEF _IMP_CREATE_DYNAMIC_METHODDEF