From 004869681a2b618d5b64fd42b8f7812f65c7dc88 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Thu, 15 Nov 2012 21:39:36 -0500 Subject: Issue #15894: Document why we don't worry about re-acquiring the global import lock after forking. --- Python/import.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 2f71b97bc8..6882b57060 100644 --- a/Python/import.c +++ b/Python/import.c @@ -202,8 +202,11 @@ _PyImport_ReInitLock(void) if (import_lock_level > 1) { /* Forked as a side effect of import */ long me = PyThread_get_thread_ident(); - PyThread_acquire_lock(import_lock, 0); - /* XXX: can the previous line fail? */ + /* The following could fail if the lock is already held, but forking as + a side-effect of an import is a) rare, b) nuts, and c) difficult to + do thanks to the lock only being held when doing individual module + locks per import. */ + PyThread_acquire_lock(import_lock, NOWAIT_LOCK); import_lock_thread = me; import_lock_level--; } else { -- cgit v1.2.1 From 838169869f9d3474c58ed662b1a51541de235c34 Mon Sep 17 00:00:00 2001 From: Andrew Svetlov Date: Fri, 14 Dec 2012 17:04:59 +0200 Subject: Issue #16421: allow to load multiple modules from the same shared object. Patch by V?clav ?milauer. --- Python/import.c | 27 +++++++++++++++++---------- 1 file changed, 17 insertions(+), 10 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 6882b57060..8e7482ac4e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -452,12 +452,12 @@ PyImport_GetMagicTag(void) /* Magic for extension modules (built-in as well as dynamically loaded). To prevent initializing an extension module more than - once, we keep a static dictionary 'extensions' keyed by module name - (for built-in modules) or by filename (for dynamically loaded - modules), containing these modules. A copy of the module's - dictionary is stored by calling _PyImport_FixupExtensionObject() - immediately after the module initialization function succeeds. A - copy can be retrieved from there by calling + once, we keep a static dictionary 'extensions' keyed by the tuple + (module name, module name) (for built-in modules) or by + (filename, module name) (for dynamically loaded modules), containing these + modules. A copy of the module's dictionary is stored by calling + _PyImport_FixupExtensionObject() immediately after the module initialization + function succeeds. A copy can be retrieved from there by calling _PyImport_FindExtensionObject(). Modules which do support multiple initialization set their m_size @@ -470,7 +470,7 @@ int _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, PyObject *filename) { - PyObject *modules, *dict; + PyObject *modules, *dict, *filename_name; struct PyModuleDef *def; if (extensions == NULL) { extensions = PyDict_New(); @@ -508,7 +508,11 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, if (def->m_base.m_copy == NULL) return -1; } - PyDict_SetItem(extensions, filename, (PyObject*)def); + filename_name = PyTuple_Pack(2,filename, name); + if (filename_name == NULL) + return -1; + if (PyDict_SetItem(extensions, filename_name, (PyObject*)def) < 0) + return -1; return 0; } @@ -528,11 +532,14 @@ _PyImport_FixupBuiltin(PyObject *mod, char *name) PyObject * _PyImport_FindExtensionObject(PyObject *name, PyObject *filename) { - PyObject *mod, *mdict; + PyObject *mod, *mdict, *filename_name; PyModuleDef* def; if (extensions == NULL) return NULL; - def = (PyModuleDef*)PyDict_GetItem(extensions, filename); + filename_name = PyTuple_Pack(2,filename, name); + if (filename_name == NULL) + return NULL; + def = (PyModuleDef*)PyDict_GetItem(extensions, filename_name); if (def == NULL) return NULL; if (def->m_size == -1) { -- cgit v1.2.1 From a0d1572bb8860a41834b2bca05748df2a28ffb5d Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sat, 15 Dec 2012 00:05:16 -0500 Subject: cleanup and fix refleaks --- Python/import.c | 20 ++++++++++++-------- 1 file changed, 12 insertions(+), 8 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 8e7482ac4e..50267895c5 100644 --- a/Python/import.c +++ b/Python/import.c @@ -470,8 +470,9 @@ int _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, PyObject *filename) { - PyObject *modules, *dict, *filename_name; + PyObject *modules, *dict, *key; struct PyModuleDef *def; + int res; if (extensions == NULL) { extensions = PyDict_New(); if (extensions == NULL) @@ -508,10 +509,12 @@ _PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, if (def->m_base.m_copy == NULL) return -1; } - filename_name = PyTuple_Pack(2,filename, name); - if (filename_name == NULL) + key = PyTuple_Pack(2, filename, name); + if (key == NULL) return -1; - if (PyDict_SetItem(extensions, filename_name, (PyObject*)def) < 0) + res = PyDict_SetItem(extensions, key, (PyObject *)def); + Py_DECREF(key); + if (res < 0) return -1; return 0; } @@ -532,14 +535,15 @@ _PyImport_FixupBuiltin(PyObject *mod, char *name) PyObject * _PyImport_FindExtensionObject(PyObject *name, PyObject *filename) { - PyObject *mod, *mdict, *filename_name; + PyObject *mod, *mdict, *key; PyModuleDef* def; if (extensions == NULL) return NULL; - filename_name = PyTuple_Pack(2,filename, name); - if (filename_name == NULL) + key = PyTuple_Pack(2, filename, name); + if (key == NULL) return NULL; - def = (PyModuleDef*)PyDict_GetItem(extensions, filename_name); + def = (PyModuleDef *)PyDict_GetItem(extensions, key); + Py_DECREF(key); if (def == NULL) return NULL; if (def->m_size == -1) { -- cgit v1.2.1 From 63c6c6e958c8f9749e69ad62e653147ff08680c1 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Mon, 18 Mar 2013 23:13:31 -0700 Subject: sprinkle const --- Python/import.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index a50ef05b07..8773be46ad 100644 --- a/Python/import.c +++ b/Python/import.c @@ -846,7 +846,7 @@ imp_fix_co_filename(PyObject *self, PyObject *args) /* Forward */ -static struct _frozen * find_frozen(PyObject *); +static const struct _frozen * find_frozen(PyObject *); /* Helper to test for built-in module */ @@ -983,10 +983,10 @@ init_builtin(PyObject *name) /* Frozen modules */ -static struct _frozen * +static const struct _frozen * find_frozen(PyObject *name) { - struct _frozen *p; + const struct _frozen *p; if (name == NULL) return NULL; @@ -1003,7 +1003,7 @@ find_frozen(PyObject *name) static PyObject * get_frozen_object(PyObject *name) { - struct _frozen *p = find_frozen(name); + const struct _frozen *p = find_frozen(name); int size; if (p == NULL) { @@ -1027,7 +1027,7 @@ get_frozen_object(PyObject *name) static PyObject * is_frozen_package(PyObject *name) { - struct _frozen *p = find_frozen(name); + const struct _frozen *p = find_frozen(name); int size; if (p == NULL) { @@ -1054,7 +1054,7 @@ is_frozen_package(PyObject *name) int PyImport_ImportFrozenModuleObject(PyObject *name) { - struct _frozen *p; + const struct _frozen *p; PyObject *co, *m, *path; int ispackage; int size; @@ -1781,7 +1781,7 @@ static PyObject * imp_is_frozen(PyObject *self, PyObject *args) { PyObject *name; - struct _frozen *p; + const struct _frozen *p; if (!PyArg_ParseTuple(args, "U:is_frozen", &name)) return NULL; p = find_frozen(name); -- cgit v1.2.1 From 05af1e7fca1463b6a0fc7a790313d93a3e68c222 Mon Sep 17 00:00:00 2001 From: Alexandre Vassalotti Date: Thu, 2 May 2013 10:44:04 -0700 Subject: Closes #17892: Fix the name of _PyObject_CallMethodObjIdArgs --- Python/import.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 805cdd7e2a..b77eda106d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -704,7 +704,7 @@ PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, "no interpreter!"); } - pathobj = _PyObject_CallMethodObjIdArgs(interp->importlib, + pathobj = _PyObject_CallMethodIdObjArgs(interp->importlib, &PyId__get_sourcefile, cpathobj, NULL); if (pathobj == NULL) @@ -1441,7 +1441,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, } if (initializing > 0) { /* _bootstrap._lock_unlock_module() releases the import lock */ - value = _PyObject_CallMethodObjIdArgs(interp->importlib, + value = _PyObject_CallMethodIdObjArgs(interp->importlib, &PyId__lock_unlock_module, abs_name, NULL); if (value == NULL) @@ -1459,7 +1459,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, } else { /* _bootstrap._find_and_load() releases the import lock */ - mod = _PyObject_CallMethodObjIdArgs(interp->importlib, + mod = _PyObject_CallMethodIdObjArgs(interp->importlib, &PyId__find_and_load, abs_name, builtins_import, NULL); if (mod == NULL) { @@ -1528,7 +1528,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, } } else { - final_mod = _PyObject_CallMethodObjIdArgs(interp->importlib, + final_mod = _PyObject_CallMethodIdObjArgs(interp->importlib, &PyId__handle_fromlist, mod, fromlist, builtins_import, NULL); -- cgit v1.2.1 From 5756d7dace379752116766907b63ac77ffc4bc98 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 6 May 2013 21:15:57 +0200 Subject: Issue #1545463: Global variables caught in reference cycles are now garbage-collected at shutdown. --- Python/import.c | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index b77eda106d..cd4fb788a1 100644 --- a/Python/import.c +++ b/Python/import.c @@ -403,6 +403,14 @@ PyImport_Cleanup(void) } } + /* Collect garbage remaining after deleting the modules. Mostly + reference cycles created by classes. */ + PyGC_Collect(); + + /* Dump GC stats before it's too late, since it uses the warnings + machinery. */ + _PyGC_DumpShutdownStats(); + /* Next, delete sys and builtins (in that order) */ value = PyDict_GetItemString(modules, "sys"); if (value != NULL && PyModule_Check(value)) { -- cgit v1.2.1 From 63d05b117ac621279bea27b1a7903d3085d18240 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 8 May 2013 13:23:25 +0200 Subject: Issue #1545463: At shutdown, defer finalization of codec modules so that stderr remains usable. (should fix Windows buildbot failures on test_gc) --- Python/import.c | 55 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 35 insertions(+), 20 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index cd4fb788a1..1fbafecc1b 100644 --- a/Python/import.c +++ b/Python/import.c @@ -295,6 +295,30 @@ static char* sys_files[] = { NULL }; +static int +is_essential_module(PyObject *name) +{ + Py_ssize_t name_len; + char *name_str = PyUnicode_AsUTF8AndSize(name, &name_len); + + if (name_str == NULL) { + PyErr_Clear(); + return 0; + } + if (strcmp(name_str, "builtins") == 0) + return 1; + if (strcmp(name_str, "sys") == 0) + return 1; + /* These are all needed for stderr to still function */ + if (strcmp(name_str, "codecs") == 0) + return 1; + if (strcmp(name_str, "_codecs") == 0) + return 1; + if (strncmp(name_str, "encodings.", 10) == 0) + return 1; + return 0; +} + /* Un-initialize things, as good as we can */ @@ -374,9 +398,7 @@ PyImport_Cleanup(void) if (value->ob_refcnt != 1) continue; if (PyUnicode_Check(key) && PyModule_Check(value)) { - if (PyUnicode_CompareWithASCIIString(key, "builtins") == 0) - continue; - if (PyUnicode_CompareWithASCIIString(key, "sys") == 0) + if (is_essential_module(key)) continue; if (Py_VerboseFlag) PySys_FormatStderr( @@ -392,9 +414,7 @@ PyImport_Cleanup(void) pos = 0; while (PyDict_Next(modules, &pos, &key, &value)) { if (PyUnicode_Check(key) && PyModule_Check(value)) { - if (PyUnicode_CompareWithASCIIString(key, "builtins") == 0) - continue; - if (PyUnicode_CompareWithASCIIString(key, "sys") == 0) + if (is_essential_module(key)) continue; if (Py_VerboseFlag) PySys_FormatStderr("# cleanup[2] %U\n", key); @@ -411,20 +431,15 @@ PyImport_Cleanup(void) machinery. */ _PyGC_DumpShutdownStats(); - /* Next, delete sys and builtins (in that order) */ - value = PyDict_GetItemString(modules, "sys"); - if (value != NULL && PyModule_Check(value)) { - if (Py_VerboseFlag) - PySys_WriteStderr("# cleanup sys\n"); - _PyModule_Clear(value); - PyDict_SetItemString(modules, "sys", Py_None); - } - value = PyDict_GetItemString(modules, "builtins"); - if (value != NULL && PyModule_Check(value)) { - if (Py_VerboseFlag) - PySys_WriteStderr("# cleanup builtins\n"); - _PyModule_Clear(value); - PyDict_SetItemString(modules, "builtins", Py_None); + /* Next, delete all remaining modules */ + pos = 0; + while (PyDict_Next(modules, &pos, &key, &value)) { + if (PyUnicode_Check(key) && PyModule_Check(value)) { + if (Py_VerboseFlag) + PySys_FormatStderr("# cleanup[3] %U\n", key); + _PyModule_Clear(value); + PyDict_SetItem(modules, key, Py_None); + } } /* Finally, clear and delete the modules directory */ -- cgit v1.2.1 From 4bb13f5d6801f4857651f13a800a964cb8b47223 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sun, 19 May 2013 01:11:58 +0200 Subject: Issue #17937: Try harder to collect cyclic garbage at shutdown. --- Python/import.c | 1 + 1 file changed, 1 insertion(+) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 1fbafecc1b..a42b0f89b0 100644 --- a/Python/import.c +++ b/Python/import.c @@ -444,6 +444,7 @@ PyImport_Cleanup(void) /* Finally, clear and delete the modules directory */ PyDict_Clear(modules); + _PyGC_CollectNoFail(); interp->modules = NULL; Py_DECREF(modules); } -- cgit v1.2.1 From 41c36b5a56ae88660fd66b03ade69aecf4c2de86 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 31 May 2013 23:18:39 -0400 Subject: Issue #18065: For frozen packages set __path__ to []. Previously __path__ was set to [__name__], but that could lead to bad results if someone managed to circumvent the frozen importer and somehow ended up with a finder that thought __name__ was a legit directory/location. --- Python/import.c | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index a42b0f89b0..0bb46d2e1e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1107,19 +1107,17 @@ PyImport_ImportFrozenModuleObject(PyObject *name) goto err_return; } if (ispackage) { - /* Set __path__ to the package name */ + /* Set __path__ to the empty list */ PyObject *d, *l; int err; m = PyImport_AddModuleObject(name); if (m == NULL) goto err_return; d = PyModule_GetDict(m); - l = PyList_New(1); + l = PyList_New(0); if (l == NULL) { goto err_return; } - Py_INCREF(name); - PyList_SET_ITEM(l, 0, name); err = PyDict_SetItemString(d, "__path__", l); Py_DECREF(l); if (err != 0) -- cgit v1.2.1 From 21d4e9534aceac9c2753b70bffe4f75ff38ec525 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 12 Jun 2013 23:29:18 -0400 Subject: Issue #15767: Touch up ModuleNotFoundError usage by import. Forgot to raise ModuleNotFoundError when None is found in sys.modules. This led to introducing the C function PyErr_SetImportErrorSubclass() to make setting ModuleNotFoundError easier. Also updated the reference docs to mention ModuleNotFoundError appropriately. Updated the docs for ModuleNotFoundError to mention the None in sys.modules case. Lastly, it was noticed that PyErr_SetImportError() was not setting an exception when returning None in one case. That issue is now fixed. --- Python/import.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 0bb46d2e1e..fad54e66ce 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1436,7 +1436,8 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, PyObject *msg = PyUnicode_FromFormat("import of %R halted; " "None in sys.modules", abs_name); if (msg != NULL) { - PyErr_SetImportError(msg, abs_name, NULL); + PyErr_SetImportErrorSubclass(PyExc_ModuleNotFoundError, msg, + abs_name, NULL); Py_DECREF(msg); } mod = NULL; -- cgit v1.2.1 From 23c2e70efd68fb16de7b6b3ad0e469784c966ace Mon Sep 17 00:00:00 2001 From: Christian Heimes Date: Sun, 23 Jun 2013 15:53:09 +0200 Subject: import.c does neither need mode_t nor _mkdir() anymore --- Python/import.c | 8 -------- 1 file changed, 8 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index fad54e66ce..b9c5924887 100644 --- a/Python/import.c +++ b/Python/import.c @@ -19,14 +19,6 @@ extern "C" { #endif -#ifdef MS_WINDOWS -/* for stat.st_mode */ -typedef unsigned short mode_t; -/* for _mkdir */ -#include -#endif - - #define CACHEDIR "__pycache__" /* See _PyImport_FixupExtensionObject() below */ -- cgit v1.2.1 From af5e4f4219618da2bfcb02b52ac7c7ac54b7542a Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Thu, 4 Jul 2013 17:48:16 -0400 Subject: Issue #15767: Revert 3a50025f1900 for ModuleNotFoundError --- Python/import.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index b9c5924887..2e5d205973 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1428,8 +1428,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, PyObject *msg = PyUnicode_FromFormat("import of %R halted; " "None in sys.modules", abs_name); if (msg != NULL) { - PyErr_SetImportErrorSubclass(PyExc_ModuleNotFoundError, msg, - abs_name, NULL); + PyErr_SetImportError(msg, abs_name, NULL); Py_DECREF(msg); } mod = NULL; -- cgit v1.2.1 From 556ce6a1ab262d2aa56c505aecf6173382ed0e3c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 16 Jul 2013 22:26:05 +0200 Subject: Issue #18408: handle PySys_GetObject() failure, raise a RuntimeError --- Python/import.c | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 2e5d205973..a1c0aee540 100644 --- a/Python/import.c +++ b/Python/import.c @@ -85,8 +85,10 @@ _PyImportZip_Init(void) int err = 0; path_hooks = PySys_GetObject("path_hooks"); - if (path_hooks == NULL) + if (path_hooks == NULL) { + PyErr_SetString(PyExc_RuntimeError, "unable to get sys.path_hooks"); goto error; + } if (Py_VerboseFlag) PySys_WriteStderr("# installing zipimport hook\n"); @@ -944,11 +946,11 @@ PyAPI_FUNC(PyObject *) PyImport_GetImporter(PyObject *path) { PyObject *importer=NULL, *path_importer_cache=NULL, *path_hooks=NULL; - if ((path_importer_cache = PySys_GetObject("path_importer_cache"))) { - if ((path_hooks = PySys_GetObject("path_hooks"))) { - importer = get_path_importer(path_importer_cache, - path_hooks, path); - } + path_importer_cache = PySys_GetObject("path_importer_cache"); + path_hooks = PySys_GetObject("path_hooks"); + if (path_importer_cache != NULL && path_hooks != NULL) { + importer = get_path_importer(path_importer_cache, + path_hooks, path); } Py_XINCREF(importer); /* get_path_importer returns a borrowed reference */ return importer; -- cgit v1.2.1 From c506ce6f93bff13d93144e71284cf0e5acf34814 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Wed, 31 Jul 2013 23:14:08 +0200 Subject: Issue #18214: Improve finalization of Python modules to avoid setting their globals to None, in most cases. --- Python/import.c | 159 ++++++++++++++++++++++++-------------------------------- 1 file changed, 68 insertions(+), 91 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index c6222ec57f..09106555a3 100644 --- a/Python/import.c +++ b/Python/import.c @@ -277,6 +277,7 @@ static char* sys_deletes[] = { "path", "argv", "ps1", "ps2", "last_type", "last_value", "last_traceback", "path_hooks", "path_importer_cache", "meta_path", + "__interactivehook__", /* misc stuff */ "flags", "float_info", NULL @@ -289,40 +290,17 @@ static char* sys_files[] = { NULL }; -static int -is_essential_module(PyObject *name) -{ - Py_ssize_t name_len; - char *name_str = PyUnicode_AsUTF8AndSize(name, &name_len); - - if (name_str == NULL) { - PyErr_Clear(); - return 0; - } - if (strcmp(name_str, "builtins") == 0) - return 1; - if (strcmp(name_str, "sys") == 0) - return 1; - /* These are all needed for stderr to still function */ - if (strcmp(name_str, "codecs") == 0) - return 1; - if (strcmp(name_str, "_codecs") == 0) - return 1; - if (strncmp(name_str, "encodings.", 10) == 0) - return 1; - return 0; -} - - /* Un-initialize things, as good as we can */ void PyImport_Cleanup(void) { - Py_ssize_t pos, ndone; + Py_ssize_t pos; PyObject *key, *value, *dict; PyInterpreterState *interp = PyThreadState_GET()->interp; PyObject *modules = interp->modules; + PyObject *builtins = interp->builtins; + PyObject *weaklist = NULL; if (modules == NULL) return; /* Already done */ @@ -333,6 +311,8 @@ PyImport_Cleanup(void) deleted *last* of all, they would come too late in the normal destruction order. Sigh. */ + /* XXX Perhaps these precautions are obsolete. Who knows? */ + value = PyDict_GetItemString(modules, "builtins"); if (value != NULL && PyModule_Check(value)) { dict = PyModule_GetDict(value); @@ -360,87 +340,84 @@ PyImport_Cleanup(void) } } - /* First, delete __main__ */ - value = PyDict_GetItemString(modules, "__main__"); - if (value != NULL && PyModule_Check(value)) { - if (Py_VerboseFlag) - PySys_WriteStderr("# cleanup __main__\n"); - _PyModule_Clear(value); - PyDict_SetItemString(modules, "__main__", Py_None); - } - - /* The special treatment of "builtins" here is because even - when it's not referenced as a module, its dictionary is - referenced by almost every module's __builtins__. Since - deleting a module clears its dictionary (even if there are - references left to it), we need to delete the "builtins" - module last. Likewise, we don't delete sys until the very - end because it is implicitly referenced (e.g. by print). - - Also note that we 'delete' modules by replacing their entry - in the modules dict with None, rather than really deleting - them; this avoids a rehash of the modules dictionary and - also marks them as "non existent" so they won't be - re-imported. */ - - /* Next, repeatedly delete modules with a reference count of - one (skipping builtins and sys) and delete them */ - do { - ndone = 0; - pos = 0; - while (PyDict_Next(modules, &pos, &key, &value)) { - if (value->ob_refcnt != 1) - continue; - if (PyUnicode_Check(key) && PyModule_Check(value)) { - if (is_essential_module(key)) - continue; - if (Py_VerboseFlag) - PySys_FormatStderr( - "# cleanup[1] %U\n", key); - _PyModule_Clear(value); - PyDict_SetItem(modules, key, Py_None); - ndone++; - } - } - } while (ndone > 0); - - /* Next, delete all modules (still skipping builtins and sys) */ + /* We prepare a list which will receive (name, weakref) tuples of + modules when they are removed from sys.modules. The name is used + for diagnosis messages (in verbose mode), while the weakref helps + detect those modules which have been held alive. */ + weaklist = PyList_New(0); + +#define STORE_MODULE_WEAKREF(mod) \ + if (weaklist != NULL) { \ + PyObject *name = PyModule_GetNameObject(mod); \ + PyObject *wr = PyWeakref_NewRef(mod, NULL); \ + if (name && wr) { \ + PyObject *tup = PyTuple_Pack(2, name, wr); \ + PyList_Append(weaklist, tup); \ + Py_XDECREF(tup); \ + } \ + Py_XDECREF(name); \ + Py_XDECREF(wr); \ + if (PyErr_Occurred()) \ + PyErr_Clear(); \ + } + + /* Remove all modules from sys.modules, hoping that garbage collection + can reclaim most of them. */ pos = 0; while (PyDict_Next(modules, &pos, &key, &value)) { - if (PyUnicode_Check(key) && PyModule_Check(value)) { - if (is_essential_module(key)) - continue; - if (Py_VerboseFlag) - PySys_FormatStderr("# cleanup[2] %U\n", key); - _PyModule_Clear(value); + if (PyModule_Check(value)) { + if (Py_VerboseFlag && PyUnicode_Check(key)) + PySys_FormatStderr("# cleanup[2] removing %U\n", key, value); + STORE_MODULE_WEAKREF(value); PyDict_SetItem(modules, key, Py_None); } } - /* Collect garbage remaining after deleting the modules. Mostly - reference cycles created by classes. */ - PyGC_Collect(); - + /* Clear the modules dict. */ + PyDict_Clear(modules); + /* Replace the interpreter's reference to builtins with an empty dict + (module globals still have a reference to the original builtins). */ + builtins = interp->builtins; + interp->builtins = PyDict_New(); + Py_DECREF(builtins); + /* Collect references */ + _PyGC_CollectNoFail(); /* Dump GC stats before it's too late, since it uses the warnings machinery. */ _PyGC_DumpShutdownStats(); - /* Next, delete all remaining modules */ - pos = 0; - while (PyDict_Next(modules, &pos, &key, &value)) { - if (PyUnicode_Check(key) && PyModule_Check(value)) { + /* Now, if there are any modules left alive, clear their globals to + minimize potential leaks. All C extension modules actually end + up here, since they are kept alive in the interpreter state. */ + if (weaklist != NULL) { + Py_ssize_t i, n; + n = PyList_GET_SIZE(weaklist); + for (i = 0; i < n; i++) { + PyObject *tup = PyList_GET_ITEM(weaklist, i); + PyObject *mod = PyWeakref_GET_OBJECT(PyTuple_GET_ITEM(tup, 1)); + if (mod == Py_None) + continue; + Py_INCREF(mod); + assert(PyModule_Check(mod)); if (Py_VerboseFlag) - PySys_FormatStderr("# cleanup[3] %U\n", key); - _PyModule_Clear(value); - PyDict_SetItem(modules, key, Py_None); + PySys_FormatStderr("# cleanup[3] wiping %U\n", + PyTuple_GET_ITEM(tup, 0), mod); + _PyModule_Clear(mod); + Py_DECREF(mod); } + Py_DECREF(weaklist); } - /* Finally, clear and delete the modules directory */ - PyDict_Clear(modules); - _PyGC_CollectNoFail(); + /* Clear and delete the modules directory. Actual modules will + still be there only if imported during the execution of some + destructor. */ interp->modules = NULL; Py_DECREF(modules); + + /* Once more */ + _PyGC_CollectNoFail(); + +#undef STORE_MODULE_WEAKREF } -- cgit v1.2.1 From 1406c9e41bf56b4ec28c417b2f79abf5c26c77b9 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Thu, 1 Aug 2013 22:07:06 +0200 Subject: Issue #10241: Clear extension module dict copies at interpreter shutdown. Patch by Neil Schemenauer, minimally modified. --- Python/import.c | 2 ++ 1 file changed, 2 insertions(+) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 09106555a3..ce09ebedd4 100644 --- a/Python/import.c +++ b/Python/import.c @@ -380,6 +380,8 @@ PyImport_Cleanup(void) builtins = interp->builtins; interp->builtins = PyDict_New(); Py_DECREF(builtins); + /* Clear module dict copies stored in the interpreter state */ + _PyState_ClearModules(); /* Collect references */ _PyGC_CollectNoFail(); /* Dump GC stats before it's too late, since it uses the warnings -- cgit v1.2.1 From 5d62cd8fe1ac1e2f8286f286530bf16e995cd34d Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Fri, 2 Aug 2013 20:39:46 +0200 Subject: Backout 62658d9d8926 (issue #10241): it causes a crash at shutdown when deallocating a Tkapp object. --- Python/import.c | 2 -- 1 file changed, 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index ce09ebedd4..09106555a3 100644 --- a/Python/import.c +++ b/Python/import.c @@ -380,8 +380,6 @@ PyImport_Cleanup(void) builtins = interp->builtins; interp->builtins = PyDict_New(); Py_DECREF(builtins); - /* Clear module dict copies stored in the interpreter state */ - _PyState_ClearModules(); /* Collect references */ _PyGC_CollectNoFail(); /* Dump GC stats before it's too late, since it uses the warnings -- cgit v1.2.1 From 7af1703c1a9afb675322700a1148a1c78e5cf0a9 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 6 Aug 2013 22:50:15 +0200 Subject: Improve verbose reporting of shutdown phase by using the "public" module name --- Python/import.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 09106555a3..be73d4bba8 100644 --- a/Python/import.c +++ b/Python/import.c @@ -345,17 +345,17 @@ PyImport_Cleanup(void) for diagnosis messages (in verbose mode), while the weakref helps detect those modules which have been held alive. */ weaklist = PyList_New(0); + if (weaklist == NULL) + PyErr_Clear(); -#define STORE_MODULE_WEAKREF(mod) \ +#define STORE_MODULE_WEAKREF(name, mod) \ if (weaklist != NULL) { \ - PyObject *name = PyModule_GetNameObject(mod); \ PyObject *wr = PyWeakref_NewRef(mod, NULL); \ if (name && wr) { \ PyObject *tup = PyTuple_Pack(2, name, wr); \ PyList_Append(weaklist, tup); \ Py_XDECREF(tup); \ } \ - Py_XDECREF(name); \ Py_XDECREF(wr); \ if (PyErr_Occurred()) \ PyErr_Clear(); \ @@ -368,7 +368,7 @@ PyImport_Cleanup(void) if (PyModule_Check(value)) { if (Py_VerboseFlag && PyUnicode_Check(key)) PySys_FormatStderr("# cleanup[2] removing %U\n", key, value); - STORE_MODULE_WEAKREF(value); + STORE_MODULE_WEAKREF(key, value); PyDict_SetItem(modules, key, Py_None); } } @@ -394,14 +394,15 @@ PyImport_Cleanup(void) n = PyList_GET_SIZE(weaklist); for (i = 0; i < n; i++) { PyObject *tup = PyList_GET_ITEM(weaklist, i); + PyObject *name = PyTuple_GET_ITEM(tup, 0); PyObject *mod = PyWeakref_GET_OBJECT(PyTuple_GET_ITEM(tup, 1)); if (mod == Py_None) continue; Py_INCREF(mod); assert(PyModule_Check(mod)); - if (Py_VerboseFlag) + if (Py_VerboseFlag && PyUnicode_Check(name)) PySys_FormatStderr("# cleanup[3] wiping %U\n", - PyTuple_GET_ITEM(tup, 0), mod); + name, mod); _PyModule_Clear(mod); Py_DECREF(mod); } -- cgit v1.2.1