From b5162583f02ef19edb3b4804cac793f1183c6283 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Feb 2011 23:12:28 +0000 Subject: Issue #3080: Mark PyWin_FindRegisteredModule() as private This function was not declared in Python public API (in any .h file) and not documented. Mark it as private to prepare a change of its API. --- Python/import.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 23752eeb72..f4015b52fa 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1547,8 +1547,8 @@ PyImport_GetImporter(PyObject *path) { pathname and an open file. Return NULL if the module is not found. */ #ifdef MS_COREDLL -extern FILE *PyWin_FindRegisteredModule(const char *, struct filedescr **, - char *, Py_ssize_t); +extern FILE *_PyWin_FindRegisteredModule(const char *, struct filedescr **, + char *, Py_ssize_t); #endif static int case_ok(char *, Py_ssize_t, Py_ssize_t, char *); @@ -1631,7 +1631,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, return &fd_builtin; } #ifdef MS_COREDLL - fp = PyWin_FindRegisteredModule(name, &fdp, buf, buflen); + fp = _PyWin_FindRegisteredModule(name, &fdp, buf, buflen); if (fp != NULL) { *p_fp = fp; return fdp; -- cgit v1.2.1 From 90268ce7e125f2d8a1d9b8f41b51d31f96b1c8fd Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 23 Feb 2011 00:02:00 +0000 Subject: Issue #3080: Mark _PyImport_FindBuiltin() argument as constant And as a consequence, mark also name argument of _PyImport_FindExtensionUnicode() constant too. But I plan to change this argument type to PyObject* later. --- Python/import.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index f4015b52fa..39cd93f7dd 100644 --- a/Python/import.c +++ b/Python/import.c @@ -636,7 +636,7 @@ _PyImport_FixupBuiltin(PyObject *mod, char *name) } PyObject * -_PyImport_FindExtensionUnicode(char *name, PyObject *filename) +_PyImport_FindExtensionUnicode(const char *name, PyObject *filename) { PyObject *mod, *mdict; PyModuleDef* def; @@ -680,7 +680,7 @@ _PyImport_FindExtensionUnicode(char *name, PyObject *filename) } PyObject * -_PyImport_FindBuiltin(char *name) +_PyImport_FindBuiltin(const char *name) { PyObject *res, *filename; filename = PyUnicode_FromString(name); -- cgit v1.2.1 From b05cc9bc8518f72fcb3e33b764f24dae32d5524f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 4 Mar 2011 12:57:09 +0000 Subject: Issue #3080: Add PyImport_AddModuleObject() and PyImport_ExecCodeModuleObject() --- Python/import.c | 87 +++++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 66 insertions(+), 21 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 39cd93f7dd..22a7c8744d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -698,18 +698,18 @@ _PyImport_FindBuiltin(const char *name) 'NEW' REFERENCE! */ PyObject * -PyImport_AddModule(const char *name) +PyImport_AddModuleObject(PyObject *name) { PyObject *modules = PyImport_GetModuleDict(); PyObject *m; - if ((m = PyDict_GetItemString(modules, name)) != NULL && + if ((m = PyDict_GetItem(modules, name)) != NULL && PyModule_Check(m)) return m; - m = PyModule_New(name); + m = PyModule_NewObject(name); if (m == NULL) return NULL; - if (PyDict_SetItemString(modules, name, m) != 0) { + if (PyDict_SetItem(modules, name, m) != 0) { Py_DECREF(m); return NULL; } @@ -718,14 +718,27 @@ PyImport_AddModule(const char *name) return m; } +PyObject * +PyImport_AddModule(const char *name) +{ + PyObject *nameobj, *module; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + module = PyImport_AddModuleObject(nameobj); + Py_DECREF(nameobj); + return module; +} + + /* Remove name from sys.modules, if it's there. */ static void -remove_module(const char *name) +remove_module(PyObject *name) { PyObject *modules = PyImport_GetModuleDict(); - if (PyDict_GetItemString(modules, name) == NULL) + if (PyDict_GetItem(modules, name) == NULL) return; - if (PyDict_DelItemString(modules, name) < 0) + if (PyDict_DelItem(modules, name) < 0) Py_FatalError("import: deleting existing key in" "sys.modules failed"); } @@ -762,11 +775,43 @@ PyImport_ExecCodeModuleEx(char *name, PyObject *co, char *pathname) PyObject * PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, char *cpathname) +{ + PyObject *m = NULL; + PyObject *nameobj, *pathobj = NULL, *cpathobj = NULL; + + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + + if (pathname != NULL) { + pathobj = PyUnicode_DecodeFSDefault(pathname); + if (pathobj == NULL) + goto error; + } else + pathobj = NULL; + if (cpathname != NULL) { + cpathobj = PyUnicode_DecodeFSDefault(cpathname); + if (cpathobj == NULL) + goto error; + } else + cpathobj = NULL; + m = PyImport_ExecCodeModuleObject(nameobj, co, pathobj, cpathobj); +error: + Py_DECREF(nameobj); + Py_XDECREF(pathobj); + Py_XDECREF(cpathobj); + return m; +} + +PyObject* +PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, + PyObject *cpathname) { PyObject *modules = PyImport_GetModuleDict(); PyObject *m, *d, *v; + PyObject *pathbytes; - m = PyImport_AddModule(name); + m = PyImport_AddModuleObject(name); if (m == NULL) return NULL; /* If the module is being reloaded, we get the old module back @@ -778,12 +823,18 @@ PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, goto error; } /* Remember the filename as the __file__ attribute */ - v = NULL; if (pathname != NULL) { - v = get_sourcefile(pathname); + pathbytes = PyUnicode_EncodeFSDefault(pathname); + if (pathbytes != NULL) { + v = get_sourcefile(PyBytes_AS_STRING(pathbytes)); + Py_DECREF(pathbytes); + } else + v = NULL; if (v == NULL) PyErr_Clear(); } + else + v = NULL; if (v == NULL) { v = ((PyCodeObject *)co)->co_filename; Py_INCREF(v); @@ -793,27 +844,21 @@ PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, Py_DECREF(v); /* Remember the pyc path name as the __cached__ attribute. */ - if (cpathname == NULL) { - v = Py_None; - Py_INCREF(v); - } - else if ((v = PyUnicode_FromString(cpathname)) == NULL) { - PyErr_Clear(); /* Not important enough to report */ + if (cpathname != NULL) + v = cpathname; + else v = Py_None; - Py_INCREF(v); - } if (PyDict_SetItemString(d, "__cached__", v) != 0) PyErr_Clear(); /* Not important enough to report */ - Py_DECREF(v); v = PyEval_EvalCode(co, d, d); if (v == NULL) goto error; Py_DECREF(v); - if ((m = PyDict_GetItemString(modules, name)) == NULL) { + if ((m = PyDict_GetItem(modules, name)) == NULL) { PyErr_Format(PyExc_ImportError, - "Loaded module %.200s not found in sys.modules", + "Loaded module %R not found in sys.modules", name); return NULL; } -- cgit v1.2.1 From 426e754b8e8cf35a60153914ae341ca57e9d5031 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 7 Mar 2011 16:57:48 +0100 Subject: import.c: replace tab by spaces --- Python/import.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 22a7c8744d..b2757dffdc 100644 --- a/Python/import.c +++ b/Python/import.c @@ -329,7 +329,7 @@ _PyImport_ReInitLock(void) /* Forked as a side effect of import */ long me = PyThread_get_thread_ident(); PyThread_acquire_lock(import_lock, 0); - /* XXX: can the previous line fail? */ + /* XXX: can the previous line fail? */ import_lock_thread = me; import_lock_level--; } else { -- cgit v1.2.1 From 13abff792323d15e50b40c4ef978441790bef7b3 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 7 Mar 2011 17:08:21 +0100 Subject: Issue #3080: PyImport_Cleanup() uses Unicode Replace strcmp() by PyUnicode_CompareWithASCIIString() --- Python/import.c | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index b2757dffdc..9366be0589 100644 --- a/Python/import.c +++ b/Python/import.c @@ -418,7 +418,6 @@ void PyImport_Cleanup(void) { Py_ssize_t pos, ndone; - char *name; PyObject *key, *value, *dict; PyInterpreterState *interp = PyThreadState_GET()->interp; PyObject *modules = interp->modules; @@ -491,14 +490,13 @@ PyImport_Cleanup(void) if (value->ob_refcnt != 1) continue; if (PyUnicode_Check(key) && PyModule_Check(value)) { - name = _PyUnicode_AsString(key); - if (strcmp(name, "builtins") == 0) + if (PyUnicode_CompareWithASCIIString(key, "builtins") == 0) continue; - if (strcmp(name, "sys") == 0) + if (PyUnicode_CompareWithASCIIString(key, "sys") == 0) continue; if (Py_VerboseFlag) - PySys_WriteStderr( - "# cleanup[1] %s\n", name); + PySys_FormatStderr( + "# cleanup[1] %U\n", key); _PyModule_Clear(value); PyDict_SetItem(modules, key, Py_None); ndone++; @@ -510,13 +508,12 @@ PyImport_Cleanup(void) pos = 0; while (PyDict_Next(modules, &pos, &key, &value)) { if (PyUnicode_Check(key) && PyModule_Check(value)) { - name = _PyUnicode_AsString(key); - if (strcmp(name, "builtins") == 0) + if (PyUnicode_CompareWithASCIIString(key, "builtins") == 0) continue; - if (strcmp(name, "sys") == 0) + if (PyUnicode_CompareWithASCIIString(key, "sys") == 0) continue; if (Py_VerboseFlag) - PySys_WriteStderr("# cleanup[2] %s\n", name); + PySys_FormatStderr("# cleanup[2] %U\n", key); _PyModule_Clear(value); PyDict_SetItem(modules, key, Py_None); } -- cgit v1.2.1 From 46cbf15e71a6cd5841e2547357932aa1f8c93936 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 20 Mar 2011 01:50:21 +0100 Subject: Issue #3080: Add PyImport_ImportFrozenModuleObject() find_frozen(), get_frozen_object(), is_frozen_package() and other functions related to frozen modules use Unicode strings instead of byte strings. --- Python/import.c | 98 +++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 61 insertions(+), 37 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 9366be0589..09fc52cbda 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1429,7 +1429,7 @@ get_sourcefile(char *file) static PyObject *load_module(char *, FILE *, char *, int, PyObject *); static struct filedescr *find_module(char *, char *, PyObject *, char *, size_t, FILE **, PyObject **); -static struct _frozen * find_frozen(char *); +static struct _frozen * find_frozen(PyObject *); /* Load a package and return its module object WITH INCREMENTED REFERENCE COUNT */ @@ -1617,6 +1617,8 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, size_t saved_namelen; char *saved_buf = NULL; #endif + PyObject *fullname_obj; + if (p_loader != NULL) *p_loader = NULL; @@ -1662,9 +1664,16 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, Py_DECREF(meta_path); } - if (find_frozen(fullname) != NULL) { - strcpy(buf, fullname); - return &fd_frozen; + if (fullname != NULL) { + fullname_obj = PyUnicode_FromString(fullname); + if (fullname == NULL) + return NULL; + if (find_frozen(fullname_obj) != NULL) { + Py_DECREF(fullname_obj); + strcpy(buf, fullname); + return &fd_frozen; + } + Py_DECREF(fullname_obj); } if (path == NULL) { @@ -2226,37 +2235,37 @@ init_builtin(char *name) /* Frozen modules */ static struct _frozen * -find_frozen(char *name) +find_frozen(PyObject *name) { struct _frozen *p; - if (!name) + if (name == NULL) return NULL; for (p = PyImport_FrozenModules; ; p++) { if (p->name == NULL) return NULL; - if (strcmp(p->name, name) == 0) + if (PyUnicode_CompareWithASCIIString(name, p->name) == 0) break; } return p; } static PyObject * -get_frozen_object(char *name) +get_frozen_object(PyObject *name) { struct _frozen *p = find_frozen(name); int size; if (p == NULL) { PyErr_Format(PyExc_ImportError, - "No such frozen object named %.200s", + "No such frozen object named %R", name); return NULL; } if (p->code == NULL) { PyErr_Format(PyExc_ImportError, - "Excluded frozen object named %.200s", + "Excluded frozen object named %R", name); return NULL; } @@ -2267,14 +2276,14 @@ get_frozen_object(char *name) } static PyObject * -is_frozen_package(char *name) +is_frozen_package(PyObject *name) { struct _frozen *p = find_frozen(name); int size; if (p == NULL) { PyErr_Format(PyExc_ImportError, - "No such frozen object named %.200s", + "No such frozen object named %R", name); return NULL; } @@ -2294,19 +2303,20 @@ is_frozen_package(char *name) This function is also used from frozenmain.c */ int -PyImport_ImportFrozenModule(char *name) +PyImport_ImportFrozenModuleObject(PyObject *name) { - struct _frozen *p = find_frozen(name); - PyObject *co; - PyObject *m; + struct _frozen *p; + PyObject *co, *m, *path; int ispackage; int size; + p = find_frozen(name); + if (p == NULL) return 0; if (p->code == NULL) { PyErr_Format(PyExc_ImportError, - "Excluded frozen object named %.200s", + "Excluded frozen object named %R", name); return -1; } @@ -2315,40 +2325,41 @@ PyImport_ImportFrozenModule(char *name) if (ispackage) size = -size; if (Py_VerboseFlag) - PySys_WriteStderr("import %s # frozen%s\n", + PySys_FormatStderr("import %U # frozen%s\n", name, ispackage ? " package" : ""); co = PyMarshal_ReadObjectFromString((char *)p->code, size); if (co == NULL) return -1; if (!PyCode_Check(co)) { PyErr_Format(PyExc_TypeError, - "frozen object %.200s is not a code object", + "frozen object %R is not a code object", name); goto err_return; } if (ispackage) { /* Set __path__ to the package name */ - PyObject *d, *s, *l; + PyObject *d, *l; int err; - m = PyImport_AddModule(name); + m = PyImport_AddModuleObject(name); if (m == NULL) goto err_return; d = PyModule_GetDict(m); - s = PyUnicode_InternFromString(name); - if (s == NULL) - goto err_return; l = PyList_New(1); if (l == NULL) { - Py_DECREF(s); goto err_return; } - PyList_SET_ITEM(l, 0, s); + Py_INCREF(name); + PyList_SET_ITEM(l, 0, name); err = PyDict_SetItemString(d, "__path__", l); Py_DECREF(l); if (err != 0) goto err_return; } - m = PyImport_ExecCodeModuleEx(name, co, ""); + path = PyUnicode_FromString(""); + if (path == NULL) + goto err_return; + m = PyImport_ExecCodeModuleObject(name, co, path, NULL); + Py_DECREF(path); if (m == NULL) goto err_return; Py_DECREF(co); @@ -2359,6 +2370,19 @@ err_return: return -1; } +int +PyImport_ImportFrozenModule(char *name) +{ + PyObject *nameobj; + int ret; + nameobj = PyUnicode_InternFromString(name); + if (nameobj == NULL) + return -1; + ret = PyImport_ImportFrozenModuleObject(nameobj); + Py_DECREF(nameobj); + return ret; +} + /* Import a module, either built-in, frozen, or external, and return its module object WITH INCREMENTED REFERENCE COUNT */ @@ -3282,19 +3306,19 @@ imp_init_builtin(PyObject *self, PyObject *args) static PyObject * imp_init_frozen(PyObject *self, PyObject *args) { - char *name; + PyObject *name; int ret; PyObject *m; - if (!PyArg_ParseTuple(args, "s:init_frozen", &name)) + if (!PyArg_ParseTuple(args, "U:init_frozen", &name)) return NULL; - ret = PyImport_ImportFrozenModule(name); + ret = PyImport_ImportFrozenModuleObject(name); if (ret < 0) return NULL; if (ret == 0) { Py_INCREF(Py_None); return Py_None; } - m = PyImport_AddModule(name); + m = PyImport_AddModuleObject(name); Py_XINCREF(m); return m; } @@ -3302,9 +3326,9 @@ imp_init_frozen(PyObject *self, PyObject *args) static PyObject * imp_get_frozen_object(PyObject *self, PyObject *args) { - char *name; + PyObject *name; - if (!PyArg_ParseTuple(args, "s:get_frozen_object", &name)) + if (!PyArg_ParseTuple(args, "U:get_frozen_object", &name)) return NULL; return get_frozen_object(name); } @@ -3312,9 +3336,9 @@ imp_get_frozen_object(PyObject *self, PyObject *args) static PyObject * imp_is_frozen_package(PyObject *self, PyObject *args) { - char *name; + PyObject *name; - if (!PyArg_ParseTuple(args, "s:is_frozen_package", &name)) + if (!PyArg_ParseTuple(args, "U:is_frozen_package", &name)) return NULL; return is_frozen_package(name); } @@ -3331,9 +3355,9 @@ imp_is_builtin(PyObject *self, PyObject *args) static PyObject * imp_is_frozen(PyObject *self, PyObject *args) { - char *name; + PyObject *name; struct _frozen *p; - if (!PyArg_ParseTuple(args, "s:is_frozen", &name)) + if (!PyArg_ParseTuple(args, "U:is_frozen", &name)) return NULL; p = find_frozen(name); return PyBool_FromLong((long) (p == NULL ? 0 : p->size)); -- cgit v1.2.1 From 17a3c1b5c7bb9b20e89c29ee9fcf6ee78a6cce82 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 7 Mar 2011 18:20:56 +0100 Subject: Issue #3080: Import builtins using Unicode strings - is_builtin(), init_builtin(), load_builtin() and other builtin related functions use Unicode strings, instead of byte strings - Rename _PyImport_FixupExtensionUnicode() to _PyImport_FixupExtensionObject() - Rename _PyImport_FindExtensionUnicode() to _PyImport_FindExtensionObject() --- Python/import.c | 112 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 60 insertions(+), 52 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 09fc52cbda..597ef439c3 100644 --- a/Python/import.c +++ b/Python/import.c @@ -117,7 +117,7 @@ typedef unsigned short mode_t; static long pyc_magic = MAGIC; static const char *pyc_tag = TAG; -/* See _PyImport_FixupExtensionUnicode() below */ +/* See _PyImport_FixupExtensionObject() below */ static PyObject *extensions = NULL; /* This table is defined in config.c: */ @@ -563,10 +563,10 @@ PyImport_GetMagicTag(void) once, we keep a static dictionary 'extensions' keyed by module name (for built-in modules) or by filename (for dynamically loaded modules), containing these modules. A copy of the module's - dictionary is stored by calling _PyImport_FixupExtensionUnicode() + dictionary is stored by calling _PyImport_FixupExtensionObject() immediately after the module initialization function succeeds. A copy can be retrieved from there by calling - _PyImport_FindExtensionUnicode(). + _PyImport_FindExtensionObject(). Modules which do support multiple initialization set their m_size field to a non-negative number (indicating the size of the @@ -575,7 +575,8 @@ PyImport_GetMagicTag(void) */ int -_PyImport_FixupExtensionUnicode(PyObject *mod, char *name, PyObject *filename) +_PyImport_FixupExtensionObject(PyObject *mod, PyObject *name, + PyObject *filename) { PyObject *modules, *dict; struct PyModuleDef *def; @@ -594,10 +595,10 @@ _PyImport_FixupExtensionUnicode(PyObject *mod, char *name, PyObject *filename) return -1; } modules = PyImport_GetModuleDict(); - if (PyDict_SetItemString(modules, name, mod) < 0) + if (PyDict_SetItem(modules, name, mod) < 0) return -1; if (_PyState_AddModule(mod, def) < 0) { - PyDict_DelItemString(modules, name); + PyDict_DelItem(modules, name); return -1; } if (def->m_size == -1) { @@ -623,17 +624,17 @@ int _PyImport_FixupBuiltin(PyObject *mod, char *name) { int res; - PyObject *filename; - filename = PyUnicode_FromString(name); - if (filename == NULL) + PyObject *nameobj; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) return -1; - res = _PyImport_FixupExtensionUnicode(mod, name, filename); - Py_DECREF(filename); + res = _PyImport_FixupExtensionObject(mod, nameobj, nameobj); + Py_DECREF(nameobj); return res; } PyObject * -_PyImport_FindExtensionUnicode(const char *name, PyObject *filename) +_PyImport_FindExtensionObject(PyObject *name, PyObject *filename) { PyObject *mod, *mdict; PyModuleDef* def; @@ -646,7 +647,7 @@ _PyImport_FindExtensionUnicode(const char *name, PyObject *filename) /* Module does not support repeated initialization */ if (def->m_base.m_copy == NULL) return NULL; - mod = PyImport_AddModule(name); + mod = PyImport_AddModuleObject(name); if (mod == NULL) return NULL; mdict = PyModule_GetDict(mod); @@ -661,16 +662,16 @@ _PyImport_FindExtensionUnicode(const char *name, PyObject *filename) mod = def->m_base.m_init(); if (mod == NULL) return NULL; - PyDict_SetItemString(PyImport_GetModuleDict(), name, mod); + PyDict_SetItem(PyImport_GetModuleDict(), name, mod); Py_DECREF(mod); } if (_PyState_AddModule(mod, def) < 0) { - PyDict_DelItemString(PyImport_GetModuleDict(), name); + PyDict_DelItem(PyImport_GetModuleDict(), name); Py_DECREF(mod); return NULL; } if (Py_VerboseFlag) - PySys_FormatStderr("import %s # previously loaded (%U)\n", + PySys_FormatStderr("import %U # previously loaded (%R)\n", name, filename); return mod; @@ -679,12 +680,12 @@ _PyImport_FindExtensionUnicode(const char *name, PyObject *filename) PyObject * _PyImport_FindBuiltin(const char *name) { - PyObject *res, *filename; - filename = PyUnicode_FromString(name); - if (filename == NULL) + PyObject *res, *nameobj; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) return NULL; - res = _PyImport_FindExtensionUnicode(name, filename); - Py_DECREF(filename); + res = _PyImport_FindExtensionObject(nameobj, nameobj); + Py_DECREF(nameobj); return res; } @@ -1491,11 +1492,12 @@ load_package(char *name, char *pathname) /* Helper to test for built-in module */ static int -is_builtin(char *name) +is_builtin(PyObject *name) { - int i; + int i, cmp; for (i = 0; PyImport_Inittab[i].name != NULL; i++) { - if (strcmp(name, PyImport_Inittab[i].name) == 0) { + cmp = PyUnicode_CompareWithASCIIString(name, PyImport_Inittab[i].name); + if (cmp == 0) { if (PyImport_Inittab[i].initfunc == NULL) return -1; else @@ -1617,7 +1619,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, size_t saved_namelen; char *saved_buf = NULL; #endif - PyObject *fullname_obj; + PyObject *fullname_obj, *nameobj; if (p_loader != NULL) *p_loader = NULL; @@ -1677,10 +1679,15 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, } if (path == NULL) { - if (is_builtin(name)) { + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + if (is_builtin(nameobj)) { + Py_DECREF(nameobj); strcpy(buf, name); return &fd_builtin; } + Py_DECREF(nameobj); #ifdef MS_COREDLL fp = _PyWin_FindRegisteredModule(name, &fdp, buf, buflen); if (fp != NULL) { @@ -2086,40 +2093,35 @@ find_init_module(char *buf) #endif /* HAVE_STAT */ -static int init_builtin(char *); /* Forward */ +static int init_builtin(PyObject *); /* Forward */ static PyObject* -load_builtin(char *name, char *pathname, int type) +load_builtin(PyObject *name, int type) { PyObject *m, *modules; int err; - if (pathname != NULL && pathname[0] != '\0') - name = pathname; - if (type == C_BUILTIN) err = init_builtin(name); else - err = PyImport_ImportFrozenModule(name); + err = PyImport_ImportFrozenModuleObject(name); if (err < 0) return NULL; if (err == 0) { PyErr_Format(PyExc_ImportError, - "Purported %s module %.200s not found", - type == C_BUILTIN ? - "builtin" : "frozen", + "Purported %s module %R not found", + type == C_BUILTIN ? "builtin" : "frozen", name); return NULL; } modules = PyImport_GetModuleDict(); - m = PyDict_GetItemString(modules, name); + m = PyDict_GetItem(modules, name); if (m == NULL) { PyErr_Format( PyExc_ImportError, - "%s module %.200s not properly initialized", - type == C_BUILTIN ? - "builtin" : "frozen", + "%s module %R not properly initialized", + type == C_BUILTIN ? "builtin" : "frozen", name); return NULL; } @@ -2168,9 +2170,15 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) break; case C_BUILTIN: - case PY_FROZEN: - m = load_builtin(name, pathname, type); + case PY_FROZEN: { + PyObject *nameobj = PyUnicode_FromString(name); + if (nameobj != NULL) { + m = load_builtin(nameobj, type); + Py_DECREF(nameobj); + } else + m = NULL; break; + } case IMP_HOOK: { if (loader == NULL) { @@ -2199,28 +2207,28 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) an exception set if the initialization failed. */ static int -init_builtin(char *name) +init_builtin(PyObject *name) { struct _inittab *p; - if (_PyImport_FindBuiltin(name) != NULL) + if (_PyImport_FindExtensionObject(name, name) != NULL) return 1; for (p = PyImport_Inittab; p->name != NULL; p++) { PyObject *mod; - if (strcmp(name, p->name) == 0) { + if (PyUnicode_CompareWithASCIIString(name, p->name) == 0) { if (p->initfunc == NULL) { PyErr_Format(PyExc_ImportError, - "Cannot re-init internal module %.200s", + "Cannot re-init internal module %R", name); return -1; } if (Py_VerboseFlag) - PySys_WriteStderr("import %s # builtin\n", name); + PySys_FormatStderr("import %U # builtin\n", name); mod = (*p->initfunc)(); if (mod == 0) return -1; - if (_PyImport_FixupBuiltin(mod, name) < 0) + if (_PyImport_FixupExtensionObject(mod, name, name) < 0) return -1; /* FixupExtension has put the module into sys.modules, so we can release our own reference. */ @@ -3286,10 +3294,10 @@ imp_find_module(PyObject *self, PyObject *args) static PyObject * imp_init_builtin(PyObject *self, PyObject *args) { - char *name; + PyObject *name; int ret; PyObject *m; - if (!PyArg_ParseTuple(args, "s:init_builtin", &name)) + if (!PyArg_ParseTuple(args, "U:init_builtin", &name)) return NULL; ret = init_builtin(name); if (ret < 0) @@ -3298,7 +3306,7 @@ imp_init_builtin(PyObject *self, PyObject *args) Py_INCREF(Py_None); return Py_None; } - m = PyImport_AddModule(name); + m = PyImport_AddModuleObject(name); Py_XINCREF(m); return m; } @@ -3346,8 +3354,8 @@ imp_is_frozen_package(PyObject *self, PyObject *args) static PyObject * imp_is_builtin(PyObject *self, PyObject *args) { - char *name; - if (!PyArg_ParseTuple(args, "s:is_builtin", &name)) + PyObject *name; + if (!PyArg_ParseTuple(args, "U:is_builtin", &name)) return NULL; return PyLong_FromLong(is_builtin(name)); } -- cgit v1.2.1 From d08ceb04095b21f699c9221e7e5d7bb2b86a20ea Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 7 Mar 2011 18:28:15 +0100 Subject: Issue #3080: Use PyUnicode_InternFromString() for builtins _PyImport_FixupBuiltin() and _PyImport_FindBuiltin() use PyUnicode_InternFromString() instead of PyUnicode_FromString(). --- Python/import.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 597ef439c3..d43b881c2f 100644 --- a/Python/import.c +++ b/Python/import.c @@ -625,7 +625,7 @@ _PyImport_FixupBuiltin(PyObject *mod, char *name) { int res; PyObject *nameobj; - nameobj = PyUnicode_FromString(name); + nameobj = PyUnicode_InternFromString(name); if (nameobj == NULL) return -1; res = _PyImport_FixupExtensionObject(mod, nameobj, nameobj); @@ -681,7 +681,7 @@ PyObject * _PyImport_FindBuiltin(const char *name) { PyObject *res, *nameobj; - nameobj = PyUnicode_FromString(name); + nameobj = PyUnicode_InternFromString(name); if (nameobj == NULL) return NULL; res = _PyImport_FindExtensionObject(nameobj, nameobj); -- cgit v1.2.1 From 4691aca1214aaa1b4433a667d875415f9a27c38e Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 8 Mar 2011 23:49:04 +0100 Subject: Issue #3080: _PyWin_FindRegisteredModule() returns the path as Unicode * Document the function * Use RegQueryValueW() instead of RegQueryValueA() * Use _Py_fopen() instead of fopen() * Allocate registry key on the heap, not on the stack, and handle memory allocation failure * Handle Python exception in find_module() --- Python/import.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index d43b881c2f..8fab810eec 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1591,8 +1591,8 @@ PyImport_GetImporter(PyObject *path) { pathname and an open file. Return NULL if the module is not found. */ #ifdef MS_COREDLL -extern FILE *_PyWin_FindRegisteredModule(const char *, struct filedescr **, - char *, Py_ssize_t); +extern FILE *_PyWin_FindRegisteredModule(PyObject *, struct filedescr **, + PyObject **p_path); #endif static int case_ok(char *, Py_ssize_t, Py_ssize_t, char *); @@ -1679,6 +1679,9 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, } if (path == NULL) { +#ifdef MS_COREDLL + PyObject *filename, *filename_bytes; +#endif nameobj = PyUnicode_FromString(name); if (nameobj == NULL) return NULL; @@ -1687,14 +1690,24 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, strcpy(buf, name); return &fd_builtin; } - Py_DECREF(nameobj); #ifdef MS_COREDLL - fp = _PyWin_FindRegisteredModule(name, &fdp, buf, buflen); + fp = _PyWin_FindRegisteredModule(nameobj, &fdp, &filename); if (fp != NULL) { + Py_DECREF(nameobj); + filename_bytes = PyUnicode_EncodeFSDefault(filename); + Py_DECREF(filename); + if (filename_bytes == NULL) + return NULL; + strncpy(buf, PyBytes_AS_STRING(filename_bytes), buflen); + buf[buflen-1] = '\0'; + Py_DECREF(filename_bytes); *p_fp = fp; return fdp; } + else if (PyErr_Occurred()) + return NULL; #endif + Py_DECREF(nameobj); path = PySys_GetObject("path"); } -- cgit v1.2.1 From 7c1cd6d18d5ac37ec86183e7fabcf35ff11e3d21 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 15:54:07 -0400 Subject: Issue #3080: _PyImport_LoadDynamicModule() uses Unicode for name and path Document also that dynamic module names are ASCII only --- Python/import.c | 54 ++++++++++++++++++++++++++++++------------------------ 1 file changed, 30 insertions(+), 24 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 8fab810eec..4e5cb2ddbf 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2173,9 +2173,21 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) break; #ifdef HAVE_DYNAMIC_LOADING - case C_EXTENSION: - m = _PyImport_LoadDynamicModule(name, pathname, fp); + case C_EXTENSION: { + PyObject *nameobj, *pathobj; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + pathobj = PyUnicode_DecodeFSDefault(pathname); + if (pathobj == NULL) { + Py_DECREF(nameobj); + return NULL; + } + m = _PyImport_LoadDynamicModule(nameobj, pathobj, fp); + Py_DECREF(nameobj); + Py_DECREF(pathobj); break; + } #endif case PKG_DIRECTORY: @@ -2185,11 +2197,10 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) case C_BUILTIN: case PY_FROZEN: { PyObject *nameobj = PyUnicode_FromString(name); - if (nameobj != NULL) { - m = load_builtin(nameobj, type); - Py_DECREF(nameobj); - } else - m = NULL; + if (nameobj == NULL) + return NULL; + m = load_builtin(nameobj, type); + Py_DECREF(nameobj); break; } @@ -3443,28 +3454,23 @@ imp_load_compiled(PyObject *self, PyObject *args) static PyObject * imp_load_dynamic(PyObject *self, PyObject *args) { - char *name; - PyObject *pathbytes; - char *pathname; - PyObject *fob = NULL; - PyObject *m; - FILE *fp = NULL; - if (!PyArg_ParseTuple(args, "sO&|O:load_dynamic", - &name, PyUnicode_FSConverter, &pathbytes, &fob)) + PyObject *name, *pathname, *fob = NULL, *mod; + FILE *fp; + + if (!PyArg_ParseTuple(args, "UO&|O:load_dynamic", + &name, PyUnicode_FSDecoder, &pathname, &fob)) return NULL; - pathname = PyBytes_AS_STRING(pathbytes); - if (fob) { - fp = get_file(pathname, fob, "r"); - if (fp == NULL) { - Py_DECREF(pathbytes); + if (fob != NULL) { + fp = get_file(NULL, fob, "r"); + if (fp == NULL) return NULL; - } } - m = _PyImport_LoadDynamicModule(name, pathname, fp); - Py_DECREF(pathbytes); + else + fp = NULL; + mod = _PyImport_LoadDynamicModule(name, pathname, fp); if (fp) fclose(fp); - return m; + return mod; } #endif /* HAVE_DYNAMIC_LOADING */ -- cgit v1.2.1 From ab8eed250aee0dfe2e0e8cc1472113d5d2b4b9c7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 12 Mar 2011 08:45:02 -0500 Subject: Issue #3080: find_module() initialize buf and *p_fp Document also the find_module() function --- Python/import.c | 42 +++++++++++++++++++++++++++++++++--------- 1 file changed, 33 insertions(+), 9 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 4e5cb2ddbf..e58d6a04be 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1443,7 +1443,7 @@ load_package(char *name, char *pathname) PyObject *path = NULL; int err; char buf[MAXPATHLEN+1]; - FILE *fp = NULL; + FILE *fp; struct filedescr *fdp; m = PyImport_AddModule(name); @@ -1464,7 +1464,6 @@ load_package(char *name, char *pathname) err = PyDict_SetItemString(d, "__path__", path); if (err != 0) goto error; - buf[0] = '\0'; fdp = find_module(name, "__init__", path, buf, sizeof(buf), &fp, NULL); if (fdp == NULL) { if (PyErr_ExceptionMatches(PyExc_ImportError)) { @@ -1599,6 +1598,31 @@ static int case_ok(char *, Py_ssize_t, Py_ssize_t, char *); static int find_init_module(char *); /* Forward */ static struct filedescr importhookdescr = {"", "", IMP_HOOK}; +/* Find a module: + + - try find_module() of each sys.meta_path hook + - try find_frozen() + - try is_builtin() + - try _PyWin_FindRegisteredModule() (Windows only) + - otherwise, call find_module_path_list() with search_path_list (if not + NULL) or sys.path + + Return: + + - &fd_builtin (C_BUILTIN) if it is a builtin + - &fd_frozen (PY_FROZEN) if it is frozen + - &fd_package (PKG_DIRECTORY) and write the filename into *buf + if it is a package + - &importhookdescr (IMP_HOOK) and write the loader into *p_loader if a + importer loader was found + - a file descriptor (PY_SOURCE, PY_COMPILED, C_EXTENSION, PY_RESOURCE or + PY_CODERESOURCE: see _PyImport_Filetab), write the filename into + *buf and the pointer to the open file into *p_fp + - NULL on error + + By default, write an empty string into *buf, and *p_fp and *p_loader (if + set) are set to NULL. Eg. *buf is an empty string for a builtin package. */ + static struct filedescr * find_module(char *fullname, char *subname, PyObject *path, char *buf, size_t buflen, FILE **p_fp, PyObject **p_loader) @@ -1621,6 +1645,8 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, #endif PyObject *fullname_obj, *nameobj; + *buf = '\0'; + *p_fp = NULL; if (p_loader != NULL) *p_loader = NULL; @@ -2962,10 +2988,10 @@ import_submodule(PyObject *mod, char *subname, char *fullname) Py_INCREF(m); } else { - PyObject *path, *loader = NULL; + PyObject *path, *loader; char buf[MAXPATHLEN+1]; struct filedescr *fdp; - FILE *fp = NULL; + FILE *fp; if (mod == Py_None) path = NULL; @@ -2978,7 +3004,6 @@ import_submodule(PyObject *mod, char *subname, char *fullname) } } - buf[0] = '\0'; fdp = find_module(fullname, subname, path, buf, MAXPATHLEN+1, &fp, &loader); Py_XDECREF(path); @@ -3012,11 +3037,11 @@ PyImport_ReloadModule(PyObject *m) PyInterpreterState *interp = PyThreadState_Get()->interp; PyObject *modules_reloading = interp->modules_reloading; PyObject *modules = PyImport_GetModuleDict(); - PyObject *path = NULL, *loader = NULL, *existing_m = NULL; + PyObject *path = NULL, *loader, *existing_m = NULL; char *name, *subname; char buf[MAXPATHLEN+1]; struct filedescr *fdp; - FILE *fp = NULL; + FILE *fp; PyObject *newm; if (modules_reloading == NULL) { @@ -3074,7 +3099,6 @@ PyImport_ReloadModule(PyObject *m) if (path == NULL) PyErr_Clear(); } - buf[0] = '\0'; fdp = find_module(name, subname, path, buf, MAXPATHLEN+1, &fp, &loader); Py_XDECREF(path); @@ -3252,7 +3276,7 @@ call_find_module(char *name, PyObject *path) PyObject *pathobj; struct filedescr *fdp; char pathname[MAXPATHLEN+1]; - FILE *fp = NULL; + FILE *fp; int fd = -1; char *found_encoding = NULL; char *encoding = NULL; -- cgit v1.2.1 From 8d647fdca120414a1901c3b1414b405ad7f415f4 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 12 Mar 2011 09:26:54 -0500 Subject: Issue #3080: Remove useless name buffer from find_module() Rename subname argument to name, and mark it as constant. --- Python/import.c | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index e58d6a04be..37cd4b81f2 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1428,7 +1428,7 @@ get_sourcefile(char *file) /* Forward */ static PyObject *load_module(char *, FILE *, char *, int, PyObject *); -static struct filedescr *find_module(char *, char *, PyObject *, +static struct filedescr *find_module(char *, const char *, PyObject *, char *, size_t, FILE **, PyObject **); static struct _frozen * find_frozen(PyObject *); @@ -1594,7 +1594,7 @@ extern FILE *_PyWin_FindRegisteredModule(PyObject *, struct filedescr **, PyObject **p_path); #endif -static int case_ok(char *, Py_ssize_t, Py_ssize_t, char *); +static int case_ok(char *, Py_ssize_t, Py_ssize_t, const char *); static int find_init_module(char *); /* Forward */ static struct filedescr importhookdescr = {"", "", IMP_HOOK}; @@ -1624,7 +1624,7 @@ static struct filedescr importhookdescr = {"", "", IMP_HOOK}; set) are set to NULL. Eg. *buf is an empty string for a builtin package. */ static struct filedescr * -find_module(char *fullname, char *subname, PyObject *path, char *buf, +find_module(char *fullname, const char *name, PyObject *path, char *buf, size_t buflen, FILE **p_fp, PyObject **p_loader) { Py_ssize_t i, npath; @@ -1637,7 +1637,6 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, static struct filedescr fd_frozen = {"", "", PY_FROZEN}; static struct filedescr fd_builtin = {"", "", C_BUILTIN}; static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; - char name[MAXPATHLEN+1]; #if defined(PYOS_OS2) size_t saved_len; size_t saved_namelen; @@ -1650,12 +1649,11 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, if (p_loader != NULL) *p_loader = NULL; - if (strlen(subname) > MAXPATHLEN) { + if (strlen(name) > MAXPATHLEN) { PyErr_SetString(PyExc_OverflowError, "module name is too long"); return NULL; } - strcpy(name, subname); /* sys.meta_path import hook */ if (p_loader != NULL) { @@ -1867,7 +1865,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, * dynamically loaded module we're going to try, * truncate the name before trying */ - if (strlen(subname) > 8) { + if (strlen(name) > 8) { /* is this an attempt to load a C extension? */ const struct filedescr *scan; scan = _PyImport_DynLoadFiletab; @@ -1880,7 +1878,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, if (scan->suffix != NULL) { /* yes, so truncate the name */ namelen = 8; - len -= strlen(subname) - namelen; + len -= strlen(name) - namelen; buf[len] = '\0'; } } @@ -1972,7 +1970,7 @@ find_module(char *fullname, char *subname, PyObject *path, char *buf, #endif static int -case_ok(char *buf, Py_ssize_t len, Py_ssize_t namelen, char *name) +case_ok(char *buf, Py_ssize_t len, Py_ssize_t namelen, const char *name) { /* Pick a platform-specific implementation; the sequence of #if's here should * match the sequence just above. -- cgit v1.2.1 From f44ffd60a466519364cc45d367f3505f1ecbe090 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 20 Mar 2011 01:34:43 +0100 Subject: Issue #3080: Create find_module_path_list() subfunction --- Python/import.c | 310 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 164 insertions(+), 146 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 37cd4b81f2..565f1336cb 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1598,168 +1598,30 @@ static int case_ok(char *, Py_ssize_t, Py_ssize_t, const char *); static int find_init_module(char *); /* Forward */ static struct filedescr importhookdescr = {"", "", IMP_HOOK}; -/* Find a module: - - - try find_module() of each sys.meta_path hook - - try find_frozen() - - try is_builtin() - - try _PyWin_FindRegisteredModule() (Windows only) - - otherwise, call find_module_path_list() with search_path_list (if not - NULL) or sys.path - - Return: - - - &fd_builtin (C_BUILTIN) if it is a builtin - - &fd_frozen (PY_FROZEN) if it is frozen - - &fd_package (PKG_DIRECTORY) and write the filename into *buf - if it is a package - - &importhookdescr (IMP_HOOK) and write the loader into *p_loader if a - importer loader was found - - a file descriptor (PY_SOURCE, PY_COMPILED, C_EXTENSION, PY_RESOURCE or - PY_CODERESOURCE: see _PyImport_Filetab), write the filename into - *buf and the pointer to the open file into *p_fp - - NULL on error - - By default, write an empty string into *buf, and *p_fp and *p_loader (if - set) are set to NULL. Eg. *buf is an empty string for a builtin package. */ - -static struct filedescr * -find_module(char *fullname, const char *name, PyObject *path, char *buf, - size_t buflen, FILE **p_fp, PyObject **p_loader) +static struct filedescr* +find_module_path_list(char *fullname, const char *name, + PyObject *search_path_list, PyObject *path_hooks, + PyObject *path_importer_cache, + char *buf, size_t buflen, + FILE **p_fp, PyObject **p_loader) { Py_ssize_t i, npath; size_t len, namelen; struct filedescr *fdp = NULL; char *filemode; FILE *fp = NULL; - PyObject *path_hooks, *path_importer_cache; struct stat statbuf; - static struct filedescr fd_frozen = {"", "", PY_FROZEN}; - static struct filedescr fd_builtin = {"", "", C_BUILTIN}; static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; #if defined(PYOS_OS2) size_t saved_len; size_t saved_namelen; char *saved_buf = NULL; #endif - PyObject *fullname_obj, *nameobj; - *buf = '\0'; - *p_fp = NULL; - if (p_loader != NULL) - *p_loader = NULL; - - if (strlen(name) > MAXPATHLEN) { - PyErr_SetString(PyExc_OverflowError, - "module name is too long"); - return NULL; - } - - /* sys.meta_path import hook */ - if (p_loader != NULL) { - PyObject *meta_path; - - meta_path = PySys_GetObject("meta_path"); - if (meta_path == NULL || !PyList_Check(meta_path)) { - PyErr_SetString(PyExc_ImportError, - "sys.meta_path must be a list of " - "import hooks"); - return NULL; - } - Py_INCREF(meta_path); /* zap guard */ - npath = PyList_Size(meta_path); - for (i = 0; i < npath; i++) { - PyObject *loader; - PyObject *hook = PyList_GetItem(meta_path, i); - loader = PyObject_CallMethod(hook, "find_module", - "sO", fullname, - path != NULL ? - path : Py_None); - if (loader == NULL) { - Py_DECREF(meta_path); - return NULL; /* true error */ - } - if (loader != Py_None) { - /* a loader was found */ - *p_loader = loader; - Py_DECREF(meta_path); - return &importhookdescr; - } - Py_DECREF(loader); - } - Py_DECREF(meta_path); - } - - if (fullname != NULL) { - fullname_obj = PyUnicode_FromString(fullname); - if (fullname == NULL) - return NULL; - if (find_frozen(fullname_obj) != NULL) { - Py_DECREF(fullname_obj); - strcpy(buf, fullname); - return &fd_frozen; - } - Py_DECREF(fullname_obj); - } - - if (path == NULL) { -#ifdef MS_COREDLL - PyObject *filename, *filename_bytes; -#endif - nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) - return NULL; - if (is_builtin(nameobj)) { - Py_DECREF(nameobj); - strcpy(buf, name); - return &fd_builtin; - } -#ifdef MS_COREDLL - fp = _PyWin_FindRegisteredModule(nameobj, &fdp, &filename); - if (fp != NULL) { - Py_DECREF(nameobj); - filename_bytes = PyUnicode_EncodeFSDefault(filename); - Py_DECREF(filename); - if (filename_bytes == NULL) - return NULL; - strncpy(buf, PyBytes_AS_STRING(filename_bytes), buflen); - buf[buflen-1] = '\0'; - Py_DECREF(filename_bytes); - *p_fp = fp; - return fdp; - } - else if (PyErr_Occurred()) - return NULL; -#endif - Py_DECREF(nameobj); - path = PySys_GetObject("path"); - } - - if (path == NULL || !PyList_Check(path)) { - PyErr_SetString(PyExc_ImportError, - "sys.path must be a list of directory names"); - return NULL; - } - - path_hooks = PySys_GetObject("path_hooks"); - if (path_hooks == NULL || !PyList_Check(path_hooks)) { - PyErr_SetString(PyExc_ImportError, - "sys.path_hooks must be a list of " - "import hooks"); - return NULL; - } - path_importer_cache = PySys_GetObject("path_importer_cache"); - if (path_importer_cache == NULL || - !PyDict_Check(path_importer_cache)) { - PyErr_SetString(PyExc_ImportError, - "sys.path_importer_cache must be a dict"); - return NULL; - } - - npath = PyList_Size(path); + npath = PyList_Size(search_path_list); namelen = strlen(name); for (i = 0; i < npath; i++) { - PyObject *v = PyList_GetItem(path, i); + PyObject *v = PyList_GetItem(search_path_list, i); PyObject *origv = v; const char *base; Py_ssize_t size; @@ -1925,6 +1787,162 @@ find_module(char *fullname, const char *name, PyObject *path, char *buf, return fdp; } +/* Find a module: + + - try find_module() of each sys.meta_path hook + - try find_frozen() + - try is_builtin() + - try _PyWin_FindRegisteredModule() (Windows only) + - otherwise, call find_module_path_list() with search_path_list (if not + NULL) or sys.path + + Return: + + - &fd_builtin (C_BUILTIN) if it is a builtin + - &fd_frozen (PY_FROZEN) if it is frozen + - &fd_package (PKG_DIRECTORY) and write the filename into *buf + if it is a package + - &importhookdescr (IMP_HOOK) and write the loader into *p_loader if a + importer loader was found + - a file descriptor (PY_SOURCE, PY_COMPILED, C_EXTENSION, PY_RESOURCE or + PY_CODERESOURCE: see _PyImport_Filetab), write the filename into + *buf and the pointer to the open file into *p_fp + - NULL on error + + By default, write an empty string into *buf, and *p_fp and *p_loader (if + set) are set to NULL. Eg. *buf is an empty string for a builtin package. */ + +static struct filedescr * +find_module(char *fullname, const char *name, PyObject *search_path_list, + char *buf, size_t buflen, FILE **p_fp, PyObject **p_loader) +{ + Py_ssize_t i, npath; + static struct filedescr fd_frozen = {"", "", PY_FROZEN}; + static struct filedescr fd_builtin = {"", "", C_BUILTIN}; + PyObject *path_hooks, *path_importer_cache; + PyObject *fullname_obj, *nameobj; + + *buf = '\0'; + *p_fp = NULL; + if (p_loader != NULL) + *p_loader = NULL; + + if (strlen(name) > MAXPATHLEN) { + PyErr_SetString(PyExc_OverflowError, + "module name is too long"); + return NULL; + } + + /* sys.meta_path import hook */ + if (p_loader != NULL) { + PyObject *meta_path; + + meta_path = PySys_GetObject("meta_path"); + if (meta_path == NULL || !PyList_Check(meta_path)) { + PyErr_SetString(PyExc_ImportError, + "sys.meta_path must be a list of " + "import hooks"); + return NULL; + } + Py_INCREF(meta_path); /* zap guard */ + npath = PyList_Size(meta_path); + for (i = 0; i < npath; i++) { + PyObject *loader; + PyObject *hook = PyList_GetItem(meta_path, i); + loader = PyObject_CallMethod(hook, "find_module", + "sO", fullname, + search_path_list != NULL ? + search_path_list : Py_None); + if (loader == NULL) { + Py_DECREF(meta_path); + return NULL; /* true error */ + } + if (loader != Py_None) { + /* a loader was found */ + *p_loader = loader; + Py_DECREF(meta_path); + return &importhookdescr; + } + Py_DECREF(loader); + } + Py_DECREF(meta_path); + } + + if (fullname != NULL) { + fullname_obj = PyUnicode_FromString(fullname); + if (fullname == NULL) + return NULL; + if (find_frozen(fullname_obj) != NULL) { + Py_DECREF(fullname_obj); + strcpy(buf, fullname); + return &fd_frozen; + } + Py_DECREF(fullname_obj); + } + + if (search_path_list == NULL) { +#ifdef MS_COREDLL + FILE *fp; + struct filedescr *fdp; + PyObject *filename, *filename_bytes; +#endif + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + if (is_builtin(nameobj)) { + Py_DECREF(nameobj); + strcpy(buf, name); + return &fd_builtin; + } +#ifdef MS_COREDLL + fp = _PyWin_FindRegisteredModule(nameobj, &fdp, &filename); + if (fp != NULL) { + Py_DECREF(nameobj); + filename_bytes = PyUnicode_EncodeFSDefault(filename); + Py_DECREF(filename); + if (filename_bytes == NULL) + return NULL; + strncpy(buf, PyBytes_AS_STRING(filename_bytes), buflen); + buf[buflen-1] = '\0'; + Py_DECREF(filename_bytes); + *p_fp = fp; + return fdp; + } + else if (PyErr_Occurred()) + return NULL; +#endif + Py_DECREF(nameobj); + search_path_list = PySys_GetObject("path"); + } + + if (search_path_list == NULL || !PyList_Check(search_path_list)) { + PyErr_SetString(PyExc_ImportError, + "sys.path must be a list of directory names"); + return NULL; + } + + path_hooks = PySys_GetObject("path_hooks"); + if (path_hooks == NULL || !PyList_Check(path_hooks)) { + PyErr_SetString(PyExc_ImportError, + "sys.path_hooks must be a list of " + "import hooks"); + return NULL; + } + path_importer_cache = PySys_GetObject("path_importer_cache"); + if (path_importer_cache == NULL || + !PyDict_Check(path_importer_cache)) { + PyErr_SetString(PyExc_ImportError, + "sys.path_importer_cache must be a dict"); + return NULL; + } + + return find_module_path_list(fullname, name, + search_path_list, path_hooks, + path_importer_cache, + buf, buflen, + p_fp, p_loader); +} + /* case_ok(char* buf, Py_ssize_t len, Py_ssize_t namelen, char* name) * The arguments here are tricky, best shown by example: * /a/b/c/d/e/f/g/h/i/j/k/some_long_module_name.py\0 -- cgit v1.2.1 From 2d77d7eabcf50177446677639435d797b85722e7 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 12 Mar 2011 16:02:28 -0500 Subject: Issue #3080: Create find_module_path() subfunction --- Python/import.c | 228 ++++++++++++++++++++++++++++++++++---------------------- 1 file changed, 139 insertions(+), 89 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 565f1336cb..59b284654e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1598,6 +1598,129 @@ static int case_ok(char *, Py_ssize_t, Py_ssize_t, const char *); static int find_init_module(char *); /* Forward */ static struct filedescr importhookdescr = {"", "", IMP_HOOK}; +/* Get the path of a module: get its importer and call importer.find_module() + hook, or check if the module if a package (if path/__init__.py exists). + + -1: error: a Python error occurred + 0: ignore: an error occurred because of invalid data, but the error is not + important enough to be reported. + 1: get path: module not found, but *buf contains its path + 2: found: *p_fd is the file descriptor (IMP_HOOK or PKG_DIRECTORY) + and *buf is the path */ + +static int +find_module_path(char *fullname, const char *name, PyObject *path, + PyObject *path_hooks, PyObject *path_importer_cache, + char *buf, size_t buflen, + PyObject **p_loader, struct filedescr **p_fd) +{ + PyObject *path_bytes; + const char *base; + Py_ssize_t len; + size_t namelen; + struct stat statbuf; + static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; + + if (PyUnicode_Check(path)) { + path_bytes = PyUnicode_EncodeFSDefault(path); + if (path_bytes == NULL) + return -1; + } + else if (PyBytes_Check(path)) { + Py_INCREF(path); + path_bytes = path; + } + else + return 0; + + namelen = strlen(name); + base = PyBytes_AS_STRING(path_bytes); + len = PyBytes_GET_SIZE(path_bytes); + if (len + 2 + namelen + MAXSUFFIXSIZE >= buflen) { + Py_DECREF(path_bytes); + return 0; /* Too long */ + } + strcpy(buf, base); + Py_DECREF(path_bytes); + + if (strlen(buf) != len) { + return 0; /* path_bytes contains '\0' */ + } + + /* sys.path_hooks import hook */ + if (p_loader != NULL) { + PyObject *importer; + + importer = get_path_importer(path_importer_cache, + path_hooks, path); + if (importer == NULL) { + return -1; + } + /* Note: importer is a borrowed reference */ + if (importer != Py_None) { + PyObject *loader; + loader = PyObject_CallMethod(importer, + "find_module", + "s", fullname); + if (loader == NULL) + return -1; /* error */ + if (loader != Py_None) { + /* a loader was found */ + *p_loader = loader; + *p_fd = &importhookdescr; + return 2; + } + Py_DECREF(loader); + return 0; + } + } + /* no hook was found, use builtin import */ + + if (len > 0 && buf[len-1] != SEP +#ifdef ALTSEP + && buf[len-1] != ALTSEP +#endif + ) + buf[len++] = SEP; + strcpy(buf+len, name); + len += namelen; + + /* Check for package import (buf holds a directory name, + and there's an __init__ module in that directory */ +#ifdef HAVE_STAT + if (stat(buf, &statbuf) == 0 && /* it exists */ + S_ISDIR(statbuf.st_mode) && /* it's a directory */ + case_ok(buf, len, namelen, name)) { /* case matches */ + if (find_init_module(buf)) { /* and has __init__.py */ + *p_fd = &fd_package; + return 2; + } + else { + int err; + PyObject *unicode = PyUnicode_DecodeFSDefault(buf); + if (unicode == NULL) + return -1; + err = PyErr_WarnFormat(PyExc_ImportWarning, 1, + "Not importing directory '%U': missing __init__.py", + unicode); + Py_DECREF(unicode); + if (err) + return -1; + } + } +#endif + return 1; +} + +/* Find a module in search_path_list. For each path, try + find_module_filename() or try each _PyImport_Filetab suffix. + + If the module is found, return a file descriptor, write the path in + *p_filename, write the pointer to the file object into *p_fp, and (if + p_loader is not NULL) the loader into *p_loader. + + Otherwise, raise an exception and return NULL. */ + static struct filedescr* find_module_path_list(char *fullname, const char *name, PyObject *search_path_list, PyObject *path_hooks, @@ -1610,8 +1733,6 @@ find_module_path_list(char *fullname, const char *name, struct filedescr *fdp = NULL; char *filemode; FILE *fp = NULL; - struct stat statbuf; - static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; #if defined(PYOS_OS2) size_t saved_len; size_t saved_namelen; @@ -1621,96 +1742,25 @@ find_module_path_list(char *fullname, const char *name, npath = PyList_Size(search_path_list); namelen = strlen(name); for (i = 0; i < npath; i++) { - PyObject *v = PyList_GetItem(search_path_list, i); - PyObject *origv = v; - const char *base; - Py_ssize_t size; - if (!v) - return NULL; - if (PyUnicode_Check(v)) { - v = PyUnicode_EncodeFSDefault(v); - if (v == NULL) - return NULL; - } - else if (!PyBytes_Check(v)) - continue; - else - Py_INCREF(v); - - base = PyBytes_AS_STRING(v); - size = PyBytes_GET_SIZE(v); - len = size; - if (len + 2 + namelen + MAXSUFFIXSIZE >= buflen) { - Py_DECREF(v); - continue; /* Too long */ - } - strcpy(buf, base); - Py_DECREF(v); + PyObject *path; + int ok; - if (strlen(buf) != len) { - continue; /* v contains '\0' */ - } - - /* sys.path_hooks import hook */ - if (p_loader != NULL) { - PyObject *importer; - - importer = get_path_importer(path_importer_cache, - path_hooks, origv); - if (importer == NULL) { - return NULL; - } - /* Note: importer is a borrowed reference */ - if (importer != Py_None) { - PyObject *loader; - loader = PyObject_CallMethod(importer, - "find_module", - "s", fullname); - if (loader == NULL) - return NULL; /* error */ - if (loader != Py_None) { - /* a loader was found */ - *p_loader = loader; - return &importhookdescr; - } - Py_DECREF(loader); - continue; - } - } - /* no hook was found, use builtin import */ + path = PyList_GetItem(search_path_list, i); + if (path == NULL) + return NULL; - if (len > 0 && buf[len-1] != SEP -#ifdef ALTSEP - && buf[len-1] != ALTSEP -#endif - ) - buf[len++] = SEP; - strcpy(buf+len, name); - len += namelen; + ok = find_module_path(fullname, name, path, + path_hooks, path_importer_cache, + buf, buflen, + p_loader, &fdp); + if (ok < 0) + return NULL; + if (ok == 0) + continue; + if (ok == 2) + return fdp; - /* Check for package import (buf holds a directory name, - and there's an __init__ module in that directory */ -#ifdef HAVE_STAT - if (stat(buf, &statbuf) == 0 && /* it exists */ - S_ISDIR(statbuf.st_mode) && /* it's a directory */ - case_ok(buf, len, namelen, name)) { /* case matches */ - if (find_init_module(buf)) { /* and has __init__.py */ - return &fd_package; - } - else { - int err; - PyObject *unicode = PyUnicode_DecodeFSDefault(buf); - if (unicode == NULL) - return NULL; - err = PyErr_WarnFormat(PyExc_ImportWarning, 1, - "Not importing directory '%U': missing __init__.py", - unicode); - Py_DECREF(unicode); - if (err) - return NULL; - } - } -#endif + len = strlen(buf); #if defined(PYOS_OS2) /* take a snapshot of the module spec for restoration * after the 8 character DLL hackery -- cgit v1.2.1 From 8509b2931d82b09524d8922b5ffd1871f5ff6311 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 13:33:46 -0400 Subject: Issue #3080: get_sourcefile(), make_source_pathname(), load_package() Use Unicode for module name and path in get_sourcefile(), make_source_pathname() and load_package() functions. --- Python/import.c | 219 +++++++++++++++++++++++++++++++++----------------------- 1 file changed, 131 insertions(+), 88 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 59b284654e..f6cde957a8 100644 --- a/Python/import.c +++ b/Python/import.c @@ -113,6 +113,8 @@ typedef unsigned short mode_t; #define MAGIC (3180 | ((long)'\r'<<16) | ((long)'\n'<<24)) #define TAG "cpython-32" #define CACHEDIR "__pycache__" +static const Py_UNICODE CACHEDIR_UNICODE[] = { + '_', '_', 'p', 'y', 'c', 'a', 'c', 'h', 'e', '_', '_', '\0'}; /* Current magic word and string tag as globals. */ static long pyc_magic = MAGIC; static const char *pyc_tag = TAG; @@ -741,8 +743,8 @@ remove_module(PyObject *name) "sys.modules failed"); } -static PyObject * get_sourcefile(char *file); -static char *make_source_pathname(char *pathname, char *buf); +static PyObject * get_sourcefile(PyObject *filename); +static PyObject *make_source_pathname(PyObject *pathname); static char *make_compiled_pathname(char *pathname, char *buf, size_t buflen, int debug); @@ -807,7 +809,6 @@ PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, { PyObject *modules = PyImport_GetModuleDict(); PyObject *m, *d, *v; - PyObject *pathbytes; m = PyImport_AddModuleObject(name); if (m == NULL) @@ -822,12 +823,7 @@ PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, } /* Remember the filename as the __file__ attribute */ if (pathname != NULL) { - pathbytes = PyUnicode_EncodeFSDefault(pathname); - if (pathbytes != NULL) { - v = get_sourcefile(PyBytes_AS_STRING(pathbytes)); - Py_DECREF(pathbytes); - } else - v = NULL; + v = get_sourcefile(pathname); if (v == NULL) PyErr_Clear(); } @@ -892,6 +888,27 @@ rightmost_sep(char *s) } +/* Like strrchr(string, '/') but searches for the rightmost of either SEP + or ALTSEP, if the latter is defined. +*/ +static Py_UNICODE* +rightmost_sep_unicode(Py_UNICODE *s) +{ + Py_UNICODE *found, c; + for (found = NULL; (c = *s); s++) { + if (c == SEP +#ifdef ALTSEP + || c == ALTSEP +#endif + ) + { + found = s; + } + } + return found; +} + + /* Given a pathname for a Python source file, fill a buffer with the pathname for the corresponding compiled file. Return the pathname for the compiled file, or NULL if there's no space in the buffer. @@ -1005,42 +1022,50 @@ make_compiled_pathname(char *pathname, char *buf, size_t buflen, int debug) source file, if the path matches the PEP 3147 format. This does not check for any file existence, however, if the pyc file name does not match PEP 3147 style, NULL is returned. buf must be at least as big as pathname; - the resulting path will always be shorter. */ + the resulting path will always be shorter. -static char * -make_source_pathname(char *pathname, char *buf) + (...)/__pycache__/foo..pyc -> (...)/foo.py */ + +static PyObject* +make_source_pathname(PyObject *pathobj) { - /* __pycache__/foo..pyc -> foo.py */ + Py_UNICODE buf[MAXPATHLEN]; + Py_UNICODE *pathname; + Py_UNICODE *left, *right, *dot0, *dot1, sep; size_t i, j; - char *left, *right, *dot0, *dot1, sep; + + if (PyUnicode_GET_SIZE(pathobj) > MAXPATHLEN) + return NULL; + pathname = PyUnicode_AS_UNICODE(pathobj); /* Look back two slashes from the end. In between these two slashes must be the string __pycache__ or this is not a PEP 3147 style path. It's possible for there to be only one slash. */ - if ((right = rightmost_sep(pathname)) == NULL) + right = rightmost_sep_unicode(pathname); + if (right == NULL) return NULL; sep = *right; *right = '\0'; - left = rightmost_sep(pathname); + left = rightmost_sep_unicode(pathname); *right = sep; if (left == NULL) left = pathname; else left++; - if (right-left != strlen(CACHEDIR) || - strncmp(left, CACHEDIR, right-left) != 0) + if (right-left != Py_UNICODE_strlen(CACHEDIR_UNICODE) || + Py_UNICODE_strncmp(left, CACHEDIR_UNICODE, right-left) != 0) return NULL; /* Now verify that the path component to the right of the last slash has two dots in it. */ - if ((dot0 = strchr(right + 1, '.')) == NULL) + if ((dot0 = Py_UNICODE_strchr(right + 1, '.')) == NULL) return NULL; - if ((dot1 = strchr(dot0 + 1, '.')) == NULL) + if ((dot1 = Py_UNICODE_strchr(dot0 + 1, '.')) == NULL) return NULL; /* Too many dots? */ - if (strchr(dot1 + 1, '.') != NULL) + if (Py_UNICODE_strchr(dot1 + 1, '.') != NULL) return NULL; /* This is a PEP 3147 path. Start by copying everything from the @@ -1048,10 +1073,11 @@ make_source_pathname(char *pathname, char *buf) copy the file's basename, removing the magic tag and adding a .py suffix. */ - strncpy(buf, pathname, (i=left-pathname)); - strncpy(buf+i, right+1, (j=dot0-right)); - strcpy(buf+i+j, "py"); - return buf; + Py_UNICODE_strncpy(buf, pathname, (i=left-pathname)); + Py_UNICODE_strncpy(buf+i, right+1, (j=dot0-right)); + buf[i+j] = 'p'; + buf[i+j+1] = 'y'; + return PyUnicode_FromUnicode(buf, i+j+2); } /* Given a pathname for a Python source file, its time of last @@ -1390,40 +1416,47 @@ load_source_module(char *name, char *pathname, FILE *fp) * Returns the path to the py file if available, else the given path */ static PyObject * -get_sourcefile(char *file) +get_sourcefile(PyObject *filename) { - char py[MAXPATHLEN + 1]; Py_ssize_t len; - PyObject *u; + Py_UNICODE *fileuni; + PyObject *py; struct stat statbuf; - if (!file || !*file) { + len = PyUnicode_GET_SIZE(filename); + if (len == 0) Py_RETURN_NONE; - } - len = strlen(file); - /* match '*.py?' */ - if (len > MAXPATHLEN || PyOS_strnicmp(&file[len-4], ".py", 3) != 0) { - return PyUnicode_DecodeFSDefault(file); - } + /* don't match *.pyc or *.pyo? */ + fileuni = PyUnicode_AS_UNICODE(filename); + if (len < 5 + || fileuni[len-4] != '.' + || (fileuni[len-3] != 'p' && fileuni[len-3] != 'P') + || (fileuni[len-2] != 'y' && fileuni[len-2] != 'Y')) + goto unchanged; /* Start by trying to turn PEP 3147 path into source path. If that * fails, just chop off the trailing character, i.e. legacy pyc path * to py. */ - if (make_source_pathname(file, py) == NULL) { - strncpy(py, file, len-1); - py[len-1] = '\0'; + py = make_source_pathname(filename); + if (py == NULL) { + PyErr_Clear(); + py = PyUnicode_FromUnicode(fileuni, len - 1); } + if (py == NULL) + goto error; - if (stat(py, &statbuf) == 0 && - S_ISREG(statbuf.st_mode)) { - u = PyUnicode_DecodeFSDefault(py); - } - else { - u = PyUnicode_DecodeFSDefault(file); - } - return u; + if (_Py_stat(py, &statbuf) == 0 && S_ISREG(statbuf.st_mode)) + return py; + Py_DECREF(py); + goto unchanged; + +error: + PyErr_Clear(); +unchanged: + Py_INCREF(filename); + return filename; } /* Forward */ @@ -1436,54 +1469,56 @@ static struct _frozen * find_frozen(PyObject *); REFERENCE COUNT */ static PyObject * -load_package(char *name, char *pathname) +load_package(PyObject *name, PyObject *pathname) { PyObject *m, *d; - PyObject *file = NULL; - PyObject *path = NULL; + PyObject *file = NULL, *path_list = NULL; int err; char buf[MAXPATHLEN+1]; - FILE *fp; + FILE *fp = NULL; struct filedescr *fdp; + char *namestr; - m = PyImport_AddModule(name); + m = PyImport_AddModuleObject(name); if (m == NULL) return NULL; if (Py_VerboseFlag) - PySys_WriteStderr("import %s # directory %s\n", + PySys_FormatStderr("import %U # directory %U\n", name, pathname); - d = PyModule_GetDict(m); file = get_sourcefile(pathname); if (file == NULL) - goto error; - path = Py_BuildValue("[O]", file); - if (path == NULL) - goto error; + return NULL; + path_list = Py_BuildValue("[O]", file); + if (path_list == NULL) { + Py_DECREF(file); + return NULL; + } + d = PyModule_GetDict(m); err = PyDict_SetItemString(d, "__file__", file); + Py_DECREF(file); if (err == 0) - err = PyDict_SetItemString(d, "__path__", path); - if (err != 0) + err = PyDict_SetItemString(d, "__path__", path_list); + if (err != 0) { + Py_DECREF(path_list); + return NULL; + } + namestr = _PyUnicode_AsString(name); + if (namestr == NULL) goto error; - fdp = find_module(name, "__init__", path, buf, sizeof(buf), &fp, NULL); + fdp = find_module(namestr, "__init__", path_list, buf, sizeof(buf), &fp, NULL); + Py_DECREF(path_list); if (fdp == NULL) { if (PyErr_ExceptionMatches(PyExc_ImportError)) { PyErr_Clear(); Py_INCREF(m); + return m; } else - m = NULL; - goto cleanup; + return NULL; } - m = load_module(name, fp, buf, fdp->type, NULL); + m = load_module(namestr, fp, buf, fdp->type, NULL); if (fp != NULL) fclose(fp); - goto cleanup; - - error: - m = NULL; - cleanup: - Py_XDECREF(path); - Py_XDECREF(file); return m; } @@ -2282,9 +2317,21 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) } #endif - case PKG_DIRECTORY: - m = load_package(name, pathname); + case PKG_DIRECTORY: { + PyObject *nameobj, *pathobj; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + pathobj = PyUnicode_DecodeFSDefault(pathname); + if (pathobj == NULL) { + Py_DECREF(nameobj); + return NULL; + } + m = load_package(nameobj, pathobj); + Py_DECREF(nameobj); + Py_DECREF(pathobj); break; + } case C_BUILTIN: case PY_FROZEN: { @@ -3637,13 +3684,12 @@ imp_load_module(PyObject *self, PyObject *args) static PyObject * imp_load_package(PyObject *self, PyObject *args) { - char *name; - PyObject *pathname; + PyObject *name, *pathname; PyObject * ret; - if (!PyArg_ParseTuple(args, "sO&:load_package", - &name, PyUnicode_FSConverter, &pathname)) + if (!PyArg_ParseTuple(args, "UO&:load_package", + &name, PyUnicode_FSDecoder, &pathname)) return NULL; - ret = load_package(name, PyBytes_AS_STRING(pathname)); + ret = load_package(name, pathname); Py_DECREF(pathname); return ret; } @@ -3716,25 +3762,22 @@ static PyObject * imp_source_from_cache(PyObject *self, PyObject *args, PyObject *kws) { static char *kwlist[] = {"path", NULL}; - - PyObject *pathname_obj; - char *pathname; - char buf[MAXPATHLEN+1]; + PyObject *pathname, *source; if (!PyArg_ParseTupleAndKeywords( args, kws, "O&", kwlist, - PyUnicode_FSConverter, &pathname_obj)) + PyUnicode_FSDecoder, &pathname)) return NULL; - pathname = PyBytes_AS_STRING(pathname_obj); - if (make_source_pathname(pathname, buf) == NULL) { - PyErr_Format(PyExc_ValueError, "Not a PEP 3147 pyc path: %s", + source = make_source_pathname(pathname); + if (source == NULL) { + PyErr_Format(PyExc_ValueError, "Not a PEP 3147 pyc path: %R", pathname); - Py_DECREF(pathname_obj); + Py_DECREF(pathname); return NULL; } - Py_DECREF(pathname_obj); - return PyUnicode_FromString(buf); + Py_DECREF(pathname); + return source; } PyDoc_STRVAR(doc_source_from_cache, -- cgit v1.2.1 From 26e6d2c66ccad73bad2edeb441f54141164d0bc1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 20 Mar 2011 00:41:24 +0100 Subject: Issue #3080: Use Unicode to import source and compiled modules * Use Unicode for module name and path in the following functions: * get_file() * load_source_module(), parse_source_module() * make_compiled_pathname(), check_compiled_module(), read_compiled_module(), load_compiled_module(), write_compiled_module(), update_compiled_module() * On Windows, use CreateDirectoryW() instead of mkdir() * update_compiled_module() cannot fail anymore --- Python/import.c | 352 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 201 insertions(+), 151 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index f6cde957a8..0efffe9149 100644 --- a/Python/import.c +++ b/Python/import.c @@ -107,8 +107,8 @@ typedef unsigned short mode_t; /* MAGIC must change whenever the bytecode emitted by the compiler may no longer be understood by older implementations of the eval loop (usually due to the addition of new opcodes) - TAG must change for each major Python release. The magic number will take - care of any bytecode changes that occur during development. + TAG and PYC_TAG_UNICODE must change for each major Python release. The magic + number will take care of any bytecode changes that occur during development. */ #define MAGIC (3180 | ((long)'\r'<<16) | ((long)'\n'<<24)) #define TAG "cpython-32" @@ -118,6 +118,8 @@ static const Py_UNICODE CACHEDIR_UNICODE[] = { /* Current magic word and string tag as globals. */ static long pyc_magic = MAGIC; static const char *pyc_tag = TAG; +static const Py_UNICODE PYC_TAG_UNICODE[] = { + 'c', 'p', 'y', 't', 'h', 'o', 'n', '-', '3', '2', '\0'}; /* See _PyImport_FixupExtensionObject() below */ static PyObject *extensions = NULL; @@ -745,8 +747,7 @@ remove_module(PyObject *name) static PyObject * get_sourcefile(PyObject *filename); static PyObject *make_source_pathname(PyObject *pathname); -static char *make_compiled_pathname(char *pathname, char *buf, size_t buflen, - int debug); +static PyObject* make_compiled_pathname(Py_UNICODE *pathname, int debug); /* Execute a code object in a module and return the module object * WITH INCREMENTED REFERENCE COUNT. If an error occurs, name is @@ -867,32 +868,11 @@ PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, } -/* Like strrchr(string, '/') but searches for the rightmost of either SEP - or ALTSEP, if the latter is defined. -*/ -static char * -rightmost_sep(char *s) -{ - char *found, c; - for (found = NULL; (c = *s); s++) { - if (c == SEP -#ifdef ALTSEP - || c == ALTSEP -#endif - ) - { - found = s; - } - } - return found; -} - - /* Like strrchr(string, '/') but searches for the rightmost of either SEP or ALTSEP, if the latter is defined. */ static Py_UNICODE* -rightmost_sep_unicode(Py_UNICODE *s) +rightmost_sep(Py_UNICODE *s) { Py_UNICODE *found, c; for (found = NULL; (c = *s); s++) { @@ -912,15 +892,18 @@ rightmost_sep_unicode(Py_UNICODE *s) /* Given a pathname for a Python source file, fill a buffer with the pathname for the corresponding compiled file. Return the pathname for the compiled file, or NULL if there's no space in the buffer. - Doesn't set an exception. */ + Doesn't set an exception. + + foo.py -> __pycache__/foo..pyc */ -static char * -make_compiled_pathname(char *pathname, char *buf, size_t buflen, int debug) +static PyObject* +make_compiled_pathname(Py_UNICODE *pathname, int debug) { - /* foo.py -> __pycache__/foo..pyc */ - size_t len = strlen(pathname); + Py_UNICODE buf[MAXPATHLEN]; + size_t buflen = (size_t)MAXPATHLEN; + size_t len = Py_UNICODE_strlen(pathname); size_t i, save; - char *pos; + Py_UNICODE *pos; int sep = SEP; /* Sanity check that the buffer has roughly enough space to hold what @@ -932,35 +915,37 @@ make_compiled_pathname(char *pathname, char *buf, size_t buflen, int debug) sanity check before writing the extension to ensure we do not overflow the buffer. */ - if (len + strlen(CACHEDIR) + strlen(pyc_tag) + 5 > buflen) + if (len + Py_UNICODE_strlen(CACHEDIR_UNICODE) + Py_UNICODE_strlen(PYC_TAG_UNICODE) + 5 > buflen) return NULL; /* Find the last path separator and copy everything from the start of the source string up to and including the separator. */ - if ((pos = rightmost_sep(pathname)) == NULL) { + pos = rightmost_sep(pathname); + if (pos == NULL) { i = 0; } else { sep = *pos; i = pos - pathname + 1; - strncpy(buf, pathname, i); + Py_UNICODE_strncpy(buf, pathname, i); } save = i; buf[i++] = '\0'; /* Add __pycache__/ */ - strcat(buf, CACHEDIR); - i += strlen(CACHEDIR) - 1; + Py_UNICODE_strcat(buf, CACHEDIR_UNICODE); + i += Py_UNICODE_strlen(CACHEDIR_UNICODE) - 1; buf[i++] = sep; buf[i++] = '\0'; /* Add the base filename, but remove the .py or .pyw extension, since the tag name must go before the extension. */ - strcat(buf, pathname + save); - if ((pos = strrchr(buf, '.')) != NULL) + Py_UNICODE_strcat(buf, pathname + save); + pos = Py_UNICODE_strrchr(buf, '.'); + if (pos != NULL) *++pos = '\0'; - strcat(buf, pyc_tag); + Py_UNICODE_strcat(buf, PYC_TAG_UNICODE); /* The length test above assumes that we're only adding one character to the end of what would normally be the extension. What if there is no extension, or the string ends in '.' or '.p', and otherwise @@ -1010,11 +995,15 @@ make_compiled_pathname(char *pathname, char *buf, size_t buflen, int debug) #if 0 printf("strlen(buf): %d; buflen: %d\n", (int)strlen(buf), (int)buflen); #endif - if (strlen(buf) + 5 > buflen) + len = Py_UNICODE_strlen(buf); + if (len + 5 > buflen) return NULL; - strcat(buf, debug ? ".pyc" : ".pyo"); - assert(strlen(buf) < buflen); - return buf; + buf[len] = '.'; len++; + buf[len] = 'p'; len++; + buf[len] = 'y'; len++; + buf[len] = debug ? 'c' : 'o'; len++; + assert(len <= buflen); + return PyUnicode_FromUnicode(buf, len); } @@ -1042,12 +1031,12 @@ make_source_pathname(PyObject *pathobj) must be the string __pycache__ or this is not a PEP 3147 style path. It's possible for there to be only one slash. */ - right = rightmost_sep_unicode(pathname); + right = rightmost_sep(pathname); if (right == NULL) return NULL; sep = *right; *right = '\0'; - left = rightmost_sep_unicode(pathname); + left = rightmost_sep(pathname); *right = sep; if (left == NULL) left = pathname; @@ -1088,31 +1077,31 @@ make_source_pathname(PyObject *pathobj) Doesn't set an exception. */ static FILE * -check_compiled_module(char *pathname, time_t mtime, char *cpathname) +check_compiled_module(PyObject *pathname, time_t mtime, PyObject *cpathname) { FILE *fp; long magic; long pyc_mtime; - fp = fopen(cpathname, "rb"); + fp = _Py_fopen(cpathname, "rb"); if (fp == NULL) return NULL; magic = PyMarshal_ReadLongFromFile(fp); if (magic != pyc_magic) { if (Py_VerboseFlag) - PySys_WriteStderr("# %s has bad magic\n", cpathname); + PySys_FormatStderr("# %R has bad magic\n", cpathname); fclose(fp); return NULL; } pyc_mtime = PyMarshal_ReadLongFromFile(fp); if (pyc_mtime != mtime) { if (Py_VerboseFlag) - PySys_WriteStderr("# %s has bad mtime\n", cpathname); + PySys_FormatStderr("# %R has bad mtime\n", cpathname); fclose(fp); return NULL; } if (Py_VerboseFlag) - PySys_WriteStderr("# %s matches %s\n", cpathname, pathname); + PySys_FormatStderr("# %R matches %R\n", cpathname, pathname); return fp; } @@ -1120,7 +1109,7 @@ check_compiled_module(char *pathname, time_t mtime, char *cpathname) /* Read a code object from a file and check it for validity */ static PyCodeObject * -read_compiled_module(char *cpathname, FILE *fp) +read_compiled_module(PyObject *cpathname, FILE *fp) { PyObject *co; @@ -1129,7 +1118,7 @@ read_compiled_module(char *cpathname, FILE *fp) return NULL; if (!PyCode_Check(co)) { PyErr_Format(PyExc_ImportError, - "Non-code object in %.200s", cpathname); + "Non-code object in %R", cpathname); Py_DECREF(co); return NULL; } @@ -1141,7 +1130,7 @@ read_compiled_module(char *cpathname, FILE *fp) module object WITH INCREMENTED REFERENCE COUNT */ static PyObject * -load_compiled_module(char *name, char *cpathname, FILE *fp) +load_compiled_module(PyObject *name, PyObject *cpathname, FILE *fp) { long magic; PyCodeObject *co; @@ -1150,7 +1139,7 @@ load_compiled_module(char *name, char *cpathname, FILE *fp) magic = PyMarshal_ReadLongFromFile(fp); if (magic != pyc_magic) { PyErr_Format(PyExc_ImportError, - "Bad magic number in %.200s", cpathname); + "Bad magic number in %R", cpathname); return NULL; } (void) PyMarshal_ReadLongFromFile(fp); @@ -1158,10 +1147,10 @@ load_compiled_module(char *name, char *cpathname, FILE *fp) if (co == NULL) return NULL; if (Py_VerboseFlag) - PySys_WriteStderr("import %s # precompiled from %s\n", - name, cpathname); - m = PyImport_ExecCodeModuleWithPathnames( - name, (PyObject *)co, cpathname, cpathname); + PySys_FormatStderr("import %U # precompiled from %R\n", + name, cpathname); + m = PyImport_ExecCodeModuleObject(name, (PyObject *)co, + cpathname, cpathname); Py_DECREF(co); return m; @@ -1170,27 +1159,37 @@ load_compiled_module(char *name, char *cpathname, FILE *fp) /* Parse a source file and return the corresponding code object */ static PyCodeObject * -parse_source_module(const char *pathname, FILE *fp) +parse_source_module(PyObject *pathname, FILE *fp) { - PyCodeObject *co = NULL; + PyCodeObject *co; + PyObject *pathbytes; mod_ty mod; PyCompilerFlags flags; - PyArena *arena = PyArena_New(); - if (arena == NULL) + PyArena *arena; + + pathbytes = PyUnicode_EncodeFSDefault(pathname); + if (pathbytes == NULL) + return NULL; + + arena = PyArena_New(); + if (arena == NULL) { + Py_DECREF(pathbytes); return NULL; + } flags.cf_flags = 0; - mod = PyParser_ASTFromFile(fp, pathname, NULL, + mod = PyParser_ASTFromFile(fp, PyBytes_AS_STRING(pathbytes), NULL, Py_file_input, 0, 0, &flags, NULL, arena); - if (mod) { - co = PyAST_Compile(mod, pathname, NULL, arena); - } + if (mod != NULL) + co = PyAST_Compile(mod, PyBytes_AS_STRING(pathbytes), NULL, arena); + else + co = NULL; + Py_DECREF(pathbytes); PyArena_Free(arena); return co; } - /* Helper to open a bytecode file for writing in exclusive mode */ static FILE * @@ -1231,10 +1230,10 @@ open_exclusive(char *filename, mode_t mode) remove the file. */ static void -write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat) +write_compiled_module(PyCodeObject *co, PyObject *cpathname, + struct stat *srcstat) { FILE *fp; - char *dirpath; time_t mtime = srcstat->st_mtime; #ifdef MS_WINDOWS /* since Windows uses different permissions */ mode_t mode = srcstat->st_mode & ~S_IEXEC; @@ -1243,39 +1242,64 @@ write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat) mode_t dirmode = (srcstat->st_mode | S_IXUSR | S_IXGRP | S_IXOTH | S_IWUSR | S_IWGRP | S_IWOTH); + PyObject *dirbytes; #endif - int saved; + PyObject *cpathbytes, *dirname; + Py_UNICODE *dirsep; + int res, ok; /* Ensure that the __pycache__ directory exists. */ - dirpath = rightmost_sep(cpathname); - if (dirpath == NULL) { + dirsep = rightmost_sep(PyUnicode_AS_UNICODE(cpathname)); + if (dirsep == NULL) { if (Py_VerboseFlag) - PySys_WriteStderr( - "# no %s path found %s\n", - CACHEDIR, cpathname); + PySys_FormatStderr("# no %s path found %R\n", CACHEDIR, cpathname); + return; + } + dirname = PyUnicode_FromUnicode(PyUnicode_AS_UNICODE(cpathname), + dirsep - PyUnicode_AS_UNICODE(cpathname)); + if (dirname == NULL) { + PyErr_Clear(); return; } - saved = *dirpath; - *dirpath = '\0'; #ifdef MS_WINDOWS - if (_mkdir(cpathname) < 0 && errno != EEXIST) { + res = CreateDirectoryW(PyUnicode_AS_UNICODE(dirname), NULL); + ok = (res != 0); + if (!ok && GetLastError() == ERROR_ALREADY_EXISTS) + ok = 1; #else - if (mkdir(cpathname, dirmode) < 0 && errno != EEXIST) { + dirbytes = PyUnicode_EncodeFSDefault(dirname); + if (dirbytes == NULL) { + PyErr_Clear(); + return; + } + res = mkdir(PyBytes_AS_STRING(dirbytes), dirmode); + Py_DECREF(dirbytes); + if (0 <= res) + ok = 1; + else + ok = (errno == EEXIST); #endif - *dirpath = saved; + if (!ok) { if (Py_VerboseFlag) - PySys_WriteStderr( - "# cannot create cache dir %s\n", cpathname); + PySys_FormatStderr("# cannot create cache dir %R\n", dirname); + Py_DECREF(dirname); + return; + } + Py_DECREF(dirname); + + cpathbytes = PyUnicode_EncodeFSDefault(cpathname); + if (cpathbytes == NULL) { + PyErr_Clear(); return; } - *dirpath = saved; - fp = open_exclusive(cpathname, mode); + fp = open_exclusive(PyBytes_AS_STRING(cpathbytes), mode); if (fp == NULL) { if (Py_VerboseFlag) - PySys_WriteStderr( - "# can't create %s\n", cpathname); + PySys_FormatStderr( + "# can't create %R\n", cpathname); + Py_DECREF(cpathbytes); return; } PyMarshal_WriteLongToFile(pyc_magic, fp, Py_MARSHAL_VERSION); @@ -1284,12 +1308,18 @@ write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat) PyMarshal_WriteObjectToFile((PyObject *)co, fp, Py_MARSHAL_VERSION); if (fflush(fp) != 0 || ferror(fp)) { if (Py_VerboseFlag) - PySys_WriteStderr("# can't write %s\n", cpathname); + PySys_FormatStderr("# can't write %R\n", cpathname); /* Don't keep partial file */ fclose(fp); - (void) unlink(cpathname); +#ifdef MS_WINDOWS + (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname)); +#else + (void) unlink(PyBytes_AS_STRING(cpathbytes)); +#endif + Py_DECREF(cpathbytes); return; } + Py_DECREF(cpathbytes); /* Now write the true mtime */ fseek(fp, 4L, 0); assert(mtime < LONG_MAX); @@ -1297,7 +1327,7 @@ write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat) fflush(fp); fclose(fp); if (Py_VerboseFlag) - PySys_WriteStderr("# wrote %s\n", cpathname); + PySys_FormatStderr("# wrote %R\n", cpathname); } static void @@ -1324,26 +1354,18 @@ update_code_filenames(PyCodeObject *co, PyObject *oldname, PyObject *newname) } } -static int -update_compiled_module(PyCodeObject *co, char *pathname) +static void +update_compiled_module(PyCodeObject *co, PyObject *newname) { - PyObject *oldname, *newname; + PyObject *oldname; - newname = PyUnicode_DecodeFSDefault(pathname); - if (newname == NULL) - return -1; - - if (!PyUnicode_Compare(co->co_filename, newname)) { - Py_DECREF(newname); - return 0; - } + if (PyUnicode_Compare(co->co_filename, newname) == 0) + return; oldname = co->co_filename; Py_INCREF(oldname); update_code_filenames(co, oldname, newname); Py_DECREF(oldname); - Py_DECREF(newname); - return 1; } /* Load a source module from a given file and return its module @@ -1351,20 +1373,19 @@ update_compiled_module(PyCodeObject *co, char *pathname) byte-compiled file, use that instead. */ static PyObject * -load_source_module(char *name, char *pathname, FILE *fp) +load_source_module(PyObject *name, PyObject *pathname, FILE *fp) { struct stat st; FILE *fpc; - char buf[MAXPATHLEN+1]; - char *cpathname; + PyObject *cpathname = NULL, *cpathbytes = NULL; PyCodeObject *co; - PyObject *m; + PyObject *m = NULL; if (fstat(fileno(fp), &st) != 0) { PyErr_Format(PyExc_RuntimeError, - "unable to get file status from '%s'", + "unable to get file status from %R", pathname); - return NULL; + goto error; } #if SIZEOF_TIME_T > 4 /* Python's .pyc timestamp handling presumes that the timestamp fits @@ -1374,41 +1395,50 @@ load_source_module(char *name, char *pathname, FILE *fp) if (st.st_mtime >> 32) { PyErr_SetString(PyExc_OverflowError, "modification time overflows a 4 byte field"); - return NULL; + goto error; } #endif cpathname = make_compiled_pathname( - pathname, buf, (size_t)MAXPATHLEN + 1, !Py_OptimizeFlag); - if (cpathname != NULL && - (fpc = check_compiled_module(pathname, st.st_mtime, cpathname))) { + PyUnicode_AS_UNICODE(pathname), + !Py_OptimizeFlag); + + if (cpathname != NULL) + fpc = check_compiled_module(pathname, st.st_mtime, cpathname); + else + fpc = NULL; + + if (fpc) { co = read_compiled_module(cpathname, fpc); fclose(fpc); if (co == NULL) - return NULL; - if (update_compiled_module(co, pathname) < 0) - return NULL; + goto error; + update_compiled_module(co, pathname); if (Py_VerboseFlag) - PySys_WriteStderr("import %s # precompiled from %s\n", - name, cpathname); - pathname = cpathname; + PySys_FormatStderr("import %U # precompiled from %R\n", + name, cpathname); + m = PyImport_ExecCodeModuleObject(name, (PyObject *)co, + cpathname, cpathname); } else { co = parse_source_module(pathname, fp); if (co == NULL) - return NULL; + goto error; if (Py_VerboseFlag) - PySys_WriteStderr("import %s # from %s\n", + PySys_FormatStderr("import %U # from %R\n", name, pathname); - if (cpathname) { + if (cpathname != NULL) { PyObject *ro = PySys_GetObject("dont_write_bytecode"); if (ro == NULL || !PyObject_IsTrue(ro)) write_compiled_module(co, cpathname, &st); } + m = PyImport_ExecCodeModuleObject(name, (PyObject *)co, + pathname, cpathname); } - m = PyImport_ExecCodeModuleWithPathnames( - name, (PyObject *)co, pathname, cpathname); Py_DECREF(co); +error: + Py_XDECREF(cpathbytes); + Py_XDECREF(cpathname); return m; } @@ -2291,13 +2321,37 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) switch (type) { - case PY_SOURCE: - m = load_source_module(name, pathname, fp); + case PY_SOURCE: { + PyObject *nameobj, *pathobj; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + pathobj = PyUnicode_DecodeFSDefault(pathname); + if (pathobj == NULL) { + Py_DECREF(nameobj); + return NULL; + } + m = load_source_module(nameobj, pathobj, fp); + Py_DECREF(nameobj); + Py_DECREF(pathobj); break; + } - case PY_COMPILED: - m = load_compiled_module(name, pathname, fp); + case PY_COMPILED: { + PyObject *nameobj, *pathobj; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + pathobj = PyUnicode_DecodeFSDefault(pathname); + if (pathobj == NULL) { + Py_DECREF(nameobj); + return NULL; + } + m = load_compiled_module(nameobj, pathobj, fp); + Py_DECREF(nameobj); + Py_DECREF(pathobj); break; + } #ifdef HAVE_DYNAMIC_LOADING case C_EXTENSION: { @@ -3533,13 +3587,13 @@ imp_is_frozen(PyObject *self, PyObject *args) } static FILE * -get_file(char *pathname, PyObject *fob, char *mode) +get_file(PyObject *pathname, PyObject *fob, char *mode) { FILE *fp; if (mode[0] == 'U') mode = "r" PY_STDIOTEXTMODE; if (fob == NULL) { - fp = fopen(pathname, mode); + fp = _Py_fopen(pathname, mode); } else { int fd = PyObject_AsFileDescriptor(fob); @@ -3565,22 +3619,21 @@ error: static PyObject * imp_load_compiled(PyObject *self, PyObject *args) { - char *name; - PyObject *pathname; + PyObject *name, *pathname; PyObject *fob = NULL; PyObject *m; FILE *fp; - if (!PyArg_ParseTuple(args, "sO&|O:load_compiled", + if (!PyArg_ParseTuple(args, "UO&|O:load_compiled", &name, - PyUnicode_FSConverter, &pathname, + PyUnicode_FSDecoder, &pathname, &fob)) return NULL; - fp = get_file(PyBytes_AS_STRING(pathname), fob, "rb"); + fp = get_file(pathname, fob, "rb"); if (fp == NULL) { Py_DECREF(pathname); return NULL; } - m = load_compiled_module(name, PyBytes_AS_STRING(pathname), fp); + m = load_compiled_module(name, pathname, fp); fclose(fp); Py_DECREF(pathname); return m; @@ -3615,22 +3668,21 @@ imp_load_dynamic(PyObject *self, PyObject *args) static PyObject * imp_load_source(PyObject *self, PyObject *args) { - char *name; - PyObject *pathname; + PyObject *name, *pathname; PyObject *fob = NULL; PyObject *m; FILE *fp; - if (!PyArg_ParseTuple(args, "sO&|O:load_source", + if (!PyArg_ParseTuple(args, "UO&|O:load_source", &name, - PyUnicode_FSConverter, &pathname, + PyUnicode_FSDecoder, &pathname, &fob)) return NULL; - fp = get_file(PyBytes_AS_STRING(pathname), fob, "r"); + fp = get_file(pathname, fob, "r"); if (fp == NULL) { Py_DECREF(pathname); return NULL; } - m = load_source_module(name, PyBytes_AS_STRING(pathname), fp); + m = load_source_module(name, pathname, fp); Py_DECREF(pathname); fclose(fp); return m; @@ -3719,33 +3771,31 @@ imp_cache_from_source(PyObject *self, PyObject *args, PyObject *kws) { static char *kwlist[] = {"path", "debug_override", NULL}; - char buf[MAXPATHLEN+1]; - PyObject *pathbytes; - char *cpathname; + PyObject *pathname, *cpathname; PyObject *debug_override = NULL; int debug = !Py_OptimizeFlag; if (!PyArg_ParseTupleAndKeywords( args, kws, "O&|O", kwlist, - PyUnicode_FSConverter, &pathbytes, &debug_override)) + PyUnicode_FSDecoder, &pathname, &debug_override)) return NULL; if (debug_override != NULL && (debug = PyObject_IsTrue(debug_override)) < 0) { - Py_DECREF(pathbytes); + Py_DECREF(pathname); return NULL; } cpathname = make_compiled_pathname( - PyBytes_AS_STRING(pathbytes), - buf, MAXPATHLEN+1, debug); - Py_DECREF(pathbytes); + PyUnicode_AS_UNICODE(pathname), + debug); + Py_DECREF(pathname); if (cpathname == NULL) { PyErr_Format(PyExc_SystemError, "path buffer too short"); return NULL; } - return PyUnicode_DecodeFSDefault(buf); + return cpathname; } PyDoc_STRVAR(doc_cache_from_source, -- cgit v1.2.1 From 1bd9d3e0561cad30f87017b5ee00be6f7c92ced8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 13 Mar 2011 21:46:30 -0400 Subject: Issue #3080: load_module() expects name and path as Unicode --- Python/import.c | 136 +++++++++++++++++++++++--------------------------------- 1 file changed, 55 insertions(+), 81 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 0efffe9149..00a9eccaf8 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1490,7 +1490,7 @@ unchanged: } /* Forward */ -static PyObject *load_module(char *, FILE *, char *, int, PyObject *); +static PyObject *load_module(PyObject *, FILE *, PyObject *, int, PyObject *); static struct filedescr *find_module(char *, const char *, PyObject *, char *, size_t, FILE **, PyObject **); static struct _frozen * find_frozen(PyObject *); @@ -1501,7 +1501,7 @@ static struct _frozen * find_frozen(PyObject *); static PyObject * load_package(PyObject *name, PyObject *pathname) { - PyObject *m, *d; + PyObject *m, *d, *bufobj; PyObject *file = NULL, *path_list = NULL; int err; char buf[MAXPATHLEN+1]; @@ -1546,7 +1546,13 @@ load_package(PyObject *name, PyObject *pathname) else return NULL; } - m = load_module(namestr, fp, buf, fdp->type, NULL); + bufobj = PyUnicode_DecodeFSDefault(buf); + if (bufobj != NULL) { + m = load_module(name, fp, bufobj, fdp->type, NULL); + Py_DECREF(bufobj); + } + else + m = NULL; if (fp != NULL) fclose(fp); return m; @@ -2303,7 +2309,7 @@ load_builtin(PyObject *name, int type) its module object WITH INCREMENTED REFERENCE COUNT */ static PyObject * -load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) +load_module(PyObject *name, FILE *fp, PyObject *pathname, int type, PyObject *loader) { PyObject *m; @@ -2313,7 +2319,7 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) case PY_COMPILED: if (fp == NULL) { PyErr_Format(PyExc_ValueError, - "file object required for import (type code %d)", + "file object required for import (type code %d)", type); return NULL; } @@ -2321,81 +2327,28 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) switch (type) { - case PY_SOURCE: { - PyObject *nameobj, *pathobj; - nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) - return NULL; - pathobj = PyUnicode_DecodeFSDefault(pathname); - if (pathobj == NULL) { - Py_DECREF(nameobj); - return NULL; - } - m = load_source_module(nameobj, pathobj, fp); - Py_DECREF(nameobj); - Py_DECREF(pathobj); + case PY_SOURCE: + m = load_source_module(name, pathname, fp); break; - } - case PY_COMPILED: { - PyObject *nameobj, *pathobj; - nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) - return NULL; - pathobj = PyUnicode_DecodeFSDefault(pathname); - if (pathobj == NULL) { - Py_DECREF(nameobj); - return NULL; - } - m = load_compiled_module(nameobj, pathobj, fp); - Py_DECREF(nameobj); - Py_DECREF(pathobj); + case PY_COMPILED: + m = load_compiled_module(name, pathname, fp); break; - } #ifdef HAVE_DYNAMIC_LOADING - case C_EXTENSION: { - PyObject *nameobj, *pathobj; - nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) - return NULL; - pathobj = PyUnicode_DecodeFSDefault(pathname); - if (pathobj == NULL) { - Py_DECREF(nameobj); - return NULL; - } - m = _PyImport_LoadDynamicModule(nameobj, pathobj, fp); - Py_DECREF(nameobj); - Py_DECREF(pathobj); + case C_EXTENSION: + m = _PyImport_LoadDynamicModule(name, pathname, fp); break; - } #endif - case PKG_DIRECTORY: { - PyObject *nameobj, *pathobj; - nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) - return NULL; - pathobj = PyUnicode_DecodeFSDefault(pathname); - if (pathobj == NULL) { - Py_DECREF(nameobj); - return NULL; - } - m = load_package(nameobj, pathobj); - Py_DECREF(nameobj); - Py_DECREF(pathobj); + case PKG_DIRECTORY: + m = load_package(name, pathname); break; - } case C_BUILTIN: - case PY_FROZEN: { - PyObject *nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) - return NULL; - m = load_builtin(nameobj, type); - Py_DECREF(nameobj); + case PY_FROZEN: + m = load_builtin(name, type); break; - } case IMP_HOOK: { if (loader == NULL) { @@ -2403,13 +2356,13 @@ load_module(char *name, FILE *fp, char *pathname, int type, PyObject *loader) "import hook without loader"); return NULL; } - m = PyObject_CallMethod(loader, "load_module", "s", name); + m = PyObject_CallMethod(loader, "load_module", "O", name); break; } default: PyErr_Format(PyExc_ImportError, - "Don't know how to import %.200s (type code %d)", + "Don't know how to import %U (type code %d)", name, type); m = NULL; @@ -3144,7 +3097,7 @@ static PyObject * import_submodule(PyObject *mod, char *subname, char *fullname) { PyObject *modules = PyImport_GetModuleDict(); - PyObject *m = NULL; + PyObject *m = NULL, *fullnameobj, *bufobj; /* Require: if mod == None: subname == fullname @@ -3181,7 +3134,19 @@ import_submodule(PyObject *mod, char *subname, char *fullname) Py_INCREF(Py_None); return Py_None; } - m = load_module(fullname, fp, buf, fdp->type, loader); + fullnameobj = PyUnicode_FromString(fullname); + if (fullnameobj != NULL) { + bufobj = PyUnicode_DecodeFSDefault(buf); + if (bufobj != NULL) { + m = load_module(fullnameobj, fp, bufobj, fdp->type, loader); + Py_DECREF(bufobj); + } + else + m = NULL; + Py_DECREF(fullnameobj); + } + else + m = NULL; Py_XDECREF(loader); if (fp) fclose(fp); @@ -3209,7 +3174,7 @@ PyImport_ReloadModule(PyObject *m) char buf[MAXPATHLEN+1]; struct filedescr *fdp; FILE *fp; - PyObject *newm; + PyObject *newm, *nameobj, *bufobj; if (modules_reloading == NULL) { Py_FatalError("PyImport_ReloadModule: " @@ -3275,7 +3240,19 @@ PyImport_ReloadModule(PyObject *m) return NULL; } - newm = load_module(name, fp, buf, fdp->type, loader); + nameobj = PyUnicode_FromString(name); + if (nameobj != NULL) { + bufobj = PyUnicode_DecodeFSDefault(buf); + if (bufobj != NULL) { + newm = load_module(nameobj, fp, bufobj, fdp->type, loader); + Py_DECREF(bufobj); + } + else + newm = NULL; + Py_DECREF(nameobj); + } + else + newm = NULL; Py_XDECREF(loader); if (fp) @@ -3691,18 +3668,15 @@ imp_load_source(PyObject *self, PyObject *args) static PyObject * imp_load_module(PyObject *self, PyObject *args) { - char *name; - PyObject *fob; - PyObject *pathname; - PyObject * ret; + PyObject *name, *fob, *pathname, *ret; char *suffix; /* Unused */ char *mode; int type; FILE *fp; - if (!PyArg_ParseTuple(args, "sOO&(ssi):load_module", + if (!PyArg_ParseTuple(args, "UOO&(ssi):load_module", &name, &fob, - PyUnicode_FSConverter, &pathname, + PyUnicode_FSDecoder, &pathname, &suffix, &mode, &type)) return NULL; if (*mode) { @@ -3726,7 +3700,7 @@ imp_load_module(PyObject *self, PyObject *args) return NULL; } } - ret = load_module(name, fp, PyBytes_AS_STRING(pathname), type, NULL); + ret = load_module(name, fp, pathname, type, NULL); Py_DECREF(pathname); if (fp) fclose(fp); -- cgit v1.2.1 From d5668295d2303564770b7dbdfbb02e7d007292bc Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 13 Mar 2011 21:57:27 -0400 Subject: Issue #3080: PyImport_ImportModuleNoBlock() uses Unicode --- Python/import.c | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 00a9eccaf8..08237aff95 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2591,8 +2591,7 @@ PyImport_ImportModule(const char *name) PyObject * PyImport_ImportModuleNoBlock(const char *name) { - PyObject *result; - PyObject *modules; + PyObject *nameobj, *modules, *result; long me; /* Try to get the module from sys.modules[name] */ @@ -2600,14 +2599,16 @@ PyImport_ImportModuleNoBlock(const char *name) if (modules == NULL) return NULL; - result = PyDict_GetItemString(modules, name); + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + result = PyDict_GetItem(modules, nameobj); if (result != NULL) { + Py_DECREF(nameobj); Py_INCREF(result); return result; } - else { - PyErr_Clear(); - } + PyErr_Clear(); #ifdef WITH_THREAD /* check the import lock * me might be -1 but I ignore the error here, the lock function @@ -2615,18 +2616,20 @@ PyImport_ImportModuleNoBlock(const char *name) me = PyThread_get_thread_ident(); if (import_lock_thread == -1 || import_lock_thread == me) { /* no thread or me is holding the lock */ - return PyImport_ImportModule(name); + result = PyImport_Import(nameobj); } else { PyErr_Format(PyExc_ImportError, - "Failed to import %.200s because the import lock" + "Failed to import %U because the import lock" "is held by another thread.", - name); - return NULL; + nameobj); + result = NULL; } #else - return PyImport_ImportModule(name); + result = PyImport_Import(nameobj); #endif + Py_DECREF(nameobj); + return result; } /* Forward declarations for helper routines */ -- cgit v1.2.1 From 93d6b25bf514369f9325ae1dd062a7ba9f1c1097 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 15 Mar 2011 09:33:57 +0100 Subject: Issue #3080: Use Unicode for the "The Magnum Opus of dotted-name import" Use Unicode for module name and paths in the following functions: * PyImport_ImportModuleLevel() * add_submodule() * ensure_from_list() * get_parent() * import_module_level() * import_submodule() * load_next() * mark_miss() --- Python/import.c | 381 +++++++++++++++++++++++++++++++++----------------------- 1 file changed, 222 insertions(+), 159 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 08237aff95..95faeee5d8 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2633,28 +2633,37 @@ PyImport_ImportModuleNoBlock(const char *name) } /* Forward declarations for helper routines */ -static PyObject *get_parent(PyObject *globals, char *buf, - Py_ssize_t *p_buflen, int level); +static PyObject *get_parent(PyObject *globals, + PyObject **p_name, + int level); static PyObject *load_next(PyObject *mod, PyObject *altmod, - char **p_name, char *buf, Py_ssize_t *p_buflen); -static int mark_miss(char *name); + PyObject *inputname, PyObject **p_outputname, + Py_UNICODE *buf, Py_ssize_t *p_buflen, + Py_ssize_t bufsize); +static int mark_miss(PyObject *name); static int ensure_fromlist(PyObject *mod, PyObject *fromlist, - char *buf, Py_ssize_t buflen, int recursive); -static PyObject * import_submodule(PyObject *mod, char *name, char *fullname); + PyObject *buf, int recursive); +static PyObject * import_submodule(PyObject *mod, PyObject *name, + PyObject *fullname); /* The Magnum Opus of dotted-name import :-) */ static PyObject * -import_module_level(char *name, PyObject *globals, PyObject *locals, +import_module_level(PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) { - char buf[MAXPATHLEN+1]; - Py_ssize_t buflen = 0; - PyObject *parent, *head, *next, *tail; + Py_UNICODE buf[MAXPATHLEN+1]; + Py_ssize_t buflen; + Py_ssize_t bufsize = MAXPATHLEN+1; + PyObject *parent, *head, *next, *tail, *inputname, *outputname; + PyObject *parent_name, *ensure_name; + const Py_UNICODE *nameunicode; - if (strchr(name, '/') != NULL -#ifdef MS_WINDOWS - || strchr(name, '\\') != NULL + nameunicode = PyUnicode_AS_UNICODE(name); + + if (Py_UNICODE_strchr(nameunicode, SEP) != NULL +#ifdef ALTSEP + || Py_UNICODE_strchr(nameunicode, ALTSEP) != NULL #endif ) { PyErr_SetString(PyExc_ImportError, @@ -2662,25 +2671,45 @@ import_module_level(char *name, PyObject *globals, PyObject *locals, return NULL; } - parent = get_parent(globals, buf, &buflen, level); + parent = get_parent(globals, &parent_name, level); if (parent == NULL) return NULL; - head = load_next(parent, level < 0 ? Py_None : parent, &name, buf, - &buflen); + buflen = PyUnicode_GET_SIZE(parent_name); + if (buflen+1 > bufsize) { + Py_DECREF(parent_name); + PyErr_SetString(PyExc_ValueError, + "Module name too long"); + return NULL; + } + Py_UNICODE_strcpy(buf, PyUnicode_AS_UNICODE(parent_name)); + Py_DECREF(parent_name); + + head = load_next(parent, level < 0 ? Py_None : parent, name, &outputname, + buf, &buflen, bufsize); if (head == NULL) return NULL; tail = head; Py_INCREF(tail); - while (name) { - next = load_next(tail, tail, &name, buf, &buflen); - Py_DECREF(tail); - if (next == NULL) { - Py_DECREF(head); - return NULL; + + if (outputname != NULL) { + while (1) { + inputname = outputname; + next = load_next(tail, tail, inputname, &outputname, + buf, &buflen, bufsize); + Py_DECREF(tail); + Py_DECREF(inputname); + if (next == NULL) { + Py_DECREF(head); + return NULL; + } + tail = next; + + if (outputname == NULL) { + break; + } } - tail = next; } if (tail == Py_None) { /* If tail is Py_None, both get_parent and load_next found @@ -2688,8 +2717,7 @@ import_module_level(char *name, PyObject *globals, PyObject *locals, doctored faulty bytecode */ Py_DECREF(tail); Py_DECREF(head); - PyErr_SetString(PyExc_ValueError, - "Empty module name"); + PyErr_SetString(PyExc_ValueError, "Empty module name"); return NULL; } @@ -2704,21 +2732,33 @@ import_module_level(char *name, PyObject *globals, PyObject *locals, } Py_DECREF(head); - if (!ensure_fromlist(tail, fromlist, buf, buflen, 0)) { + + ensure_name = PyUnicode_FromUnicode(buf, Py_UNICODE_strlen(buf)); + if (ensure_name == NULL) { + Py_DECREF(tail); + return NULL; + } + if (!ensure_fromlist(tail, fromlist, ensure_name, 0)) { Py_DECREF(tail); + Py_DECREF(ensure_name); return NULL; } + Py_DECREF(ensure_name); return tail; } PyObject * PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals, - PyObject *fromlist, int level) + PyObject *fromlist, int level) { - PyObject *result; + PyObject *nameobj, *result; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; _PyImport_AcquireLock(); - result = import_module_level(name, globals, locals, fromlist, level); + result = import_module_level(nameobj, globals, locals, fromlist, level); + Py_DECREF(nameobj); if (_PyImport_ReleaseLock() < 0) { Py_XDECREF(result); PyErr_SetString(PyExc_RuntimeError, @@ -2733,15 +2773,20 @@ PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals, sys.modules entry for foo.bar. If globals is from a package's __init__.py, the package's entry in sys.modules is returned, as a borrowed reference. - The *name* of the returned package is returned in buf, with the length of - the name in *p_buflen. + The name of the returned package is returned in *p_name. If globals doesn't come from a package or a module in a package, or a corresponding entry is not found in sys.modules, Py_None is returned. */ static PyObject * -get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) +get_parent(PyObject *globals, + PyObject **p_name, + int level) { + Py_UNICODE name[MAXPATHLEN+1]; + const Py_ssize_t bufsize = MAXPATHLEN+1; + PyObject *nameobj; + static PyObject *namestr = NULL; static PyObject *pathstr = NULL; static PyObject *pkgstr = NULL; @@ -2749,7 +2794,7 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) int orig_level = level; if (globals == NULL || !PyDict_Check(globals) || !level) - return Py_None; + goto return_none; if (namestr == NULL) { namestr = PyUnicode_InternFromString("__name__"); @@ -2767,55 +2812,46 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) return NULL; } - *buf = '\0'; - *p_buflen = 0; pkgname = PyDict_GetItem(globals, pkgstr); if ((pkgname != NULL) && (pkgname != Py_None)) { /* __package__ is set, so use it */ - char *pkgname_str; - Py_ssize_t len; - if (!PyUnicode_Check(pkgname)) { PyErr_SetString(PyExc_ValueError, "__package__ set to non-string"); return NULL; } - pkgname_str = _PyUnicode_AsStringAndSize(pkgname, &len); - if (len == 0) { + if (PyUnicode_GET_SIZE(pkgname) == 0) { if (level > 0) { PyErr_SetString(PyExc_ValueError, "Attempted relative import in non-package"); return NULL; } - return Py_None; + goto return_none; } - if (len > MAXPATHLEN) { + if (PyUnicode_GET_SIZE(pkgname)+1 > bufsize) { PyErr_SetString(PyExc_ValueError, "Package name too long"); return NULL; } - strcpy(buf, pkgname_str); + Py_UNICODE_strcpy(name, PyUnicode_AS_UNICODE(pkgname)); } else { /* __package__ not set, so figure it out and set it */ modname = PyDict_GetItem(globals, namestr); if (modname == NULL || !PyUnicode_Check(modname)) - return Py_None; + goto return_none; modpath = PyDict_GetItem(globals, pathstr); if (modpath != NULL) { /* __path__ is set, so modname is already the package name */ - char *modname_str; - Py_ssize_t len; int error; - modname_str = _PyUnicode_AsStringAndSize(modname, &len); - if (len > MAXPATHLEN) { + if (PyUnicode_GET_SIZE(modname)+1 > bufsize) { PyErr_SetString(PyExc_ValueError, "Module name too long"); return NULL; } - strcpy(buf, modname_str); + Py_UNICODE_strcpy(name, PyUnicode_AS_UNICODE(modname)); error = PyDict_SetItem(globals, pkgstr, modname); if (error) { PyErr_SetString(PyExc_ValueError, @@ -2824,9 +2860,9 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) } } else { /* Normal module, so work out the package name if any */ - char *start = _PyUnicode_AsString(modname); - char *lastdot = strrchr(start, '.'); - size_t len; + Py_UNICODE *start = PyUnicode_AS_UNICODE(modname); + Py_UNICODE *lastdot = Py_UNICODE_strrchr(start, '.'); + Py_ssize_t len; int error; if (lastdot == NULL && level > 0) { PyErr_SetString(PyExc_ValueError, @@ -2840,17 +2876,17 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) "Could not set __package__"); return NULL; } - return Py_None; + goto return_none; } len = lastdot - start; - if (len >= MAXPATHLEN) { + if (len+1 > bufsize) { PyErr_SetString(PyExc_ValueError, "Module name too long"); return NULL; } - strncpy(buf, start, len); - buf[len] = '\0'; - pkgname = PyUnicode_FromString(buf); + Py_UNICODE_strncpy(name, start, len); + name[len] = '\0'; + pkgname = PyUnicode_FromUnicode(name, len); if (pkgname == NULL) { return NULL; } @@ -2864,7 +2900,7 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) } } while (--level > 0) { - char *dot = strrchr(buf, '.'); + Py_UNICODE *dot = Py_UNICODE_strrchr(name, '.'); if (dot == NULL) { PyErr_SetString(PyExc_ValueError, "Attempted relative import beyond " @@ -2873,137 +2909,181 @@ get_parent(PyObject *globals, char *buf, Py_ssize_t *p_buflen, int level) } *dot = '\0'; } - *p_buflen = strlen(buf); + + nameobj = PyUnicode_FromUnicode(name, Py_UNICODE_strlen(name)); + if (nameobj == NULL) + return NULL; modules = PyImport_GetModuleDict(); - parent = PyDict_GetItemString(modules, buf); + parent = PyDict_GetItem(modules, nameobj); if (parent == NULL) { - if (orig_level < 1) { - PyObject *err_msg = PyBytes_FromFormat( - "Parent module '%.200s' not found " - "while handling absolute import", buf); - if (err_msg == NULL) { - return NULL; - } - if (!PyErr_WarnEx(PyExc_RuntimeWarning, - PyBytes_AsString(err_msg), 1)) { - *buf = '\0'; - *p_buflen = 0; - parent = Py_None; - } - Py_DECREF(err_msg); - } else { + int err; + + if (orig_level >= 1) { PyErr_Format(PyExc_SystemError, - "Parent module '%.200s' not loaded, " - "cannot perform relative import", buf); + "Parent module %R not loaded, " + "cannot perform relative import", nameobj); + Py_DECREF(nameobj); + return NULL; } + + err = PyErr_WarnFormat( + PyExc_RuntimeWarning, 1, + "Parent module %R not found while handling absolute import", + nameobj); + Py_DECREF(nameobj); + if (err) + return NULL; + + goto return_none; } + *p_name = nameobj; return parent; /* We expect, but can't guarantee, if parent != None, that: - - parent.__name__ == buf + - parent.__name__ == name - parent.__dict__ is globals If this is violated... Who cares? */ + +return_none: + nameobj = PyUnicode_FromUnicode(NULL, 0); + if (nameobj == NULL) + return NULL; + *p_name = nameobj; + return Py_None; } /* altmod is either None or same as mod */ static PyObject * -load_next(PyObject *mod, PyObject *altmod, char **p_name, char *buf, - Py_ssize_t *p_buflen) +load_next(PyObject *mod, PyObject *altmod, + PyObject *inputname, PyObject **p_outputname, + Py_UNICODE *buf, Py_ssize_t *p_buflen, Py_ssize_t bufsize) { - char *name = *p_name; - char *dot = strchr(name, '.'); - size_t len; - char *p; - PyObject *result; + const Py_UNICODE *dot; + Py_ssize_t len; + Py_UNICODE *p; + PyObject *fullname, *name, *result, *mark_name; + const Py_UNICODE *nameuni; + + *p_outputname = NULL; - if (strlen(name) == 0) { + if (PyUnicode_GET_SIZE(inputname) == 0) { /* completely empty module name should only happen in 'from . import' (or '__import__("")')*/ Py_INCREF(mod); - *p_name = NULL; return mod; } - if (dot == NULL) { - *p_name = NULL; - len = strlen(name); - } - else { - *p_name = dot+1; - len = dot-name; + nameuni = PyUnicode_AS_UNICODE(inputname); + if (nameuni == NULL) + return NULL; + + dot = Py_UNICODE_strchr(nameuni, '.'); + if (dot != NULL) { + len = dot - nameuni; + if (len == 0) { + PyErr_SetString(PyExc_ValueError, + "Empty module name"); + return NULL; + } } - if (len == 0) { + else + len = PyUnicode_GET_SIZE(inputname); + + if (*p_buflen+len+1 >= bufsize) { PyErr_SetString(PyExc_ValueError, - "Empty module name"); + "Module name too long"); return NULL; } p = buf + *p_buflen; - if (p != buf) + if (p != buf) { *p++ = '.'; - if (p+len-buf >= MAXPATHLEN) { - PyErr_SetString(PyExc_ValueError, - "Module name too long"); - return NULL; + *p_buflen += 1; } - strncpy(p, name, len); + Py_UNICODE_strncpy(p, nameuni, len); p[len] = '\0'; - *p_buflen = p+len-buf; + *p_buflen += len; - result = import_submodule(mod, p, buf); + fullname = PyUnicode_FromUnicode(buf, *p_buflen); + if (fullname == NULL) + return NULL; + name = PyUnicode_FromUnicode(p, len); + if (name == NULL) { + Py_DECREF(fullname); + return NULL; + } + result = import_submodule(mod, name, fullname); + Py_DECREF(fullname); if (result == Py_None && altmod != mod) { Py_DECREF(result); /* Here, altmod must be None and mod must not be None */ - result = import_submodule(altmod, p, p); + result = import_submodule(altmod, name, name); + Py_DECREF(name); if (result != NULL && result != Py_None) { - if (mark_miss(buf) != 0) { + mark_name = PyUnicode_FromUnicode(buf, *p_buflen); + if (mark_name == NULL) { + Py_DECREF(result); + return NULL; + } + if (mark_miss(mark_name) != 0) { Py_DECREF(result); + Py_DECREF(mark_name); return NULL; } - strncpy(buf, name, len); + Py_DECREF(mark_name); + Py_UNICODE_strncpy(buf, nameuni, len); buf[len] = '\0'; *p_buflen = len; } } + else + Py_DECREF(name); if (result == NULL) return NULL; if (result == Py_None) { Py_DECREF(result); PyErr_Format(PyExc_ImportError, - "No module named %.200s", name); + "No module named %R", inputname); return NULL; } + if (dot != NULL) { + *p_outputname = PyUnicode_FromUnicode(dot+1, Py_UNICODE_strlen(dot+1)); + if (*p_outputname == NULL) { + Py_DECREF(result); + return NULL; + } + } + return result; } static int -mark_miss(char *name) +mark_miss(PyObject *name) { PyObject *modules = PyImport_GetModuleDict(); - return PyDict_SetItemString(modules, name, Py_None); + return PyDict_SetItem(modules, name, Py_None); } static int -ensure_fromlist(PyObject *mod, PyObject *fromlist, char *buf, Py_ssize_t buflen, +ensure_fromlist(PyObject *mod, PyObject *fromlist, PyObject *name, int recursive) { int i; + PyObject *fullname; + Py_ssize_t fromlist_len; if (!PyObject_HasAttrString(mod, "__path__")) return 1; - for (i = 0; ; i++) { + fromlist_len = PySequence_Size(fromlist); + + for (i = 0; i < fromlist_len; i++) { PyObject *item = PySequence_GetItem(fromlist, i); int hasit; - if (item == NULL) { - if (PyErr_ExceptionMatches(PyExc_IndexError)) { - PyErr_Clear(); - return 1; - } + if (item == NULL) return 0; - } if (!PyUnicode_Check(item)) { PyErr_SetString(PyExc_TypeError, "Item in ``from list'' not a string"); @@ -3020,7 +3100,7 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, char *buf, Py_ssize_t buflen, if (all == NULL) PyErr_Clear(); else { - int ret = ensure_fromlist(mod, all, buf, buflen, 1); + int ret = ensure_fromlist(mod, all, name, 1); Py_DECREF(all); if (!ret) return 0; @@ -3029,27 +3109,14 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, char *buf, Py_ssize_t buflen, } hasit = PyObject_HasAttr(mod, item); if (!hasit) { - PyObject *item8; - char *subname; PyObject *submod; - char *p; - item8 = PyUnicode_EncodeFSDefault(item); - if (!item8) { - PyErr_SetString(PyExc_ValueError, "Cannot encode path item"); - return 0; + fullname = PyUnicode_FromFormat("%U.%U", name, item); + if (fullname != NULL) { + submod = import_submodule(mod, item, fullname); + Py_DECREF(fullname); } - subname = PyBytes_AS_STRING(item8); - if (buflen + strlen(subname) >= MAXPATHLEN) { - PyErr_SetString(PyExc_ValueError, - "Module name too long"); - Py_DECREF(item); - return 0; - } - p = buf + buflen; - *p++ = '.'; - strcpy(p, subname); - submod = import_submodule(mod, subname, buf); - Py_DECREF(item8); + else + submod = NULL; Py_XDECREF(submod); if (submod == NULL) { Py_DECREF(item); @@ -3059,12 +3126,12 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, char *buf, Py_ssize_t buflen, Py_DECREF(item); } - /* NOTREACHED */ + return 1; } static int -add_submodule(PyObject *mod, PyObject *submod, char *fullname, char *subname, - PyObject *modules) +add_submodule(PyObject *mod, PyObject *submod, PyObject *fullname, + PyObject *subname, PyObject *modules) { if (mod == Py_None) return 1; @@ -3075,7 +3142,7 @@ add_submodule(PyObject *mod, PyObject *submod, char *fullname, char *subname, load failed with a SyntaxError -- then there's no trace in sys.modules. In that case, of course, do nothing extra.) */ if (submod == NULL) { - submod = PyDict_GetItemString(modules, fullname); + submod = PyDict_GetItem(modules, fullname); if (submod == NULL) return 1; } @@ -3086,28 +3153,28 @@ add_submodule(PyObject *mod, PyObject *submod, char *fullname, char *subname, PyObject *dict = PyModule_GetDict(mod); if (!dict) return 0; - if (PyDict_SetItemString(dict, subname, submod) < 0) + if (PyDict_SetItem(dict, subname, submod) < 0) return 0; } else { - if (PyObject_SetAttrString(mod, subname, submod) < 0) + if (PyObject_SetAttr(mod, subname, submod) < 0) return 0; } return 1; } static PyObject * -import_submodule(PyObject *mod, char *subname, char *fullname) +import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) { PyObject *modules = PyImport_GetModuleDict(); - PyObject *m = NULL, *fullnameobj, *bufobj; + PyObject *m = NULL, *bufobj; /* Require: if mod == None: subname == fullname else: mod.__name__ + "." + subname == fullname */ - if ((m = PyDict_GetItemString(modules, fullname)) != NULL) { + if ((m = PyDict_GetItem(modules, fullname)) != NULL) { Py_INCREF(m); } else { @@ -3127,7 +3194,9 @@ import_submodule(PyObject *mod, char *subname, char *fullname) } } - fdp = find_module(fullname, subname, path, buf, MAXPATHLEN+1, + fdp = find_module(_PyUnicode_AsString(fullname), + _PyUnicode_AsString(subname), + path, buf, MAXPATHLEN+1, &fp, &loader); Py_XDECREF(path); if (fdp == NULL) { @@ -3137,23 +3206,17 @@ import_submodule(PyObject *mod, char *subname, char *fullname) Py_INCREF(Py_None); return Py_None; } - fullnameobj = PyUnicode_FromString(fullname); - if (fullnameobj != NULL) { - bufobj = PyUnicode_DecodeFSDefault(buf); - if (bufobj != NULL) { - m = load_module(fullnameobj, fp, bufobj, fdp->type, loader); - Py_DECREF(bufobj); - } - else - m = NULL; - Py_DECREF(fullnameobj); + bufobj = PyUnicode_DecodeFSDefault(buf); + if (bufobj != NULL) { + m = load_module(fullname, fp, bufobj, fdp->type, loader); + Py_DECREF(bufobj); } else m = NULL; Py_XDECREF(loader); if (fp) fclose(fp); - if (!add_submodule(mod, m, fullname, subname, modules)) { + if (m != NULL && !add_submodule(mod, m, fullname, subname, modules)) { Py_XDECREF(m); m = NULL; } -- cgit v1.2.1 From 74c6f0ba2d358a8d6717772150740b5f49142711 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 13 Mar 2011 22:38:06 -0400 Subject: Issue #3080: Use %R to format module name in error messages %R format instead of %U --- Python/import.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 95faeee5d8..90ba4b5e22 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2620,7 +2620,7 @@ PyImport_ImportModuleNoBlock(const char *name) } else { PyErr_Format(PyExc_ImportError, - "Failed to import %U because the import lock" + "Failed to import %R because the import lock" "is held by another thread.", nameobj); result = NULL; @@ -3285,7 +3285,7 @@ PyImport_ReloadModule(PyObject *m) parent = PyDict_GetItem(modules, parentname); if (parent == NULL) { PyErr_Format(PyExc_ImportError, - "reload(): parent %U not in sys.modules", + "reload(): parent %R not in sys.modules", parentname); Py_DECREF(parentname); imp_modules_reloading_clear(); -- cgit v1.2.1 From ec2c770704d651436f9217bf78ef289f63162209 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 13 Mar 2011 22:38:38 -0400 Subject: Issue #3080: Reindent and simplify import_submodule() --- Python/import.c | 81 +++++++++++++++++++++++++++------------------------------ 1 file changed, 39 insertions(+), 42 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 90ba4b5e22..cce2de1b44 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2779,9 +2779,7 @@ PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals, corresponding entry is not found in sys.modules, Py_None is returned. */ static PyObject * -get_parent(PyObject *globals, - PyObject **p_name, - int level) +get_parent(PyObject *globals, PyObject **p_name, int level) { Py_UNICODE name[MAXPATHLEN+1]; const Py_ssize_t bufsize = MAXPATHLEN+1; @@ -3167,7 +3165,10 @@ static PyObject * import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) { PyObject *modules = PyImport_GetModuleDict(); - PyObject *m = NULL, *bufobj; + PyObject *m = NULL, *bufobj, *path, *loader; + char buf[MAXPATHLEN+1]; + struct filedescr *fdp; + FILE *fp; /* Require: if mod == None: subname == fullname @@ -3176,52 +3177,48 @@ import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) if ((m = PyDict_GetItem(modules, fullname)) != NULL) { Py_INCREF(m); + return m; } - else { - PyObject *path, *loader; - char buf[MAXPATHLEN+1]; - struct filedescr *fdp; - FILE *fp; - - if (mod == Py_None) - path = NULL; - else { - path = PyObject_GetAttrString(mod, "__path__"); - if (path == NULL) { - PyErr_Clear(); - Py_INCREF(Py_None); - return Py_None; - } - } - fdp = find_module(_PyUnicode_AsString(fullname), - _PyUnicode_AsString(subname), - path, buf, MAXPATHLEN+1, - &fp, &loader); - Py_XDECREF(path); - if (fdp == NULL) { - if (!PyErr_ExceptionMatches(PyExc_ImportError)) - return NULL; + if (mod == Py_None) + path = NULL; + else { + path = PyObject_GetAttrString(mod, "__path__"); + if (path == NULL) { PyErr_Clear(); Py_INCREF(Py_None); return Py_None; } - bufobj = PyUnicode_DecodeFSDefault(buf); - if (bufobj != NULL) { - m = load_module(fullname, fp, bufobj, fdp->type, loader); - Py_DECREF(bufobj); - } - else - m = NULL; - Py_XDECREF(loader); - if (fp) - fclose(fp); - if (m != NULL && !add_submodule(mod, m, fullname, subname, modules)) { - Py_XDECREF(m); - m = NULL; - } } + fdp = find_module(_PyUnicode_AsString(fullname), + _PyUnicode_AsString(subname), + path, buf, MAXPATHLEN+1, + &fp, &loader); + Py_XDECREF(path); + if (fdp == NULL) { + if (!PyErr_ExceptionMatches(PyExc_ImportError)) + return NULL; + PyErr_Clear(); + Py_INCREF(Py_None); + return Py_None; + } + bufobj = PyUnicode_DecodeFSDefault(buf); + if (bufobj != NULL) { + m = load_module(fullname, fp, bufobj, fdp->type, loader); + Py_DECREF(bufobj); + } + else + m = NULL; + Py_XDECREF(loader); + if (fp) + fclose(fp); + if (m == NULL) + return NULL; + if (!add_submodule(mod, m, fullname, subname, modules)) { + Py_XDECREF(m); + return NULL; + } return m; } -- cgit v1.2.1 From 7b12e183cf51fb59b0d62f26e37f91ec881a0dd9 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 13 Mar 2011 23:11:02 -0400 Subject: Issue #3080: Drop OS/2 support for the import machinery Sorry Andrew I MacIntyre! --- Python/import.c | 52 ---------------------------------------------------- 1 file changed, 52 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index cce2de1b44..e6d7c060fe 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1804,11 +1804,6 @@ find_module_path_list(char *fullname, const char *name, struct filedescr *fdp = NULL; char *filemode; FILE *fp = NULL; -#if defined(PYOS_OS2) - size_t saved_len; - size_t saved_namelen; - char *saved_buf = NULL; -#endif npath = PyList_Size(search_path_list); namelen = strlen(name); @@ -1832,40 +1827,7 @@ find_module_path_list(char *fullname, const char *name, return fdp; len = strlen(buf); -#if defined(PYOS_OS2) - /* take a snapshot of the module spec for restoration - * after the 8 character DLL hackery - */ - saved_buf = strdup(buf); - saved_len = len; - saved_namelen = namelen; -#endif /* PYOS_OS2 */ for (fdp = _PyImport_Filetab; fdp->suffix != NULL; fdp++) { -#if defined(PYOS_OS2) && defined(HAVE_DYNAMIC_LOADING) - /* OS/2 limits DLLs to 8 character names (w/o - extension) - * so if the name is longer than that and its a - * dynamically loaded module we're going to try, - * truncate the name before trying - */ - if (strlen(name) > 8) { - /* is this an attempt to load a C extension? */ - const struct filedescr *scan; - scan = _PyImport_DynLoadFiletab; - while (scan->suffix != NULL) { - if (!strcmp(scan->suffix, fdp->suffix)) - break; - else - scan++; - } - if (scan->suffix != NULL) { - /* yes, so truncate the name */ - namelen = 8; - len -= strlen(name) - namelen; - buf[len] = '\0'; - } - } -#endif /* PYOS_OS2 */ strcpy(buf+len, fdp->suffix); if (Py_VerboseFlag > 1) PySys_WriteStderr("# trying %s\n", buf); @@ -1881,21 +1843,7 @@ find_module_path_list(char *fullname, const char *name, fp = NULL; } } -#if defined(PYOS_OS2) - /* restore the saved snapshot */ - strcpy(buf, saved_buf); - len = saved_len; - namelen = saved_namelen; -#endif - } -#if defined(PYOS_OS2) - /* don't need/want the module name snapshot anymore */ - if (saved_buf) - { - free(saved_buf); - saved_buf = NULL; } -#endif if (fp != NULL) break; } -- cgit v1.2.1 From e7fd2eaecbfdc259fc9ae199608d33d721d88f57 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 09:21:33 -0400 Subject: Issue #3080: find_module() expects module fullname and subname as Unicode And PyImport_ReloadModule() uses Unicode for the module name. --- Python/import.c | 170 ++++++++++++++++++++++++++++---------------------------- 1 file changed, 84 insertions(+), 86 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index e6d7c060fe..a3f263146c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1491,7 +1491,7 @@ unchanged: /* Forward */ static PyObject *load_module(PyObject *, FILE *, PyObject *, int, PyObject *); -static struct filedescr *find_module(char *, const char *, PyObject *, +static struct filedescr *find_module(PyObject *, PyObject *, PyObject *, char *, size_t, FILE **, PyObject **); static struct _frozen * find_frozen(PyObject *); @@ -1507,7 +1507,13 @@ load_package(PyObject *name, PyObject *pathname) char buf[MAXPATHLEN+1]; FILE *fp = NULL; struct filedescr *fdp; - char *namestr; + static PyObject *initstr = NULL; + + if (initstr == NULL) { + initstr = PyUnicode_InternFromString("__init__"); + if (initstr == NULL) + return NULL; + } m = PyImport_AddModuleObject(name); if (m == NULL) @@ -1532,10 +1538,7 @@ load_package(PyObject *name, PyObject *pathname) Py_DECREF(path_list); return NULL; } - namestr = _PyUnicode_AsString(name); - if (namestr == NULL) - goto error; - fdp = find_module(namestr, "__init__", path_list, buf, sizeof(buf), &fp, NULL); + fdp = find_module(name, initstr, path_list, buf, sizeof(buf), &fp, NULL); Py_DECREF(path_list); if (fdp == NULL) { if (PyErr_ExceptionMatches(PyExc_ImportError)) { @@ -1680,7 +1683,7 @@ static struct filedescr importhookdescr = {"", "", IMP_HOOK}; and *buf is the path */ static int -find_module_path(char *fullname, const char *name, PyObject *path, +find_module_path(PyObject *fullname, PyObject *name, PyObject *path, PyObject *path_hooks, PyObject *path_importer_cache, char *buf, size_t buflen, PyObject **p_loader, struct filedescr **p_fd) @@ -1691,6 +1694,7 @@ find_module_path(char *fullname, const char *name, PyObject *path, size_t namelen; struct stat statbuf; static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; + char *namestr; if (PyUnicode_Check(path)) { path_bytes = PyUnicode_EncodeFSDefault(path); @@ -1704,7 +1708,8 @@ find_module_path(char *fullname, const char *name, PyObject *path, else return 0; - namelen = strlen(name); + namestr = _PyUnicode_AsString(name); + namelen = strlen(namestr); base = PyBytes_AS_STRING(path_bytes); len = PyBytes_GET_SIZE(path_bytes); if (len + 2 + namelen + MAXSUFFIXSIZE >= buflen) { @@ -1731,8 +1736,7 @@ find_module_path(char *fullname, const char *name, PyObject *path, if (importer != Py_None) { PyObject *loader; loader = PyObject_CallMethod(importer, - "find_module", - "s", fullname); + "find_module", "O", fullname); if (loader == NULL) return -1; /* error */ if (loader != Py_None) { @@ -1753,7 +1757,7 @@ find_module_path(char *fullname, const char *name, PyObject *path, #endif ) buf[len++] = SEP; - strcpy(buf+len, name); + strcpy(buf+len, namestr); len += namelen; /* Check for package import (buf holds a directory name, @@ -1761,7 +1765,7 @@ find_module_path(char *fullname, const char *name, PyObject *path, #ifdef HAVE_STAT if (stat(buf, &statbuf) == 0 && /* it exists */ S_ISDIR(statbuf.st_mode) && /* it's a directory */ - case_ok(buf, len, namelen, name)) { /* case matches */ + case_ok(buf, len, namelen, namestr)) { /* case matches */ if (find_init_module(buf)) { /* and has __init__.py */ *p_fd = &fd_package; return 2; @@ -1784,7 +1788,7 @@ find_module_path(char *fullname, const char *name, PyObject *path, } /* Find a module in search_path_list. For each path, try - find_module_filename() or try each _PyImport_Filetab suffix. + find_module_path() or try each _PyImport_Filetab suffix. If the module is found, return a file descriptor, write the path in *p_filename, write the pointer to the file object into *p_fp, and (if @@ -1793,7 +1797,7 @@ find_module_path(char *fullname, const char *name, PyObject *path, Otherwise, raise an exception and return NULL. */ static struct filedescr* -find_module_path_list(char *fullname, const char *name, +find_module_path_list(PyObject *fullname, PyObject *name, PyObject *search_path_list, PyObject *path_hooks, PyObject *path_importer_cache, char *buf, size_t buflen, @@ -1804,9 +1808,11 @@ find_module_path_list(char *fullname, const char *name, struct filedescr *fdp = NULL; char *filemode; FILE *fp = NULL; + char *namestr; npath = PyList_Size(search_path_list); - namelen = strlen(name); + namestr = _PyUnicode_AsString(name); + namelen = strlen(namestr); for (i = 0; i < npath; i++) { PyObject *path; int ok; @@ -1836,7 +1842,7 @@ find_module_path_list(char *fullname, const char *name, filemode = "r" PY_STDIOTEXTMODE; fp = fopen(buf, filemode); if (fp != NULL) { - if (case_ok(buf, len, namelen, name)) + if (case_ok(buf, len, namelen, namestr)) break; else { /* continue search */ fclose(fp); @@ -1849,7 +1855,7 @@ find_module_path_list(char *fullname, const char *name, } if (fp == NULL) { PyErr_Format(PyExc_ImportError, - "No module named %.200s", name); + "No module named %U", name); return NULL; } *p_fp = fp; @@ -1882,21 +1888,20 @@ find_module_path_list(char *fullname, const char *name, set) are set to NULL. Eg. *buf is an empty string for a builtin package. */ static struct filedescr * -find_module(char *fullname, const char *name, PyObject *search_path_list, +find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, char *buf, size_t buflen, FILE **p_fp, PyObject **p_loader) { Py_ssize_t i, npath; static struct filedescr fd_frozen = {"", "", PY_FROZEN}; static struct filedescr fd_builtin = {"", "", C_BUILTIN}; PyObject *path_hooks, *path_importer_cache; - PyObject *fullname_obj, *nameobj; *buf = '\0'; *p_fp = NULL; if (p_loader != NULL) *p_loader = NULL; - if (strlen(name) > MAXPATHLEN) { + if (PyUnicode_GET_SIZE(name) > MAXPATHLEN) { PyErr_SetString(PyExc_OverflowError, "module name is too long"); return NULL; @@ -1919,7 +1924,7 @@ find_module(char *fullname, const char *name, PyObject *search_path_list, PyObject *loader; PyObject *hook = PyList_GetItem(meta_path, i); loader = PyObject_CallMethod(hook, "find_module", - "sO", fullname, + "OO", fullname, search_path_list != NULL ? search_path_list : Py_None); if (loader == NULL) { @@ -1937,16 +1942,9 @@ find_module(char *fullname, const char *name, PyObject *search_path_list, Py_DECREF(meta_path); } - if (fullname != NULL) { - fullname_obj = PyUnicode_FromString(fullname); - if (fullname == NULL) - return NULL; - if (find_frozen(fullname_obj) != NULL) { - Py_DECREF(fullname_obj); - strcpy(buf, fullname); - return &fd_frozen; - } - Py_DECREF(fullname_obj); + if (find_frozen(fullname) != NULL) { + strcpy(buf, _PyUnicode_AsString(fullname)); + return &fd_frozen; } if (search_path_list == NULL) { @@ -1955,18 +1953,13 @@ find_module(char *fullname, const char *name, PyObject *search_path_list, struct filedescr *fdp; PyObject *filename, *filename_bytes; #endif - nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) - return NULL; - if (is_builtin(nameobj)) { - Py_DECREF(nameobj); - strcpy(buf, name); + if (is_builtin(name)) { + strcpy(buf, _PyUnicode_AsString(name)); return &fd_builtin; } #ifdef MS_COREDLL - fp = _PyWin_FindRegisteredModule(nameobj, &fdp, &filename); + fp = _PyWin_FindRegisteredModule(name, &fdp, &filename); if (fp != NULL) { - Py_DECREF(nameobj); filename_bytes = PyUnicode_EncodeFSDefault(filename); Py_DECREF(filename); if (filename_bytes == NULL) @@ -1980,7 +1973,6 @@ find_module(char *fullname, const char *name, PyObject *search_path_list, else if (PyErr_Occurred()) return NULL; #endif - Py_DECREF(nameobj); search_path_list = PySys_GetObject("path"); } @@ -3139,8 +3131,8 @@ import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) } } - fdp = find_module(_PyUnicode_AsString(fullname), - _PyUnicode_AsString(subname), + fdp = find_module(fullname, + subname, path, buf, MAXPATHLEN+1, &fp, &loader); Py_XDECREF(path); @@ -3180,12 +3172,13 @@ PyImport_ReloadModule(PyObject *m) PyInterpreterState *interp = PyThreadState_Get()->interp; PyObject *modules_reloading = interp->modules_reloading; PyObject *modules = PyImport_GetModuleDict(); - PyObject *path = NULL, *loader, *existing_m = NULL; - char *name, *subname; char buf[MAXPATHLEN+1]; + PyObject *path = NULL, *loader = NULL, *existing_m = NULL; + PyObject *nameobj, *bufobj, *subnameobj; + Py_UNICODE *name, *subname; struct filedescr *fdp; - FILE *fp; - PyObject *newm, *nameobj, *bufobj; + FILE *fp = NULL; + PyObject *newm = NULL; if (modules_reloading == NULL) { Py_FatalError("PyImport_ReloadModule: " @@ -3198,69 +3191,75 @@ PyImport_ReloadModule(PyObject *m) "reload() argument must be module"); return NULL; } - name = (char*)PyModule_GetName(m); - if (name == NULL) + nameobj = PyModule_GetNameObject(m); + if (nameobj == NULL) return NULL; - if (m != PyDict_GetItemString(modules, name)) { + if (m != PyDict_GetItem(modules, nameobj)) { PyErr_Format(PyExc_ImportError, - "reload(): module %.200s not in sys.modules", - name); + "reload(): module %U not in sys.modules", + nameobj); + Py_DECREF(nameobj); return NULL; } - existing_m = PyDict_GetItemString(modules_reloading, name); + existing_m = PyDict_GetItem(modules_reloading, nameobj); if (existing_m != NULL) { /* Due to a recursive reload, this module is already being reloaded. */ + Py_DECREF(nameobj); Py_INCREF(existing_m); return existing_m; } - if (PyDict_SetItemString(modules_reloading, name, m) < 0) + if (PyDict_SetItem(modules_reloading, nameobj, m) < 0) { + Py_DECREF(nameobj); return NULL; + } - subname = strrchr(name, '.'); - if (subname == NULL) - subname = name; + name = PyUnicode_AS_UNICODE(nameobj); + subname = Py_UNICODE_strrchr(name, '.'); + if (subname == NULL) { + Py_INCREF(nameobj); + subnameobj = nameobj; + } else { PyObject *parentname, *parent; - parentname = PyUnicode_FromStringAndSize(name, (subname-name)); + Py_ssize_t len; + len = subname - name; + parentname = PyUnicode_FromUnicode(name, len); if (parentname == NULL) { - imp_modules_reloading_clear(); - return NULL; + goto error; } parent = PyDict_GetItem(modules, parentname); if (parent == NULL) { PyErr_Format(PyExc_ImportError, - "reload(): parent %R not in sys.modules", + "reload(): parent %U not in sys.modules", parentname); Py_DECREF(parentname); - imp_modules_reloading_clear(); - return NULL; + goto error; } Py_DECREF(parentname); - subname++; path = PyObject_GetAttrString(parent, "__path__"); if (path == NULL) PyErr_Clear(); + subname++; + len = PyUnicode_GET_SIZE(nameobj) - (len + 1); + subnameobj = PyUnicode_FromUnicode(subname, len); } - fdp = find_module(name, subname, path, buf, MAXPATHLEN+1, &fp, &loader); + if (subnameobj == NULL) + goto error; + fdp = find_module(nameobj, subnameobj, + path, buf, MAXPATHLEN+1, &fp, &loader); + Py_DECREF(subnameobj); Py_XDECREF(path); if (fdp == NULL) { Py_XDECREF(loader); - imp_modules_reloading_clear(); - return NULL; + goto error; } - nameobj = PyUnicode_FromString(name); - if (nameobj != NULL) { - bufobj = PyUnicode_DecodeFSDefault(buf); - if (bufobj != NULL) { - newm = load_module(nameobj, fp, bufobj, fdp->type, loader); - Py_DECREF(bufobj); - } - else - newm = NULL; - Py_DECREF(nameobj); + bufobj = PyUnicode_DecodeFSDefault(buf); + if (bufobj != NULL) { + newm = load_module(nameobj, fp, bufobj, fdp->type, loader); + Py_DECREF(bufobj); } else newm = NULL; @@ -3274,9 +3273,12 @@ PyImport_ReloadModule(PyObject *m) * going to return NULL in this case regardless of whether * replacing name succeeds, so the return value is ignored. */ - PyDict_SetItemString(modules, name, m); + PyDict_SetItem(modules, nameobj, m); } + +error: imp_modules_reloading_clear(); + Py_DECREF(nameobj); return newm; } @@ -3424,7 +3426,7 @@ imp_get_suffixes(PyObject *self, PyObject *noargs) } static PyObject * -call_find_module(char *name, PyObject *path) +call_find_module(PyObject *name, PyObject *path) { extern int fclose(FILE *); PyObject *fob, *ret; @@ -3483,15 +3485,11 @@ call_find_module(char *name, PyObject *path) static PyObject * imp_find_module(PyObject *self, PyObject *args) { - PyObject *name; - PyObject *ret, *path = NULL; - if (!PyArg_ParseTuple(args, "O&|O:find_module", - PyUnicode_FSConverter, &name, - &path)) + PyObject *name, *path_list = NULL; + if (!PyArg_ParseTuple(args, "U|O:find_module", + &name, &path_list)) return NULL; - ret = call_find_module(PyBytes_AS_STRING(name), path); - Py_DECREF(name); - return ret; + return call_find_module(name, path_list); } static PyObject * -- cgit v1.2.1 From c23cd9690f4b004e965101735800e7c352fcfc78 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 13:22:54 -0400 Subject: Issue #3080: Rename some path variables to path_list --- Python/import.c | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index a3f263146c..73d38fe10f 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1538,7 +1538,8 @@ load_package(PyObject *name, PyObject *pathname) Py_DECREF(path_list); return NULL; } - fdp = find_module(name, initstr, path_list, buf, sizeof(buf), &fp, NULL); + fdp = find_module(name, initstr, path_list, + buf, sizeof(buf), &fp, NULL); Py_DECREF(path_list); if (fdp == NULL) { if (PyErr_ExceptionMatches(PyExc_ImportError)) { @@ -3105,7 +3106,7 @@ static PyObject * import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) { PyObject *modules = PyImport_GetModuleDict(); - PyObject *m = NULL, *bufobj, *path, *loader; + PyObject *m = NULL, *bufobj, *path_list, *loader; char buf[MAXPATHLEN+1]; struct filedescr *fdp; FILE *fp; @@ -3121,21 +3122,19 @@ import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) } if (mod == Py_None) - path = NULL; + path_list = NULL; else { - path = PyObject_GetAttrString(mod, "__path__"); - if (path == NULL) { + path_list = PyObject_GetAttrString(mod, "__path__"); + if (path_list == NULL) { PyErr_Clear(); Py_INCREF(Py_None); return Py_None; } } - fdp = find_module(fullname, - subname, - path, buf, MAXPATHLEN+1, - &fp, &loader); - Py_XDECREF(path); + fdp = find_module(fullname, subname, path_list, + buf, MAXPATHLEN+1, &fp, &loader); + Py_XDECREF(path_list); if (fdp == NULL) { if (!PyErr_ExceptionMatches(PyExc_ImportError)) return NULL; @@ -3173,7 +3172,7 @@ PyImport_ReloadModule(PyObject *m) PyObject *modules_reloading = interp->modules_reloading; PyObject *modules = PyImport_GetModuleDict(); char buf[MAXPATHLEN+1]; - PyObject *path = NULL, *loader = NULL, *existing_m = NULL; + PyObject *path_list = NULL, *loader = NULL, *existing_m = NULL; PyObject *nameobj, *bufobj, *subnameobj; Py_UNICODE *name, *subname; struct filedescr *fdp; @@ -3237,8 +3236,8 @@ PyImport_ReloadModule(PyObject *m) goto error; } Py_DECREF(parentname); - path = PyObject_GetAttrString(parent, "__path__"); - if (path == NULL) + path_list = PyObject_GetAttrString(parent, "__path__"); + if (path_list == NULL) PyErr_Clear(); subname++; len = PyUnicode_GET_SIZE(nameobj) - (len + 1); @@ -3246,10 +3245,10 @@ PyImport_ReloadModule(PyObject *m) } if (subnameobj == NULL) goto error; - fdp = find_module(nameobj, subnameobj, - path, buf, MAXPATHLEN+1, &fp, &loader); + fdp = find_module(nameobj, subnameobj, path_list, + buf, MAXPATHLEN+1, &fp, &loader); Py_DECREF(subnameobj); - Py_XDECREF(path); + Py_XDECREF(path_list); if (fdp == NULL) { Py_XDECREF(loader); @@ -3426,7 +3425,7 @@ imp_get_suffixes(PyObject *self, PyObject *noargs) } static PyObject * -call_find_module(PyObject *name, PyObject *path) +call_find_module(PyObject *name, PyObject *path_list) { extern int fclose(FILE *); PyObject *fob, *ret; @@ -3439,9 +3438,10 @@ call_find_module(PyObject *name, PyObject *path) char *encoding = NULL; pathname[0] = '\0'; - if (path == Py_None) - path = NULL; - fdp = find_module(NULL, name, path, pathname, MAXPATHLEN+1, &fp, NULL); + if (path_list == Py_None) + path_list = NULL; + fdp = find_module(NULL, name, path_list, + pathname, MAXPATHLEN+1, &fp, NULL); if (fdp == NULL) return NULL; if (fp != NULL) { -- cgit v1.2.1 From d5658be50acead958aef78e5c41c8d4d4bd0e857 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 13:40:04 -0400 Subject: Issue #3080: find_module() sets an empty path for builtin and frozen modules --- Python/import.c | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 73d38fe10f..2c21112c4b 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1943,10 +1943,8 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, Py_DECREF(meta_path); } - if (find_frozen(fullname) != NULL) { - strcpy(buf, _PyUnicode_AsString(fullname)); + if (find_frozen(fullname) != NULL) return &fd_frozen; - } if (search_path_list == NULL) { #ifdef MS_COREDLL @@ -1954,10 +1952,8 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, struct filedescr *fdp; PyObject *filename, *filename_bytes; #endif - if (is_builtin(name)) { - strcpy(buf, _PyUnicode_AsString(name)); + if (is_builtin(name)) return &fd_builtin; - } #ifdef MS_COREDLL fp = _PyWin_FindRegisteredModule(name, &fdp, &filename); if (fp != NULL) { -- cgit v1.2.1 From 2c6f2066619d885ebc3fd2c23dd6ca17f18ba71c Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 14:04:10 -0400 Subject: Issue #3080: Refactor find_module_path(), use return instead of break Prepare also the API change of case_ok() --- Python/import.c | 77 +++++++++++++++++++++++++++------------------------------ 1 file changed, 37 insertions(+), 40 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 2c21112c4b..979058e583 100644 --- a/Python/import.c +++ b/Python/import.c @@ -144,6 +144,8 @@ static const struct filedescr _PyImport_StandardFiletab[] = { {0, 0} }; +static PyObject *initstr = NULL; + /* Initialize things */ @@ -155,6 +157,10 @@ _PyImport_Init(void) int countD = 0; int countS = 0; + initstr = PyUnicode_InternFromString("__init__"); + if (initstr == NULL) + Py_FatalError("Can't initialize import variables"); + /* prepare _PyImport_Filetab: copy entries from _PyImport_DynLoadFiletab and _PyImport_StandardFiletab. */ @@ -1507,13 +1513,6 @@ load_package(PyObject *name, PyObject *pathname) char buf[MAXPATHLEN+1]; FILE *fp = NULL; struct filedescr *fdp; - static PyObject *initstr = NULL; - - if (initstr == NULL) { - initstr = PyUnicode_InternFromString("__init__"); - if (initstr == NULL) - return NULL; - } m = PyImport_AddModuleObject(name); if (m == NULL) @@ -1765,23 +1764,25 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, and there's an __init__ module in that directory */ #ifdef HAVE_STAT if (stat(buf, &statbuf) == 0 && /* it exists */ - S_ISDIR(statbuf.st_mode) && /* it's a directory */ - case_ok(buf, len, namelen, namestr)) { /* case matches */ - if (find_init_module(buf)) { /* and has __init__.py */ - *p_fd = &fd_package; - return 2; - } - else { - int err; - PyObject *unicode = PyUnicode_DecodeFSDefault(buf); - if (unicode == NULL) - return -1; - err = PyErr_WarnFormat(PyExc_ImportWarning, 1, - "Not importing directory '%U': missing __init__.py", - unicode); - Py_DECREF(unicode); - if (err) - return -1; + S_ISDIR(statbuf.st_mode)) /* it's a directory */ + { + if (case_ok(buf, len, namelen, namestr)) { /* case matches */ + if (find_init_module(buf)) { /* and has __init__.py */ + *p_fd = &fd_package; + return 2; + } + else { + int err; + PyObject *unicode = PyUnicode_DecodeFSDefault(buf); + if (unicode == NULL) + return -1; + err = PyErr_WarnFormat(PyExc_ImportWarning, 1, + "Not importing directory '%U': missing __init__.py", + unicode); + Py_DECREF(unicode); + if (err) + return -1; + } } } #endif @@ -1842,25 +1843,21 @@ find_module_path_list(PyObject *fullname, PyObject *name, if (filemode[0] == 'U') filemode = "r" PY_STDIOTEXTMODE; fp = fopen(buf, filemode); - if (fp != NULL) { - if (case_ok(buf, len, namelen, namestr)) - break; - else { /* continue search */ - fclose(fp); - fp = NULL; - } + if (fp == NULL) + continue; + + if (case_ok(buf, len, namelen, namestr)) { + *p_fp = fp; + return fdp; } + + fclose(fp); + fp = NULL; } - if (fp != NULL) - break; - } - if (fp == NULL) { - PyErr_Format(PyExc_ImportError, - "No module named %U", name); - return NULL; } - *p_fp = fp; - return fdp; + PyErr_Format(PyExc_ImportError, + "No module named %U", name); + return NULL; } /* Find a module: -- cgit v1.2.1 From 87e72d7343dc9ff213470b182fbba2bd73e7a6d1 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 20 Mar 2011 03:07:28 +0100 Subject: Issue #3080: find_init_module() expects Unicode --- Python/import.c | 88 +++++++++++++++++++++++++++++---------------------------- 1 file changed, 45 insertions(+), 43 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 979058e583..8a242a57eb 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1668,8 +1668,9 @@ extern FILE *_PyWin_FindRegisteredModule(PyObject *, struct filedescr **, PyObject **p_path); #endif +/* Forward */ static int case_ok(char *, Py_ssize_t, Py_ssize_t, const char *); -static int find_init_module(char *); /* Forward */ +static int find_init_module(PyObject *); static struct filedescr importhookdescr = {"", "", IMP_HOOK}; /* Get the path of a module: get its importer and call importer.find_module() @@ -1766,24 +1767,27 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, if (stat(buf, &statbuf) == 0 && /* it exists */ S_ISDIR(statbuf.st_mode)) /* it's a directory */ { + PyObject *bufobj = PyUnicode_DecodeFSDefault(buf); + if (bufobj == NULL) + return -1; if (case_ok(buf, len, namelen, namestr)) { /* case matches */ - if (find_init_module(buf)) { /* and has __init__.py */ + if (find_init_module(bufobj)) { /* and has __init__.py */ + Py_DECREF(bufobj); *p_fd = &fd_package; return 2; } else { int err; - PyObject *unicode = PyUnicode_DecodeFSDefault(buf); - if (unicode == NULL) - return -1; err = PyErr_WarnFormat(PyExc_ImportWarning, 1, - "Not importing directory '%U': missing __init__.py", - unicode); - Py_DECREF(unicode); - if (err) + "Not importing directory %R: missing __init__.py", + bufobj); + if (err) { + Py_DECREF(bufobj); return -1; + } } } + Py_DECREF(bufobj); } #endif return 1; @@ -2154,49 +2158,47 @@ case_ok(char *buf, Py_ssize_t len, Py_ssize_t namelen, const char *name) #ifdef HAVE_STAT -/* Helper to look for __init__.py or __init__.py[co] in potential package */ +/* Helper to look for __init__.py or __init__.py[co] in potential package. + Return 1 if __init__ was found, 0 if not, or -1 on error. */ static int -find_init_module(char *buf) +find_init_module(PyObject *directory) { - const size_t save_len = strlen(buf); - size_t i = save_len; - char *pname; /* pointer to start of __init__ */ + size_t len; struct stat statbuf; - -/* For calling case_ok(buf, len, namelen, name): - * /a/b/c/d/e/f/g/h/i/j/k/some_long_module_name.py\0 - * ^ ^ ^ ^ - * |--------------------- buf ---------------------| - * |------------------- len ------------------| - * |------ name -------| - * |----- namelen -----| - */ - if (save_len + 13 >= MAXPATHLEN) - return 0; - buf[i++] = SEP; - pname = buf + i; - strcpy(pname, "__init__.py"); - if (stat(buf, &statbuf) == 0) { - if (case_ok(buf, - save_len + 9, /* len("/__init__") */ - 8, /* len("__init__") */ - pname)) { - buf[save_len] = '\0'; + PyObject *filename; + int match; + char *filestr; + size_t filelen; + + len = PyUnicode_GET_SIZE(directory); + filename = PyUnicode_FromFormat("%U%c__init__.py", directory, SEP); + if (filename == NULL) + return -1; + if (_Py_stat(filename, &statbuf) == 0) { + /* 9=len("/__init__") */ + filestr = _PyUnicode_AsString(filename); + filelen = strlen(filestr); + if (case_ok(filestr, filelen-9, 8, "__init__")) { + Py_DECREF(filename); return 1; } } - i += strlen(pname); - strcpy(buf+i, Py_OptimizeFlag ? "o" : "c"); - if (stat(buf, &statbuf) == 0) { - if (case_ok(buf, - save_len + 9, /* len("/__init__") */ - 8, /* len("__init__") */ - pname)) { - buf[save_len] = '\0'; + Py_DECREF(filename); + + filename = PyUnicode_FromFormat("%U%c__init__.py%c", + directory, SEP, Py_OptimizeFlag ? 'o' : 'c'); + if (filename == NULL) + return -1; + if (_Py_stat(filename, &statbuf) == 0) { + /* 9=len("/__init__") */ + filestr = _PyUnicode_AsString(filename); + filelen = strlen(filestr); + if (case_ok(filestr, filelen-9, 8, "__init__")) { + Py_DECREF(filename); return 1; } } - buf[save_len] = '\0'; + Py_DECREF(filename); return 0; } -- cgit v1.2.1 From 7bca4dba35971a2705590aee419a18ad29fefd06 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 14:34:13 -0400 Subject: Issue #3080: case_ok() expects Unicode strings --- Python/import.c | 181 +++++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 132 insertions(+), 49 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 8a242a57eb..bba0120321 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1669,7 +1669,7 @@ extern FILE *_PyWin_FindRegisteredModule(PyObject *, struct filedescr **, #endif /* Forward */ -static int case_ok(char *, Py_ssize_t, Py_ssize_t, const char *); +static int case_ok(PyObject *, Py_ssize_t, PyObject *); static int find_init_module(PyObject *); static struct filedescr importhookdescr = {"", "", IMP_HOOK}; @@ -1767,12 +1767,20 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, if (stat(buf, &statbuf) == 0 && /* it exists */ S_ISDIR(statbuf.st_mode)) /* it's a directory */ { - PyObject *bufobj = PyUnicode_DecodeFSDefault(buf); - if (bufobj == NULL) + int match; + PyObject *filename; + + filename = PyUnicode_DecodeFSDefault(buf); + if (filename == NULL) + return -1; + match = case_ok(filename, 0, name); + if (match < 0) { + Py_DECREF(filename); return -1; - if (case_ok(buf, len, namelen, namestr)) { /* case matches */ - if (find_init_module(bufobj)) { /* and has __init__.py */ - Py_DECREF(bufobj); + } + if (match) { /* case matches */ + if (find_init_module(filename)) { /* and has __init__.py */ + Py_DECREF(filename); *p_fd = &fd_package; return 2; } @@ -1780,14 +1788,14 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, int err; err = PyErr_WarnFormat(PyExc_ImportWarning, 1, "Not importing directory %R: missing __init__.py", - bufobj); + filename); if (err) { - Py_DECREF(bufobj); + Py_DECREF(filename); return -1; } } } - Py_DECREF(bufobj); + Py_DECREF(filename); } #endif return 1; @@ -1815,6 +1823,8 @@ find_module_path_list(PyObject *fullname, PyObject *name, char *filemode; FILE *fp = NULL; char *namestr; + PyObject *filename; + int match; npath = PyList_Size(search_path_list); namestr = _PyUnicode_AsString(name); @@ -1840,20 +1850,34 @@ find_module_path_list(PyObject *fullname, PyObject *name, len = strlen(buf); for (fdp = _PyImport_Filetab; fdp->suffix != NULL; fdp++) { - strcpy(buf+len, fdp->suffix); - if (Py_VerboseFlag > 1) - PySys_WriteStderr("# trying %s\n", buf); filemode = fdp->mode; if (filemode[0] == 'U') filemode = "r" PY_STDIOTEXTMODE; - fp = fopen(buf, filemode); - if (fp == NULL) - continue; - if (case_ok(buf, len, namelen, namestr)) { + strcpy(buf+len, fdp->suffix); + filename = PyUnicode_DecodeFSDefault(buf); + if (filename == NULL) + return NULL; + + if (Py_VerboseFlag > 1) + PySys_FormatStderr("# trying %R\n", filename); + + fp = _Py_fopen(filename, filemode); + if (fp == NULL) { + Py_DECREF(filename); + continue; + } + match = case_ok(filename, -(Py_ssize_t)strlen(fdp->suffix), name); + if (match < 0) { + Py_DECREF(filename); + return NULL; + } + if (match) { + Py_DECREF(filename); *p_fp = fp; return fdp; } + Py_DECREF(filename); fclose(fp); fp = NULL; @@ -2002,7 +2026,7 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, p_fp, p_loader); } -/* case_ok(char* buf, Py_ssize_t len, Py_ssize_t namelen, char* name) +/* case_bytes(char* buf, Py_ssize_t len, Py_ssize_t namelen, char* name) * The arguments here are tricky, best shown by example: * /a/b/c/d/e/f/g/h/i/j/k/some_long_module_name.py\0 * ^ ^ ^ ^ @@ -2016,18 +2040,18 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, * We've already done a successful stat() or fopen() on buf, so know that * there's some match, possibly case-insensitive. * - * case_ok() is to return 1 if there's a case-sensitive match for - * name, else 0. case_ok() is also to return 1 if envar PYTHONCASEOK + * case_bytes() is to return 1 if there's a case-sensitive match for + * name, else 0. case_bytes() is also to return 1 if envar PYTHONCASEOK * exists. * - * case_ok() is used to implement case-sensitive import semantics even + * case_bytes() is used to implement case-sensitive import semantics even * on platforms with case-insensitive filesystems. It's trivial to implement * for case-sensitive filesystems. It's pretty much a cross-platform * nightmare for systems with case-insensitive filesystems. */ /* First we may need a pile of platform-specific header files; the sequence - * of #if's here should match the sequence in the body of case_ok(). + * of #if's here should match the sequence in the body of case_bytes(). */ #if defined(MS_WINDOWS) #include @@ -2046,33 +2070,24 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, #include #endif +#if defined(DJGPP) \ + || ((defined(__MACH__) && defined(__APPLE__) || defined(__CYGWIN__)) \ + && defined(HAVE_DIRENT_H)) \ + || defined(PYOS_OS2) +# define USE_CASE_OK_BYTES +#endif + + +#ifdef USE_CASE_OK_BYTES static int -case_ok(char *buf, Py_ssize_t len, Py_ssize_t namelen, const char *name) +case_bytes(char *buf, Py_ssize_t len, Py_ssize_t namelen, const char *name) { /* Pick a platform-specific implementation; the sequence of #if's here should * match the sequence just above. */ -/* MS_WINDOWS */ -#if defined(MS_WINDOWS) - WIN32_FIND_DATA data; - HANDLE h; - - if (Py_GETENV("PYTHONCASEOK") != NULL) - return 1; - - h = FindFirstFile(buf, &data); - if (h == INVALID_HANDLE_VALUE) { - PyErr_Format(PyExc_NameError, - "Can't find file for module %.100s\n(filename %.300s)", - name, buf); - return 0; - } - FindClose(h); - return strncmp(data.cFileName, name, namelen) == 0; - /* DJGPP */ -#elif defined(DJGPP) +#if defined(DJGPP) struct ffblk ffblk; int done; @@ -2151,6 +2166,70 @@ case_ok(char *buf, Py_ssize_t len, Py_ssize_t namelen, const char *name) /* assuming it's a case-sensitive filesystem, so there's nothing to do! */ #else +# error "USE_CASE_OK_BYTES is not correctly defined" +#endif +} +#endif + +/* + * Check if a filename case matchs the name case. We've already done a + * successful stat() or fopen() on buf, so know that there's some match, + * possibly case-insensitive. + * + * case_ok() is to return 1 if there's a case-sensitive match for name, 0 if it + * the filename doesn't match, or -1 on error. case_ok() is also to return 1 + * if envar PYTHONCASEOK exists. + * + * case_ok() is used to implement case-sensitive import semantics even + * on platforms with case-insensitive filesystems. It's trivial to implement + * for case-sensitive filesystems. It's pretty much a cross-platform + * nightmare for systems with case-insensitive filesystems. + */ + +static int +case_ok(PyObject *filename, Py_ssize_t prefix_delta, PyObject *name) +{ +#ifdef MS_WINDOWS + WIN32_FIND_DATAW data; + HANDLE h; + int cmp; + + if (Py_GETENV("PYTHONCASEOK") != NULL) + return 1; + + h = FindFirstFileW(PyUnicode_AS_UNICODE(filename), &data); + if (h == INVALID_HANDLE_VALUE) { + PyErr_Format(PyExc_NameError, + "Can't find file for module %R\n(filename %R)", + name, filename); + return 0; + } + FindClose(h); + cmp = wcsncmp(data.cFileName, + PyUnicode_AS_UNICODE(name), + PyUnicode_GET_SIZE(name)); + return cmp == 0; +#elif defined(USE_CASE_OK_BYTES) + int match; + PyObject *filebytes, *namebytes; + filebytes = PyUnicode_EncodeFSDefault(filename); + if (filebytes == NULL) + return -1; + namebytes = PyUnicode_EncodeFSDefault(name); + if (namebytes == NULL) { + Py_DECREF(filebytes); + return -1; + } + match = case_bytes( + PyBytes_AS_STRING(filebytes), + PyBytes_GET_SIZE(filebytes) + prefix_delta, + PyBytes_AS_STRING(namebytes), + PyBytes_GET_SIZE(namebytes)); + Py_DECREF(filebytes); + Py_DECREF(namebytes); + return match; +#else + /* assuming it's a case-sensitive filesystem, so there's nothing to do! */ return 1; #endif @@ -2167,8 +2246,6 @@ find_init_module(PyObject *directory) struct stat statbuf; PyObject *filename; int match; - char *filestr; - size_t filelen; len = PyUnicode_GET_SIZE(directory); filename = PyUnicode_FromFormat("%U%c__init__.py", directory, SEP); @@ -2176,9 +2253,12 @@ find_init_module(PyObject *directory) return -1; if (_Py_stat(filename, &statbuf) == 0) { /* 9=len("/__init__") */ - filestr = _PyUnicode_AsString(filename); - filelen = strlen(filestr); - if (case_ok(filestr, filelen-9, 8, "__init__")) { + match = case_ok(filename, 9, initstr); + if (match < 0) { + Py_DECREF(filename); + return -1; + } + if (match) { Py_DECREF(filename); return 1; } @@ -2191,9 +2271,12 @@ find_init_module(PyObject *directory) return -1; if (_Py_stat(filename, &statbuf) == 0) { /* 9=len("/__init__") */ - filestr = _PyUnicode_AsString(filename); - filelen = strlen(filestr); - if (case_ok(filestr, filelen-9, 8, "__init__")) { + match = case_ok(filename, 9, initstr); + if (match < 0) { + Py_DECREF(filename); + return -1; + } + if (match) { Py_DECREF(filename); return 1; } -- cgit v1.2.1 From 02352565abd1eed9eb32c5c273bc640869f2100b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 15:19:39 -0400 Subject: Issue #3080: find_module() returns the path as Unicode --- Python/import.c | 171 ++++++++++++++++++++++++-------------------------------- 1 file changed, 74 insertions(+), 97 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index bba0120321..5dcbf17710 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1498,7 +1498,7 @@ unchanged: /* Forward */ static PyObject *load_module(PyObject *, FILE *, PyObject *, int, PyObject *); static struct filedescr *find_module(PyObject *, PyObject *, PyObject *, - char *, size_t, FILE **, PyObject **); + PyObject **, FILE **, PyObject **); static struct _frozen * find_frozen(PyObject *); /* Load a package and return its module object WITH INCREMENTED @@ -1510,7 +1510,6 @@ load_package(PyObject *name, PyObject *pathname) PyObject *m, *d, *bufobj; PyObject *file = NULL, *path_list = NULL; int err; - char buf[MAXPATHLEN+1]; FILE *fp = NULL; struct filedescr *fdp; @@ -1538,7 +1537,7 @@ load_package(PyObject *name, PyObject *pathname) return NULL; } fdp = find_module(name, initstr, path_list, - buf, sizeof(buf), &fp, NULL); + &bufobj, &fp, NULL); Py_DECREF(path_list); if (fdp == NULL) { if (PyErr_ExceptionMatches(PyExc_ImportError)) { @@ -1549,13 +1548,8 @@ load_package(PyObject *name, PyObject *pathname) else return NULL; } - bufobj = PyUnicode_DecodeFSDefault(buf); - if (bufobj != NULL) { - m = load_module(name, fp, bufobj, fdp->type, NULL); - Py_DECREF(bufobj); - } - else - m = NULL; + m = load_module(name, fp, bufobj, fdp->type, NULL); + Py_XDECREF(bufobj); if (fp != NULL) fclose(fp); return m; @@ -1686,43 +1680,40 @@ static struct filedescr importhookdescr = {"", "", IMP_HOOK}; static int find_module_path(PyObject *fullname, PyObject *name, PyObject *path, PyObject *path_hooks, PyObject *path_importer_cache, - char *buf, size_t buflen, - PyObject **p_loader, struct filedescr **p_fd) + PyObject **p_path, PyObject **p_loader, struct filedescr **p_fd) { - PyObject *path_bytes; - const char *base; + Py_UNICODE buf[MAXPATHLEN+1]; + Py_ssize_t buflen = MAXPATHLEN+1; + PyObject *path_unicode, *filename; + const Py_UNICODE *base; Py_ssize_t len; - size_t namelen; struct stat statbuf; static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; - char *namestr; if (PyUnicode_Check(path)) { - path_bytes = PyUnicode_EncodeFSDefault(path); - if (path_bytes == NULL) - return -1; + Py_INCREF(path); + path_unicode = path; } else if (PyBytes_Check(path)) { - Py_INCREF(path); - path_bytes = path; + path_unicode = PyUnicode_DecodeFSDefaultAndSize( + PyBytes_AS_STRING(path), PyBytes_GET_SIZE(path)); + if (path_unicode == NULL) + return -1; } else return 0; - namestr = _PyUnicode_AsString(name); - namelen = strlen(namestr); - base = PyBytes_AS_STRING(path_bytes); - len = PyBytes_GET_SIZE(path_bytes); - if (len + 2 + namelen + MAXSUFFIXSIZE >= buflen) { - Py_DECREF(path_bytes); + base = PyUnicode_AS_UNICODE(path_unicode); + len = PyUnicode_GET_SIZE(path_unicode); + if (len + 2 + PyUnicode_GET_SIZE(name) + MAXSUFFIXSIZE >= buflen) { + Py_DECREF(path_unicode); return 0; /* Too long */ } - strcpy(buf, base); - Py_DECREF(path_bytes); + Py_UNICODE_strcpy(buf, PyUnicode_AS_UNICODE(path_unicode)); + Py_DECREF(path_unicode); - if (strlen(buf) != len) { - return 0; /* path_bytes contains '\0' */ - } + if (Py_UNICODE_strlen(buf) != len) + return 0; /* path contains '\0' */ /* sys.path_hooks import hook */ if (p_loader != NULL) { @@ -1758,21 +1749,21 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, #endif ) buf[len++] = SEP; - strcpy(buf+len, namestr); - len += namelen; + Py_UNICODE_strcpy(buf+len, PyUnicode_AS_UNICODE(name)); + len += PyUnicode_GET_SIZE(name); + + filename = PyUnicode_FromUnicode(buf, len); + if (filename == NULL) + return -1; /* Check for package import (buf holds a directory name, and there's an __init__ module in that directory */ #ifdef HAVE_STAT - if (stat(buf, &statbuf) == 0 && /* it exists */ + if (_Py_stat(filename, &statbuf) == 0 && /* it exists */ S_ISDIR(statbuf.st_mode)) /* it's a directory */ { int match; - PyObject *filename; - filename = PyUnicode_DecodeFSDefault(buf); - if (filename == NULL) - return -1; match = case_ok(filename, 0, name); if (match < 0) { Py_DECREF(filename); @@ -1780,7 +1771,7 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, } if (match) { /* case matches */ if (find_init_module(filename)) { /* and has __init__.py */ - Py_DECREF(filename); + *p_path = filename; *p_fd = &fd_package; return 2; } @@ -1795,9 +1786,9 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, } } } - Py_DECREF(filename); } #endif + *p_path = filename; return 1; } @@ -1814,21 +1805,16 @@ static struct filedescr* find_module_path_list(PyObject *fullname, PyObject *name, PyObject *search_path_list, PyObject *path_hooks, PyObject *path_importer_cache, - char *buf, size_t buflen, - FILE **p_fp, PyObject **p_loader) + PyObject **p_path, FILE **p_fp, PyObject **p_loader) { Py_ssize_t i, npath; - size_t len, namelen; struct filedescr *fdp = NULL; char *filemode; FILE *fp = NULL; - char *namestr; - PyObject *filename; + PyObject *prefix, *filename; int match; npath = PyList_Size(search_path_list); - namestr = _PyUnicode_AsString(name); - namelen = strlen(namestr); for (i = 0; i < npath; i++) { PyObject *path; int ok; @@ -1837,27 +1823,29 @@ find_module_path_list(PyObject *fullname, PyObject *name, if (path == NULL) return NULL; + prefix = NULL; ok = find_module_path(fullname, name, path, path_hooks, path_importer_cache, - buf, buflen, - p_loader, &fdp); + &prefix, p_loader, &fdp); if (ok < 0) return NULL; if (ok == 0) continue; - if (ok == 2) + if (ok == 2) { + *p_path = prefix; return fdp; + } - len = strlen(buf); for (fdp = _PyImport_Filetab; fdp->suffix != NULL; fdp++) { filemode = fdp->mode; if (filemode[0] == 'U') filemode = "r" PY_STDIOTEXTMODE; - strcpy(buf+len, fdp->suffix); - filename = PyUnicode_DecodeFSDefault(buf); - if (filename == NULL) + filename = PyUnicode_FromFormat("%U%s", prefix, fdp->suffix); + if (filename == NULL) { + Py_DECREF(prefix); return NULL; + } if (Py_VerboseFlag > 1) PySys_FormatStderr("# trying %R\n", filename); @@ -1869,11 +1857,13 @@ find_module_path_list(PyObject *fullname, PyObject *name, } match = case_ok(filename, -(Py_ssize_t)strlen(fdp->suffix), name); if (match < 0) { + Py_DECREF(prefix); Py_DECREF(filename); return NULL; } if (match) { - Py_DECREF(filename); + Py_DECREF(prefix); + *p_path = filename; *p_fp = fp; return fdp; } @@ -1882,6 +1872,7 @@ find_module_path_list(PyObject *fullname, PyObject *name, fclose(fp); fp = NULL; } + Py_DECREF(prefix); } PyErr_Format(PyExc_ImportError, "No module named %U", name); @@ -1897,32 +1888,35 @@ find_module_path_list(PyObject *fullname, PyObject *name, - otherwise, call find_module_path_list() with search_path_list (if not NULL) or sys.path + fullname can be NULL, but only if p_loader is NULL. + Return: - &fd_builtin (C_BUILTIN) if it is a builtin - &fd_frozen (PY_FROZEN) if it is frozen - - &fd_package (PKG_DIRECTORY) and write the filename into *buf + - &fd_package (PKG_DIRECTORY) and write the filename into *p_path if it is a package - &importhookdescr (IMP_HOOK) and write the loader into *p_loader if a importer loader was found - a file descriptor (PY_SOURCE, PY_COMPILED, C_EXTENSION, PY_RESOURCE or PY_CODERESOURCE: see _PyImport_Filetab), write the filename into - *buf and the pointer to the open file into *p_fp + *p_path and the pointer to the open file into *p_fp - NULL on error - By default, write an empty string into *buf, and *p_fp and *p_loader (if - set) are set to NULL. Eg. *buf is an empty string for a builtin package. */ + By default, *p_path, *p_fp and *p_loader (if set) are set to NULL. + Eg. *p_path is set to NULL for a builtin package. +*/ static struct filedescr * find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, - char *buf, size_t buflen, FILE **p_fp, PyObject **p_loader) + PyObject **p_path, FILE **p_fp, PyObject **p_loader) { Py_ssize_t i, npath; static struct filedescr fd_frozen = {"", "", PY_FROZEN}; static struct filedescr fd_builtin = {"", "", C_BUILTIN}; PyObject *path_hooks, *path_importer_cache; - *buf = '\0'; + *p_path = NULL; *p_fp = NULL; if (p_loader != NULL) *p_loader = NULL; @@ -1975,20 +1969,12 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, #ifdef MS_COREDLL FILE *fp; struct filedescr *fdp; - PyObject *filename, *filename_bytes; #endif if (is_builtin(name)) return &fd_builtin; #ifdef MS_COREDLL - fp = _PyWin_FindRegisteredModule(name, &fdp, &filename); + fp = _PyWin_FindRegisteredModule(name, &fdp, p_path); if (fp != NULL) { - filename_bytes = PyUnicode_EncodeFSDefault(filename); - Py_DECREF(filename); - if (filename_bytes == NULL) - return NULL; - strncpy(buf, PyBytes_AS_STRING(filename_bytes), buflen); - buf[buflen-1] = '\0'; - Py_DECREF(filename_bytes); *p_fp = fp; return fdp; } @@ -2022,8 +2008,7 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, return find_module_path_list(fullname, name, search_path_list, path_hooks, path_importer_cache, - buf, buflen, - p_fp, p_loader); + p_path, p_fp, p_loader); } /* case_bytes(char* buf, Py_ssize_t len, Py_ssize_t namelen, char* name) @@ -3185,7 +3170,6 @@ import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) { PyObject *modules = PyImport_GetModuleDict(); PyObject *m = NULL, *bufobj, *path_list, *loader; - char buf[MAXPATHLEN+1]; struct filedescr *fdp; FILE *fp; @@ -3211,7 +3195,7 @@ import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) } fdp = find_module(fullname, subname, path_list, - buf, MAXPATHLEN+1, &fp, &loader); + &bufobj, &fp, &loader); Py_XDECREF(path_list); if (fdp == NULL) { if (!PyErr_ExceptionMatches(PyExc_ImportError)) @@ -3220,13 +3204,8 @@ import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) Py_INCREF(Py_None); return Py_None; } - bufobj = PyUnicode_DecodeFSDefault(buf); - if (bufobj != NULL) { - m = load_module(fullname, fp, bufobj, fdp->type, loader); - Py_DECREF(bufobj); - } - else - m = NULL; + m = load_module(fullname, fp, bufobj, fdp->type, loader); + Py_XDECREF(bufobj); Py_XDECREF(loader); if (fp) fclose(fp); @@ -3249,7 +3228,6 @@ PyImport_ReloadModule(PyObject *m) PyInterpreterState *interp = PyThreadState_Get()->interp; PyObject *modules_reloading = interp->modules_reloading; PyObject *modules = PyImport_GetModuleDict(); - char buf[MAXPATHLEN+1]; PyObject *path_list = NULL, *loader = NULL, *existing_m = NULL; PyObject *nameobj, *bufobj, *subnameobj; Py_UNICODE *name, *subname; @@ -3324,7 +3302,7 @@ PyImport_ReloadModule(PyObject *m) if (subnameobj == NULL) goto error; fdp = find_module(nameobj, subnameobj, path_list, - buf, MAXPATHLEN+1, &fp, &loader); + &bufobj, &fp, &loader); Py_DECREF(subnameobj); Py_XDECREF(path_list); @@ -3333,13 +3311,8 @@ PyImport_ReloadModule(PyObject *m) goto error; } - bufobj = PyUnicode_DecodeFSDefault(buf); - if (bufobj != NULL) { - newm = load_module(nameobj, fp, bufobj, fdp->type, loader); - Py_DECREF(bufobj); - } - else - newm = NULL; + newm = load_module(nameobj, fp, bufobj, fdp->type, loader); + Py_XDECREF(bufobj); Py_XDECREF(loader); if (fp) @@ -3509,17 +3482,15 @@ call_find_module(PyObject *name, PyObject *path_list) PyObject *fob, *ret; PyObject *pathobj; struct filedescr *fdp; - char pathname[MAXPATHLEN+1]; FILE *fp; int fd = -1; char *found_encoding = NULL; char *encoding = NULL; - pathname[0] = '\0'; if (path_list == Py_None) path_list = NULL; fdp = find_module(NULL, name, path_list, - pathname, MAXPATHLEN+1, &fp, NULL); + &pathobj, &fp, NULL); if (fdp == NULL) return NULL; if (fp != NULL) { @@ -3535,14 +3506,17 @@ call_find_module(PyObject *name, PyObject *path_list) memory. */ found_encoding = PyTokenizer_FindEncoding(fd); lseek(fd, 0, 0); /* Reset position */ - if (found_encoding == NULL && PyErr_Occurred()) + if (found_encoding == NULL && PyErr_Occurred()) { + Py_XDECREF(pathobj); return NULL; + } encoding = (found_encoding != NULL) ? found_encoding : (char*)PyUnicode_GetDefaultEncoding(); } - fob = PyFile_FromFd(fd, pathname, fdp->mode, -1, + fob = PyFile_FromFd(fd, NULL, fdp->mode, -1, (char*)encoding, NULL, NULL, 1); if (fob == NULL) { + Py_XDECREF(pathobj); close(fd); PyMem_FREE(found_encoding); return NULL; @@ -3552,7 +3526,10 @@ call_find_module(PyObject *name, PyObject *path_list) fob = Py_None; Py_INCREF(fob); } - pathobj = PyUnicode_DecodeFSDefault(pathname); + if (pathobj == NULL) { + Py_INCREF(Py_None); + pathobj = Py_None; + } ret = Py_BuildValue("NN(ssi)", fob, pathobj, fdp->suffix, fdp->mode, fdp->type); PyMem_FREE(found_encoding); -- cgit v1.2.1 From 8784da32ff0f704a48459abf452eba8345f0d053 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 14:53:28 -0400 Subject: Issue #3080: imp.new_module() uses Unicode --- Python/import.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 5dcbf17710..3237ddc84e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3787,10 +3787,10 @@ imp_load_package(PyObject *self, PyObject *args) static PyObject * imp_new_module(PyObject *self, PyObject *args) { - char *name; - if (!PyArg_ParseTuple(args, "s:new_module", &name)) + PyObject *name; + if (!PyArg_ParseTuple(args, "U:new_module", &name)) return NULL; - return PyModule_New(name); + return PyModule_NewObject(name); } static PyObject * -- cgit v1.2.1 From 24cf6f1d864c6c51a299ed32ba9e1fe8701a4b54 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 15:05:12 -0400 Subject: Fix imp.cache_from_source() if the directory name contains a dot If the directory name contains a dot but not the filename, don't strip at the dot. --- Python/import.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 3237ddc84e..e5cb627003 100644 --- a/Python/import.c +++ b/Python/import.c @@ -943,12 +943,12 @@ make_compiled_pathname(Py_UNICODE *pathname, int debug) Py_UNICODE_strcat(buf, CACHEDIR_UNICODE); i += Py_UNICODE_strlen(CACHEDIR_UNICODE) - 1; buf[i++] = sep; - buf[i++] = '\0'; + buf[i] = '\0'; /* Add the base filename, but remove the .py or .pyw extension, since the tag name must go before the extension. */ Py_UNICODE_strcat(buf, pathname + save); - pos = Py_UNICODE_strrchr(buf, '.'); + pos = Py_UNICODE_strrchr(buf + i, '.'); if (pos != NULL) *++pos = '\0'; Py_UNICODE_strcat(buf, PYC_TAG_UNICODE); -- cgit v1.2.1 From 0b5200ff91987d460004543f52dc7443f57a6b36 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 15:15:47 -0400 Subject: Issue #3080: Use repr() to format the module name on error --- Python/import.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index e5cb627003..65944473cc 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1431,7 +1431,7 @@ load_source_module(PyObject *name, PyObject *pathname, FILE *fp) goto error; if (Py_VerboseFlag) PySys_FormatStderr("import %U # from %R\n", - name, pathname); + name, pathname); if (cpathname != NULL) { PyObject *ro = PySys_GetObject("dont_write_bytecode"); if (ro == NULL || !PyObject_IsTrue(ro)) @@ -1517,8 +1517,8 @@ load_package(PyObject *name, PyObject *pathname) if (m == NULL) return NULL; if (Py_VerboseFlag) - PySys_FormatStderr("import %U # directory %U\n", - name, pathname); + PySys_FormatStderr("import %U # directory %R\n", + name, pathname); file = get_sourcefile(pathname); if (file == NULL) return NULL; @@ -1875,7 +1875,7 @@ find_module_path_list(PyObject *fullname, PyObject *name, Py_DECREF(prefix); } PyErr_Format(PyExc_ImportError, - "No module named %U", name); + "No module named %R", name); return NULL; } @@ -2366,7 +2366,7 @@ load_module(PyObject *name, FILE *fp, PyObject *pathname, int type, PyObject *lo default: PyErr_Format(PyExc_ImportError, - "Don't know how to import %U (type code %d)", + "Don't know how to import %R (type code %d)", name, type); m = NULL; @@ -3251,7 +3251,7 @@ PyImport_ReloadModule(PyObject *m) return NULL; if (m != PyDict_GetItem(modules, nameobj)) { PyErr_Format(PyExc_ImportError, - "reload(): module %U not in sys.modules", + "reload(): module %R not in sys.modules", nameobj); Py_DECREF(nameobj); return NULL; @@ -3286,7 +3286,7 @@ PyImport_ReloadModule(PyObject *m) parent = PyDict_GetItem(modules, parentname); if (parent == NULL) { PyErr_Format(PyExc_ImportError, - "reload(): parent %U not in sys.modules", + "reload(): parent %R not in sys.modules", parentname); Py_DECREF(parentname); goto error; -- cgit v1.2.1 From 67a022a7f5cdf458fa1e76d7b5482fec1784890f Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 14 Mar 2011 15:54:52 -0400 Subject: Issue #3080: Add PyImport_ImportModuleLevelObject() function Use it for the builtin __import__ function. --- Python/import.c | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 65944473cc..7f9042ac01 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2753,25 +2753,37 @@ import_module_level(PyObject *name, PyObject *globals, PyObject *locals, } PyObject * -PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals, - PyObject *fromlist, int level) +PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, + PyObject *locals, PyObject *fromlist, + int level) { - PyObject *nameobj, *result; - nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) - return NULL; + PyObject *mod; _PyImport_AcquireLock(); - result = import_module_level(nameobj, globals, locals, fromlist, level); - Py_DECREF(nameobj); + mod = import_module_level(name, globals, locals, fromlist, level); if (_PyImport_ReleaseLock() < 0) { - Py_XDECREF(result); + Py_XDECREF(mod); PyErr_SetString(PyExc_RuntimeError, "not holding the import lock"); return NULL; } - return result; + return mod; +} + +PyObject * +PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals, + PyObject *fromlist, int level) +{ + PyObject *nameobj, *mod; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) + return NULL; + mod = PyImport_ImportModuleLevelObject(nameobj, globals, locals, + fromlist, level); + Py_DECREF(nameobj); + return mod; } + /* Return the package that an import is being performed in. If globals comes from the module foo.bar.bat (not itself a package), this returns the sys.modules entry for foo.bar. If globals is from a package's __init__.py, -- cgit v1.2.1 From 83ee336336ccdc6b1fcc97ce0f693409ecf4b60a Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 20 Mar 2011 04:28:55 +0100 Subject: Issue #3080: Fix case_ok() using case_bytes() Invert name and namelen arguments. --- Python/import.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 7f9042ac01..0194d89956 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2208,8 +2208,8 @@ case_ok(PyObject *filename, Py_ssize_t prefix_delta, PyObject *name) match = case_bytes( PyBytes_AS_STRING(filebytes), PyBytes_GET_SIZE(filebytes) + prefix_delta, - PyBytes_AS_STRING(namebytes), - PyBytes_GET_SIZE(namebytes)); + PyBytes_GET_SIZE(namebytes), + PyBytes_AS_STRING(namebytes)); Py_DECREF(filebytes); Py_DECREF(namebytes); return match; -- cgit v1.2.1 From f8715e031dd6628c290bfc9341b092063449a6f8 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 20 Mar 2011 04:58:29 +0100 Subject: Issue #3080: Fix call to case_ok() in find_init_module() --- Python/import.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 0194d89956..38960b165d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2237,8 +2237,8 @@ find_init_module(PyObject *directory) if (filename == NULL) return -1; if (_Py_stat(filename, &statbuf) == 0) { - /* 9=len("/__init__") */ - match = case_ok(filename, 9, initstr); + /* 3=len(".py") */ + match = case_ok(filename, -3, initstr); if (match < 0) { Py_DECREF(filename); return -1; @@ -2255,8 +2255,8 @@ find_init_module(PyObject *directory) if (filename == NULL) return -1; if (_Py_stat(filename, &statbuf) == 0) { - /* 9=len("/__init__") */ - match = case_ok(filename, 9, initstr); + /* 4=len(".pyc") */ + match = case_ok(filename, -4, initstr); if (match < 0) { Py_DECREF(filename); return -1; -- cgit v1.2.1 From 6d71d6018e93b53f8af5ed0d268a4c834a5cdda2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 20 Mar 2011 22:37:17 +0100 Subject: Issue #3080: imp.load_module() accepts None for the module path imp.find_module() returns None as module path for builtin and frozen builtins. --- Python/import.c | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 38960b165d..abc3906719 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3744,17 +3744,22 @@ imp_load_source(PyObject *self, PyObject *args) static PyObject * imp_load_module(PyObject *self, PyObject *args) { - PyObject *name, *fob, *pathname, *ret; + PyObject *name, *fob, *pathname, *pathname_obj, *ret; char *suffix; /* Unused */ char *mode; int type; FILE *fp; - if (!PyArg_ParseTuple(args, "UOO&(ssi):load_module", - &name, &fob, - PyUnicode_FSDecoder, &pathname, - &suffix, &mode, &type)) + if (!PyArg_ParseTuple(args, "UOO(ssi):load_module", + &name, &fob, &pathname_obj, &suffix, &mode, &type)) return NULL; + if (pathname_obj != Py_None) { + if (!PyUnicode_FSDecoder(pathname_obj, &pathname)) + return NULL; + } + else + pathname = NULL; + if (*mode) { /* Mode must start with 'r' or 'U' and must not contain '+'. Implicit in this test is the assumption that the mode @@ -3763,7 +3768,7 @@ imp_load_module(PyObject *self, PyObject *args) if (!(*mode == 'r' || *mode == 'U') || strchr(mode, '+')) { PyErr_Format(PyExc_ValueError, "invalid file open mode %.200s", mode); - Py_DECREF(pathname); + Py_XDECREF(pathname); return NULL; } } @@ -3772,12 +3777,12 @@ imp_load_module(PyObject *self, PyObject *args) else { fp = get_file(NULL, fob, mode); if (fp == NULL) { - Py_DECREF(pathname); + Py_XDECREF(pathname); return NULL; } } ret = load_module(name, fp, pathname, type, NULL); - Py_DECREF(pathname); + Py_XDECREF(pathname); if (fp) fclose(fp); return ret; -- cgit v1.2.1 From 05292378fc049f9a5ffa9480d0ad27b25f2c5528 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2011 01:22:27 +0100 Subject: Issue #3080: On DJGPP, case_bytes() returns -1 to signal an error if the file cannot be found. --- Python/import.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index abc3906719..3b8ce931d9 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2084,7 +2084,7 @@ case_bytes(char *buf, Py_ssize_t len, Py_ssize_t namelen, const char *name) PyErr_Format(PyExc_NameError, "Can't find file for module %.100s\n(filename %.300s)", name, buf); - return 0; + return -1; } return strncmp(ffblk.ff_name, name, namelen) == 0; -- cgit v1.2.1 From aad56d7f47263bddcaca2765d03df11d81bc0926 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 22 Mar 2011 10:46:35 +0100 Subject: Issue #11630, issue #3080: Fix refleak introduced by ef2b6305d395 --- Python/import.c | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 3b8ce931d9..907ccd797c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3705,12 +3705,15 @@ imp_load_dynamic(PyObject *self, PyObject *args) return NULL; if (fob != NULL) { fp = get_file(NULL, fob, "r"); - if (fp == NULL) + if (fp == NULL) { + Py_DECREF(pathname); return NULL; + } } else fp = NULL; mod = _PyImport_LoadDynamicModule(name, pathname, fp); + Py_DECREF(pathname); if (fp) fclose(fp); return mod; -- cgit v1.2.1 From 725c21b3a4aec4426ccaa4c0425ee4e777a9eb35 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Wed, 23 Mar 2011 16:14:42 -0700 Subject: Make importlib compatible with __import__ by "fixing" code.co_filename paths. __import__ does a little trick when importing from bytecode by back-patching the co_filename paths to point to the file location where the code object was loaded from, *not* where the code object was originally created. This allows co_filename to point to a valid path. Problem is that co_filename is immutable from Python, so a private function -- imp._fix_co_filename() -- had to be introduced in order to get things working properly. Originally the plan was to add a file argument to marshal.loads(), but that failed as the algorithm used by __import__ is not fully recursive as one might expect, so to be fully backwards-compatible the code used by __import__ needed to be exposed. This closes issue #6811 by taking a different approach than outlined in the issue. --- Python/import.c | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 907ccd797c..b074b834e9 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1374,6 +1374,32 @@ update_compiled_module(PyCodeObject *co, PyObject *newname) Py_DECREF(oldname); } +static PyObject * +imp_fix_co_filename(PyObject *self, PyObject *args) +{ + PyObject *co; + PyObject *file_path; + + if (!PyArg_ParseTuple(args, "OO:_fix_co_filename", &co, &file_path)) + return NULL; + + if (!PyCode_Check(co)) { + PyErr_SetString(PyExc_TypeError, + "first argument must be a code object"); + return NULL; + } + + if (!PyUnicode_Check(file_path)) { + PyErr_SetString(PyExc_TypeError, + "second argument must be a string"); + return NULL; + } + + update_compiled_module((PyCodeObject*)co, file_path); + + Py_RETURN_NONE; +} + /* Load a source module from a given file and return its module object WITH INCREMENTED REFERENCE COUNT. If there's a matching byte-compiled file, use that instead. */ @@ -3976,6 +4002,7 @@ static PyMethodDef imp_methods[] = { #endif {"load_package", imp_load_package, METH_VARARGS}, {"load_source", imp_load_source, METH_VARARGS}, + {"_fix_co_filename", imp_fix_co_filename, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; -- cgit v1.2.1 From f41c2fdbb05ad29ad7c2a6a65dfac2e9d919f022 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 5 Apr 2011 01:48:03 +0200 Subject: Issue #9319: Include the filename in "Non-UTF8 code ..." syntax error. --- Python/import.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index b074b834e9..4159a8ece6 100644 --- a/Python/import.c +++ b/Python/import.c @@ -124,12 +124,12 @@ static const Py_UNICODE PYC_TAG_UNICODE[] = { /* See _PyImport_FixupExtensionObject() below */ static PyObject *extensions = NULL; +/* Function from Parser/tokenizer.c */ +extern char * PyTokenizer_FindEncodingFilename(int, PyObject *); + /* This table is defined in config.c: */ extern struct _inittab _PyImport_Inittab[]; -/* Method from Parser/tokenizer.c */ -extern char * PyTokenizer_FindEncoding(int); - struct _inittab *PyImport_Inittab = _PyImport_Inittab; /* these tables define the module suffixes that Python recognizes */ @@ -3540,9 +3540,9 @@ call_find_module(PyObject *name, PyObject *path_list) } if (fd != -1) { if (strchr(fdp->mode, 'b') == NULL) { - /* PyTokenizer_FindEncoding() returns PyMem_MALLOC'ed + /* PyTokenizer_FindEncodingFilename() returns PyMem_MALLOC'ed memory. */ - found_encoding = PyTokenizer_FindEncoding(fd); + found_encoding = PyTokenizer_FindEncodingFilename(fd, pathobj); lseek(fd, 0, 0); /* Reset position */ if (found_encoding == NULL && PyErr_Occurred()) { Py_XDECREF(pathobj); -- cgit v1.2.1 From 0cd7388f789c312864700c151157a939f07cc090 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Wed, 20 Apr 2011 03:27:51 +0200 Subject: Close #11619: write_compiled_module() doesn't encode the filename Reimplement open_exclusive() using _wopen() to avoid encoding the filename to the filesystem encoding: use the Unicode version of the Windows API. --- Python/import.c | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 4159a8ece6..5360d57c51 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1198,6 +1198,7 @@ parse_source_module(PyObject *pathname, FILE *fp) /* Helper to open a bytecode file for writing in exclusive mode */ +#ifndef MS_WINDOWS static FILE * open_exclusive(char *filename, mode_t mode) { @@ -1228,6 +1229,7 @@ open_exclusive(char *filename, mode_t mode) return fopen(filename, "wb"); #endif } +#endif /* Write a compiled module to a file, placing the time of last @@ -1250,7 +1252,12 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, S_IWUSR | S_IWGRP | S_IWOTH); PyObject *dirbytes; #endif - PyObject *cpathbytes, *dirname; +#ifdef MS_WINDOWS + int fd; +#else + PyObject *cpathbytes; +#endif + PyObject *dirname; Py_UNICODE *dirsep; int res, ok; @@ -1294,6 +1301,16 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, } Py_DECREF(dirname); +#ifdef MS_WINDOWS + (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname)); + fd = _wopen(PyUnicode_AS_UNICODE(cpathname), + O_EXCL | O_CREAT | O_WRONLY | O_TRUNC | O_BINARY, + mode); + if (0 <= fd) + fp = fdopen(fd, "wb"); + else + fp = NULL; +#else cpathbytes = PyUnicode_EncodeFSDefault(cpathname); if (cpathbytes == NULL) { PyErr_Clear(); @@ -1301,11 +1318,14 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, } fp = open_exclusive(PyBytes_AS_STRING(cpathbytes), mode); +#endif if (fp == NULL) { if (Py_VerboseFlag) PySys_FormatStderr( "# can't create %R\n", cpathname); +#ifndef MS_WINDOWS Py_DECREF(cpathbytes); +#endif return; } PyMarshal_WriteLongToFile(pyc_magic, fp, Py_MARSHAL_VERSION); @@ -1321,11 +1341,13 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname)); #else (void) unlink(PyBytes_AS_STRING(cpathbytes)); -#endif Py_DECREF(cpathbytes); +#endif return; } +#ifndef MS_WINDOWS Py_DECREF(cpathbytes); +#endif /* Now write the true mtime */ fseek(fp, 4L, 0); assert(mtime < LONG_MAX); -- cgit v1.2.1 From bf30ea7ad386f8851207ea9d410a3215b03d31af Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Wed, 25 May 2011 09:29:00 -0500 Subject: make PyImport_ImportModuleLevel's first arg const like similiar functions (closes #12173) --- Python/import.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 5360d57c51..bfb976c77c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2818,7 +2818,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, } PyObject * -PyImport_ImportModuleLevel(char *name, PyObject *globals, PyObject *locals, +PyImport_ImportModuleLevel(const char *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) { PyObject *nameobj, *mod; -- cgit v1.2.1 From 54ecf696bb489a653324f5b8a20018495b8f0142 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 26 May 2011 13:47:08 +0200 Subject: Remove useless assignments Warnings found by the the Clang Static Analyzer. --- Python/import.c | 4 ---- 1 file changed, 4 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index bfb976c77c..1f28d2233d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1733,7 +1733,6 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, Py_UNICODE buf[MAXPATHLEN+1]; Py_ssize_t buflen = MAXPATHLEN+1; PyObject *path_unicode, *filename; - const Py_UNICODE *base; Py_ssize_t len; struct stat statbuf; static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; @@ -1751,7 +1750,6 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, else return 0; - base = PyUnicode_AS_UNICODE(path_unicode); len = PyUnicode_GET_SIZE(path_unicode); if (len + 2 + PyUnicode_GET_SIZE(name) + MAXSUFFIXSIZE >= buflen) { Py_DECREF(path_unicode); @@ -2275,12 +2273,10 @@ case_ok(PyObject *filename, Py_ssize_t prefix_delta, PyObject *name) static int find_init_module(PyObject *directory) { - size_t len; struct stat statbuf; PyObject *filename; int match; - len = PyUnicode_GET_SIZE(directory); filename = PyUnicode_FromFormat("%U%c__init__.py", directory, SEP); if (filename == NULL) return -1; -- cgit v1.2.1 From dbea271885eccda0db2e5248fd932cc56ec8a495 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Fri, 3 Jun 2011 17:50:16 -0500 Subject: some horrible preprocessing tricks to automatically update the tag --- Python/import.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 1f28d2233d..aaf649f858 100644 --- a/Python/import.c +++ b/Python/import.c @@ -110,8 +110,12 @@ typedef unsigned short mode_t; TAG and PYC_TAG_UNICODE must change for each major Python release. The magic number will take care of any bytecode changes that occur during development. */ +#define QUOTE(arg) #arg +#define STRIFY(name) QUOTE(name) +#define MAJOR STRIFY(PY_MAJOR_VERSION) +#define MINOR STRIFY(PY_MINOR_VERSION) #define MAGIC (3180 | ((long)'\r'<<16) | ((long)'\n'<<24)) -#define TAG "cpython-32" +#define TAG "cpython-" MAJOR MINOR; #define CACHEDIR "__pycache__" static const Py_UNICODE CACHEDIR_UNICODE[] = { '_', '_', 'p', 'y', 'c', 'a', 'c', 'h', 'e', '_', '_', '\0'}; @@ -119,7 +123,11 @@ static const Py_UNICODE CACHEDIR_UNICODE[] = { static long pyc_magic = MAGIC; static const char *pyc_tag = TAG; static const Py_UNICODE PYC_TAG_UNICODE[] = { - 'c', 'p', 'y', 't', 'h', 'o', 'n', '-', '3', '2', '\0'}; + 'c', 'p', 'y', 't', 'h', 'o', 'n', '-', PY_MAJOR_VERSION + 48, PY_MINOR_VERSION + 48, '\0'}; +#undef QUOTE +#undef STRIFY +#undef MAJOR +#undef MINOR /* See _PyImport_FixupExtensionObject() below */ static PyObject *extensions = NULL; -- cgit v1.2.1 From 18d2c7a85fe49dbac1177fffb6517b8093f8e23c Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sun, 19 Jun 2011 19:54:45 -0500 Subject: bump magic for super closure change --- Python/import.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index aaf649f858..9ec8cd8b84 100644 --- a/Python/import.c +++ b/Python/import.c @@ -102,6 +102,7 @@ typedef unsigned short mode_t; Python 3.2a1: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR) tag: cpython-32 Python 3.2a2 3180 (add DELETE_DEREF) + Python 3.3a0 3190 __class__ super closure changed */ /* MAGIC must change whenever the bytecode emitted by the compiler may no @@ -114,7 +115,7 @@ typedef unsigned short mode_t; #define STRIFY(name) QUOTE(name) #define MAJOR STRIFY(PY_MAJOR_VERSION) #define MINOR STRIFY(PY_MINOR_VERSION) -#define MAGIC (3180 | ((long)'\r'<<16) | ((long)'\n'<<24)) +#define MAGIC (3190 | ((long)'\r'<<16) | ((long)'\n'<<24)) #define TAG "cpython-" MAJOR MINOR; #define CACHEDIR "__pycache__" static const Py_UNICODE CACHEDIR_UNICODE[] = { -- cgit v1.2.1 From 1a17e5818c9c58989ffdd4d0e2931db9c66579a0 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 20 Jun 2011 15:01:10 +0200 Subject: find_module_path_list() fails if _Py_fopen() failed and raised an exception (UnicodeEncodeError). --- Python/import.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 9ec8cd8b84..de5dc0ca41 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1908,6 +1908,10 @@ find_module_path_list(PyObject *fullname, PyObject *name, fp = _Py_fopen(filename, filemode); if (fp == NULL) { Py_DECREF(filename); + if (PyErr_Occurred()) { + Py_DECREF(prefix); + return NULL; + } continue; } match = case_ok(filename, -(Py_ssize_t)strlen(fdp->suffix), name); -- cgit v1.2.1 From fab579dcbba91dc35f5f2b25fcf221736fff5e4b Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Mon, 20 Jun 2011 15:16:55 +0200 Subject: call_find_module() handles dup() failure: raise an OSError exception --- Python/import.c | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index de5dc0ca41..19e975a593 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3567,6 +3567,10 @@ call_find_module(PyObject *name, PyObject *path_list) if (fd != -1) fd = dup(fd); fclose(fp); + if (fd == -1) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } fp = NULL; } if (fd != -1) { -- cgit v1.2.1 From 9ce343f33c35b14fe616a15cea33f8f84ccbff85 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 15 Sep 2011 19:50:01 +0200 Subject: import.c: remove now useless arbitrary limit --- Python/import.c | 6 ------ 1 file changed, 6 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 7902721a92..adfd2cc5df 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1980,12 +1980,6 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, if (p_loader != NULL) *p_loader = NULL; - if (PyUnicode_GET_SIZE(name) > MAXPATHLEN) { - PyErr_SetString(PyExc_OverflowError, - "module name is too long"); - return NULL; - } - /* sys.meta_path import hook */ if (p_loader != NULL) { PyObject *meta_path; -- cgit v1.2.1 From d1d013c01c268d869597b35cbcd8b5d7c5baf2ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Wed, 28 Sep 2011 07:41:54 +0200 Subject: Implement PEP 393. --- Python/import.c | 335 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 190 insertions(+), 145 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 24df985d46..4afdb7058c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -118,12 +118,12 @@ typedef unsigned short mode_t; #define MAGIC (3190 | ((long)'\r'<<16) | ((long)'\n'<<24)) #define TAG "cpython-" MAJOR MINOR; #define CACHEDIR "__pycache__" -static const Py_UNICODE CACHEDIR_UNICODE[] = { +static const Py_UCS4 CACHEDIR_UNICODE[] = { '_', '_', 'p', 'y', 'c', 'a', 'c', 'h', 'e', '_', '_', '\0'}; /* Current magic word and string tag as globals. */ static long pyc_magic = MAGIC; static const char *pyc_tag = TAG; -static const Py_UNICODE PYC_TAG_UNICODE[] = { +static const Py_UCS4 PYC_TAG_UNICODE[] = { 'c', 'p', 'y', 't', 'h', 'o', 'n', '-', PY_MAJOR_VERSION + 48, PY_MINOR_VERSION + 48, '\0'}; #undef QUOTE #undef STRIFY @@ -762,7 +762,7 @@ remove_module(PyObject *name) static PyObject * get_sourcefile(PyObject *filename); static PyObject *make_source_pathname(PyObject *pathname); -static PyObject* make_compiled_pathname(Py_UNICODE *pathname, int debug); +static PyObject* make_compiled_pathname(PyObject *pathname, int debug); /* Execute a code object in a module and return the module object * WITH INCREMENTED REFERENCE COUNT. If an error occurs, name is @@ -886,10 +886,10 @@ PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, /* Like strrchr(string, '/') but searches for the rightmost of either SEP or ALTSEP, if the latter is defined. */ -static Py_UNICODE* -rightmost_sep(Py_UNICODE *s) +static Py_UCS4* +rightmost_sep(Py_UCS4 *s) { - Py_UNICODE *found, c; + Py_UCS4 *found, c; for (found = NULL; (c = *s); s++) { if (c == SEP #ifdef ALTSEP @@ -912,15 +912,21 @@ rightmost_sep(Py_UNICODE *s) foo.py -> __pycache__/foo..pyc */ static PyObject* -make_compiled_pathname(Py_UNICODE *pathname, int debug) +make_compiled_pathname(PyObject *pathstr, int debug) { - Py_UNICODE buf[MAXPATHLEN]; + Py_UCS4 *pathname; + Py_UCS4 buf[MAXPATHLEN]; size_t buflen = (size_t)MAXPATHLEN; - size_t len = Py_UNICODE_strlen(pathname); + size_t len; size_t i, save; - Py_UNICODE *pos; + Py_UCS4 *pos; int sep = SEP; + pathname = PyUnicode_AsUCS4Copy(pathstr); + if (!pathname) + return NULL; + len = Py_UCS4_strlen(pathname); + /* Sanity check that the buffer has roughly enough space to hold what will eventually be the full path to the compiled file. The 5 extra bytes include the slash afer __pycache__, the two extra dots, the @@ -930,8 +936,10 @@ make_compiled_pathname(Py_UNICODE *pathname, int debug) sanity check before writing the extension to ensure we do not overflow the buffer. */ - if (len + Py_UNICODE_strlen(CACHEDIR_UNICODE) + Py_UNICODE_strlen(PYC_TAG_UNICODE) + 5 > buflen) + if (len + Py_UCS4_strlen(CACHEDIR_UNICODE) + Py_UCS4_strlen(PYC_TAG_UNICODE) + 5 > buflen) { + PyMem_Free(pathname); return NULL; + } /* Find the last path separator and copy everything from the start of the source string up to and including the separator. @@ -943,24 +951,28 @@ make_compiled_pathname(Py_UNICODE *pathname, int debug) else { sep = *pos; i = pos - pathname + 1; - Py_UNICODE_strncpy(buf, pathname, i); + Py_UCS4_strncpy(buf, pathname, i); } save = i; buf[i++] = '\0'; /* Add __pycache__/ */ - Py_UNICODE_strcat(buf, CACHEDIR_UNICODE); - i += Py_UNICODE_strlen(CACHEDIR_UNICODE) - 1; + Py_UCS4_strcat(buf, CACHEDIR_UNICODE); + i += Py_UCS4_strlen(CACHEDIR_UNICODE) - 1; buf[i++] = sep; buf[i] = '\0'; /* Add the base filename, but remove the .py or .pyw extension, since the tag name must go before the extension. */ - Py_UNICODE_strcat(buf, pathname + save); - pos = Py_UNICODE_strrchr(buf + i, '.'); + Py_UCS4_strcat(buf, pathname + save); + pos = Py_UCS4_strrchr(buf + i, '.'); if (pos != NULL) *++pos = '\0'; - Py_UNICODE_strcat(buf, PYC_TAG_UNICODE); + + /* pathname is not used from here on. */ + PyMem_Free(pathname); + + Py_UCS4_strcat(buf, PYC_TAG_UNICODE); /* The length test above assumes that we're only adding one character to the end of what would normally be the extension. What if there is no extension, or the string ends in '.' or '.p', and otherwise @@ -1010,7 +1022,7 @@ make_compiled_pathname(Py_UNICODE *pathname, int debug) #if 0 printf("strlen(buf): %d; buflen: %d\n", (int)strlen(buf), (int)buflen); #endif - len = Py_UNICODE_strlen(buf); + len = Py_UCS4_strlen(buf); if (len + 5 > buflen) return NULL; buf[len] = '.'; len++; @@ -1018,7 +1030,7 @@ make_compiled_pathname(Py_UNICODE *pathname, int debug) buf[len] = 'y'; len++; buf[len] = debug ? 'c' : 'o'; len++; assert(len <= buflen); - return PyUnicode_FromUnicode(buf, len); + return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, len); } @@ -1033,14 +1045,16 @@ make_compiled_pathname(Py_UNICODE *pathname, int debug) static PyObject* make_source_pathname(PyObject *pathobj) { - Py_UNICODE buf[MAXPATHLEN]; - Py_UNICODE *pathname; - Py_UNICODE *left, *right, *dot0, *dot1, sep; + Py_UCS4 buf[MAXPATHLEN]; + Py_UCS4 *pathname; + Py_UCS4 *left, *right, *dot0, *dot1, sep; size_t i, j; - if (PyUnicode_GET_SIZE(pathobj) > MAXPATHLEN) + if (PyUnicode_GET_LENGTH(pathobj) > MAXPATHLEN) + return NULL; + pathname = PyUnicode_AsUCS4Copy(pathobj); + if (!pathname) return NULL; - pathname = PyUnicode_AS_UNICODE(pathobj); /* Look back two slashes from the end. In between these two slashes must be the string __pycache__ or this is not a PEP 3147 style @@ -1057,31 +1071,35 @@ make_source_pathname(PyObject *pathobj) left = pathname; else left++; - if (right-left != Py_UNICODE_strlen(CACHEDIR_UNICODE) || - Py_UNICODE_strncmp(left, CACHEDIR_UNICODE, right-left) != 0) - return NULL; + if (right-left != Py_UCS4_strlen(CACHEDIR_UNICODE) || + Py_UCS4_strncmp(left, CACHEDIR_UNICODE, right-left) != 0) + goto error; /* Now verify that the path component to the right of the last slash has two dots in it. */ - if ((dot0 = Py_UNICODE_strchr(right + 1, '.')) == NULL) - return NULL; - if ((dot1 = Py_UNICODE_strchr(dot0 + 1, '.')) == NULL) - return NULL; + if ((dot0 = Py_UCS4_strchr(right + 1, '.')) == NULL) + goto error; + if ((dot1 = Py_UCS4_strchr(dot0 + 1, '.')) == NULL) + goto error; /* Too many dots? */ - if (Py_UNICODE_strchr(dot1 + 1, '.') != NULL) - return NULL; + if (Py_UCS4_strchr(dot1 + 1, '.') != NULL) + goto error; /* This is a PEP 3147 path. Start by copying everything from the start of pathname up to and including the leftmost slash. Then copy the file's basename, removing the magic tag and adding a .py suffix. */ - Py_UNICODE_strncpy(buf, pathname, (i=left-pathname)); - Py_UNICODE_strncpy(buf+i, right+1, (j=dot0-right)); + Py_UCS4_strncpy(buf, pathname, (i=left-pathname)); + Py_UCS4_strncpy(buf+i, right+1, (j=dot0-right)); buf[i+j] = 'p'; buf[i+j+1] = 'y'; - return PyUnicode_FromUnicode(buf, i+j+2); + PyMem_Free(pathname); + return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, i+j+2); + error: + PyMem_Free(pathname); + return NULL; } /* Given a pathname for a Python source file, its time of last @@ -1250,6 +1268,7 @@ static void write_compiled_module(PyCodeObject *co, PyObject *cpathname, struct stat *srcstat) { + Py_UCS4 *cpathname_ucs4; FILE *fp; time_t mtime = srcstat->st_mtime; #ifdef MS_WINDOWS /* since Windows uses different permissions */ @@ -1267,18 +1286,23 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, PyObject *cpathbytes; #endif PyObject *dirname; - Py_UNICODE *dirsep; + Py_UCS4 *dirsep; int res, ok; /* Ensure that the __pycache__ directory exists. */ - dirsep = rightmost_sep(PyUnicode_AS_UNICODE(cpathname)); + cpathname_ucs4 = PyUnicode_AsUCS4Copy(cpathname); + if (!cpathname_ucs4) + return; + dirsep = rightmost_sep(cpathname_ucs4); if (dirsep == NULL) { if (Py_VerboseFlag) PySys_FormatStderr("# no %s path found %R\n", CACHEDIR, cpathname); return; } - dirname = PyUnicode_FromUnicode(PyUnicode_AS_UNICODE(cpathname), - dirsep - PyUnicode_AS_UNICODE(cpathname)); + dirname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + cpathname_ucs4, + dirsep - cpathname_ucs4); + PyMem_Free(cpathname_ucs4); if (dirname == NULL) { PyErr_Clear(); return; @@ -1461,9 +1485,7 @@ load_source_module(PyObject *name, PyObject *pathname, FILE *fp) goto error; } #endif - cpathname = make_compiled_pathname( - PyUnicode_AS_UNICODE(pathname), - !Py_OptimizeFlag); + cpathname = make_compiled_pathname(pathname, !Py_OptimizeFlag); if (cpathname != NULL) fpc = check_compiled_module(pathname, st.st_mtime, cpathname); @@ -1512,16 +1534,18 @@ static PyObject * get_sourcefile(PyObject *filename) { Py_ssize_t len; - Py_UNICODE *fileuni; + Py_UCS4 *fileuni; PyObject *py; struct stat statbuf; - len = PyUnicode_GET_SIZE(filename); + len = PyUnicode_GET_LENGTH(filename); if (len == 0) Py_RETURN_NONE; /* don't match *.pyc or *.pyo? */ - fileuni = PyUnicode_AS_UNICODE(filename); + fileuni = PyUnicode_AsUCS4Copy(filename); + if (!fileuni) + return NULL; if (len < 5 || fileuni[len-4] != '.' || (fileuni[len-3] != 'p' && fileuni[len-3] != 'P') @@ -1535,7 +1559,7 @@ get_sourcefile(PyObject *filename) py = make_source_pathname(filename); if (py == NULL) { PyErr_Clear(); - py = PyUnicode_FromUnicode(fileuni, len - 1); + py = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, fileuni, len - 1); } if (py == NULL) goto error; @@ -1548,6 +1572,7 @@ get_sourcefile(PyObject *filename) error: PyErr_Clear(); unchanged: + PyMem_Free(fileuni); Py_INCREF(filename); return filename; } @@ -1739,8 +1764,7 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, PyObject *path_hooks, PyObject *path_importer_cache, PyObject **p_path, PyObject **p_loader, struct filedescr **p_fd) { - Py_UNICODE buf[MAXPATHLEN+1]; - Py_ssize_t buflen = MAXPATHLEN+1; + Py_UCS4 buf[MAXPATHLEN+1]; PyObject *path_unicode, *filename; Py_ssize_t len; struct stat statbuf; @@ -1759,15 +1783,15 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, else return 0; - len = PyUnicode_GET_SIZE(path_unicode); - if (len + 2 + PyUnicode_GET_SIZE(name) + MAXSUFFIXSIZE >= buflen) { + len = PyUnicode_GET_LENGTH(path_unicode); + if (!PyUnicode_AsUCS4(path_unicode, buf, PY_ARRAY_LENGTH(buf), 1)) { Py_DECREF(path_unicode); - return 0; /* Too long */ + PyErr_Clear(); + return 0; } - Py_UNICODE_strcpy(buf, PyUnicode_AS_UNICODE(path_unicode)); Py_DECREF(path_unicode); - if (Py_UNICODE_strlen(buf) != len) + if (Py_UCS4_strlen(buf) != len) return 0; /* path contains '\0' */ /* sys.path_hooks import hook */ @@ -1804,10 +1828,14 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, #endif ) buf[len++] = SEP; - Py_UNICODE_strcpy(buf+len, PyUnicode_AS_UNICODE(name)); - len += PyUnicode_GET_SIZE(name); + if (!PyUnicode_AsUCS4(name, buf+len, PY_ARRAY_LENGTH(buf)-len, 1)) { + PyErr_Clear(); + return 0; + } + len += PyUnicode_GET_LENGTH(name); - filename = PyUnicode_FromUnicode(buf, len); + filename = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + buf, len); if (filename == NULL) return -1; @@ -1989,6 +2017,12 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, if (p_loader != NULL) *p_loader = NULL; + if (PyUnicode_GET_LENGTH(name) > MAXPATHLEN) { + PyErr_SetString(PyExc_OverflowError, + "module name is too long"); + return NULL; + } + /* sys.meta_path import hook */ if (p_loader != NULL) { PyObject *meta_path; @@ -2704,7 +2738,7 @@ static PyObject *get_parent(PyObject *globals, int level); static PyObject *load_next(PyObject *mod, PyObject *altmod, PyObject *inputname, PyObject **p_outputname, - Py_UNICODE *buf, Py_ssize_t *p_buflen, + Py_UCS4 *buf, Py_ssize_t *p_buflen, Py_ssize_t bufsize); static int mark_miss(PyObject *name); static int ensure_fromlist(PyObject *mod, PyObject *fromlist, @@ -2718,37 +2752,47 @@ static PyObject * import_module_level(PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) { - Py_UNICODE buf[MAXPATHLEN+1]; + Py_UCS4 buf[MAXPATHLEN+1]; Py_ssize_t buflen; Py_ssize_t bufsize = MAXPATHLEN+1; PyObject *parent, *head, *next, *tail, *inputname, *outputname; PyObject *parent_name, *ensure_name; - const Py_UNICODE *nameunicode; + Py_ssize_t sep, altsep; - nameunicode = PyUnicode_AS_UNICODE(name); + if (PyUnicode_READY(name)) + return NULL; - if (Py_UNICODE_strchr(nameunicode, SEP) != NULL + sep = PyUnicode_FindChar(name, SEP, 0, PyUnicode_GET_LENGTH(name), 1); + if (sep == -2) + return NULL; #ifdef ALTSEP - || Py_UNICODE_strchr(nameunicode, ALTSEP) != NULL + altsep = PyUnicode_FindChar(name, ALTSEP, 0, PyUnicode_GET_LENGTH(name), 1); + if (altsep == -2) + return NULL; +#else + altsep = -1; #endif - ) { + if (sep != -1 || altsep != -1) + { PyErr_SetString(PyExc_ImportError, "Import by filename is not supported."); return NULL; } parent = get_parent(globals, &parent_name, level); - if (parent == NULL) + if (parent == NULL) { return NULL; + } - buflen = PyUnicode_GET_SIZE(parent_name); - if (buflen+1 > bufsize) { + if (PyUnicode_READY(parent_name)) + return NULL; + buflen = PyUnicode_GET_LENGTH(parent_name); + if (!PyUnicode_AsUCS4(parent_name, buf, PY_ARRAY_LENGTH(buf), 1)) { Py_DECREF(parent_name); PyErr_SetString(PyExc_ValueError, "Module name too long"); return NULL; } - Py_UNICODE_strcpy(buf, PyUnicode_AS_UNICODE(parent_name)); Py_DECREF(parent_name); head = load_next(parent, level < 0 ? Py_None : parent, name, &outputname, @@ -2799,7 +2843,8 @@ import_module_level(PyObject *name, PyObject *globals, PyObject *locals, Py_DECREF(head); - ensure_name = PyUnicode_FromUnicode(buf, Py_UNICODE_strlen(buf)); + ensure_name = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + buf, Py_UCS4_strlen(buf)); if (ensure_name == NULL) { Py_DECREF(tail); return NULL; @@ -2859,8 +2904,6 @@ PyImport_ImportModuleLevel(const char *name, PyObject *globals, PyObject *locals static PyObject * get_parent(PyObject *globals, PyObject **p_name, int level) { - Py_UNICODE name[MAXPATHLEN+1]; - const Py_ssize_t bufsize = MAXPATHLEN+1; PyObject *nameobj; static PyObject *namestr = NULL; @@ -2897,7 +2940,7 @@ get_parent(PyObject *globals, PyObject **p_name, int level) "__package__ set to non-string"); return NULL; } - if (PyUnicode_GET_SIZE(pkgname) == 0) { + if (PyUnicode_GET_LENGTH(pkgname) == 0) { if (level > 0) { PyErr_SetString(PyExc_ValueError, "Attempted relative import in non-package"); @@ -2905,12 +2948,8 @@ get_parent(PyObject *globals, PyObject **p_name, int level) } goto return_none; } - if (PyUnicode_GET_SIZE(pkgname)+1 > bufsize) { - PyErr_SetString(PyExc_ValueError, - "Package name too long"); - return NULL; - } - Py_UNICODE_strcpy(name, PyUnicode_AS_UNICODE(pkgname)); + Py_INCREF(pkgname); + nameobj = pkgname; } else { /* __package__ not set, so figure it out and set it */ modname = PyDict_GetItem(globals, namestr); @@ -2922,74 +2961,71 @@ get_parent(PyObject *globals, PyObject **p_name, int level) /* __path__ is set, so modname is already the package name */ int error; - if (PyUnicode_GET_SIZE(modname)+1 > bufsize) { - PyErr_SetString(PyExc_ValueError, - "Module name too long"); - return NULL; - } - Py_UNICODE_strcpy(name, PyUnicode_AS_UNICODE(modname)); error = PyDict_SetItem(globals, pkgstr, modname); if (error) { PyErr_SetString(PyExc_ValueError, "Could not set __package__"); return NULL; } + Py_INCREF(modname); + nameobj = modname; } else { /* Normal module, so work out the package name if any */ - Py_UNICODE *start = PyUnicode_AS_UNICODE(modname); - Py_UNICODE *lastdot = Py_UNICODE_strrchr(start, '.'); Py_ssize_t len; - int error; - if (lastdot == NULL && level > 0) { - PyErr_SetString(PyExc_ValueError, - "Attempted relative import in non-package"); + len = PyUnicode_FindChar(modname, '.', + 0, PyUnicode_GET_LENGTH(modname), -1); + if (len == -2) return NULL; - } - if (lastdot == NULL) { - error = PyDict_SetItem(globals, pkgstr, Py_None); - if (error) { + if (len < 0) { + if (level > 0) { + PyErr_SetString(PyExc_ValueError, + "Attempted relative import in non-package"); + return NULL; + } + if (PyDict_SetItem(globals, pkgstr, Py_None)) { PyErr_SetString(PyExc_ValueError, "Could not set __package__"); return NULL; } goto return_none; } - len = lastdot - start; - if (len+1 > bufsize) { + pkgname = PyUnicode_Substring(modname, 0, len); + if (pkgname == NULL) + return NULL; + if (PyDict_SetItem(globals, pkgstr, pkgname)) { + Py_DECREF(pkgname); PyErr_SetString(PyExc_ValueError, - "Module name too long"); + "Could not set __package__"); return NULL; } - Py_UNICODE_strncpy(name, start, len); - name[len] = '\0'; - pkgname = PyUnicode_FromUnicode(name, len); - if (pkgname == NULL) { + nameobj = pkgname; + } + } + if (level > 1) { + Py_ssize_t dot, end = PyUnicode_GET_LENGTH(nameobj); + PyObject *newname; + while (--level > 0) { + dot = PyUnicode_FindChar(nameobj, '.', 0, end, -1); + if (dot == -2) { + Py_DECREF(nameobj); return NULL; } - error = PyDict_SetItem(globals, pkgstr, pkgname); - Py_DECREF(pkgname); - if (error) { + if (dot < 0) { + Py_DECREF(nameobj); PyErr_SetString(PyExc_ValueError, - "Could not set __package__"); + "Attempted relative import beyond " + "toplevel package"); return NULL; } + end = dot; } - } - while (--level > 0) { - Py_UNICODE *dot = Py_UNICODE_strrchr(name, '.'); - if (dot == NULL) { - PyErr_SetString(PyExc_ValueError, - "Attempted relative import beyond " - "toplevel package"); + newname = PyUnicode_Substring(nameobj, 0, end); + Py_DECREF(nameobj); + if (newname == NULL) return NULL; - } - *dot = '\0'; + nameobj = newname; } - nameobj = PyUnicode_FromUnicode(name, Py_UNICODE_strlen(name)); - if (nameobj == NULL) - return NULL; - modules = PyImport_GetModuleDict(); parent = PyDict_GetItem(modules, nameobj); if (parent == NULL) { @@ -3021,7 +3057,7 @@ get_parent(PyObject *globals, PyObject **p_name, int level) If this is violated... Who cares? */ return_none: - nameobj = PyUnicode_FromUnicode(NULL, 0); + nameobj = PyUnicode_New(0, 0); if (nameobj == NULL) return NULL; *p_name = nameobj; @@ -3032,28 +3068,28 @@ return_none: static PyObject * load_next(PyObject *mod, PyObject *altmod, PyObject *inputname, PyObject **p_outputname, - Py_UNICODE *buf, Py_ssize_t *p_buflen, Py_ssize_t bufsize) + Py_UCS4 *buf, Py_ssize_t *p_buflen, Py_ssize_t bufsize) { - const Py_UNICODE *dot; + Py_UCS4 *dot; Py_ssize_t len; - Py_UNICODE *p; + Py_UCS4 *p; PyObject *fullname, *name, *result, *mark_name; - const Py_UNICODE *nameuni; + const Py_UCS4 *nameuni; *p_outputname = NULL; - if (PyUnicode_GET_SIZE(inputname) == 0) { + if (PyUnicode_GET_LENGTH(inputname) == 0) { /* completely empty module name should only happen in 'from . import' (or '__import__("")')*/ Py_INCREF(mod); return mod; } - nameuni = PyUnicode_AS_UNICODE(inputname); + nameuni = PyUnicode_AsUCS4Copy(inputname); if (nameuni == NULL) return NULL; - dot = Py_UNICODE_strchr(nameuni, '.'); + dot = Py_UCS4_strchr(nameuni, '.'); if (dot != NULL) { len = dot - nameuni; if (len == 0) { @@ -3063,7 +3099,7 @@ load_next(PyObject *mod, PyObject *altmod, } } else - len = PyUnicode_GET_SIZE(inputname); + len = PyUnicode_GET_LENGTH(inputname); if (*p_buflen+len+1 >= bufsize) { PyErr_SetString(PyExc_ValueError, @@ -3076,14 +3112,16 @@ load_next(PyObject *mod, PyObject *altmod, *p++ = '.'; *p_buflen += 1; } - Py_UNICODE_strncpy(p, nameuni, len); + Py_UCS4_strncpy(p, nameuni, len); p[len] = '\0'; *p_buflen += len; - fullname = PyUnicode_FromUnicode(buf, *p_buflen); + fullname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + buf, *p_buflen); if (fullname == NULL) return NULL; - name = PyUnicode_FromUnicode(p, len); + name = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + p, len); if (name == NULL) { Py_DECREF(fullname); return NULL; @@ -3096,7 +3134,8 @@ load_next(PyObject *mod, PyObject *altmod, result = import_submodule(altmod, name, name); Py_DECREF(name); if (result != NULL && result != Py_None) { - mark_name = PyUnicode_FromUnicode(buf, *p_buflen); + mark_name = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + buf, *p_buflen); if (mark_name == NULL) { Py_DECREF(result); return NULL; @@ -3107,7 +3146,7 @@ load_next(PyObject *mod, PyObject *altmod, return NULL; } Py_DECREF(mark_name); - Py_UNICODE_strncpy(buf, nameuni, len); + Py_UCS4_strncpy(buf, nameuni, len); buf[len] = '\0'; *p_buflen = len; } @@ -3125,7 +3164,8 @@ load_next(PyObject *mod, PyObject *altmod, } if (dot != NULL) { - *p_outputname = PyUnicode_FromUnicode(dot+1, Py_UNICODE_strlen(dot+1)); + *p_outputname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + dot+1, Py_UCS4_strlen(dot+1)); if (*p_outputname == NULL) { Py_DECREF(result); return NULL; @@ -3166,7 +3206,7 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, PyObject *name, Py_DECREF(item); return 0; } - if (PyUnicode_AS_UNICODE(item)[0] == '*') { + if (PyUnicode_READ_CHAR(item, 0) == '*') { PyObject *all; Py_DECREF(item); /* See if the package defines __all__ */ @@ -3304,7 +3344,7 @@ PyImport_ReloadModule(PyObject *m) PyObject *modules = PyImport_GetModuleDict(); PyObject *path_list = NULL, *loader = NULL, *existing_m = NULL; PyObject *nameobj, *bufobj, *subnameobj; - Py_UNICODE *name, *subname; + Py_UCS4 *name = NULL, *subname; struct filedescr *fdp; FILE *fp = NULL; PyObject *newm = NULL; @@ -3321,7 +3361,7 @@ PyImport_ReloadModule(PyObject *m) return NULL; } nameobj = PyModule_GetNameObject(m); - if (nameobj == NULL) + if (nameobj == NULL || PyUnicode_READY(nameobj) == -1) return NULL; if (m != PyDict_GetItem(modules, nameobj)) { PyErr_Format(PyExc_ImportError, @@ -3343,8 +3383,12 @@ PyImport_ReloadModule(PyObject *m) return NULL; } - name = PyUnicode_AS_UNICODE(nameobj); - subname = Py_UNICODE_strrchr(name, '.'); + name = PyUnicode_AsUCS4Copy(nameobj); + if (!name) { + Py_DECREF(nameobj); + return NULL; + } + subname = Py_UCS4_strrchr(name, '.'); if (subname == NULL) { Py_INCREF(nameobj); subnameobj = nameobj; @@ -3353,7 +3397,8 @@ PyImport_ReloadModule(PyObject *m) PyObject *parentname, *parent; Py_ssize_t len; len = subname - name; - parentname = PyUnicode_FromUnicode(name, len); + parentname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + name, len); if (parentname == NULL) { goto error; } @@ -3370,8 +3415,9 @@ PyImport_ReloadModule(PyObject *m) if (path_list == NULL) PyErr_Clear(); subname++; - len = PyUnicode_GET_SIZE(nameobj) - (len + 1); - subnameobj = PyUnicode_FromUnicode(subname, len); + len = PyUnicode_GET_LENGTH(nameobj) - (len + 1); + subnameobj = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, + subname, len); } if (subnameobj == NULL) goto error; @@ -3403,6 +3449,7 @@ PyImport_ReloadModule(PyObject *m) error: imp_modules_reloading_clear(); Py_DECREF(nameobj); + PyMem_Free(name); return newm; } @@ -3910,9 +3957,7 @@ imp_cache_from_source(PyObject *self, PyObject *args, PyObject *kws) return NULL; } - cpathname = make_compiled_pathname( - PyUnicode_AS_UNICODE(pathname), - debug); + cpathname = make_compiled_pathname(pathname, debug); Py_DECREF(pathname); if (cpathname == NULL) { @@ -4105,7 +4150,7 @@ NullImporter_init(NullImporter *self, PyObject *args, PyObject *kwds) &pathobj)) return -1; - if (PyUnicode_GET_SIZE(pathobj) == 0) { + if (PyUnicode_GET_LENGTH(pathobj) == 0) { PyErr_SetString(PyExc_ImportError, "empty pathname"); return -1; } -- cgit v1.2.1 From 593041f3e6ac869b37c7da4355b69b1adc173b37 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Wed, 28 Sep 2011 21:49:49 +0200 Subject: Rename new macros to conform to naming rules (function macros have "Py" prefix, not "PY"). --- Python/import.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 4afdb7058c..9f38061723 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1784,7 +1784,7 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, return 0; len = PyUnicode_GET_LENGTH(path_unicode); - if (!PyUnicode_AsUCS4(path_unicode, buf, PY_ARRAY_LENGTH(buf), 1)) { + if (!PyUnicode_AsUCS4(path_unicode, buf, Py_ARRAY_LENGTH(buf), 1)) { Py_DECREF(path_unicode); PyErr_Clear(); return 0; @@ -1828,7 +1828,7 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, #endif ) buf[len++] = SEP; - if (!PyUnicode_AsUCS4(name, buf+len, PY_ARRAY_LENGTH(buf)-len, 1)) { + if (!PyUnicode_AsUCS4(name, buf+len, Py_ARRAY_LENGTH(buf)-len, 1)) { PyErr_Clear(); return 0; } @@ -2787,7 +2787,7 @@ import_module_level(PyObject *name, PyObject *globals, PyObject *locals, if (PyUnicode_READY(parent_name)) return NULL; buflen = PyUnicode_GET_LENGTH(parent_name); - if (!PyUnicode_AsUCS4(parent_name, buf, PY_ARRAY_LENGTH(buf), 1)) { + if (!PyUnicode_AsUCS4(parent_name, buf, Py_ARRAY_LENGTH(buf), 1)) { Py_DECREF(parent_name); PyErr_SetString(PyExc_ValueError, "Module name too long"); -- cgit v1.2.1 From 2ab1e534bf781097c59588983df13f737c912576 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sat, 1 Oct 2011 16:35:40 +0200 Subject: Issue 13085: Fix some memory leaks. Patch by Stefan Krah. --- Python/import.c | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 9f38061723..901b733480 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1564,8 +1564,10 @@ get_sourcefile(PyObject *filename) if (py == NULL) goto error; - if (_Py_stat(py, &statbuf) == 0 && S_ISREG(statbuf.st_mode)) + if (_Py_stat(py, &statbuf) == 0 && S_ISREG(statbuf.st_mode)) { + PyMem_Free(fileuni); return py; + } Py_DECREF(py); goto unchanged; @@ -3074,7 +3076,7 @@ load_next(PyObject *mod, PyObject *altmod, Py_ssize_t len; Py_UCS4 *p; PyObject *fullname, *name, *result, *mark_name; - const Py_UCS4 *nameuni; + Py_UCS4 *nameuni; *p_outputname = NULL; @@ -3095,7 +3097,7 @@ load_next(PyObject *mod, PyObject *altmod, if (len == 0) { PyErr_SetString(PyExc_ValueError, "Empty module name"); - return NULL; + goto error; } } else @@ -3104,7 +3106,7 @@ load_next(PyObject *mod, PyObject *altmod, if (*p_buflen+len+1 >= bufsize) { PyErr_SetString(PyExc_ValueError, "Module name too long"); - return NULL; + goto error; } p = buf + *p_buflen; @@ -3119,12 +3121,12 @@ load_next(PyObject *mod, PyObject *altmod, fullname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, *p_buflen); if (fullname == NULL) - return NULL; + goto error; name = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, p, len); if (name == NULL) { Py_DECREF(fullname); - return NULL; + goto error; } result = import_submodule(mod, name, fullname); Py_DECREF(fullname); @@ -3138,12 +3140,12 @@ load_next(PyObject *mod, PyObject *altmod, buf, *p_buflen); if (mark_name == NULL) { Py_DECREF(result); - return NULL; + goto error; } if (mark_miss(mark_name) != 0) { Py_DECREF(result); Py_DECREF(mark_name); - return NULL; + goto error; } Py_DECREF(mark_name); Py_UCS4_strncpy(buf, nameuni, len); @@ -3154,13 +3156,13 @@ load_next(PyObject *mod, PyObject *altmod, else Py_DECREF(name); if (result == NULL) - return NULL; + goto error; if (result == Py_None) { Py_DECREF(result); PyErr_Format(PyExc_ImportError, "No module named %R", inputname); - return NULL; + goto error; } if (dot != NULL) { @@ -3168,11 +3170,17 @@ load_next(PyObject *mod, PyObject *altmod, dot+1, Py_UCS4_strlen(dot+1)); if (*p_outputname == NULL) { Py_DECREF(result); - return NULL; + goto error; } } +out: + PyMem_Free(nameuni); return result; + +error: + PyMem_Free(nameuni); + return NULL; } static int -- cgit v1.2.1 From 7a98f5f5ced85e78dc79169bfaeaa1b3adfa0596 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Sun, 2 Oct 2011 13:19:16 -0400 Subject: remove unused label --- Python/import.c | 1 - 1 file changed, 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 901b733480..9f094c0da0 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3174,7 +3174,6 @@ load_next(PyObject *mod, PyObject *altmod, } } -out: PyMem_Free(nameuni); return result; -- cgit v1.2.1 From 0aeb34c31cc7bfc0c0fd5595b017e3f9dc0a93ef Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 6 Oct 2011 02:39:42 +0200 Subject: Fix find_module_path(): make the string ready --- Python/import.c | 3 +++ 1 file changed, 3 insertions(+) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 9f094c0da0..5f84ac2f74 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1785,6 +1785,9 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, else return 0; + if (PyUnicode_READY(path_unicode)) + return -1; + len = PyUnicode_GET_LENGTH(path_unicode); if (!PyUnicode_AsUCS4(path_unicode, buf, Py_ARRAY_LENGTH(buf), 1)) { Py_DECREF(path_unicode); -- cgit v1.2.1 From dcacb24f386ae1b107462181af42b6826ec15fb7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 9 Oct 2011 10:38:36 +0200 Subject: =?UTF-8?q?Add=20API=20for=20static=20strings,=20primarily=20good?= =?UTF-8?q?=20for=20identifiers.=20Thanks=20to=20Konrad=20Sch=C3=B6bel=20a?= =?UTF-8?q?nd=20Jasper=20Schulz=20for=20helping=20with=20the=20mass-editin?= =?UTF-8?q?g.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Python/import.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 5f84ac2f74..6eca90a0f7 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1801,6 +1801,7 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, /* sys.path_hooks import hook */ if (p_loader != NULL) { + _Py_identifier(find_module); PyObject *importer; importer = get_path_importer(path_importer_cache, @@ -1811,8 +1812,8 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, /* Note: importer is a borrowed reference */ if (importer != Py_None) { PyObject *loader; - loader = PyObject_CallMethod(importer, - "find_module", "O", fullname); + loader = _PyObject_CallMethodId(importer, + &PyId_find_module, "O", fullname); if (loader == NULL) return -1; /* error */ if (loader != Py_None) { @@ -2030,6 +2031,7 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, /* sys.meta_path import hook */ if (p_loader != NULL) { + _Py_identifier(find_module); PyObject *meta_path; meta_path = PySys_GetObject("meta_path"); @@ -2044,7 +2046,7 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, for (i = 0; i < npath; i++) { PyObject *loader; PyObject *hook = PyList_GetItem(meta_path, i); - loader = PyObject_CallMethod(hook, "find_module", + loader = _PyObject_CallMethodId(hook, &PyId_find_module, "OO", fullname, search_path_list != NULL ? search_path_list : Py_None); @@ -2454,12 +2456,13 @@ load_module(PyObject *name, FILE *fp, PyObject *pathname, int type, PyObject *lo break; case IMP_HOOK: { + _Py_identifier(load_module); if (loader == NULL) { PyErr_SetString(PyExc_ImportError, "import hook without loader"); return NULL; } - m = PyObject_CallMethod(loader, "load_module", "O", name); + m = _PyObject_CallMethodId(loader, &PyId_load_module, "O", name); break; } -- cgit v1.2.1 From c3eb93fe1ce5f5ff2666fb9f34a6fd7c46279554 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Mon, 10 Oct 2011 18:11:30 +0200 Subject: Use identifier API for PyObject_GetAttrString. --- Python/import.c | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 6eca90a0f7..3d6c6ed1d4 100644 --- a/Python/import.c +++ b/Python/import.c @@ -154,7 +154,7 @@ static const struct filedescr _PyImport_StandardFiletab[] = { }; static PyObject *initstr = NULL; - +_Py_identifier(__path__); /* Initialize things */ @@ -248,8 +248,9 @@ _PyImportHooks_Init(void) PySys_WriteStderr("# can't import zipimport\n"); } else { - PyObject *zipimporter = PyObject_GetAttrString(zimpimport, - "zipimporter"); + _Py_identifier(zipimporter); + PyObject *zipimporter = _PyObject_GetAttrId(zimpimport, + &PyId_zipimporter); Py_DECREF(zimpimport); if (zipimporter == NULL) { PyErr_Clear(); /* No zipimporter object -- okay */ @@ -3203,7 +3204,7 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, PyObject *name, PyObject *fullname; Py_ssize_t fromlist_len; - if (!PyObject_HasAttrString(mod, "__path__")) + if (!_PyObject_HasAttrId(mod, &PyId___path__)) return 1; fromlist_len = PySequence_Size(fromlist); @@ -3221,11 +3222,12 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, PyObject *name, } if (PyUnicode_READ_CHAR(item, 0) == '*') { PyObject *all; + _Py_identifier(__all__); Py_DECREF(item); /* See if the package defines __all__ */ if (recursive) continue; /* Avoid endless recursion */ - all = PyObject_GetAttrString(mod, "__all__"); + all = _PyObject_GetAttrId(mod, &PyId___all__); if (all == NULL) PyErr_Clear(); else { @@ -3313,7 +3315,7 @@ import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) if (mod == Py_None) path_list = NULL; else { - path_list = PyObject_GetAttrString(mod, "__path__"); + path_list = _PyObject_GetAttrId(mod, &PyId___path__); if (path_list == NULL) { PyErr_Clear(); Py_INCREF(Py_None); @@ -3424,7 +3426,7 @@ PyImport_ReloadModule(PyObject *m) goto error; } Py_DECREF(parentname); - path_list = PyObject_GetAttrString(parent, "__path__"); + path_list = _PyObject_GetAttrId(parent, &PyId___path__); if (path_list == NULL) PyErr_Clear(); subname++; -- cgit v1.2.1 From 9fde12ef0a3cbe4260897de13f054ec630456992 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 11 Oct 2011 21:55:01 +0200 Subject: Use PyUnicode_AsUnicodeAndSize() instead of PyUnicode_GET_SIZE() --- Python/import.c | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 3d6c6ed1d4..6f564a65da 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2282,6 +2282,8 @@ case_ok(PyObject *filename, Py_ssize_t prefix_delta, PyObject *name) WIN32_FIND_DATAW data; HANDLE h; int cmp; + wchar_t *wname; + Py_ssizet wname_len; if (Py_GETENV("PYTHONCASEOK") != NULL) return 1; @@ -2294,9 +2296,12 @@ case_ok(PyObject *filename, Py_ssize_t prefix_delta, PyObject *name) return 0; } FindClose(h); - cmp = wcsncmp(data.cFileName, - PyUnicode_AS_UNICODE(name), - PyUnicode_GET_SIZE(name)); + + wname = PyUnicode_AsUnicodeAndSize(name, &wname_len); + if (wname == NULL) + return -1; + + cmp = wcsncmp(data.cFileName, wname, wname_len); return cmp == 0; #elif defined(USE_CASE_OK_BYTES) int match; -- cgit v1.2.1 From f716b4e95cf727dc9fe4a592c38e86b73ca16c74 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Tue, 11 Oct 2011 22:27:13 +0200 Subject: Fix typo in import.c --- Python/import.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 6f564a65da..d937cc165a 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2283,7 +2283,7 @@ case_ok(PyObject *filename, Py_ssize_t prefix_delta, PyObject *name) HANDLE h; int cmp; wchar_t *wname; - Py_ssizet wname_len; + Py_ssize_t wname_len; if (Py_GETENV("PYTHONCASEOK") != NULL) return 1; -- cgit v1.2.1 From f5ad3b280b43227eb0e3fa63d89490a58ba9c28b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Fri, 14 Oct 2011 10:20:37 +0200 Subject: Rename _Py_identifier to _Py_IDENTIFIER. --- Python/import.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index d937cc165a..6e3e0f181e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -154,7 +154,7 @@ static const struct filedescr _PyImport_StandardFiletab[] = { }; static PyObject *initstr = NULL; -_Py_identifier(__path__); +_Py_IDENTIFIER(__path__); /* Initialize things */ @@ -248,7 +248,7 @@ _PyImportHooks_Init(void) PySys_WriteStderr("# can't import zipimport\n"); } else { - _Py_identifier(zipimporter); + _Py_IDENTIFIER(zipimporter); PyObject *zipimporter = _PyObject_GetAttrId(zimpimport, &PyId_zipimporter); Py_DECREF(zimpimport); @@ -1802,7 +1802,7 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, /* sys.path_hooks import hook */ if (p_loader != NULL) { - _Py_identifier(find_module); + _Py_IDENTIFIER(find_module); PyObject *importer; importer = get_path_importer(path_importer_cache, @@ -2032,7 +2032,7 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, /* sys.meta_path import hook */ if (p_loader != NULL) { - _Py_identifier(find_module); + _Py_IDENTIFIER(find_module); PyObject *meta_path; meta_path = PySys_GetObject("meta_path"); @@ -2462,7 +2462,7 @@ load_module(PyObject *name, FILE *fp, PyObject *pathname, int type, PyObject *lo break; case IMP_HOOK: { - _Py_identifier(load_module); + _Py_IDENTIFIER(load_module); if (loader == NULL) { PyErr_SetString(PyExc_ImportError, "import hook without loader"); @@ -3227,7 +3227,7 @@ ensure_fromlist(PyObject *mod, PyObject *fromlist, PyObject *name, } if (PyUnicode_READ_CHAR(item, 0) == '*') { PyObject *all; - _Py_identifier(__all__); + _Py_IDENTIFIER(__all__); Py_DECREF(item); /* See if the package defines __all__ */ if (recursive) -- cgit v1.2.1 From 5263d9e0be0bf742276426378f96f8b6e6687872 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 17 Oct 2011 19:28:44 +0200 Subject: Issue #13146: Writing a pyc file is now atomic under POSIX. --- Python/import.c | 45 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 36 insertions(+), 9 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 6e3e0f181e..0db7e43d29 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1284,7 +1284,8 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, #ifdef MS_WINDOWS int fd; #else - PyObject *cpathbytes; + PyObject *cpathbytes, *cpathbytes_tmp; + Py_ssize_t cpathbytes_len; #endif PyObject *dirname; Py_UCS4 *dirsep; @@ -1345,13 +1346,25 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, else fp = NULL; #else + /* Under POSIX, we first write to a tmp file and then take advantage + of atomic renaming. */ cpathbytes = PyUnicode_EncodeFSDefault(cpathname); if (cpathbytes == NULL) { PyErr_Clear(); return; } + cpathbytes_len = PyBytes_GET_SIZE(cpathbytes); + cpathbytes_tmp = PyBytes_FromStringAndSize(NULL, cpathbytes_len + 4); + if (cpathbytes_tmp == NULL) { + Py_DECREF(cpathbytes); + PyErr_Clear(); + return; + } + memcpy(PyBytes_AS_STRING(cpathbytes_tmp), PyBytes_AS_STRING(cpathbytes), + cpathbytes_len); + memcpy(PyBytes_AS_STRING(cpathbytes_tmp) + cpathbytes_len, ".tmp", 4); - fp = open_exclusive(PyBytes_AS_STRING(cpathbytes), mode); + fp = open_exclusive(PyBytes_AS_STRING(cpathbytes_tmp), mode); #endif if (fp == NULL) { if (Py_VerboseFlag) @@ -1359,6 +1372,7 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, "# can't create %R\n", cpathname); #ifndef MS_WINDOWS Py_DECREF(cpathbytes); + Py_DECREF(cpathbytes_tmp); #endif return; } @@ -1366,6 +1380,11 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, /* First write a 0 for mtime */ PyMarshal_WriteLongToFile(0L, fp, Py_MARSHAL_VERSION); PyMarshal_WriteObjectToFile((PyObject *)co, fp, Py_MARSHAL_VERSION); + fflush(fp); + /* Now write the true mtime */ + fseek(fp, 4L, 0); + assert(mtime < LONG_MAX); + PyMarshal_WriteLongToFile((long)mtime, fp, Py_MARSHAL_VERSION); if (fflush(fp) != 0 || ferror(fp)) { if (Py_VerboseFlag) PySys_FormatStderr("# can't write %R\n", cpathname); @@ -1374,20 +1393,28 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, #ifdef MS_WINDOWS (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname)); #else - (void) unlink(PyBytes_AS_STRING(cpathbytes)); + (void) unlink(PyBytes_AS_STRING(cpathbytes_tmp)); Py_DECREF(cpathbytes); + Py_DECREF(cpathbytes_tmp); #endif return; } + fclose(fp); + /* Under POSIX, do an atomic rename */ #ifndef MS_WINDOWS + if (rename(PyBytes_AS_STRING(cpathbytes_tmp), + PyBytes_AS_STRING(cpathbytes))) { + if (Py_VerboseFlag) + PySys_FormatStderr("# can't write %R\n", cpathname); + /* Don't keep tmp file */ + unlink(PyBytes_AS_STRING(cpathbytes_tmp)); + Py_DECREF(cpathbytes); + Py_DECREF(cpathbytes_tmp); + return; + } Py_DECREF(cpathbytes); + Py_DECREF(cpathbytes_tmp); #endif - /* Now write the true mtime */ - fseek(fp, 4L, 0); - assert(mtime < LONG_MAX); - PyMarshal_WriteLongToFile((long)mtime, fp, Py_MARSHAL_VERSION); - fflush(fp); - fclose(fp); if (Py_VerboseFlag) PySys_FormatStderr("# wrote %R\n", cpathname); } -- cgit v1.2.1 From 787a4931e7a65741deb81c532dfb184af13a0e20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 23 Oct 2011 17:29:08 +0200 Subject: Reformulate make_compiled_pathname in terms of unicode objects. --- Python/import.c | 177 +++++++++++++++++++------------------------------------- 1 file changed, 61 insertions(+), 116 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 0db7e43d29..1101c920f5 100644 --- a/Python/import.c +++ b/Python/import.c @@ -904,6 +904,25 @@ rightmost_sep(Py_UCS4 *s) return found; } +/* Like rightmost_sep, but operate on unicode objects. */ +static Py_ssize_t +rightmost_sep_obj(PyObject* o) +{ + Py_ssize_t found, i; + Py_UCS4 c; + for (found = -1, i = 0; i < PyUnicode_GET_LENGTH(o); i++) { + c = PyUnicode_READ_CHAR(o, i); + if (c == SEP +#ifdef ALTSEP + || c == ALTSEP +#endif + ) + { + found = i; + } + } + return found; +} /* Given a pathname for a Python source file, fill a buffer with the pathname for the corresponding compiled file. Return the pathname @@ -915,123 +934,49 @@ rightmost_sep(Py_UCS4 *s) static PyObject* make_compiled_pathname(PyObject *pathstr, int debug) { - Py_UCS4 *pathname; - Py_UCS4 buf[MAXPATHLEN]; - size_t buflen = (size_t)MAXPATHLEN; - size_t len; - size_t i, save; - Py_UCS4 *pos; - int sep = SEP; - - pathname = PyUnicode_AsUCS4Copy(pathstr); - if (!pathname) - return NULL; - len = Py_UCS4_strlen(pathname); - - /* Sanity check that the buffer has roughly enough space to hold what - will eventually be the full path to the compiled file. The 5 extra - bytes include the slash afer __pycache__, the two extra dots, the - extra trailing character ('c' or 'o') and null. This isn't exact - because the contents of the buffer can affect how many actual - characters of the string get into the buffer. We'll do a final - sanity check before writing the extension to ensure we do not - overflow the buffer. - */ - if (len + Py_UCS4_strlen(CACHEDIR_UNICODE) + Py_UCS4_strlen(PYC_TAG_UNICODE) + 5 > buflen) { - PyMem_Free(pathname); - return NULL; - } - - /* Find the last path separator and copy everything from the start of - the source string up to and including the separator. - */ - pos = rightmost_sep(pathname); - if (pos == NULL) { - i = 0; - } - else { - sep = *pos; - i = pos - pathname + 1; - Py_UCS4_strncpy(buf, pathname, i); - } - - save = i; - buf[i++] = '\0'; - /* Add __pycache__/ */ - Py_UCS4_strcat(buf, CACHEDIR_UNICODE); - i += Py_UCS4_strlen(CACHEDIR_UNICODE) - 1; - buf[i++] = sep; - buf[i] = '\0'; - /* Add the base filename, but remove the .py or .pyw extension, since - the tag name must go before the extension. - */ - Py_UCS4_strcat(buf, pathname + save); - pos = Py_UCS4_strrchr(buf + i, '.'); - if (pos != NULL) - *++pos = '\0'; - - /* pathname is not used from here on. */ - PyMem_Free(pathname); - - Py_UCS4_strcat(buf, PYC_TAG_UNICODE); - /* The length test above assumes that we're only adding one character - to the end of what would normally be the extension. What if there - is no extension, or the string ends in '.' or '.p', and otherwise - fills the buffer? By appending 4 more characters onto the string - here, we could overrun the buffer. - - As a simple example, let's say buflen=32 and the input string is - 'xxx.py'. strlen() would be 6 and the test above would yield: - - (6 + 11 + 10 + 5 == 32) > 32 - - which is false and so the name mangling would continue. This would - be fine because we'd end up with this string in buf: - - __pycache__/xxx.cpython-32.pyc\0 - - strlen(of that) == 30 + the nul fits inside a 32 character buffer. - We can even handle an input string of say 'xxxxx' above because - that's (5 + 11 + 10 + 5 == 31) > 32 which is also false. Name - mangling that yields: - - __pycache__/xxxxxcpython-32.pyc\0 - - which is 32 characters including the nul, and thus fits in the - buffer. However, an input string of 'xxxxxx' would yield a result - string of: - - __pycache__/xxxxxxcpython-32.pyc\0 - - which is 33 characters long (including the nul), thus overflowing - the buffer, even though the first test would fail, i.e.: the input - string is also 6 characters long, so 32 > 32 is false. - - The reason the first test fails but we still overflow the buffer is - that the test above only expects to add one extra character to be - added to the extension, and here we're adding three (pyc). We - don't add the first dot, so that reclaims one of expected - positions, leaving us overflowing by 1 byte (3 extra - 1 reclaimed - dot - 1 expected extra == 1 overflowed). - - The best we can do is ensure that we still have enough room in the - target buffer before we write the extension. Because it's always - only the extension that can cause the overflow, and never the other - path bytes we've written, it's sufficient to just do one more test - here. Still, the assertion that follows can't hurt. - */ -#if 0 - printf("strlen(buf): %d; buflen: %d\n", (int)strlen(buf), (int)buflen); -#endif - len = Py_UCS4_strlen(buf); - if (len + 5 > buflen) + PyObject *result; + Py_ssize_t fname, ext, len, i, pos, taglen; + Py_ssize_t pycache_len = sizeof("__pycache__/") - 1; + int kind; + void *data; + + /* Compute the output string size. */ + len = PyUnicode_GET_LENGTH(pathstr); + /* If there is no separator, this returns -1, so + lastsep will be 0. */ + fname = rightmost_sep_obj(pathstr) + 1; + ext = fname - 1; + for(i = fname; i < len; i++) + if (PyUnicode_READ_CHAR(pathstr, i) == '.') + ext = i + 1; + if (ext < fname) + /* No dot in filename; use entire filename */ + ext = len; + + /* result = pathstr[:fname] + "__pycache__" + SEP + + pathstr[fname:ext] + tag + ".py[co]" */ + taglen = strlen(pyc_tag); + result = PyUnicode_New(ext + pycache_len + taglen + 4, + PyUnicode_MAX_CHAR_VALUE(pathstr)); + if (!result) return NULL; - buf[len] = '.'; len++; - buf[len] = 'p'; len++; - buf[len] = 'y'; len++; - buf[len] = debug ? 'c' : 'o'; len++; - assert(len <= buflen); - return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, len); + kind = PyUnicode_KIND(result); + data = PyUnicode_DATA(result); + PyUnicode_CopyCharacters(result, 0, pathstr, 0, fname); + pos = fname; + for (i = 0; i < pycache_len - 1; i++) + PyUnicode_WRITE(kind, data, pos++, "__pycache__"[i]); + PyUnicode_WRITE(kind, data, pos++, SEP); + PyUnicode_CopyCharacters(result, pos, pathstr, + fname, ext - fname); + pos += ext - fname; + for (i = 0; pyc_tag[i]; i++) + PyUnicode_WRITE(kind, data, pos++, pyc_tag[i]); + PyUnicode_WRITE(kind, data, pos++, '.'); + PyUnicode_WRITE(kind, data, pos++, 'p'); + PyUnicode_WRITE(kind, data, pos++, 'y'); + PyUnicode_WRITE(kind, data, pos++, debug ? 'c' : 'o'); + return result; } -- cgit v1.2.1 From 17bab0f4a5891ac06360ae32cc1b1f9ab746cf52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 23 Oct 2011 17:35:46 +0200 Subject: Add ready checks for make_compiled_pathname. --- Python/import.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 1101c920f5..5dcaf3a40e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -929,7 +929,10 @@ rightmost_sep_obj(PyObject* o) for the compiled file, or NULL if there's no space in the buffer. Doesn't set an exception. - foo.py -> __pycache__/foo..pyc */ + foo.py -> __pycache__/foo..pyc + + pathstr is assumed to be "ready". +*/ static PyObject* make_compiled_pathname(PyObject *pathstr, int debug) @@ -1458,6 +1461,8 @@ load_source_module(PyObject *name, PyObject *pathname, FILE *fp) goto error; } #endif + if (PyUnicode_READY(pathname) < 0) + return NULL; cpathname = make_compiled_pathname(pathname, !Py_OptimizeFlag); if (cpathname != NULL) @@ -3949,6 +3954,9 @@ imp_cache_from_source(PyObject *self, PyObject *args, PyObject *kws) return NULL; } + if (PyUnicode_READY(pathname) < 0) + return NULL; + cpathname = make_compiled_pathname(pathname, debug); Py_DECREF(pathname); -- cgit v1.2.1 From ff4c4b6f318abc223cbae8920ee2286d111d8bce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 23 Oct 2011 18:05:06 +0200 Subject: Rewrite make_source_pathname using Unicode API. --- Python/import.c | 79 ++++++++++++++++++++++++++++++--------------------------- 1 file changed, 41 insertions(+), 38 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 5dcaf3a40e..a20871efcc 100644 --- a/Python/import.c +++ b/Python/import.c @@ -906,11 +906,11 @@ rightmost_sep(Py_UCS4 *s) /* Like rightmost_sep, but operate on unicode objects. */ static Py_ssize_t -rightmost_sep_obj(PyObject* o) +rightmost_sep_obj(PyObject* o, Py_ssize_t start, Py_ssize_t end) { Py_ssize_t found, i; Py_UCS4 c; - for (found = -1, i = 0; i < PyUnicode_GET_LENGTH(o); i++) { + for (found = -1, i = start; i < end; i++) { c = PyUnicode_READ_CHAR(o, i); if (c == SEP #ifdef ALTSEP @@ -947,7 +947,7 @@ make_compiled_pathname(PyObject *pathstr, int debug) len = PyUnicode_GET_LENGTH(pathstr); /* If there is no separator, this returns -1, so lastsep will be 0. */ - fname = rightmost_sep_obj(pathstr) + 1; + fname = rightmost_sep_obj(pathstr, 0, len) + 1; ext = fname - 1; for(i = fname; i < len; i++) if (PyUnicode_READ_CHAR(pathstr, i) == '.') @@ -992,63 +992,66 @@ make_compiled_pathname(PyObject *pathstr, int debug) (...)/__pycache__/foo..pyc -> (...)/foo.py */ static PyObject* -make_source_pathname(PyObject *pathobj) +make_source_pathname(PyObject *path) { - Py_UCS4 buf[MAXPATHLEN]; - Py_UCS4 *pathname; - Py_UCS4 *left, *right, *dot0, *dot1, sep; - size_t i, j; + Py_ssize_t left, right, dot0, dot1, len; + Py_ssize_t i, j; + PyObject *result; + int kind; + void *data; - if (PyUnicode_GET_LENGTH(pathobj) > MAXPATHLEN) - return NULL; - pathname = PyUnicode_AsUCS4Copy(pathobj); - if (!pathname) + len = PyUnicode_GET_LENGTH(path); + if (len > MAXPATHLEN) return NULL; /* Look back two slashes from the end. In between these two slashes must be the string __pycache__ or this is not a PEP 3147 style path. It's possible for there to be only one slash. */ - right = rightmost_sep(pathname); - if (right == NULL) + right = rightmost_sep_obj(path, 0, len); + if (right == -1) return NULL; - sep = *right; - *right = '\0'; - left = rightmost_sep(pathname); - *right = sep; - if (left == NULL) - left = pathname; + left = rightmost_sep_obj(path, 0, right); + if (left == -1) + left = 0; else left++; - if (right-left != Py_UCS4_strlen(CACHEDIR_UNICODE) || - Py_UCS4_strncmp(left, CACHEDIR_UNICODE, right-left) != 0) - goto error; + if (right-left != sizeof(CACHEDIR)-1) + return NULL; + for (i = 0; i < sizeof(CACHEDIR)-1; i++) + if (PyUnicode_READ_CHAR(path, left+i) != CACHEDIR[i]) + return NULL; /* Now verify that the path component to the right of the last slash has two dots in it. */ - if ((dot0 = Py_UCS4_strchr(right + 1, '.')) == NULL) - goto error; - if ((dot1 = Py_UCS4_strchr(dot0 + 1, '.')) == NULL) - goto error; + dot0 = PyUnicode_FindChar(path, '.', right+1, len, 1); + if (dot0 < 0) + return NULL; + dot1 = PyUnicode_FindChar(path, '.', dot0+1, len, 1); + if (dot1 < 0) + return NULL; /* Too many dots? */ - if (Py_UCS4_strchr(dot1 + 1, '.') != NULL) - goto error; + if (PyUnicode_FindChar(path, '.', dot1+1, len, 1) != -1) + return NULL; /* This is a PEP 3147 path. Start by copying everything from the start of pathname up to and including the leftmost slash. Then copy the file's basename, removing the magic tag and adding a .py suffix. */ - Py_UCS4_strncpy(buf, pathname, (i=left-pathname)); - Py_UCS4_strncpy(buf+i, right+1, (j=dot0-right)); - buf[i+j] = 'p'; - buf[i+j+1] = 'y'; - PyMem_Free(pathname); - return PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, buf, i+j+2); - error: - PyMem_Free(pathname); - return NULL; + result = PyUnicode_New(left + (dot0-right) + 2, + PyUnicode_MAX_CHAR_VALUE(path)); + if (!result) + return NULL; + kind = PyUnicode_KIND(result); + data = PyUnicode_DATA(result); + PyUnicode_CopyCharacters(result, 0, path, 0, (i = left)); + PyUnicode_CopyCharacters(result, left, path, right+1, + (j = dot0-right)); + PyUnicode_WRITE(kind, data, i+j, 'p'); + PyUnicode_WRITE(kind, data, i+j+1, 'y'); + return result; } /* Given a pathname for a Python source file, its time of last -- cgit v1.2.1 From c7e9dd65640704b9ef0174a6c5fc9d82fbebf492 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 23 Oct 2011 18:08:20 +0200 Subject: Drop unused macros. Use CACHEDIR consistently. --- Python/import.c | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index a20871efcc..ebaaeacebd 100644 --- a/Python/import.c +++ b/Python/import.c @@ -108,8 +108,8 @@ typedef unsigned short mode_t; /* MAGIC must change whenever the bytecode emitted by the compiler may no longer be understood by older implementations of the eval loop (usually due to the addition of new opcodes) - TAG and PYC_TAG_UNICODE must change for each major Python release. The magic - number will take care of any bytecode changes that occur during development. + TAG must change for each major Python release. The magic number will take + care of any bytecode changes that occur during development. */ #define QUOTE(arg) #arg #define STRIFY(name) QUOTE(name) @@ -118,13 +118,9 @@ typedef unsigned short mode_t; #define MAGIC (3190 | ((long)'\r'<<16) | ((long)'\n'<<24)) #define TAG "cpython-" MAJOR MINOR; #define CACHEDIR "__pycache__" -static const Py_UCS4 CACHEDIR_UNICODE[] = { - '_', '_', 'p', 'y', 'c', 'a', 'c', 'h', 'e', '_', '_', '\0'}; /* Current magic word and string tag as globals. */ static long pyc_magic = MAGIC; static const char *pyc_tag = TAG; -static const Py_UCS4 PYC_TAG_UNICODE[] = { - 'c', 'p', 'y', 't', 'h', 'o', 'n', '-', PY_MAJOR_VERSION + 48, PY_MINOR_VERSION + 48, '\0'}; #undef QUOTE #undef STRIFY #undef MAJOR @@ -939,7 +935,7 @@ make_compiled_pathname(PyObject *pathstr, int debug) { PyObject *result; Py_ssize_t fname, ext, len, i, pos, taglen; - Py_ssize_t pycache_len = sizeof("__pycache__/") - 1; + Py_ssize_t pycache_len = sizeof(CACHEDIR) - 1; int kind; void *data; @@ -968,7 +964,7 @@ make_compiled_pathname(PyObject *pathstr, int debug) PyUnicode_CopyCharacters(result, 0, pathstr, 0, fname); pos = fname; for (i = 0; i < pycache_len - 1; i++) - PyUnicode_WRITE(kind, data, pos++, "__pycache__"[i]); + PyUnicode_WRITE(kind, data, pos++, CACHEDIR[i]); PyUnicode_WRITE(kind, data, pos++, SEP); PyUnicode_CopyCharacters(result, pos, pathstr, fname, ext - fname); -- cgit v1.2.1 From 012400c9fe4281ff97e7dee256cd2a05649a54ce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 23 Oct 2011 18:41:56 +0200 Subject: Fix off-by-one error. --- Python/import.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index ebaaeacebd..736ce7717c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -955,7 +955,7 @@ make_compiled_pathname(PyObject *pathstr, int debug) /* result = pathstr[:fname] + "__pycache__" + SEP + pathstr[fname:ext] + tag + ".py[co]" */ taglen = strlen(pyc_tag); - result = PyUnicode_New(ext + pycache_len + taglen + 4, + result = PyUnicode_New(ext + pycache_len + 1 + taglen + 4, PyUnicode_MAX_CHAR_VALUE(pathstr)); if (!result) return NULL; @@ -963,7 +963,7 @@ make_compiled_pathname(PyObject *pathstr, int debug) data = PyUnicode_DATA(result); PyUnicode_CopyCharacters(result, 0, pathstr, 0, fname); pos = fname; - for (i = 0; i < pycache_len - 1; i++) + for (i = 0; i < pycache_len; i++) PyUnicode_WRITE(kind, data, pos++, CACHEDIR[i]); PyUnicode_WRITE(kind, data, pos++, SEP); PyUnicode_CopyCharacters(result, pos, pathstr, -- cgit v1.2.1 From 9d5ef173055b3857a637b716f029b0e81d172c29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 23 Oct 2011 18:45:16 +0200 Subject: Rewrite find_module_path using unicode API. --- Python/import.c | 83 +++++++++++++++++++++++++++++++++------------------------ 1 file changed, 48 insertions(+), 35 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 736ce7717c..e21ae39359 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1743,11 +1743,11 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, PyObject *path_hooks, PyObject *path_importer_cache, PyObject **p_path, PyObject **p_loader, struct filedescr **p_fd) { - Py_UCS4 buf[MAXPATHLEN+1]; - PyObject *path_unicode, *filename; - Py_ssize_t len; + PyObject *path_unicode, *filename = NULL; + Py_ssize_t len, pos; struct stat statbuf; static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; + int result, addsep; if (PyUnicode_Check(path)) { Py_INCREF(path); @@ -1766,15 +1766,10 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, return -1; len = PyUnicode_GET_LENGTH(path_unicode); - if (!PyUnicode_AsUCS4(path_unicode, buf, Py_ARRAY_LENGTH(buf), 1)) { - Py_DECREF(path_unicode); - PyErr_Clear(); - return 0; + if (PyUnicode_FindChar(path_unicode, 0, 0, len, 1) != -1) { + result = 0; + goto out; /* path contains '\0' */ } - Py_DECREF(path_unicode); - - if (Py_UCS4_strlen(buf) != len) - return 0; /* path contains '\0' */ /* sys.path_hooks import hook */ if (p_loader != NULL) { @@ -1784,43 +1779,54 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, importer = get_path_importer(path_importer_cache, path_hooks, path); if (importer == NULL) { - return -1; + result = -1; + goto out; } /* Note: importer is a borrowed reference */ if (importer != Py_None) { PyObject *loader; loader = _PyObject_CallMethodId(importer, &PyId_find_module, "O", fullname); - if (loader == NULL) - return -1; /* error */ + if (loader == NULL) { + result = -1; /* error */ + goto out; + } if (loader != Py_None) { /* a loader was found */ *p_loader = loader; *p_fd = &importhookdescr; - return 2; + result = 2; + goto out; } Py_DECREF(loader); - return 0; + result = 0; + goto out; } } /* no hook was found, use builtin import */ - if (len > 0 && buf[len-1] != SEP + addsep = 0; + if (len > 0 && PyUnicode_READ_CHAR(path_unicode, len-1) != SEP #ifdef ALTSEP - && buf[len-1] != ALTSEP + && PyUnicode_READ_CHAR(path_unicode, len-1) != ALTSEP #endif ) - buf[len++] = SEP; - if (!PyUnicode_AsUCS4(name, buf+len, Py_ARRAY_LENGTH(buf)-len, 1)) { - PyErr_Clear(); - return 0; - } - len += PyUnicode_GET_LENGTH(name); - - filename = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - buf, len); - if (filename == NULL) - return -1; + addsep = 1; + filename = PyUnicode_New(len + PyUnicode_GET_LENGTH(name) + addsep, + Py_MAX(PyUnicode_MAX_CHAR_VALUE(path_unicode), + PyUnicode_MAX_CHAR_VALUE(name))); + if (filename == NULL) { + result = -1; + goto out; + } + PyUnicode_CopyCharacters(filename, 0, path_unicode, 0, len); + pos = len; + if (addsep) + PyUnicode_WRITE(PyUnicode_KIND(filename), + PyUnicode_DATA(filename), + pos++, SEP); + PyUnicode_CopyCharacters(filename, pos, name, 0, + PyUnicode_GET_LENGTH(name)); /* Check for package import (buf holds a directory name, and there's an __init__ module in that directory */ @@ -1832,14 +1838,16 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, match = case_ok(filename, 0, name); if (match < 0) { - Py_DECREF(filename); - return -1; + result = -1; + goto out; } if (match) { /* case matches */ if (find_init_module(filename)) { /* and has __init__.py */ *p_path = filename; + filename = NULL; *p_fd = &fd_package; - return 2; + result = 2; + goto out; } else { int err; @@ -1847,15 +1855,20 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, "Not importing directory %R: missing __init__.py", filename); if (err) { - Py_DECREF(filename); - return -1; + result = -1; + goto out; } } } } #endif *p_path = filename; - return 1; + filename = NULL; + result = 1; + out: + Py_DECREF(path_unicode); + Py_XDECREF(filename); + return result; } /* Find a module in search_path_list. For each path, try -- cgit v1.2.1 From 9ab76944255d2498c50789b1d0582aec1e417ad9 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 25 Oct 2011 00:21:02 +0200 Subject: Fix test_imp failure under Windows --- Python/import.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index e21ae39359..3842c63c2e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -938,12 +938,19 @@ make_compiled_pathname(PyObject *pathstr, int debug) Py_ssize_t pycache_len = sizeof(CACHEDIR) - 1; int kind; void *data; + Py_UCS4 lastsep; /* Compute the output string size. */ len = PyUnicode_GET_LENGTH(pathstr); /* If there is no separator, this returns -1, so - lastsep will be 0. */ + fname will be 0. */ fname = rightmost_sep_obj(pathstr, 0, len) + 1; + /* Windows: re-use the last separator character (/ or \\) when + appending the __pycache__ path. */ + if (fname > 0) + lastsep = PyUnicode_READ_CHAR(pathstr, fname -1); + else + lastsep = SEP; ext = fname - 1; for(i = fname; i < len; i++) if (PyUnicode_READ_CHAR(pathstr, i) == '.') @@ -965,7 +972,7 @@ make_compiled_pathname(PyObject *pathstr, int debug) pos = fname; for (i = 0; i < pycache_len; i++) PyUnicode_WRITE(kind, data, pos++, CACHEDIR[i]); - PyUnicode_WRITE(kind, data, pos++, SEP); + PyUnicode_WRITE(kind, data, pos++, lastsep); PyUnicode_CopyCharacters(result, pos, pathstr, fname, ext - fname); pos += ext - fname; -- cgit v1.2.1 From 7336fba07c5e34404a0d72c4b24e78a3d83ac1cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 30 Oct 2011 09:07:07 +0100 Subject: Port PyImport_ReloadModule to Unicode API. --- Python/import.c | 60 +++++++++++++++++++++++++-------------------------------- 1 file changed, 26 insertions(+), 34 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 3842c63c2e..294eb2bb33 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3359,8 +3359,8 @@ PyImport_ReloadModule(PyObject *m) PyObject *modules_reloading = interp->modules_reloading; PyObject *modules = PyImport_GetModuleDict(); PyObject *path_list = NULL, *loader = NULL, *existing_m = NULL; - PyObject *nameobj, *bufobj, *subnameobj; - Py_UCS4 *name = NULL, *subname; + PyObject *name, *bufobj, *subname; + Py_ssize_t subname_start; struct filedescr *fdp; FILE *fp = NULL; PyObject *newm = NULL; @@ -3376,45 +3376,39 @@ PyImport_ReloadModule(PyObject *m) "reload() argument must be module"); return NULL; } - nameobj = PyModule_GetNameObject(m); - if (nameobj == NULL || PyUnicode_READY(nameobj) == -1) + name = PyModule_GetNameObject(m); + if (name == NULL || PyUnicode_READY(name) == -1) return NULL; - if (m != PyDict_GetItem(modules, nameobj)) { + if (m != PyDict_GetItem(modules, name)) { PyErr_Format(PyExc_ImportError, "reload(): module %R not in sys.modules", - nameobj); - Py_DECREF(nameobj); + name); + Py_DECREF(name); return NULL; } - existing_m = PyDict_GetItem(modules_reloading, nameobj); + existing_m = PyDict_GetItem(modules_reloading, name); if (existing_m != NULL) { /* Due to a recursive reload, this module is already being reloaded. */ - Py_DECREF(nameobj); + Py_DECREF(name); Py_INCREF(existing_m); return existing_m; } - if (PyDict_SetItem(modules_reloading, nameobj, m) < 0) { - Py_DECREF(nameobj); + if (PyDict_SetItem(modules_reloading, name, m) < 0) { + Py_DECREF(name); return NULL; } - name = PyUnicode_AsUCS4Copy(nameobj); - if (!name) { - Py_DECREF(nameobj); - return NULL; - } - subname = Py_UCS4_strrchr(name, '.'); - if (subname == NULL) { - Py_INCREF(nameobj); - subnameobj = nameobj; + subname_start = PyUnicode_FindChar(name, '.', 0, + PyUnicode_GET_LENGTH(name), -1); + if (subname_start == -1) { + Py_INCREF(name); + subname = name; } else { PyObject *parentname, *parent; Py_ssize_t len; - len = subname - name; - parentname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - name, len); + parentname = PyUnicode_Substring(name, 0, subname_start); if (parentname == NULL) { goto error; } @@ -3430,16 +3424,15 @@ PyImport_ReloadModule(PyObject *m) path_list = _PyObject_GetAttrId(parent, &PyId___path__); if (path_list == NULL) PyErr_Clear(); - subname++; - len = PyUnicode_GET_LENGTH(nameobj) - (len + 1); - subnameobj = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - subname, len); + subname_start++; + len = PyUnicode_GET_LENGTH(name) - (subname_start + 1); + subname = PyUnicode_Substring(name, subname_start, len); } - if (subnameobj == NULL) + if (subname == NULL) goto error; - fdp = find_module(nameobj, subnameobj, path_list, + fdp = find_module(name, subname, path_list, &bufobj, &fp, &loader); - Py_DECREF(subnameobj); + Py_DECREF(subname); Py_XDECREF(path_list); if (fdp == NULL) { @@ -3447,7 +3440,7 @@ PyImport_ReloadModule(PyObject *m) goto error; } - newm = load_module(nameobj, fp, bufobj, fdp->type, loader); + newm = load_module(name, fp, bufobj, fdp->type, loader); Py_XDECREF(bufobj); Py_XDECREF(loader); @@ -3459,13 +3452,12 @@ PyImport_ReloadModule(PyObject *m) * going to return NULL in this case regardless of whether * replacing name succeeds, so the return value is ignored. */ - PyDict_SetItem(modules, nameobj, m); + PyDict_SetItem(modules, name, m); } error: imp_modules_reloading_clear(); - Py_DECREF(nameobj); - PyMem_Free(name); + Py_DECREF(name); return newm; } -- cgit v1.2.1 From 2593ff3b46728f698ea5a5307fcc6263012b358b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= Date: Sun, 30 Oct 2011 23:50:02 +0100 Subject: Port import_module_level to Unicode API. --- Python/import.c | 164 +++++++++++++++++++++----------------------------------- 1 file changed, 61 insertions(+), 103 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 8e3aa4b921..03e9986126 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2753,8 +2753,7 @@ static PyObject *get_parent(PyObject *globals, int level); static PyObject *load_next(PyObject *mod, PyObject *altmod, PyObject *inputname, PyObject **p_outputname, - Py_UCS4 *buf, Py_ssize_t *p_buflen, - Py_ssize_t bufsize); + PyObject **p_prefix); static int mark_miss(PyObject *name); static int ensure_fromlist(PyObject *mod, PyObject *fromlist, PyObject *buf, int recursive); @@ -2767,11 +2766,11 @@ static PyObject * import_module_level(PyObject *name, PyObject *globals, PyObject *locals, PyObject *fromlist, int level) { - Py_UCS4 buf[MAXPATHLEN+1]; - Py_ssize_t buflen; - Py_ssize_t bufsize = MAXPATHLEN+1; - PyObject *parent, *head, *next, *tail, *inputname, *outputname; - PyObject *parent_name, *ensure_name; + PyObject *parent, *next, *inputname, *outputname; + PyObject *head = NULL; + PyObject *tail = NULL; + PyObject *prefix = NULL; + PyObject *result = NULL; Py_ssize_t sep, altsep; if (PyUnicode_READY(name)) @@ -2794,26 +2793,18 @@ import_module_level(PyObject *name, PyObject *globals, PyObject *locals, return NULL; } - parent = get_parent(globals, &parent_name, level); + parent = get_parent(globals, &prefix, level); if (parent == NULL) { return NULL; } - if (PyUnicode_READY(parent_name)) + if (PyUnicode_READY(prefix)) return NULL; - buflen = PyUnicode_GET_LENGTH(parent_name); - if (!PyUnicode_AsUCS4(parent_name, buf, Py_ARRAY_LENGTH(buf), 1)) { - Py_DECREF(parent_name); - PyErr_SetString(PyExc_ValueError, - "Module name too long"); - return NULL; - } - Py_DECREF(parent_name); head = load_next(parent, level < 0 ? Py_None : parent, name, &outputname, - buf, &buflen, bufsize); + &prefix); if (head == NULL) - return NULL; + goto out; tail = head; Py_INCREF(tail); @@ -2822,13 +2813,11 @@ import_module_level(PyObject *name, PyObject *globals, PyObject *locals, while (1) { inputname = outputname; next = load_next(tail, tail, inputname, &outputname, - buf, &buflen, bufsize); - Py_DECREF(tail); - Py_DECREF(inputname); - if (next == NULL) { - Py_DECREF(head); - return NULL; - } + &prefix); + Py_CLEAR(tail); + Py_CLEAR(inputname); + if (next == NULL) + goto out; tail = next; if (outputname == NULL) { @@ -2840,10 +2829,8 @@ import_module_level(PyObject *name, PyObject *globals, PyObject *locals, /* If tail is Py_None, both get_parent and load_next found an empty module name: someone called __import__("") or doctored faulty bytecode */ - Py_DECREF(tail); - Py_DECREF(head); PyErr_SetString(PyExc_ValueError, "Empty module name"); - return NULL; + goto out; } if (fromlist != NULL) { @@ -2852,26 +2839,21 @@ import_module_level(PyObject *name, PyObject *globals, PyObject *locals, } if (fromlist == NULL) { - Py_DECREF(tail); - return head; - } - - Py_DECREF(head); - - ensure_name = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - buf, Py_UCS4_strlen(buf)); - if (ensure_name == NULL) { - Py_DECREF(tail); - return NULL; - } - if (!ensure_fromlist(tail, fromlist, ensure_name, 0)) { - Py_DECREF(tail); - Py_DECREF(ensure_name); - return NULL; + result = head; + Py_INCREF(result); + goto out; } - Py_DECREF(ensure_name); - return tail; + if (!ensure_fromlist(tail, fromlist, prefix, 0)) + goto out; + + result = tail; + Py_INCREF(result); + out: + Py_XDECREF(head); + Py_XDECREF(tail); + Py_XDECREF(prefix); + return result; } PyObject * @@ -3083,91 +3065,67 @@ return_none: static PyObject * load_next(PyObject *mod, PyObject *altmod, PyObject *inputname, PyObject **p_outputname, - Py_UCS4 *buf, Py_ssize_t *p_buflen, Py_ssize_t bufsize) + PyObject **p_prefix) { - Py_UCS4 *dot; + Py_ssize_t dot; Py_ssize_t len; - Py_UCS4 *p; - PyObject *fullname, *name, *result, *mark_name; - Py_UCS4 *nameuni; + PyObject *fullname, *name = NULL, *result; *p_outputname = NULL; - if (PyUnicode_GET_LENGTH(inputname) == 0) { + len = PyUnicode_GET_LENGTH(inputname); + if (len == 0) { /* completely empty module name should only happen in 'from . import' (or '__import__("")')*/ Py_INCREF(mod); return mod; } - nameuni = PyUnicode_AsUCS4Copy(inputname); - if (nameuni == NULL) - return NULL; - dot = Py_UCS4_strchr(nameuni, '.'); - if (dot != NULL) { - len = dot - nameuni; + dot = PyUnicode_FindChar(inputname, '.', 0, len, 1); + if (dot >= 0) { + len = dot; if (len == 0) { PyErr_SetString(PyExc_ValueError, "Empty module name"); goto error; } } - else - len = PyUnicode_GET_LENGTH(inputname); - if (*p_buflen+len+1 >= bufsize) { - PyErr_SetString(PyExc_ValueError, - "Module name too long"); + /* name = inputname[:len] */ + name = PyUnicode_Substring(inputname, 0, len); + if (name == NULL) goto error; - } - p = buf + *p_buflen; - if (p != buf) { - *p++ = '.'; - *p_buflen += 1; + if (PyUnicode_GET_LENGTH(*p_prefix)) { + /* fullname = prefix + "." + name */ + fullname = PyUnicode_FromFormat("%U.%U", *p_prefix, name); + if (fullname == NULL) + goto error; } - Py_UCS4_strncpy(p, nameuni, len); - p[len] = '\0'; - *p_buflen += len; - - fullname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - buf, *p_buflen); - if (fullname == NULL) - goto error; - name = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - p, len); - if (name == NULL) { - Py_DECREF(fullname); - goto error; + else { + fullname = name; + Py_INCREF(fullname); } + result = import_submodule(mod, name, fullname); - Py_DECREF(fullname); + Py_DECREF(*p_prefix); + /* Transfer reference. */ + *p_prefix = fullname; if (result == Py_None && altmod != mod) { Py_DECREF(result); /* Here, altmod must be None and mod must not be None */ result = import_submodule(altmod, name, name); - Py_DECREF(name); if (result != NULL && result != Py_None) { - mark_name = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - buf, *p_buflen); - if (mark_name == NULL) { + if (mark_miss(*p_prefix) != 0) { Py_DECREF(result); goto error; } - if (mark_miss(mark_name) != 0) { - Py_DECREF(result); - Py_DECREF(mark_name); - goto error; - } - Py_DECREF(mark_name); - Py_UCS4_strncpy(buf, nameuni, len); - buf[len] = '\0'; - *p_buflen = len; + Py_DECREF(*p_prefix); + *p_prefix = name; + Py_INCREF(*p_prefix); } } - else - Py_DECREF(name); if (result == NULL) goto error; @@ -3178,20 +3136,20 @@ load_next(PyObject *mod, PyObject *altmod, goto error; } - if (dot != NULL) { - *p_outputname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - dot+1, Py_UCS4_strlen(dot+1)); + if (dot >= 0) { + *p_outputname = PyUnicode_Substring(inputname, dot+1, + PyUnicode_GET_LENGTH(inputname)); if (*p_outputname == NULL) { Py_DECREF(result); goto error; } } - PyMem_Free(nameuni); + Py_DECREF(name); return result; error: - PyMem_Free(nameuni); + Py_XDECREF(name); return NULL; } -- cgit v1.2.1 From 0c820a9c9584c838de7ba3abea17c76a03bd405b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Mon, 31 Oct 2011 20:47:31 +0100 Subject: Issue #13303: Fix a race condition in the bytecode file creation. --- Python/import.c | 50 ++++++++------------------------------------------ 1 file changed, 8 insertions(+), 42 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 03e9986126..701041b2f2 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1183,42 +1183,6 @@ parse_source_module(PyObject *pathname, FILE *fp) return co; } -/* Helper to open a bytecode file for writing in exclusive mode */ - -#ifndef MS_WINDOWS -static FILE * -open_exclusive(char *filename, mode_t mode) -{ -#if defined(O_EXCL)&&defined(O_CREAT)&&defined(O_WRONLY)&&defined(O_TRUNC) - /* Use O_EXCL to avoid a race condition when another process tries to - write the same file. When that happens, our open() call fails, - which is just fine (since it's only a cache). - XXX If the file exists and is writable but the directory is not - writable, the file will never be written. Oh well. - */ - int fd; - (void) unlink(filename); - fd = open(filename, O_EXCL|O_CREAT|O_WRONLY|O_TRUNC -#ifdef O_BINARY - |O_BINARY /* necessary for Windows */ -#endif -#ifdef __VMS - , mode, "ctxt=bin", "shr=nil" -#else - , mode -#endif - ); - if (fd < 0) - return NULL; - return fdopen(fd, "wb"); -#else - /* Best we can do -- on Windows this can't happen anyway */ - return fopen(filename, "wb"); -#endif -} -#endif - - /* Write a compiled module to a file, placing the time of last modification of its source into the header. Errors are ignored, if a write error occurs an attempt is made to @@ -1234,15 +1198,13 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, #ifdef MS_WINDOWS /* since Windows uses different permissions */ mode_t mode = srcstat->st_mode & ~S_IEXEC; #else - mode_t mode = srcstat->st_mode & ~S_IXUSR & ~S_IXGRP & ~S_IXOTH; mode_t dirmode = (srcstat->st_mode | S_IXUSR | S_IXGRP | S_IXOTH | S_IWUSR | S_IWGRP | S_IWOTH); PyObject *dirbytes; #endif -#ifdef MS_WINDOWS int fd; -#else +#ifndef MS_WINDOWS PyObject *cpathbytes, *cpathbytes_tmp; Py_ssize_t cpathbytes_len; #endif @@ -1313,7 +1275,7 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, return; } cpathbytes_len = PyBytes_GET_SIZE(cpathbytes); - cpathbytes_tmp = PyBytes_FromStringAndSize(NULL, cpathbytes_len + 4); + cpathbytes_tmp = PyBytes_FromStringAndSize(NULL, cpathbytes_len + 6); if (cpathbytes_tmp == NULL) { Py_DECREF(cpathbytes); PyErr_Clear(); @@ -1321,9 +1283,13 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, } memcpy(PyBytes_AS_STRING(cpathbytes_tmp), PyBytes_AS_STRING(cpathbytes), cpathbytes_len); - memcpy(PyBytes_AS_STRING(cpathbytes_tmp) + cpathbytes_len, ".tmp", 4); + memcpy(PyBytes_AS_STRING(cpathbytes_tmp) + cpathbytes_len, "XXXXXX", 6); - fp = open_exclusive(PyBytes_AS_STRING(cpathbytes_tmp), mode); + fd = mkstemp(PyBytes_AS_STRING(cpathbytes_tmp)); + if (0 <= fd) + fp = fdopen(fd, "wb"); + else + fp = NULL; #endif if (fp == NULL) { if (Py_VerboseFlag) -- cgit v1.2.1 From 17ec0be2a18cb0c8d6dc40f0bf0205642e47c31d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Thu, 10 Nov 2011 19:12:29 +0100 Subject: Issue #13303: Fix bytecode file default permission. --- Python/import.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 5c33b6be57..ae1101e4ec 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1202,12 +1202,10 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, S_IXUSR | S_IXGRP | S_IXOTH | S_IWUSR | S_IWGRP | S_IWOTH); PyObject *dirbytes; -#endif - int fd; -#ifndef MS_WINDOWS PyObject *cpathbytes, *cpathbytes_tmp; Py_ssize_t cpathbytes_len; #endif + int fd; PyObject *dirname; Py_UCS4 *dirsep; int res, ok; @@ -1275,7 +1273,7 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, return; } cpathbytes_len = PyBytes_GET_SIZE(cpathbytes); - cpathbytes_tmp = PyBytes_FromStringAndSize(NULL, cpathbytes_len + 6); + cpathbytes_tmp = PyBytes_FromStringAndSize(NULL, cpathbytes_len + 4); if (cpathbytes_tmp == NULL) { Py_DECREF(cpathbytes); PyErr_Clear(); @@ -1283,9 +1281,10 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, } memcpy(PyBytes_AS_STRING(cpathbytes_tmp), PyBytes_AS_STRING(cpathbytes), cpathbytes_len); - memcpy(PyBytes_AS_STRING(cpathbytes_tmp) + cpathbytes_len, "XXXXXX", 6); + memcpy(PyBytes_AS_STRING(cpathbytes_tmp) + cpathbytes_len, ".tmp", 4); - fd = mkstemp(PyBytes_AS_STRING(cpathbytes_tmp)); + fd = open(PyBytes_AS_STRING(cpathbytes_tmp), + O_CREAT | O_EXCL | O_WRONLY, 0666); if (0 <= fd) fp = fdopen(fd, "wb"); else -- cgit v1.2.1 From 1a238237f0304a03e73e464b069a2a4aec4beb3c Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 15 Nov 2011 19:15:19 +0100 Subject: Issue #13392: Writing a pyc file should now be atomic under Windows as well. --- Python/import.c | 48 ++++++++++++++++++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 10 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index ae1101e4ec..3c3e504e28 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1197,6 +1197,8 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, time_t mtime = srcstat->st_mtime; #ifdef MS_WINDOWS /* since Windows uses different permissions */ mode_t mode = srcstat->st_mode & ~S_IEXEC; + PyObject *cpathname_tmp; + Py_ssize_t cpathname_len; #else mode_t dirmode = (srcstat->st_mode | S_IXUSR | S_IXGRP | S_IXOTH | @@ -1255,18 +1257,29 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, } Py_DECREF(dirname); + /* We first write to a tmp file and then take advantage + of atomic renaming (which *should* be true even under Windows). */ #ifdef MS_WINDOWS - (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname)); - fd = _wopen(PyUnicode_AS_UNICODE(cpathname), - O_EXCL | O_CREAT | O_WRONLY | O_TRUNC | O_BINARY, - mode); + cpathname_len = PyUnicode_GET_LENGTH(cpathname); + cpathname_tmp = PyUnicode_New(cpathname_len + 4, + PyUnicode_MAX_CHAR_VALUE(cpathname)); + if (cpathname_tmp == NULL) { + PyErr_Clear(); + return; + } + PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 0, '.'); + PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 1, 't'); + PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 2, 'm'); + PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 3, 'p'); + (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname_tmp)); + fd = _wopen(PyUnicode_AS_UNICODE(cpathname_tmp), + O_EXCL | O_CREAT | O_WRONLY | O_BINARY, + mode); if (0 <= fd) fp = fdopen(fd, "wb"); else fp = NULL; #else - /* Under POSIX, we first write to a tmp file and then take advantage - of atomic renaming. */ cpathbytes = PyUnicode_EncodeFSDefault(cpathname); if (cpathbytes == NULL) { PyErr_Clear(); @@ -1294,7 +1307,9 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, if (Py_VerboseFlag) PySys_FormatStderr( "# can't create %R\n", cpathname); -#ifndef MS_WINDOWS +#ifdef MS_WINDOWS + Py_DECREF(cpathname_tmp); +#else Py_DECREF(cpathbytes); Py_DECREF(cpathbytes_tmp); #endif @@ -1315,7 +1330,8 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, /* Don't keep partial file */ fclose(fp); #ifdef MS_WINDOWS - (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname)); + (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname_tmp)); + Py_DECREF(cpathname_tmp); #else (void) unlink(PyBytes_AS_STRING(cpathbytes_tmp)); Py_DECREF(cpathbytes); @@ -1324,8 +1340,20 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, return; } fclose(fp); - /* Under POSIX, do an atomic rename */ -#ifndef MS_WINDOWS + /* Do a (hopefully) atomic rename */ +#ifdef MS_WINDOWS + if (!MoveFileExW(PyUnicode_AS_UNICODE(cpathname_tmp), + PyUnicode_AS_UNICODE(cpathname), + MOVEFILE_REPLACE_EXISTING)) { + if (Py_VerboseFlag) + PySys_FormatStderr("# can't write %R\n", cpathname); + /* Don't keep tmp file */ + (void) DeleteFileW(PyUnicode_AS_UNICODE(cpathname_tmp)); + Py_DECREF(cpathname_tmp); + return; + } + Py_DECREF(cpathname_tmp); +#else if (rename(PyBytes_AS_STRING(cpathbytes_tmp), PyBytes_AS_STRING(cpathbytes))) { if (Py_VerboseFlag) -- cgit v1.2.1 From 1a11faf52590fae1e61f304835b27c42ed21b1dd Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 15 Nov 2011 20:40:55 +0100 Subject: Fix regression under Windows following b75b41237380 (from issue #13392) --- Python/import.c | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 3c3e504e28..64197d7a48 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1267,6 +1267,11 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, PyErr_Clear(); return; } + if (PyUnicode_CopyCharacters(cpathname_tmp, 0, + cpathname, 0, cpathname_len) < 0) { + PyErr_Clear(); + return; + } PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 0, '.'); PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 1, 't'); PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 2, 'm'); -- cgit v1.2.1 From 23ad9bf93bf81d89142e55ca1cfb04f9c3ffc837 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 15 Nov 2011 22:27:32 +0100 Subject: Use PyUnicode_FromFormat() to create the temporary file name. Also, as in importlib, append the id of an object to make the file name pseudo-random. --- Python/import.c | 41 ++++++++++++++--------------------------- 1 file changed, 14 insertions(+), 27 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 64197d7a48..b1eca64f4d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1195,17 +1195,15 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, Py_UCS4 *cpathname_ucs4; FILE *fp; time_t mtime = srcstat->st_mtime; + PyObject *cpathname_tmp; #ifdef MS_WINDOWS /* since Windows uses different permissions */ mode_t mode = srcstat->st_mode & ~S_IEXEC; - PyObject *cpathname_tmp; - Py_ssize_t cpathname_len; #else mode_t dirmode = (srcstat->st_mode | S_IXUSR | S_IXGRP | S_IXOTH | S_IWUSR | S_IWGRP | S_IWOTH); PyObject *dirbytes; PyObject *cpathbytes, *cpathbytes_tmp; - Py_ssize_t cpathbytes_len; #endif int fd; PyObject *dirname; @@ -1258,24 +1256,18 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, Py_DECREF(dirname); /* We first write to a tmp file and then take advantage - of atomic renaming (which *should* be true even under Windows). */ -#ifdef MS_WINDOWS - cpathname_len = PyUnicode_GET_LENGTH(cpathname); - cpathname_tmp = PyUnicode_New(cpathname_len + 4, - PyUnicode_MAX_CHAR_VALUE(cpathname)); + of atomic renaming (which *should* be true even under Windows). + As in importlib, we use id(something) to generate a pseudo-random + filename. mkstemp() can't be used since it doesn't allow specifying + the file access permissions. + */ + cpathname_tmp = PyUnicode_FromFormat("%U.%zd", + cpathname, (Py_ssize_t) co); if (cpathname_tmp == NULL) { PyErr_Clear(); return; } - if (PyUnicode_CopyCharacters(cpathname_tmp, 0, - cpathname, 0, cpathname_len) < 0) { - PyErr_Clear(); - return; - } - PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 0, '.'); - PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 1, 't'); - PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 2, 'm'); - PyUnicode_WriteChar(cpathname_tmp, cpathname_len + 3, 'p'); +#ifdef MS_WINDOWS (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname_tmp)); fd = _wopen(PyUnicode_AS_UNICODE(cpathname_tmp), O_EXCL | O_CREAT | O_WRONLY | O_BINARY, @@ -1285,22 +1277,17 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, else fp = NULL; #else - cpathbytes = PyUnicode_EncodeFSDefault(cpathname); - if (cpathbytes == NULL) { + cpathbytes_tmp = PyUnicode_EncodeFSDefault(cpathname_tmp); + Py_DECREF(cpathname_tmp); + if (cpathbytes_tmp == NULL) { PyErr_Clear(); return; } - cpathbytes_len = PyBytes_GET_SIZE(cpathbytes); - cpathbytes_tmp = PyBytes_FromStringAndSize(NULL, cpathbytes_len + 4); - if (cpathbytes_tmp == NULL) { - Py_DECREF(cpathbytes); + cpathbytes = PyUnicode_EncodeFSDefault(cpathname); + if (cpathbytes == NULL) { PyErr_Clear(); return; } - memcpy(PyBytes_AS_STRING(cpathbytes_tmp), PyBytes_AS_STRING(cpathbytes), - cpathbytes_len); - memcpy(PyBytes_AS_STRING(cpathbytes_tmp) + cpathbytes_len, ".tmp", 4); - fd = open(PyBytes_AS_STRING(cpathbytes_tmp), O_CREAT | O_EXCL | O_WRONLY, 0666); if (0 <= fd) -- cgit v1.2.1 From 84acdc7c2fbb43c63eab2518445c5a7771755037 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 17 Nov 2011 00:45:54 +0100 Subject: Catch PyUnicode_AS_UNICODE() errors --- Python/import.c | 47 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 33 insertions(+), 14 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index b1eca64f4d..6c6d1f697a 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1198,6 +1198,7 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, PyObject *cpathname_tmp; #ifdef MS_WINDOWS /* since Windows uses different permissions */ mode_t mode = srcstat->st_mode & ~S_IEXEC; + wchar_t *wdirname, *wpathname, *wpathname_tmp; #else mode_t dirmode = (srcstat->st_mode | S_IXUSR | S_IXGRP | S_IXOTH | @@ -1230,7 +1231,12 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, } #ifdef MS_WINDOWS - res = CreateDirectoryW(PyUnicode_AS_UNICODE(dirname), NULL); + wdirname = PyUnicode_AsUnicode(dirname); + if (wdirname == NULL) { + PyErr_Clear(); + return; + } + res = CreateDirectoryW(wdirname, NULL); ok = (res != 0); if (!ok && GetLastError() == ERROR_ALREADY_EXISTS) ok = 1; @@ -1268,8 +1274,19 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, return; } #ifdef MS_WINDOWS - (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname_tmp)); - fd = _wopen(PyUnicode_AS_UNICODE(cpathname_tmp), + wpathname = PyUnicode_AsUnicode(cpathname); + if (wpathname == NULL) { + PyErr_Clear(); + return; + } + wpathname_tmp = PyUnicode_AsUnicode(cpathname_tmp); + if (wpathname_tmp == NULL) { + PyErr_Clear(); + return; + } + + (void)DeleteFileW(wpathname_tmp); + fd = _wopen(wpathname_tmp, O_EXCL | O_CREAT | O_WRONLY | O_BINARY, mode); if (0 <= fd) @@ -1322,7 +1339,7 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, /* Don't keep partial file */ fclose(fp); #ifdef MS_WINDOWS - (void)DeleteFileW(PyUnicode_AS_UNICODE(cpathname_tmp)); + (void)DeleteFileW(wpathname_tmp); Py_DECREF(cpathname_tmp); #else (void) unlink(PyBytes_AS_STRING(cpathbytes_tmp)); @@ -1334,13 +1351,11 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, fclose(fp); /* Do a (hopefully) atomic rename */ #ifdef MS_WINDOWS - if (!MoveFileExW(PyUnicode_AS_UNICODE(cpathname_tmp), - PyUnicode_AS_UNICODE(cpathname), - MOVEFILE_REPLACE_EXISTING)) { + if (!MoveFileExW(wpathname_tmp, wpathname, MOVEFILE_REPLACE_EXISTING)) { if (Py_VerboseFlag) PySys_FormatStderr("# can't write %R\n", cpathname); /* Don't keep tmp file */ - (void) DeleteFileW(PyUnicode_AS_UNICODE(cpathname_tmp)); + (void) DeleteFileW(wpathname_tmp); Py_DECREF(cpathname_tmp); return; } @@ -1819,10 +1834,10 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, PyUnicode_CopyCharacters(filename, 0, path_unicode, 0, len); pos = len; if (addsep) - PyUnicode_WRITE(PyUnicode_KIND(filename), + PyUnicode_WRITE(PyUnicode_KIND(filename), PyUnicode_DATA(filename), pos++, SEP); - PyUnicode_CopyCharacters(filename, pos, name, 0, + PyUnicode_CopyCharacters(filename, pos, name, 0, PyUnicode_GET_LENGTH(name)); /* Check for package import (buf holds a directory name, @@ -2268,18 +2283,22 @@ case_ok(PyObject *filename, Py_ssize_t prefix_delta, PyObject *name) WIN32_FIND_DATAW data; HANDLE h; int cmp; - wchar_t *wname; + wchar_t *wfilename, *wname; Py_ssize_t wname_len; if (Py_GETENV("PYTHONCASEOK") != NULL) return 1; - h = FindFirstFileW(PyUnicode_AS_UNICODE(filename), &data); + wfilename = PyUnicode_AsUnicode(filename); + if (wfilename == NULL) + return -1; + + h = FindFirstFileW(wfilename, &data); if (h == INVALID_HANDLE_VALUE) { PyErr_Format(PyExc_NameError, "Can't find file for module %R\n(filename %R)", name, filename); - return 0; + return -1; } FindClose(h); @@ -2831,7 +2850,7 @@ import_module_level(PyObject *name, PyObject *globals, PyObject *locals, if (!ensure_fromlist(tail, fromlist, prefix, 0)) goto out; - + result = tail; Py_INCREF(result); out: -- cgit v1.2.1 From 949c3f094c436fea35325980527d63d0d9eabd21 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Fri, 25 Nov 2011 18:56:07 +0100 Subject: PEP 3155 / issue #13448: Qualified name for classes and functions. --- Python/import.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 6c6d1f697a..7199230098 100644 --- a/Python/import.c +++ b/Python/import.c @@ -103,6 +103,7 @@ typedef unsigned short mode_t; tag: cpython-32 Python 3.2a2 3180 (add DELETE_DEREF) Python 3.3a0 3190 __class__ super closure changed + Python 3.3a0 3200 (__qualname__ added) */ /* MAGIC must change whenever the bytecode emitted by the compiler may no @@ -115,7 +116,7 @@ typedef unsigned short mode_t; #define STRIFY(name) QUOTE(name) #define MAJOR STRIFY(PY_MAJOR_VERSION) #define MINOR STRIFY(PY_MINOR_VERSION) -#define MAGIC (3190 | ((long)'\r'<<16) | ((long)'\n'<<24)) +#define MAGIC (3200 | ((long)'\r'<<16) | ((long)'\n'<<24)) #define TAG "cpython-" MAJOR MINOR; #define CACHEDIR "__pycache__" /* Current magic word and string tag as globals. */ -- cgit v1.2.1 From 07c717edd69dd5e05901335996d83fe11d6a24e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Wed, 7 Dec 2011 19:16:01 +0100 Subject: Issue #11051: Reduce the number of syscalls per import. --- Python/import.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 7199230098..9773687f62 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1944,8 +1944,7 @@ find_module_path_list(PyObject *fullname, PyObject *name, if (Py_VerboseFlag > 1) PySys_FormatStderr("# trying %R\n", filename); - if (_Py_stat(filename, &statbuf) == 0 && /* it exists */ - S_ISDIR(statbuf.st_mode)) /* it's a directory */ + if (_Py_stat(filename, &statbuf) != 0 || S_ISDIR(statbuf.st_mode)) { Py_DECREF(filename); continue; -- cgit v1.2.1 From d07179846c02fb4a745fd29f7ddfcfc8f51e366b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Charles-Fran=C3=A7ois=20Natali?= Date: Wed, 7 Dec 2011 23:17:58 +0100 Subject: Followup to a541bda2f5e2: Add a short comment. --- Python/import.c | 1 + 1 file changed, 1 insertion(+) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 9773687f62..47839c543d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1946,6 +1946,7 @@ find_module_path_list(PyObject *fullname, PyObject *name, if (_Py_stat(filename, &statbuf) != 0 || S_ISDIR(statbuf.st_mode)) { + /* it doesn't exist, or it's a directory */ Py_DECREF(filename); continue; } -- cgit v1.2.1 From 219d85b51cf4eed90b0cd74706d6b55b3753c604 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sun, 18 Dec 2011 20:47:30 +0100 Subject: import.c now catchs _Py_stat() exceptions _Py_stat() now returns -2 if an exception was raised. --- Python/import.c | 38 +++++++++++++++++++++++++++++++------- 1 file changed, 31 insertions(+), 7 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 47839c543d..34d1a26dc7 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1527,6 +1527,7 @@ get_sourcefile(PyObject *filename) Py_UCS4 *fileuni; PyObject *py; struct stat statbuf; + int err; len = PyUnicode_GET_LENGTH(filename); if (len == 0) @@ -1554,7 +1555,10 @@ get_sourcefile(PyObject *filename) if (py == NULL) goto error; - if (_Py_stat(py, &statbuf) == 0 && S_ISREG(statbuf.st_mode)) { + err = _Py_stat(py, &statbuf); + if (err == -2) + goto error; + if (err == 0 && S_ISREG(statbuf.st_mode)) { PyMem_Free(fileuni); return py; } @@ -1760,7 +1764,7 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, Py_ssize_t len, pos; struct stat statbuf; static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; - int result, addsep; + int err, result, addsep; if (PyUnicode_Check(path)) { Py_INCREF(path); @@ -1844,7 +1848,12 @@ find_module_path(PyObject *fullname, PyObject *name, PyObject *path, /* Check for package import (buf holds a directory name, and there's an __init__ module in that directory */ #ifdef HAVE_STAT - if (_Py_stat(filename, &statbuf) == 0 && /* it exists */ + err = _Py_stat(filename, &statbuf); + if (err == -2) { + result = -1; + goto out; + } + if (err == 0 && /* it exists */ S_ISDIR(statbuf.st_mode)) /* it's a directory */ { int match; @@ -1905,6 +1914,7 @@ find_module_path_list(PyObject *fullname, PyObject *name, FILE *fp = NULL; PyObject *prefix, *filename; int match; + int err; npath = PyList_Size(search_path_list); for (i = 0; i < npath; i++) { @@ -1944,8 +1954,13 @@ find_module_path_list(PyObject *fullname, PyObject *name, if (Py_VerboseFlag > 1) PySys_FormatStderr("# trying %R\n", filename); - if (_Py_stat(filename, &statbuf) != 0 || S_ISDIR(statbuf.st_mode)) - { + err = _Py_stat(filename, &statbuf); + if (err == -2) { + Py_DECREF(prefix); + Py_DECREF(filename); + return NULL; + } + if (err != 0 || S_ISDIR(statbuf.st_mode)) { /* it doesn't exist, or it's a directory */ Py_DECREF(filename); continue; @@ -2345,11 +2360,15 @@ find_init_module(PyObject *directory) struct stat statbuf; PyObject *filename; int match; + int err; filename = PyUnicode_FromFormat("%U%c__init__.py", directory, SEP); if (filename == NULL) return -1; - if (_Py_stat(filename, &statbuf) == 0) { + err = _Py_stat(filename, &statbuf); + if (err == -2) + return -1; + if (err == 0) { /* 3=len(".py") */ match = case_ok(filename, -3, initstr); if (match < 0) { @@ -2367,7 +2386,12 @@ find_init_module(PyObject *directory) directory, SEP, Py_OptimizeFlag ? 'o' : 'c'); if (filename == NULL) return -1; - if (_Py_stat(filename, &statbuf) == 0) { + err = _Py_stat(filename, &statbuf); + if (err == -2) { + Py_DECREF(filename); + return -1; + } + if (err == 0) { /* 4=len(".pyc") */ match = case_ok(filename, -4, initstr); if (match < 0) { -- cgit v1.2.1 From 1434e995657f5525a55584535d023dbe0ce2af24 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Fri, 13 Jan 2012 18:52:16 +0100 Subject: Issue #13645: pyc files now contain the size of the corresponding source code, to avoid timestamp collisions (especially on filesystems with a low timestamp resolution) when checking for freshness of the bytecode. --- Python/import.c | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index c7541316b5..4efc36923a 100644 --- a/Python/import.c +++ b/Python/import.c @@ -104,6 +104,7 @@ typedef unsigned short mode_t; Python 3.2a2 3180 (add DELETE_DEREF) Python 3.3a0 3190 __class__ super closure changed Python 3.3a0 3200 (__qualname__ added) + 3210 (added size modulo 2**32 to the pyc header) */ /* MAGIC must change whenever the bytecode emitted by the compiler may no @@ -116,7 +117,7 @@ typedef unsigned short mode_t; #define STRIFY(name) QUOTE(name) #define MAJOR STRIFY(PY_MAJOR_VERSION) #define MINOR STRIFY(PY_MINOR_VERSION) -#define MAGIC (3200 | ((long)'\r'<<16) | ((long)'\n'<<24)) +#define MAGIC (3210 | ((long)'\r'<<16) | ((long)'\n'<<24)) #define TAG "cpython-" MAJOR MINOR; #define CACHEDIR "__pycache__" /* Current magic word and string tag as globals. */ @@ -1071,11 +1072,12 @@ make_source_pathname(PyObject *path) Doesn't set an exception. */ static FILE * -check_compiled_module(PyObject *pathname, time_t mtime, PyObject *cpathname) +check_compiled_module(PyObject *pathname, struct stat *srcstat, PyObject *cpathname) { FILE *fp; long magic; long pyc_mtime; + long pyc_size; fp = _Py_fopen(cpathname, "rb"); if (fp == NULL) @@ -1088,12 +1090,19 @@ check_compiled_module(PyObject *pathname, time_t mtime, PyObject *cpathname) return NULL; } pyc_mtime = PyMarshal_ReadLongFromFile(fp); - if (pyc_mtime != mtime) { + if (pyc_mtime != srcstat->st_mtime) { if (Py_VerboseFlag) PySys_FormatStderr("# %R has bad mtime\n", cpathname); fclose(fp); return NULL; } + pyc_size = PyMarshal_ReadLongFromFile(fp); + if (pyc_size != (srcstat->st_size & 0xFFFFFFFF)) { + if (Py_VerboseFlag) + PySys_FormatStderr("# %R has bad size\n", cpathname); + fclose(fp); + return NULL; + } if (Py_VerboseFlag) PySys_FormatStderr("# %R matches %R\n", cpathname, pathname); return fp; @@ -1136,6 +1145,8 @@ load_compiled_module(PyObject *name, PyObject *cpathname, FILE *fp) "Bad magic number in %R", cpathname); return NULL; } + /* Skip mtime and size */ + (void) PyMarshal_ReadLongFromFile(fp); (void) PyMarshal_ReadLongFromFile(fp); co = read_compiled_module(cpathname, fp); if (co == NULL) @@ -1196,6 +1207,7 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, Py_UCS4 *cpathname_ucs4; FILE *fp; time_t mtime = srcstat->st_mtime; + long size = srcstat->st_size & 0xFFFFFFFF; PyObject *cpathname_tmp; #ifdef MS_WINDOWS /* since Windows uses different permissions */ mode_t mode = srcstat->st_mode & ~S_IEXEC; @@ -1326,14 +1338,16 @@ write_compiled_module(PyCodeObject *co, PyObject *cpathname, return; } PyMarshal_WriteLongToFile(pyc_magic, fp, Py_MARSHAL_VERSION); - /* First write a 0 for mtime */ + /* First write a 0 for mtime and size */ + PyMarshal_WriteLongToFile(0L, fp, Py_MARSHAL_VERSION); PyMarshal_WriteLongToFile(0L, fp, Py_MARSHAL_VERSION); PyMarshal_WriteObjectToFile((PyObject *)co, fp, Py_MARSHAL_VERSION); fflush(fp); - /* Now write the true mtime */ + /* Now write the true mtime and size */ fseek(fp, 4L, 0); assert(mtime < LONG_MAX); PyMarshal_WriteLongToFile((long)mtime, fp, Py_MARSHAL_VERSION); + PyMarshal_WriteLongToFile(size, fp, Py_MARSHAL_VERSION); if (fflush(fp) != 0 || ferror(fp)) { if (Py_VerboseFlag) PySys_FormatStderr("# can't write %R\n", cpathname); @@ -1478,7 +1492,7 @@ load_source_module(PyObject *name, PyObject *pathname, FILE *fp) cpathname = make_compiled_pathname(pathname, !Py_OptimizeFlag); if (cpathname != NULL) - fpc = check_compiled_module(pathname, st.st_mtime, cpathname); + fpc = check_compiled_module(pathname, &st, cpathname); else fpc = NULL; -- cgit v1.2.1 From c6d22e93db4b5e0cdb94ae023a4b7e43693dfed2 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 26 Jan 2012 00:31:49 +0100 Subject: Remove now useless arbitrary limit of module name length --- Python/import.c | 6 ------ 1 file changed, 6 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index d5b89d5966..8bd7a61722 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2052,12 +2052,6 @@ find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, if (p_loader != NULL) *p_loader = NULL; - if (PyUnicode_GET_LENGTH(name) > MAXPATHLEN) { - PyErr_SetString(PyExc_OverflowError, - "module name is too long"); - return NULL; - } - /* sys.meta_path import hook */ if (p_loader != NULL) { _Py_IDENTIFIER(find_module); -- cgit v1.2.1 From 0ea564cff248528b899a26f75b173f35b55d505a Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Thu, 15 Mar 2012 15:37:39 -0500 Subject: perform yield from delegation by repeating YIELD_FROM opcode (closes #14230) This allows generators that are using yield from to be seen by debuggers. It also kills the f_yieldfrom field on frame objects. Patch mostly from Mark Shannon with a few tweaks by me. --- Python/import.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 4871b99a93..14148c7a1c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -104,7 +104,8 @@ typedef unsigned short mode_t; Python 3.2a2 3180 (add DELETE_DEREF) Python 3.3a0 3190 __class__ super closure changed Python 3.3a0 3200 (__qualname__ added) - 3210 (added size modulo 2**32 to the pyc header) + Python 3.3a1 3210 (added size modulo 2**32 to the pyc header) + 3220 (changed PEP 380 implementation) */ /* MAGIC must change whenever the bytecode emitted by the compiler may no -- cgit v1.2.1 From 06b08c0eb2d160a859326db57c12ea099d74a13c Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Thu, 15 Mar 2012 15:40:37 -0500 Subject: fix comment --- Python/import.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 14148c7a1c..c0f89aaa1b 100644 --- a/Python/import.c +++ b/Python/import.c @@ -104,8 +104,8 @@ typedef unsigned short mode_t; Python 3.2a2 3180 (add DELETE_DEREF) Python 3.3a0 3190 __class__ super closure changed Python 3.3a0 3200 (__qualname__ added) - Python 3.3a1 3210 (added size modulo 2**32 to the pyc header) - 3220 (changed PEP 380 implementation) + 3210 (added size modulo 2**32 to the pyc header) + Python 3.3a1 3220 (changed PEP 380 implementation) */ /* MAGIC must change whenever the bytecode emitted by the compiler may no -- cgit v1.2.1 From d8c00dc5e03601e3959e3147dbf9c527e1d3b648 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Fri, 16 Mar 2012 09:39:12 -0500 Subject: in 72556ff86828, I should have updated the magic as well as the comment (#14230) --- Python/import.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index c0f89aaa1b..144ca71117 100644 --- a/Python/import.c +++ b/Python/import.c @@ -118,7 +118,7 @@ typedef unsigned short mode_t; #define STRIFY(name) QUOTE(name) #define MAJOR STRIFY(PY_MAJOR_VERSION) #define MINOR STRIFY(PY_MINOR_VERSION) -#define MAGIC (3210 | ((long)'\r'<<16) | ((long)'\n'<<24)) +#define MAGIC (3220 | ((long)'\r'<<16) | ((long)'\n'<<24)) #define TAG "cpython-" MAJOR MINOR; #define CACHEDIR "__pycache__" /* Current magic word and string tag as globals. */ -- cgit v1.2.1 From e6ecc6d13e7b3c963d6cabfdacca8138f968e970 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sat, 14 Apr 2012 14:10:13 -0400 Subject: Issue #2377: Make importlib the implementation of __import__(). importlib._bootstrap is now frozen into Python/importlib.h and stored as _frozen_importlib in sys.modules. Py_Initialize() loads the frozen code along with sys and imp and then uses _frozen_importlib._install() to set builtins.__import__() w/ _frozen_importlib.__import__(). --- Python/import.c | 766 ++++++++++++++++++-------------------------------------- 1 file changed, 244 insertions(+), 522 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 144ca71117..87c1214c7e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -203,17 +203,13 @@ _PyImport_Init(void) void _PyImportHooks_Init(void) { - PyObject *v, *path_hooks = NULL, *zimpimport; + PyObject *v, *path_hooks = NULL; int err = 0; - /* adding sys.path_hooks and sys.path_importer_cache, setting up - zipimport */ if (PyType_Ready(&PyNullImporter_Type) < 0) goto error; - if (Py_VerboseFlag) - PySys_WriteStderr("# installing zipimport hook\n"); - + /* adding sys.path_hooks and sys.path_importer_cache */ v = PyList_New(0); if (v == NULL) goto error; @@ -234,11 +230,26 @@ _PyImportHooks_Init(void) err = PySys_SetObject("path_hooks", path_hooks); if (err) { error: - PyErr_Print(); - Py_FatalError("initializing sys.meta_path, sys.path_hooks, " - "path_importer_cache, or NullImporter failed" - ); + PyErr_Print(); + Py_FatalError("initializing sys.meta_path, sys.path_hooks, " + "path_importer_cache, or NullImporter failed" + ); } + Py_DECREF(path_hooks); +} + +void +_PyImportZip_Init(void) +{ + PyObject *path_hooks, *zimpimport; + int err = 0; + + path_hooks = PySys_GetObject("path_hooks"); + if (path_hooks == NULL) + goto error; + + if (Py_VerboseFlag) + PySys_WriteStderr("# installing zipimport hook\n"); zimpimport = PyImport_ImportModule("zipimport"); if (zimpimport == NULL) { @@ -261,14 +272,20 @@ _PyImportHooks_Init(void) /* sys.path_hooks.append(zipimporter) */ err = PyList_Append(path_hooks, zipimporter); Py_DECREF(zipimporter); - if (err) + if (err < 0) { goto error; + } if (Py_VerboseFlag) PySys_WriteStderr( "# installed zipimport hook\n"); } } - Py_DECREF(path_hooks); + + return; + + error: + PyErr_Print(); + Py_FatalError("initializing zipimport or NullImporter failed"); } /* Locking primitives to prevent parallel imports of the same module @@ -2542,8 +2559,6 @@ init_builtin(PyObject *name) name); return -1; } - if (Py_VerboseFlag) - PySys_FormatStderr("import %U # builtin\n", name); mod = (*p->initfunc)(); if (mod == 0) return -1; @@ -2654,9 +2669,6 @@ PyImport_ImportFrozenModuleObject(PyObject *name) ispackage = (size < 0); if (ispackage) size = -size; - if (Py_VerboseFlag) - PySys_FormatStderr("import %U # frozen%s\n", - name, ispackage ? " package" : ""); co = PyMarshal_ReadObjectFromString((char *)p->code, size); if (co == NULL) return -1; @@ -2786,568 +2798,278 @@ PyImport_ImportModuleNoBlock(const char *name) return result; } -/* Forward declarations for helper routines */ -static PyObject *get_parent(PyObject *globals, - PyObject **p_name, - int level); -static PyObject *load_next(PyObject *mod, PyObject *altmod, - PyObject *inputname, PyObject **p_outputname, - PyObject **p_prefix); -static int mark_miss(PyObject *name); -static int ensure_fromlist(PyObject *mod, PyObject *fromlist, - PyObject *buf, int recursive); -static PyObject * import_submodule(PyObject *mod, PyObject *name, - PyObject *fullname); - -/* The Magnum Opus of dotted-name import :-) */ -static PyObject * -import_module_level(PyObject *name, PyObject *globals, PyObject *locals, - PyObject *fromlist, int level) +PyObject * +PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, + PyObject *locals, PyObject *given_fromlist, + int level) { - PyObject *parent, *next, *inputname, *outputname; - PyObject *head = NULL; - PyObject *tail = NULL; - PyObject *prefix = NULL; - PyObject *result = NULL; - Py_ssize_t sep, altsep; - - if (PyUnicode_READY(name)) - return NULL; - - sep = PyUnicode_FindChar(name, SEP, 0, PyUnicode_GET_LENGTH(name), 1); - if (sep == -2) - return NULL; -#ifdef ALTSEP - altsep = PyUnicode_FindChar(name, ALTSEP, 0, PyUnicode_GET_LENGTH(name), 1); - if (altsep == -2) - return NULL; -#else - altsep = -1; -#endif - if (sep != -1 || altsep != -1) - { - PyErr_SetString(PyExc_ImportError, - "Import by filename is not supported."); - return NULL; - } - - parent = get_parent(globals, &prefix, level); - if (parent == NULL) { - return NULL; - } - - if (PyUnicode_READY(prefix)) - return NULL; - - head = load_next(parent, level < 0 ? Py_None : parent, name, &outputname, - &prefix); - if (head == NULL) - goto out; - - tail = head; - Py_INCREF(tail); - - if (outputname != NULL) { - while (1) { - inputname = outputname; - next = load_next(tail, tail, inputname, &outputname, - &prefix); - Py_CLEAR(tail); - Py_CLEAR(inputname); - if (next == NULL) - goto out; - tail = next; + _Py_IDENTIFIER(__import__); + _Py_IDENTIFIER(__package__); + _Py_IDENTIFIER(__path__); + _Py_IDENTIFIER(__name__); + _Py_IDENTIFIER(_find_and_load); + _Py_IDENTIFIER(_handle_fromlist); + _Py_static_string(single_dot, "."); + PyObject *abs_name = NULL; + PyObject *builtins_import = NULL; + PyObject *final_mod = NULL; + PyObject *mod = NULL; + PyObject *package = NULL; + PyObject *globals = NULL; + PyObject *fromlist = NULL; + PyInterpreterState *interp = PyThreadState_GET()->interp; - if (outputname == NULL) { - break; - } + /* Make sure to use default values so as to not have + PyObject_CallMethodObjArgs() truncate the parameter list because of a + NULL argument. */ + if (given_globals == NULL) { + globals = PyDict_New(); + if (globals == NULL) { + goto error; } } - if (tail == Py_None) { - /* If tail is Py_None, both get_parent and load_next found - an empty module name: someone called __import__("") or - doctored faulty bytecode */ - PyErr_SetString(PyExc_ValueError, "Empty module name"); - goto out; + else { + /* Only have to care what given_globals is if it will be used + fortsomething. */ + if (level > 0 && !PyDict_Check(given_globals)) { + PyErr_SetString(PyExc_TypeError, "globals must be a dict"); + goto error; + } + globals = given_globals; + Py_INCREF(globals); } - if (fromlist != NULL) { - if (fromlist == Py_None || !PyObject_IsTrue(fromlist)) - fromlist = NULL; + if (given_fromlist == NULL) { + fromlist = PyList_New(0); + if (fromlist == NULL) { + goto error; + } } - - if (fromlist == NULL) { - result = head; - Py_INCREF(result); - goto out; + else { + fromlist = given_fromlist; + Py_INCREF(fromlist); } - - if (!ensure_fromlist(tail, fromlist, prefix, 0)) - goto out; - - result = tail; - Py_INCREF(result); - out: - Py_XDECREF(head); - Py_XDECREF(tail); - Py_XDECREF(prefix); - return result; -} - -PyObject * -PyImport_ImportModuleLevelObject(PyObject *name, PyObject *globals, - PyObject *locals, PyObject *fromlist, - int level) -{ - PyObject *mod; - _PyImport_AcquireLock(); - mod = import_module_level(name, globals, locals, fromlist, level); - if (_PyImport_ReleaseLock() < 0) { - Py_XDECREF(mod); - PyErr_SetString(PyExc_RuntimeError, - "not holding the import lock"); - return NULL; + if (name == NULL) { + PyErr_SetString(PyExc_ValueError, "Empty module name"); + goto error; } - return mod; -} -PyObject * -PyImport_ImportModuleLevel(const char *name, PyObject *globals, PyObject *locals, - PyObject *fromlist, int level) -{ - PyObject *nameobj, *mod; - nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) - return NULL; - mod = PyImport_ImportModuleLevelObject(nameobj, globals, locals, - fromlist, level); - Py_DECREF(nameobj); - return mod; -} + /* The below code is importlib.__import__() & _gcd_import(), ported to C + for added performance. */ - -/* Return the package that an import is being performed in. If globals comes - from the module foo.bar.bat (not itself a package), this returns the - sys.modules entry for foo.bar. If globals is from a package's __init__.py, - the package's entry in sys.modules is returned, as a borrowed reference. - - The name of the returned package is returned in *p_name. - - If globals doesn't come from a package or a module in a package, or a - corresponding entry is not found in sys.modules, Py_None is returned. -*/ -static PyObject * -get_parent(PyObject *globals, PyObject **p_name, int level) -{ - PyObject *nameobj; - - static PyObject *namestr = NULL; - static PyObject *pathstr = NULL; - static PyObject *pkgstr = NULL; - PyObject *pkgname, *modname, *modpath, *modules, *parent; - int orig_level = level; - - if (globals == NULL || !PyDict_Check(globals) || !level) - goto return_none; - - if (namestr == NULL) { - namestr = PyUnicode_InternFromString("__name__"); - if (namestr == NULL) - return NULL; + if (!PyUnicode_Check(name)) { + PyErr_SetString(PyExc_TypeError, "module name must be a string"); + goto error; } - if (pathstr == NULL) { - pathstr = PyUnicode_InternFromString("__path__"); - if (pathstr == NULL) - return NULL; + else if (PyUnicode_READY(name) < 0) { + goto error; } - if (pkgstr == NULL) { - pkgstr = PyUnicode_InternFromString("__package__"); - if (pkgstr == NULL) - return NULL; + if (level < 0) { + PyErr_SetString(PyExc_ValueError, "level must be >= 0"); + goto error; } - - pkgname = PyDict_GetItem(globals, pkgstr); - - if ((pkgname != NULL) && (pkgname != Py_None)) { - /* __package__ is set, so use it */ - if (!PyUnicode_Check(pkgname)) { - PyErr_SetString(PyExc_ValueError, - "__package__ set to non-string"); - return NULL; - } - if (PyUnicode_GET_LENGTH(pkgname) == 0) { - if (level > 0) { - PyErr_SetString(PyExc_ValueError, - "Attempted relative import in non-package"); - return NULL; + else if (level > 0) { + package = _PyDict_GetItemId(globals, &PyId___package__); + if (package != NULL && package != Py_None) { + Py_INCREF(package); + if (!PyUnicode_Check(package)) { + PyErr_SetString(PyExc_TypeError, "package must be a string"); + goto error; } - goto return_none; } - Py_INCREF(pkgname); - nameobj = pkgname; - } else { - /* __package__ not set, so figure it out and set it */ - modname = PyDict_GetItem(globals, namestr); - if (modname == NULL || !PyUnicode_Check(modname)) - goto return_none; - - modpath = PyDict_GetItem(globals, pathstr); - if (modpath != NULL) { - /* __path__ is set, so modname is already the package name */ - int error; - - error = PyDict_SetItem(globals, pkgstr, modname); - if (error) { - PyErr_SetString(PyExc_ValueError, - "Could not set __package__"); - return NULL; + else { + package = _PyDict_GetItemIdWithError(globals, &PyId___name__); + if (package == NULL) { + goto error; } - Py_INCREF(modname); - nameobj = modname; - } else { - /* Normal module, so work out the package name if any */ - Py_ssize_t len; - len = PyUnicode_FindChar(modname, '.', - 0, PyUnicode_GET_LENGTH(modname), -1); - if (len == -2) - return NULL; - if (len < 0) { - if (level > 0) { - PyErr_SetString(PyExc_ValueError, - "Attempted relative import in non-package"); - return NULL; + else if (!PyUnicode_Check(package)) { + PyErr_SetString(PyExc_TypeError, "__name__ must be a string"); + } + Py_INCREF(package); + + if (_PyDict_GetItemId(globals, &PyId___path__) == NULL) { + PyObject *borrowed_dot = _PyUnicode_FromId(&single_dot); + if (borrowed_dot == NULL) { + goto error; } - if (PyDict_SetItem(globals, pkgstr, Py_None)) { - PyErr_SetString(PyExc_ValueError, - "Could not set __package__"); - return NULL; + PyObject *partition = PyUnicode_RPartition(package, + borrowed_dot); + Py_DECREF(package); + if (partition == NULL) { + goto error; } - goto return_none; + package = PyTuple_GET_ITEM(partition, 0); + Py_INCREF(package); + Py_DECREF(partition); } - pkgname = PyUnicode_Substring(modname, 0, len); - if (pkgname == NULL) - return NULL; - if (PyDict_SetItem(globals, pkgstr, pkgname)) { - Py_DECREF(pkgname); - PyErr_SetString(PyExc_ValueError, - "Could not set __package__"); - return NULL; - } - nameobj = pkgname; } - } - if (level > 1) { - Py_ssize_t dot, end = PyUnicode_GET_LENGTH(nameobj); - PyObject *newname; - while (--level > 0) { - dot = PyUnicode_FindChar(nameobj, '.', 0, end, -1); - if (dot == -2) { - Py_DECREF(nameobj); - return NULL; - } - if (dot < 0) { - Py_DECREF(nameobj); - PyErr_SetString(PyExc_ValueError, - "Attempted relative import beyond " - "toplevel package"); - return NULL; - } - end = dot; - } - newname = PyUnicode_Substring(nameobj, 0, end); - Py_DECREF(nameobj); - if (newname == NULL) - return NULL; - nameobj = newname; - } - modules = PyImport_GetModuleDict(); - parent = PyDict_GetItem(modules, nameobj); - if (parent == NULL) { - int err; - - if (orig_level >= 1) { + if (PyDict_GetItem(interp->modules, package) == NULL) { PyErr_Format(PyExc_SystemError, - "Parent module %R not loaded, " - "cannot perform relative import", nameobj); - Py_DECREF(nameobj); - return NULL; + "Parent module %R not loaded, cannot perform relative " + "import", package); + goto error; } - - err = PyErr_WarnFormat( - PyExc_RuntimeWarning, 1, - "Parent module %R not found while handling absolute import", - nameobj); - Py_DECREF(nameobj); - if (err) - return NULL; - - goto return_none; } - *p_name = nameobj; - return parent; - /* We expect, but can't guarantee, if parent != None, that: - - parent.__name__ == name - - parent.__dict__ is globals - If this is violated... Who cares? */ - -return_none: - nameobj = PyUnicode_New(0, 0); - if (nameobj == NULL) - return NULL; - *p_name = nameobj; - return Py_None; -} - -/* altmod is either None or same as mod */ -static PyObject * -load_next(PyObject *mod, PyObject *altmod, - PyObject *inputname, PyObject **p_outputname, - PyObject **p_prefix) -{ - Py_ssize_t dot; - Py_ssize_t len; - PyObject *fullname, *name = NULL, *result; - - *p_outputname = NULL; - - len = PyUnicode_GET_LENGTH(inputname); - if (len == 0) { - /* completely empty module name should only happen in - 'from . import' (or '__import__("")')*/ - Py_INCREF(mod); - return mod; + else { /* level == 0 */ + if (PyUnicode_GET_LENGTH(name) == 0) { + PyErr_SetString(PyExc_ValueError, "Empty module name"); + goto error; + } + package = Py_None; + Py_INCREF(package); } + if (level > 0) { + Py_ssize_t last_dot = PyUnicode_GET_LENGTH(package); + PyObject *base = NULL; + int level_up = 1; - dot = PyUnicode_FindChar(inputname, '.', 0, len, 1); - if (dot >= 0) { - len = dot; - if (len == 0) { - PyErr_SetString(PyExc_ValueError, - "Empty module name"); - goto error; + for (level_up = 1; level_up < level; level_up += 1) { + last_dot = PyUnicode_FindChar(package, '.', 0, last_dot, -1); + if (last_dot == -2) { + goto error; + } + else if (last_dot == -1) { + PyErr_SetString(PyExc_ValueError, + "attempted relative import beyond top-level " + "package"); + goto error; + } } - } + base = PyUnicode_Substring(package, 0, last_dot); + if (PyUnicode_GET_LENGTH(name) > 0) { + PyObject *borrowed_dot = NULL; + PyObject *seq = PyTuple_Pack(2, base, name); - /* name = inputname[:len] */ - name = PyUnicode_Substring(inputname, 0, len); - if (name == NULL) - goto error; + borrowed_dot = _PyUnicode_FromId(&single_dot); + if (borrowed_dot == NULL || seq == NULL) { + goto error; + } - if (PyUnicode_GET_LENGTH(*p_prefix)) { - /* fullname = prefix + "." + name */ - fullname = PyUnicode_FromFormat("%U.%U", *p_prefix, name); - if (fullname == NULL) - goto error; - } - else { - fullname = name; - Py_INCREF(fullname); - } - - result = import_submodule(mod, name, fullname); - Py_DECREF(*p_prefix); - /* Transfer reference. */ - *p_prefix = fullname; - if (result == Py_None && altmod != mod) { - Py_DECREF(result); - /* Here, altmod must be None and mod must not be None */ - result = import_submodule(altmod, name, name); - if (result != NULL && result != Py_None) { - if (mark_miss(*p_prefix) != 0) { - Py_DECREF(result); + abs_name = PyUnicode_Join(borrowed_dot, seq); + Py_DECREF(seq); + if (abs_name == NULL) { goto error; } - Py_DECREF(*p_prefix); - *p_prefix = name; - Py_INCREF(*p_prefix); + } + else { + abs_name = base; } } - if (result == NULL) - goto error; - - if (result == Py_None) { - Py_DECREF(result); - PyErr_Format(PyExc_ImportError, - "No module named %R", inputname); - goto error; + else { + abs_name = name; + Py_INCREF(abs_name); } - if (dot >= 0) { - *p_outputname = PyUnicode_Substring(inputname, dot+1, - PyUnicode_GET_LENGTH(inputname)); - if (*p_outputname == NULL) { - Py_DECREF(result); - goto error; +#if WITH_THREAD + _PyImport_AcquireLock(); +#endif + /* From this point forward, goto error_with_unlock! */ + if (PyDict_Check(globals)) { + builtins_import = _PyDict_GetItemId(globals, &PyId___import__); + } + if (builtins_import == NULL) { + builtins_import = _PyDict_GetItemId(interp->builtins, &PyId___import__); + if (builtins_import == NULL) { + Py_FatalError("__import__ missing"); } } + Py_INCREF(builtins_import); - Py_DECREF(name); - return result; - -error: - Py_XDECREF(name); - return NULL; -} + mod = PyDict_GetItem(interp->modules, abs_name); + if (mod == Py_None) { + PyErr_Format(PyExc_ImportError, + "import of %R halted; None in sys.modules", abs_name); + goto error_with_unlock; + } + else if (mod != NULL) { + Py_INCREF(mod); + } + else { + mod = _PyObject_CallMethodObjIdArgs(interp->importlib, + &PyId__find_and_load, abs_name, + builtins_import, NULL); + if (mod == NULL) { + goto error_with_unlock; + } + } -static int -mark_miss(PyObject *name) -{ - PyObject *modules = PyImport_GetModuleDict(); - return PyDict_SetItem(modules, name, Py_None); -} + if (PyObject_Not(fromlist)) { + if (level == 0 || PyUnicode_GET_LENGTH(name) > 0) { + PyObject *front = NULL; + PyObject *borrowed_dot = _PyUnicode_FromId(&single_dot); -static int -ensure_fromlist(PyObject *mod, PyObject *fromlist, PyObject *name, - int recursive) -{ - int i; - PyObject *fullname; - Py_ssize_t fromlist_len; + if (borrowed_dot == NULL) { + goto error_with_unlock; + } - if (!_PyObject_HasAttrId(mod, &PyId___path__)) - return 1; + PyObject *partition = PyUnicode_Partition(name, borrowed_dot); + if (partition == NULL) { + goto error_with_unlock; + } - fromlist_len = PySequence_Size(fromlist); + front = PyTuple_GET_ITEM(partition, 0); + Py_INCREF(front); + Py_DECREF(partition); - for (i = 0; i < fromlist_len; i++) { - PyObject *item = PySequence_GetItem(fromlist, i); - int hasit; - if (item == NULL) - return 0; - if (!PyUnicode_Check(item)) { - PyErr_SetString(PyExc_TypeError, - "Item in ``from list'' not a string"); - Py_DECREF(item); - return 0; - } - if (PyUnicode_READ_CHAR(item, 0) == '*') { - PyObject *all; - _Py_IDENTIFIER(__all__); - Py_DECREF(item); - /* See if the package defines __all__ */ - if (recursive) - continue; /* Avoid endless recursion */ - all = _PyObject_GetAttrId(mod, &PyId___all__); - if (all == NULL) - PyErr_Clear(); + if (level == 0) { + final_mod = PyDict_GetItemWithError(interp->modules, front); + Py_DECREF(front); + Py_XINCREF(final_mod); + } else { - int ret = ensure_fromlist(mod, all, name, 1); - Py_DECREF(all); - if (!ret) - return 0; + Py_ssize_t cut_off = PyUnicode_GetLength(name) - + PyUnicode_GetLength(front); + Py_ssize_t abs_name_len = PyUnicode_GetLength(abs_name); + PyObject *to_return = PyUnicode_Substring(name, 0, + abs_name_len - cut_off); + + final_mod = PyDict_GetItem(interp->modules, to_return); + Py_INCREF(final_mod); + Py_DECREF(to_return); } - continue; } - hasit = PyObject_HasAttr(mod, item); - if (!hasit) { - PyObject *submod; - fullname = PyUnicode_FromFormat("%U.%U", name, item); - if (fullname != NULL) { - submod = import_submodule(mod, item, fullname); - Py_DECREF(fullname); - } - else - submod = NULL; - Py_XDECREF(submod); - if (submod == NULL) { - Py_DECREF(item); - return 0; - } + else { + final_mod = mod; + Py_INCREF(mod); } - Py_DECREF(item); - } - - return 1; -} - -static int -add_submodule(PyObject *mod, PyObject *submod, PyObject *fullname, - PyObject *subname, PyObject *modules) -{ - if (mod == Py_None) - return 1; - /* Irrespective of the success of this load, make a - reference to it in the parent package module. A copy gets - saved in the modules dictionary under the full name, so get a - reference from there, if need be. (The exception is when the - load failed with a SyntaxError -- then there's no trace in - sys.modules. In that case, of course, do nothing extra.) */ - if (submod == NULL) { - submod = PyDict_GetItem(modules, fullname); - if (submod == NULL) - return 1; - } - if (PyModule_Check(mod)) { - /* We can't use setattr here since it can give a - * spurious warning if the submodule name shadows a - * builtin name */ - PyObject *dict = PyModule_GetDict(mod); - if (!dict) - return 0; - if (PyDict_SetItem(dict, subname, submod) < 0) - return 0; } else { - if (PyObject_SetAttr(mod, subname, submod) < 0) - return 0; + final_mod = _PyObject_CallMethodObjIdArgs(interp->importlib, + &PyId__handle_fromlist, mod, + fromlist, builtins_import, + NULL); } - return 1; + error_with_unlock: +#if WITH_THREAD + if (_PyImport_ReleaseLock() < 0) { + PyErr_SetString(PyExc_RuntimeError, "not holding the import lock"); + } +#endif + error: + Py_XDECREF(abs_name); + Py_XDECREF(builtins_import); + Py_XDECREF(mod); + Py_XDECREF(package); + Py_XDECREF(globals); + Py_XDECREF(fromlist); + return final_mod; } -static PyObject * -import_submodule(PyObject *mod, PyObject *subname, PyObject *fullname) +PyObject * +PyImport_ImportModuleLevel(const char *name, PyObject *globals, PyObject *locals, + PyObject *fromlist, int level) { - PyObject *modules = PyImport_GetModuleDict(); - PyObject *m = NULL, *bufobj, *path_list, *loader; - struct filedescr *fdp; - FILE *fp; - - /* Require: - if mod == None: subname == fullname - else: mod.__name__ + "." + subname == fullname - */ - - if ((m = PyDict_GetItem(modules, fullname)) != NULL) { - Py_INCREF(m); - return m; - } - - if (mod == Py_None) - path_list = NULL; - else { - path_list = _PyObject_GetAttrId(mod, &PyId___path__); - if (path_list == NULL) { - PyErr_Clear(); - Py_INCREF(Py_None); - return Py_None; - } - } - - fdp = find_module(fullname, subname, path_list, - &bufobj, &fp, &loader); - Py_XDECREF(path_list); - if (fdp == NULL) { - if (!PyErr_ExceptionMatches(PyExc_ImportError)) - return NULL; - PyErr_Clear(); - Py_INCREF(Py_None); - return Py_None; - } - m = load_module(fullname, fp, bufobj, fdp->type, loader); - Py_XDECREF(bufobj); - Py_XDECREF(loader); - if (fp) - fclose(fp); - if (m == NULL) - return NULL; - if (!add_submodule(mod, m, fullname, subname, modules)) { - Py_XDECREF(m); + PyObject *nameobj, *mod; + nameobj = PyUnicode_FromString(name); + if (nameobj == NULL) return NULL; - } - return m; + mod = PyImport_ImportModuleLevelObject(nameobj, globals, locals, + fromlist, level); + Py_DECREF(nameobj); + return mod; } -- cgit v1.2.1 From d23854c2d242ca58e31af9d37638e8a20d85ad58 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sat, 14 Apr 2012 14:23:49 -0400 Subject: Undo a C99 idiom. --- Python/import.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 87c1214c7e..083146b155 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2889,12 +2889,12 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, Py_INCREF(package); if (_PyDict_GetItemId(globals, &PyId___path__) == NULL) { + PyObject *partition = NULL; PyObject *borrowed_dot = _PyUnicode_FromId(&single_dot); if (borrowed_dot == NULL) { goto error; } - PyObject *partition = PyUnicode_RPartition(package, - borrowed_dot); + partition = PyUnicode_RPartition(package, borrowed_dot); Py_DECREF(package); if (partition == NULL) { goto error; -- cgit v1.2.1 From 29f1fb195df1165ebc3116bc3e9807622d68fa56 Mon Sep 17 00:00:00 2001 From: Brian Curtin Date: Sat, 14 Apr 2012 14:19:33 -0500 Subject: Fix Windows build --- Python/import.c | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 083146b155..be83ab4e1c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2963,7 +2963,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, Py_INCREF(abs_name); } -#if WITH_THREAD +#ifdef WITH_THREAD _PyImport_AcquireLock(); #endif /* From this point forward, goto error_with_unlock! */ @@ -2999,13 +2999,14 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, if (PyObject_Not(fromlist)) { if (level == 0 || PyUnicode_GET_LENGTH(name) > 0) { PyObject *front = NULL; + PyObject *partition = NULL; PyObject *borrowed_dot = _PyUnicode_FromId(&single_dot); if (borrowed_dot == NULL) { goto error_with_unlock; } - PyObject *partition = PyUnicode_Partition(name, borrowed_dot); + partition = PyUnicode_Partition(name, borrowed_dot); if (partition == NULL) { goto error_with_unlock; } @@ -3043,7 +3044,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, NULL); } error_with_unlock: -#if WITH_THREAD +#ifdef WITH_THREAD if (_PyImport_ReleaseLock() < 0) { PyErr_SetString(PyExc_RuntimeError, "not holding the import lock"); } -- cgit v1.2.1 From 312c8a1bbef75dc3656917e7a3565437e3062146 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sat, 14 Apr 2012 21:50:00 -0400 Subject: Handle importing pkg.mod by executing __import__('mod', {'__packaging__': 'pkg', level=1) w/o properly (and thus not segfaulting). --- Python/import.c | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index be83ab4e1c..5136d98ac7 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3016,20 +3016,33 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, Py_DECREF(partition); if (level == 0) { - final_mod = PyDict_GetItemWithError(interp->modules, front); + final_mod = PyDict_GetItem(interp->modules, front); Py_DECREF(front); - Py_XINCREF(final_mod); + if (final_mod == NULL) { + PyErr_Format(PyExc_KeyError, + "%R not in sys.modules as expected", front); + } + else { + Py_INCREF(final_mod); + } } else { Py_ssize_t cut_off = PyUnicode_GetLength(name) - PyUnicode_GetLength(front); Py_ssize_t abs_name_len = PyUnicode_GetLength(abs_name); - PyObject *to_return = PyUnicode_Substring(name, 0, + PyObject *to_return = PyUnicode_Substring(abs_name, 0, abs_name_len - cut_off); final_mod = PyDict_GetItem(interp->modules, to_return); - Py_INCREF(final_mod); Py_DECREF(to_return); + if (final_mod == NULL) { + PyErr_Format(PyExc_KeyError, + "%R not in sys.modules as expected", + to_return); + } + else { + Py_INCREF(final_mod); + } } } else { -- cgit v1.2.1 From 1d6e335a95def2f9b4c7855ad0cd84c0c387ae45 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sun, 15 Apr 2012 14:15:31 -0400 Subject: Set ImportError.name when raising the exception in the case of None found in sys.modules. --- Python/import.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 5136d98ac7..d4f57835d3 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2980,8 +2980,11 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, mod = PyDict_GetItem(interp->modules, abs_name); if (mod == Py_None) { - PyErr_Format(PyExc_ImportError, - "import of %R halted; None in sys.modules", abs_name); + PyObject *msg = PyUnicode_FromFormat("import of %R halted; " + "None in sys.modules", abs_name); + if (msg != NULL) { + PyErr_SetFromImportErrorWithName(msg, abs_name); + } goto error_with_unlock; } else if (mod != NULL) { -- cgit v1.2.1 From 4d6e5e7766952c0591f3a9cc0cdbc0b400e5e457 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sun, 15 Apr 2012 15:24:04 -0400 Subject: Issue #14582: Import returns the module returned by a loader instead of sys.modules when possible. This is being done for two reasons. One is to gain a little bit of performance by skipping an unnecessary dict lookup in sys.modules. But the other (and main) reason is to be a little bit more clear in how things should work from the perspective of import's interactions with loaders. Otherwise loaders can easily forget to return the module even though PEP 302 explicitly states they are expected to return the module they loaded. --- Python/import.c | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index d4f57835d3..e96c7e4bc6 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3019,15 +3019,22 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, Py_DECREF(partition); if (level == 0) { - final_mod = PyDict_GetItem(interp->modules, front); - Py_DECREF(front); - if (final_mod == NULL) { - PyErr_Format(PyExc_KeyError, - "%R not in sys.modules as expected", front); + if (PyUnicode_GET_LENGTH(name) == + PyUnicode_GET_LENGTH(front)) { + final_mod = mod; } else { - Py_INCREF(final_mod); + final_mod = PyDict_GetItem(interp->modules, front); + if (final_mod == NULL) { + PyErr_Format(PyExc_KeyError, + "%R not in sys.modules as expected", front); + } + } + Py_DECREF(front); + if (final_mod == NULL) { + goto error_with_unlock; } + Py_INCREF(final_mod); } else { Py_ssize_t cut_off = PyUnicode_GetLength(name) - -- cgit v1.2.1 From 6d17cfc162c054f5c39bd2c7dbde2a6513924722 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sun, 15 Apr 2012 16:08:47 -0400 Subject: Issue #13959: Rename imp to _imp and add Lib/imp.py and begin rewriting functionality in pure Python. To start, imp.new_module() has been rewritten in pure Python, put into importlib (privately) and then publicly exposed in imp. --- Python/import.c | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index e96c7e4bc6..a930382fc7 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3684,15 +3684,6 @@ imp_load_package(PyObject *self, PyObject *args) return ret; } -static PyObject * -imp_new_module(PyObject *self, PyObject *args) -{ - PyObject *name; - if (!PyArg_ParseTuple(args, "U:new_module", &name)) - return NULL; - return PyModule_NewObject(name); -} - static PyObject * imp_reload(PyObject *self, PyObject *v) { @@ -3781,8 +3772,7 @@ does not conform to PEP 3147 format, ValueError will be raised."); /* Doc strings */ PyDoc_STRVAR(doc_imp, -"This module provides the components needed to build your own\n\ -__import__ function. Undocumented functions are obsolete."); +"(Extremely) low-level import machinery bits as used by importlib and imp."); PyDoc_STRVAR(doc_find_module, "find_module(name, [path]) -> (file, filename, (suffix, mode, type))\n\ @@ -3809,11 +3799,6 @@ PyDoc_STRVAR(doc_get_suffixes, Return a list of (suffix, mode, type) tuples describing the files\n\ that find_module() looks for."); -PyDoc_STRVAR(doc_new_module, -"new_module(name) -> module\n\ -Create a new module. Do not enter it in sys.modules.\n\ -The module name must include the full package name, if any."); - PyDoc_STRVAR(doc_lock_held, "lock_held() -> boolean\n\ Return True if the import lock is currently held, else False.\n\ @@ -3837,7 +3822,6 @@ static PyMethodDef imp_methods[] = { {"get_tag", imp_get_tag, METH_NOARGS, doc_get_tag}, {"get_suffixes", imp_get_suffixes, METH_NOARGS, doc_get_suffixes}, {"load_module", imp_load_module, METH_VARARGS, doc_load_module}, - {"new_module", imp_new_module, METH_VARARGS, doc_new_module}, {"lock_held", imp_lock_held, METH_NOARGS, doc_lock_held}, {"acquire_lock", imp_acquire_lock, METH_NOARGS, doc_acquire_lock}, {"release_lock", imp_release_lock, METH_NOARGS, doc_release_lock}, @@ -4005,7 +3989,7 @@ PyTypeObject PyNullImporter_Type = { static struct PyModuleDef impmodule = { PyModuleDef_HEAD_INIT, - "imp", + "_imp", doc_imp, 0, imp_methods, -- cgit v1.2.1 From accab4318fbf81aed2975225dd45fb3b2950d67f Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sun, 15 Apr 2012 17:56:09 -0400 Subject: Issue #13959: Simplify imp.reload() by relying on a module's __loader__. Since import now sets __loader__ on all modules it creates and imp.reload() already relied on the attribute for modules that import didn't create, the only potential compatibility issue is if people were deleting the attribute on modules and expecting imp.reload() to continue to work. --- Python/import.c | 44 ++++++++++---------------------------------- 1 file changed, 10 insertions(+), 34 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index a930382fc7..747b0ddff4 100644 --- a/Python/import.c +++ b/Python/import.c @@ -153,7 +153,6 @@ static const struct filedescr _PyImport_StandardFiletab[] = { }; static PyObject *initstr = NULL; -_Py_IDENTIFIER(__path__); /* Initialize things */ @@ -3106,12 +3105,12 @@ PyImport_ReloadModule(PyObject *m) PyInterpreterState *interp = PyThreadState_Get()->interp; PyObject *modules_reloading = interp->modules_reloading; PyObject *modules = PyImport_GetModuleDict(); - PyObject *path_list = NULL, *loader = NULL, *existing_m = NULL; - PyObject *name, *bufobj, *subname; + PyObject *loader = NULL, *existing_m = NULL; + PyObject *name; Py_ssize_t subname_start; - struct filedescr *fdp; - FILE *fp = NULL; PyObject *newm = NULL; + _Py_IDENTIFIER(__loader__); + _Py_IDENTIFIER(load_module); if (modules_reloading == NULL) { Py_FatalError("PyImport_ReloadModule: " @@ -3149,51 +3148,28 @@ PyImport_ReloadModule(PyObject *m) subname_start = PyUnicode_FindChar(name, '.', 0, PyUnicode_GET_LENGTH(name), -1); - if (subname_start == -1) { - Py_INCREF(name); - subname = name; - } - else { + if (subname_start != -1) { PyObject *parentname, *parent; - Py_ssize_t len; parentname = PyUnicode_Substring(name, 0, subname_start); if (parentname == NULL) { goto error; } parent = PyDict_GetItem(modules, parentname); + Py_XDECREF(parent); if (parent == NULL) { PyErr_Format(PyExc_ImportError, "reload(): parent %R not in sys.modules", parentname); - Py_DECREF(parentname); goto error; } - Py_DECREF(parentname); - path_list = _PyObject_GetAttrId(parent, &PyId___path__); - if (path_list == NULL) - PyErr_Clear(); - subname_start++; - len = PyUnicode_GET_LENGTH(name) - (subname_start + 1); - subname = PyUnicode_Substring(name, subname_start, len); } - if (subname == NULL) - goto error; - fdp = find_module(name, subname, path_list, - &bufobj, &fp, &loader); - Py_DECREF(subname); - Py_XDECREF(path_list); - if (fdp == NULL) { - Py_XDECREF(loader); + loader = _PyObject_GetAttrId(m, &PyId___loader__); + if (loader == NULL) { goto error; } - - newm = load_module(name, fp, bufobj, fdp->type, loader); - Py_XDECREF(bufobj); - Py_XDECREF(loader); - - if (fp) - fclose(fp); + newm = _PyObject_CallMethodId(loader, &PyId_load_module, "O", name); + Py_DECREF(loader); if (newm == NULL) { /* load_module probably removed name from modules because of * the error. Put back the original module object. We're -- cgit v1.2.1 From 1aa20525c9c533e58932c23f8686f6b4e628965e Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sun, 15 Apr 2012 20:25:23 -0400 Subject: Issue #13959: Re-implement imp.load_module() in imp.py. --- Python/import.c | 53 ----------------------------------------------------- 1 file changed, 53 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 747b0ddff4..06830fe2cb 100644 --- a/Python/import.c +++ b/Python/import.c @@ -3600,53 +3600,6 @@ imp_load_source(PyObject *self, PyObject *args) return m; } -static PyObject * -imp_load_module(PyObject *self, PyObject *args) -{ - PyObject *name, *fob, *pathname, *pathname_obj, *ret; - char *suffix; /* Unused */ - char *mode; - int type; - FILE *fp; - - if (!PyArg_ParseTuple(args, "UOO(ssi):load_module", - &name, &fob, &pathname_obj, &suffix, &mode, &type)) - return NULL; - if (pathname_obj != Py_None) { - if (!PyUnicode_FSDecoder(pathname_obj, &pathname)) - return NULL; - } - else - pathname = NULL; - - if (*mode) { - /* Mode must start with 'r' or 'U' and must not contain '+'. - Implicit in this test is the assumption that the mode - may contain other modifiers like 'b' or 't'. */ - - if (!(*mode == 'r' || *mode == 'U') || strchr(mode, '+')) { - PyErr_Format(PyExc_ValueError, - "invalid file open mode %.200s", mode); - Py_XDECREF(pathname); - return NULL; - } - } - if (fob == Py_None) - fp = NULL; - else { - fp = get_file(NULL, fob, mode); - if (fp == NULL) { - Py_XDECREF(pathname); - return NULL; - } - } - ret = load_module(name, fp, pathname, type, NULL); - Py_XDECREF(pathname); - if (fp) - fclose(fp); - return ret; -} - static PyObject * imp_load_package(PyObject *self, PyObject *args) { @@ -3757,11 +3710,6 @@ built-in, frozen or special module and continue search in sys.path.\n\ The module name cannot contain '.'; to search for a submodule of a\n\ package, pass the submodule name and the package's __path__."); -PyDoc_STRVAR(doc_load_module, -"load_module(name, file, filename, (suffix, mode, type)) -> module\n\ -Load a module, given information returned by find_module().\n\ -The module name must include the full package name, if any."); - PyDoc_STRVAR(doc_get_magic, "get_magic() -> string\n\ Return the magic number for .pyc or .pyo files."); @@ -3797,7 +3745,6 @@ static PyMethodDef imp_methods[] = { {"get_magic", imp_get_magic, METH_NOARGS, doc_get_magic}, {"get_tag", imp_get_tag, METH_NOARGS, doc_get_tag}, {"get_suffixes", imp_get_suffixes, METH_NOARGS, doc_get_suffixes}, - {"load_module", imp_load_module, METH_VARARGS, doc_load_module}, {"lock_held", imp_lock_held, METH_NOARGS, doc_lock_held}, {"acquire_lock", imp_acquire_lock, METH_NOARGS, doc_acquire_lock}, {"release_lock", imp_release_lock, METH_NOARGS, doc_release_lock}, -- cgit v1.2.1 From b7a6769987b1f9cee26a973a603ce6d8b0205a5a Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sun, 15 Apr 2012 22:28:28 -0400 Subject: Issue #13959: Re-implement imp.load_package() in imp.py. Thanks to Eric Snow for helping with imp.load_module() (previous commit) which led to the removal of a bunch of C code. --- Python/import.c | 171 -------------------------------------------------------- 1 file changed, 171 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 06830fe2cb..1b93b7c096 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1602,65 +1602,10 @@ unchanged: } /* Forward */ -static PyObject *load_module(PyObject *, FILE *, PyObject *, int, PyObject *); static struct filedescr *find_module(PyObject *, PyObject *, PyObject *, PyObject **, FILE **, PyObject **); static struct _frozen * find_frozen(PyObject *); -/* Load a package and return its module object WITH INCREMENTED - REFERENCE COUNT */ - -static PyObject * -load_package(PyObject *name, PyObject *pathname) -{ - PyObject *m, *d, *bufobj; - PyObject *file = NULL, *path_list = NULL; - int err; - FILE *fp = NULL; - struct filedescr *fdp; - - m = PyImport_AddModuleObject(name); - if (m == NULL) - return NULL; - if (Py_VerboseFlag) - PySys_FormatStderr("import %U # directory %R\n", - name, pathname); - file = get_sourcefile(pathname); - if (file == NULL) - return NULL; - path_list = Py_BuildValue("[O]", file); - if (path_list == NULL) { - Py_DECREF(file); - return NULL; - } - d = PyModule_GetDict(m); - err = PyDict_SetItemString(d, "__file__", file); - Py_DECREF(file); - if (err == 0) - err = PyDict_SetItemString(d, "__path__", path_list); - if (err != 0) { - Py_DECREF(path_list); - return NULL; - } - fdp = find_module(name, initstr, path_list, - &bufobj, &fp, NULL); - Py_DECREF(path_list); - if (fdp == NULL) { - if (PyErr_ExceptionMatches(PyExc_ImportError)) { - PyErr_Clear(); - Py_INCREF(m); - return m; - } - else - return NULL; - } - m = load_module(name, fp, bufobj, fdp->type, NULL); - Py_XDECREF(bufobj); - if (fp != NULL) - fclose(fp); - return m; -} - /* Helper to test for built-in module */ @@ -2434,108 +2379,6 @@ find_init_module(PyObject *directory) static int init_builtin(PyObject *); /* Forward */ -static PyObject* -load_builtin(PyObject *name, int type) -{ - PyObject *m, *modules; - int err; - - if (type == C_BUILTIN) - err = init_builtin(name); - else - err = PyImport_ImportFrozenModuleObject(name); - if (err < 0) - return NULL; - if (err == 0) { - PyErr_Format(PyExc_ImportError, - "Purported %s module %R not found", - type == C_BUILTIN ? "builtin" : "frozen", - name); - return NULL; - } - - modules = PyImport_GetModuleDict(); - m = PyDict_GetItem(modules, name); - if (m == NULL) { - PyErr_Format( - PyExc_ImportError, - "%s module %R not properly initialized", - type == C_BUILTIN ? "builtin" : "frozen", - name); - return NULL; - } - Py_INCREF(m); - return m; -} - -/* Load an external module using the default search path and return - its module object WITH INCREMENTED REFERENCE COUNT */ - -static PyObject * -load_module(PyObject *name, FILE *fp, PyObject *pathname, int type, PyObject *loader) -{ - PyObject *m; - - /* First check that there's an open file (if we need one) */ - switch (type) { - case PY_SOURCE: - case PY_COMPILED: - if (fp == NULL) { - PyErr_Format(PyExc_ValueError, - "file object required for import (type code %d)", - type); - return NULL; - } - } - - switch (type) { - - case PY_SOURCE: - m = load_source_module(name, pathname, fp); - break; - - case PY_COMPILED: - m = load_compiled_module(name, pathname, fp); - break; - -#ifdef HAVE_DYNAMIC_LOADING - case C_EXTENSION: - m = _PyImport_LoadDynamicModule(name, pathname, fp); - break; -#endif - - case PKG_DIRECTORY: - m = load_package(name, pathname); - break; - - case C_BUILTIN: - case PY_FROZEN: - m = load_builtin(name, type); - break; - - case IMP_HOOK: { - _Py_IDENTIFIER(load_module); - if (loader == NULL) { - PyErr_SetString(PyExc_ImportError, - "import hook without loader"); - return NULL; - } - m = _PyObject_CallMethodId(loader, &PyId_load_module, "O", name); - break; - } - - default: - PyErr_Format(PyExc_ImportError, - "Don't know how to import %R (type code %d)", - name, type); - m = NULL; - - } - - return m; -} - - /* Initialize a built-in module. Return 1 for success, 0 if the module is not found, and -1 with an exception set if the initialization failed. */ @@ -3600,19 +3443,6 @@ imp_load_source(PyObject *self, PyObject *args) return m; } -static PyObject * -imp_load_package(PyObject *self, PyObject *args) -{ - PyObject *name, *pathname; - PyObject * ret; - if (!PyArg_ParseTuple(args, "UO&:load_package", - &name, PyUnicode_FSDecoder, &pathname)) - return NULL; - ret = load_package(name, pathname); - Py_DECREF(pathname); - return ret; -} - static PyObject * imp_reload(PyObject *self, PyObject *v) { @@ -3764,7 +3594,6 @@ static PyMethodDef imp_methods[] = { #ifdef HAVE_DYNAMIC_LOADING {"load_dynamic", imp_load_dynamic, METH_VARARGS}, #endif - {"load_package", imp_load_package, METH_VARARGS}, {"load_source", imp_load_source, METH_VARARGS}, {"_fix_co_filename", imp_fix_co_filename, METH_VARARGS}, {NULL, NULL} /* sentinel */ -- cgit v1.2.1 From 7e1cc9ac0bcb3416e62b4c3a5841bfdf4f6318e9 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 16 Apr 2012 18:48:49 +0200 Subject: Fix reference loss on Py_None when None is encountered in sys.modules. --- Python/import.c | 1 + 1 file changed, 1 insertion(+) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 1b93b7c096..bd0d18ac31 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2827,6 +2827,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, if (msg != NULL) { PyErr_SetFromImportErrorWithName(msg, abs_name); } + mod = NULL; goto error_with_unlock; } else if (mod != NULL) { -- cgit v1.2.1 From 92e8b174fc2d8893cd6c710f4e4af397425aee73 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 16 Apr 2012 21:52:45 +0200 Subject: Fix a refleak --- Python/import.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index bd0d18ac31..a71bc3756d 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2782,10 +2782,11 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, } base = PyUnicode_Substring(package, 0, last_dot); if (PyUnicode_GET_LENGTH(name) > 0) { - PyObject *borrowed_dot = NULL; - PyObject *seq = PyTuple_Pack(2, base, name); + PyObject *borrowed_dot, *seq = NULL; borrowed_dot = _PyUnicode_FromId(&single_dot); + seq = PyTuple_Pack(2, base, name); + Py_DECREF(base); if (borrowed_dot == NULL || seq == NULL) { goto error; } -- cgit v1.2.1 From 8f692771f066cef9711f5bb63f22254a85a33913 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 16 Apr 2012 22:06:21 +0200 Subject: Fix another refleak --- Python/import.c | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index a71bc3756d..1b232fc874 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2881,11 +2881,15 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, Py_INCREF(final_mod); } else { - Py_ssize_t cut_off = PyUnicode_GetLength(name) - - PyUnicode_GetLength(front); - Py_ssize_t abs_name_len = PyUnicode_GetLength(abs_name); + Py_ssize_t cut_off = PyUnicode_GET_LENGTH(name) - + PyUnicode_GET_LENGTH(front); + Py_ssize_t abs_name_len = PyUnicode_GET_LENGTH(abs_name); PyObject *to_return = PyUnicode_Substring(abs_name, 0, abs_name_len - cut_off); + Py_DECREF(front); + if (to_return == NULL) { + goto error_with_unlock; + } final_mod = PyDict_GetItem(interp->modules, to_return); Py_DECREF(to_return); -- cgit v1.2.1 From a045ed1a8ada767dc4cbce8168b5953238da8cbe Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 16 Apr 2012 22:11:25 -0400 Subject: Issue #13959: Re-implement imp.load_source() in imp.py. --- Python/import.c | 390 -------------------------------------------------------- 1 file changed, 390 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 1b232fc874..584f30eb38 100644 --- a/Python/import.c +++ b/Python/import.c @@ -904,26 +904,6 @@ PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, } -/* Like strrchr(string, '/') but searches for the rightmost of either SEP - or ALTSEP, if the latter is defined. -*/ -static Py_UCS4* -rightmost_sep(Py_UCS4 *s) -{ - Py_UCS4 *found, c; - for (found = NULL; (c = *s); s++) { - if (c == SEP -#ifdef ALTSEP - || c == ALTSEP -#endif - ) - { - found = s; - } - } - return found; -} - /* Like rightmost_sep, but operate on unicode objects. */ static Py_ssize_t rightmost_sep_obj(PyObject* o, Py_ssize_t start, Py_ssize_t end) @@ -1081,50 +1061,6 @@ make_source_pathname(PyObject *path) return result; } -/* Given a pathname for a Python source file, its time of last - modification, and a pathname for a compiled file, check whether the - compiled file represents the same version of the source. If so, - return a FILE pointer for the compiled file, positioned just after - the header; if not, return NULL. - Doesn't set an exception. */ - -static FILE * -check_compiled_module(PyObject *pathname, struct stat *srcstat, PyObject *cpathname) -{ - FILE *fp; - long magic; - long pyc_mtime; - long pyc_size; - - fp = _Py_fopen(cpathname, "rb"); - if (fp == NULL) - return NULL; - magic = PyMarshal_ReadLongFromFile(fp); - if (magic != pyc_magic) { - if (Py_VerboseFlag) - PySys_FormatStderr("# %R has bad magic\n", cpathname); - fclose(fp); - return NULL; - } - pyc_mtime = PyMarshal_ReadLongFromFile(fp); - if (pyc_mtime != srcstat->st_mtime) { - if (Py_VerboseFlag) - PySys_FormatStderr("# %R has bad mtime\n", cpathname); - fclose(fp); - return NULL; - } - pyc_size = PyMarshal_ReadLongFromFile(fp); - if (pyc_size != (srcstat->st_size & 0xFFFFFFFF)) { - if (Py_VerboseFlag) - PySys_FormatStderr("# %R has bad size\n", cpathname); - fclose(fp); - return NULL; - } - if (Py_VerboseFlag) - PySys_FormatStderr("# %R matches %R\n", cpathname, pathname); - return fp; -} - /* Read a code object from a file and check it for validity */ @@ -1178,238 +1114,6 @@ load_compiled_module(PyObject *name, PyObject *cpathname, FILE *fp) return m; } -/* Parse a source file and return the corresponding code object */ - -static PyCodeObject * -parse_source_module(PyObject *pathname, FILE *fp) -{ - PyCodeObject *co; - PyObject *pathbytes; - mod_ty mod; - PyCompilerFlags flags; - PyArena *arena; - - pathbytes = PyUnicode_EncodeFSDefault(pathname); - if (pathbytes == NULL) - return NULL; - - arena = PyArena_New(); - if (arena == NULL) { - Py_DECREF(pathbytes); - return NULL; - } - - flags.cf_flags = 0; - mod = PyParser_ASTFromFile(fp, PyBytes_AS_STRING(pathbytes), NULL, - Py_file_input, 0, 0, &flags, - NULL, arena); - if (mod != NULL) - co = PyAST_Compile(mod, PyBytes_AS_STRING(pathbytes), NULL, arena); - else - co = NULL; - Py_DECREF(pathbytes); - PyArena_Free(arena); - return co; -} - -/* Write a compiled module to a file, placing the time of last - modification of its source into the header. - Errors are ignored, if a write error occurs an attempt is made to - remove the file. */ - -static void -write_compiled_module(PyCodeObject *co, PyObject *cpathname, - struct stat *srcstat) -{ - Py_UCS4 *cpathname_ucs4; - FILE *fp; - time_t mtime = srcstat->st_mtime; - long size = srcstat->st_size & 0xFFFFFFFF; - PyObject *cpathname_tmp; -#ifdef MS_WINDOWS /* since Windows uses different permissions */ - mode_t mode = srcstat->st_mode & ~S_IEXEC; - wchar_t *wdirname, *wpathname, *wpathname_tmp; -#else - mode_t dirmode = (srcstat->st_mode | - S_IXUSR | S_IXGRP | S_IXOTH | - S_IWUSR | S_IWGRP | S_IWOTH); - PyObject *dirbytes; - PyObject *cpathbytes, *cpathbytes_tmp; -#endif - int fd; - PyObject *dirname; - Py_UCS4 *dirsep; - int res, ok; - - /* Ensure that the __pycache__ directory exists. */ - cpathname_ucs4 = PyUnicode_AsUCS4Copy(cpathname); - if (!cpathname_ucs4) - return; - dirsep = rightmost_sep(cpathname_ucs4); - if (dirsep == NULL) { - if (Py_VerboseFlag) - PySys_FormatStderr("# no %s path found %R\n", CACHEDIR, cpathname); - return; - } - dirname = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, - cpathname_ucs4, - dirsep - cpathname_ucs4); - PyMem_Free(cpathname_ucs4); - if (dirname == NULL) { - PyErr_Clear(); - return; - } - -#ifdef MS_WINDOWS - wdirname = PyUnicode_AsUnicode(dirname); - if (wdirname == NULL) { - PyErr_Clear(); - return; - } - res = CreateDirectoryW(wdirname, NULL); - ok = (res != 0); - if (!ok && GetLastError() == ERROR_ALREADY_EXISTS) - ok = 1; -#else - dirbytes = PyUnicode_EncodeFSDefault(dirname); - if (dirbytes == NULL) { - PyErr_Clear(); - return; - } - res = mkdir(PyBytes_AS_STRING(dirbytes), dirmode); - Py_DECREF(dirbytes); - if (0 <= res) - ok = 1; - else - ok = (errno == EEXIST); -#endif - if (!ok) { - if (Py_VerboseFlag) - PySys_FormatStderr("# cannot create cache dir %R\n", dirname); - Py_DECREF(dirname); - return; - } - Py_DECREF(dirname); - - /* We first write to a tmp file and then take advantage - of atomic renaming (which *should* be true even under Windows). - As in importlib, we use id(something) to generate a pseudo-random - filename. mkstemp() can't be used since it doesn't allow specifying - the file access permissions. - */ - cpathname_tmp = PyUnicode_FromFormat("%U.%zd", - cpathname, (Py_ssize_t) co); - if (cpathname_tmp == NULL) { - PyErr_Clear(); - return; - } -#ifdef MS_WINDOWS - wpathname = PyUnicode_AsUnicode(cpathname); - if (wpathname == NULL) { - PyErr_Clear(); - return; - } - wpathname_tmp = PyUnicode_AsUnicode(cpathname_tmp); - if (wpathname_tmp == NULL) { - PyErr_Clear(); - return; - } - - (void)DeleteFileW(wpathname_tmp); - fd = _wopen(wpathname_tmp, - O_EXCL | O_CREAT | O_WRONLY | O_BINARY, - mode); - if (0 <= fd) - fp = fdopen(fd, "wb"); - else - fp = NULL; -#else - cpathbytes_tmp = PyUnicode_EncodeFSDefault(cpathname_tmp); - Py_DECREF(cpathname_tmp); - if (cpathbytes_tmp == NULL) { - PyErr_Clear(); - return; - } - cpathbytes = PyUnicode_EncodeFSDefault(cpathname); - if (cpathbytes == NULL) { - PyErr_Clear(); - return; - } - fd = open(PyBytes_AS_STRING(cpathbytes_tmp), - O_CREAT | O_EXCL | O_WRONLY, 0666); - if (0 <= fd) - fp = fdopen(fd, "wb"); - else - fp = NULL; -#endif - if (fp == NULL) { - if (Py_VerboseFlag) - PySys_FormatStderr( - "# can't create %R\n", cpathname); -#ifdef MS_WINDOWS - Py_DECREF(cpathname_tmp); -#else - Py_DECREF(cpathbytes); - Py_DECREF(cpathbytes_tmp); -#endif - return; - } - PyMarshal_WriteLongToFile(pyc_magic, fp, Py_MARSHAL_VERSION); - /* First write a 0 for mtime and size */ - PyMarshal_WriteLongToFile(0L, fp, Py_MARSHAL_VERSION); - PyMarshal_WriteLongToFile(0L, fp, Py_MARSHAL_VERSION); - PyMarshal_WriteObjectToFile((PyObject *)co, fp, Py_MARSHAL_VERSION); - fflush(fp); - /* Now write the true mtime and size (as 32-bit fields) */ - fseek(fp, 4L, 0); - assert(mtime <= 0xFFFFFFFF); - PyMarshal_WriteLongToFile((long)mtime, fp, Py_MARSHAL_VERSION); - PyMarshal_WriteLongToFile(size, fp, Py_MARSHAL_VERSION); - if (fflush(fp) != 0 || ferror(fp)) { - if (Py_VerboseFlag) - PySys_FormatStderr("# can't write %R\n", cpathname); - /* Don't keep partial file */ - fclose(fp); -#ifdef MS_WINDOWS - (void)DeleteFileW(wpathname_tmp); - Py_DECREF(cpathname_tmp); -#else - (void) unlink(PyBytes_AS_STRING(cpathbytes_tmp)); - Py_DECREF(cpathbytes); - Py_DECREF(cpathbytes_tmp); -#endif - return; - } - fclose(fp); - /* Do a (hopefully) atomic rename */ -#ifdef MS_WINDOWS - if (!MoveFileExW(wpathname_tmp, wpathname, MOVEFILE_REPLACE_EXISTING)) { - if (Py_VerboseFlag) - PySys_FormatStderr("# can't write %R\n", cpathname); - /* Don't keep tmp file */ - (void) DeleteFileW(wpathname_tmp); - Py_DECREF(cpathname_tmp); - return; - } - Py_DECREF(cpathname_tmp); -#else - if (rename(PyBytes_AS_STRING(cpathbytes_tmp), - PyBytes_AS_STRING(cpathbytes))) { - if (Py_VerboseFlag) - PySys_FormatStderr("# can't write %R\n", cpathname); - /* Don't keep tmp file */ - unlink(PyBytes_AS_STRING(cpathbytes_tmp)); - Py_DECREF(cpathbytes); - Py_DECREF(cpathbytes_tmp); - return; - } - Py_DECREF(cpathbytes); - Py_DECREF(cpathbytes_tmp); -#endif - if (Py_VerboseFlag) - PySys_FormatStderr("# wrote %R\n", cpathname); -} - static void update_code_filenames(PyCodeObject *co, PyObject *oldname, PyObject *newname) { @@ -1474,76 +1178,6 @@ imp_fix_co_filename(PyObject *self, PyObject *args) Py_RETURN_NONE; } -/* Load a source module from a given file and return its module - object WITH INCREMENTED REFERENCE COUNT. If there's a matching - byte-compiled file, use that instead. */ - -static PyObject * -load_source_module(PyObject *name, PyObject *pathname, FILE *fp) -{ - struct stat st; - FILE *fpc; - PyObject *cpathname = NULL, *cpathbytes = NULL; - PyCodeObject *co; - PyObject *m = NULL; - - if (fstat(fileno(fp), &st) != 0) { - PyErr_Format(PyExc_RuntimeError, - "unable to get file status from %R", - pathname); - goto error; - } - if (sizeof st.st_mtime > 4) { - /* Python's .pyc timestamp handling presumes that the timestamp fits - in 4 bytes. Since the code only does an equality comparison, - ordering is not important and we can safely ignore the higher bits - (collisions are extremely unlikely). - */ - st.st_mtime &= 0xFFFFFFFF; - } - if (PyUnicode_READY(pathname) < 0) - return NULL; - cpathname = make_compiled_pathname(pathname, !Py_OptimizeFlag); - - if (cpathname != NULL) - fpc = check_compiled_module(pathname, &st, cpathname); - else - fpc = NULL; - - if (fpc) { - co = read_compiled_module(cpathname, fpc); - fclose(fpc); - if (co == NULL) - goto error; - update_compiled_module(co, pathname); - if (Py_VerboseFlag) - PySys_FormatStderr("import %U # precompiled from %R\n", - name, cpathname); - m = PyImport_ExecCodeModuleObject(name, (PyObject *)co, - cpathname, cpathname); - } - else { - co = parse_source_module(pathname, fp); - if (co == NULL) - goto error; - if (Py_VerboseFlag) - PySys_FormatStderr("import %U # from %R\n", - name, pathname); - if (cpathname != NULL) { - PyObject *ro = PySys_GetObject("dont_write_bytecode"); - if (ro == NULL || !PyObject_IsTrue(ro)) - write_compiled_module(co, cpathname, &st); - } - m = PyImport_ExecCodeModuleObject(name, (PyObject *)co, - pathname, cpathname); - } - Py_DECREF(co); - -error: - Py_XDECREF(cpathbytes); - Py_XDECREF(cpathname); - return m; -} /* Get source file -> unicode or None * Returns the path to the py file if available, else the given path @@ -3426,29 +3060,6 @@ imp_load_dynamic(PyObject *self, PyObject *args) #endif /* HAVE_DYNAMIC_LOADING */ -static PyObject * -imp_load_source(PyObject *self, PyObject *args) -{ - PyObject *name, *pathname; - PyObject *fob = NULL; - PyObject *m; - FILE *fp; - if (!PyArg_ParseTuple(args, "UO&|O:load_source", - &name, - PyUnicode_FSDecoder, &pathname, - &fob)) - return NULL; - fp = get_file(pathname, fob, "r"); - if (fp == NULL) { - Py_DECREF(pathname); - return NULL; - } - m = load_source_module(name, pathname, fp); - Py_DECREF(pathname); - fclose(fp); - return m; -} - static PyObject * imp_reload(PyObject *self, PyObject *v) { @@ -3600,7 +3211,6 @@ static PyMethodDef imp_methods[] = { #ifdef HAVE_DYNAMIC_LOADING {"load_dynamic", imp_load_dynamic, METH_VARARGS}, #endif - {"load_source", imp_load_source, METH_VARARGS}, {"_fix_co_filename", imp_fix_co_filename, METH_VARARGS}, {NULL, NULL} /* sentinel */ }; -- cgit v1.2.1 From 5d8520121e1ea38bcfe37258449a946ee328cac1 Mon Sep 17 00:00:00 2001 From: Brian Curtin Date: Tue, 17 Apr 2012 16:57:09 -0500 Subject: Fix #14600. Correct reference handling and naming of ImportError convenience function --- Python/import.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 584f30eb38..f3de7d8262 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2460,7 +2460,8 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, PyObject *msg = PyUnicode_FromFormat("import of %R halted; " "None in sys.modules", abs_name); if (msg != NULL) { - PyErr_SetFromImportErrorWithName(msg, abs_name); + PyErr_SetImportError(msg, abs_name, NULL); + Py_DECREF(msg); } mod = NULL; goto error_with_unlock; -- cgit v1.2.1 From 9459c0f7879ddbb51a8e8da936c1c41113b077fa Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Tue, 17 Apr 2012 19:05:11 -0400 Subject: Issue #14592: A relative import will raise a KeyError if __package__ or __name__ are not set in globals. Thanks to Stefan Behnel for the bug report. --- Python/import.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index f3de7d8262..3e620b3706 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2355,8 +2355,9 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, } } else { - package = _PyDict_GetItemIdWithError(globals, &PyId___name__); + package = _PyDict_GetItemId(globals, &PyId___name__); if (package == NULL) { + PyErr_SetString(PyExc_KeyError, "'__name__' not in globals"); goto error; } else if (!PyUnicode_Check(package)) { -- cgit v1.2.1 From 8544db0e8a92dab860693980b6325b3cafe4d4af Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Tue, 17 Apr 2012 19:14:26 -0400 Subject: Issue #13959: Re-implement imp.load_compiled() in imp.py. --- Python/import.c | 76 --------------------------------------------------------- 1 file changed, 76 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 3e620b3706..07c3d6f40c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1062,58 +1062,6 @@ make_source_pathname(PyObject *path) } -/* Read a code object from a file and check it for validity */ - -static PyCodeObject * -read_compiled_module(PyObject *cpathname, FILE *fp) -{ - PyObject *co; - - co = PyMarshal_ReadLastObjectFromFile(fp); - if (co == NULL) - return NULL; - if (!PyCode_Check(co)) { - PyErr_Format(PyExc_ImportError, - "Non-code object in %R", cpathname); - Py_DECREF(co); - return NULL; - } - return (PyCodeObject *)co; -} - - -/* Load a module from a compiled file, execute it, and return its - module object WITH INCREMENTED REFERENCE COUNT */ - -static PyObject * -load_compiled_module(PyObject *name, PyObject *cpathname, FILE *fp) -{ - long magic; - PyCodeObject *co; - PyObject *m; - - magic = PyMarshal_ReadLongFromFile(fp); - if (magic != pyc_magic) { - PyErr_Format(PyExc_ImportError, - "Bad magic number in %R", cpathname); - return NULL; - } - /* Skip mtime and size */ - (void) PyMarshal_ReadLongFromFile(fp); - (void) PyMarshal_ReadLongFromFile(fp); - co = read_compiled_module(cpathname, fp); - if (co == NULL) - return NULL; - if (Py_VerboseFlag) - PySys_FormatStderr("import %U # precompiled from %R\n", - name, cpathname); - m = PyImport_ExecCodeModuleObject(name, (PyObject *)co, - cpathname, cpathname); - Py_DECREF(co); - - return m; -} - static void update_code_filenames(PyCodeObject *co, PyObject *oldname, PyObject *newname) { @@ -3010,29 +2958,6 @@ get_file(PyObject *pathname, PyObject *fob, char *mode) } } -static PyObject * -imp_load_compiled(PyObject *self, PyObject *args) -{ - PyObject *name, *pathname; - PyObject *fob = NULL; - PyObject *m; - FILE *fp; - if (!PyArg_ParseTuple(args, "UO&|O:load_compiled", - &name, - PyUnicode_FSDecoder, &pathname, - &fob)) - return NULL; - fp = get_file(pathname, fob, "rb"); - if (fp == NULL) { - Py_DECREF(pathname); - return NULL; - } - m = load_compiled_module(name, pathname, fp); - fclose(fp); - Py_DECREF(pathname); - return m; -} - #ifdef HAVE_DYNAMIC_LOADING static PyObject * @@ -3209,7 +3134,6 @@ static PyMethodDef imp_methods[] = { {"init_frozen", imp_init_frozen, METH_VARARGS}, {"is_builtin", imp_is_builtin, METH_VARARGS}, {"is_frozen", imp_is_frozen, METH_VARARGS}, - {"load_compiled", imp_load_compiled, METH_VARARGS}, #ifdef HAVE_DYNAMIC_LOADING {"load_dynamic", imp_load_dynamic, METH_VARARGS}, #endif -- cgit v1.2.1 From dfd572c80d2888ed84d050b9cbbaef1eacf7506f Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Wed, 18 Apr 2012 10:55:43 -0400 Subject: rollback 005fd1fe31ab (see #14609 and #14582) Being able to overload a sys.module entry during import of a module was broken by this changeset. --- Python/import.c | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 07c3d6f40c..11d58aefe4 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2447,22 +2447,15 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, Py_DECREF(partition); if (level == 0) { - if (PyUnicode_GET_LENGTH(name) == - PyUnicode_GET_LENGTH(front)) { - final_mod = mod; - } - else { - final_mod = PyDict_GetItem(interp->modules, front); - if (final_mod == NULL) { - PyErr_Format(PyExc_KeyError, - "%R not in sys.modules as expected", front); - } - } + final_mod = PyDict_GetItem(interp->modules, front); Py_DECREF(front); if (final_mod == NULL) { - goto error_with_unlock; + PyErr_Format(PyExc_KeyError, + "%R not in sys.modules as expected", front); + } + else { + Py_INCREF(final_mod); } - Py_INCREF(final_mod); } else { Py_ssize_t cut_off = PyUnicode_GET_LENGTH(name) - -- cgit v1.2.1 From 20a5054c215951276c9a2f74ef2da0468f5c6a6c Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 20 Apr 2012 21:44:46 -0400 Subject: Issue #13959: Re-implement imp.cache_from_source() in Lib/imp.py. --- Python/import.c | 111 -------------------------------------------------------- 1 file changed, 111 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 11d58aefe4..7298ca51e6 100644 --- a/Python/import.c +++ b/Python/import.c @@ -783,7 +783,6 @@ remove_module(PyObject *name) static PyObject * get_sourcefile(PyObject *filename); static PyObject *make_source_pathname(PyObject *pathname); -static PyObject* make_compiled_pathname(PyObject *pathname, int debug); /* Execute a code object in a module and return the module object * WITH INCREMENTED REFERENCE COUNT. If an error occurs, name is @@ -924,71 +923,6 @@ rightmost_sep_obj(PyObject* o, Py_ssize_t start, Py_ssize_t end) return found; } -/* Given a pathname for a Python source file, fill a buffer with the - pathname for the corresponding compiled file. Return the pathname - for the compiled file, or NULL if there's no space in the buffer. - Doesn't set an exception. - - foo.py -> __pycache__/foo..pyc - - pathstr is assumed to be "ready". -*/ - -static PyObject* -make_compiled_pathname(PyObject *pathstr, int debug) -{ - PyObject *result; - Py_ssize_t fname, ext, len, i, pos, taglen; - Py_ssize_t pycache_len = sizeof(CACHEDIR) - 1; - int kind; - void *data; - Py_UCS4 lastsep; - - /* Compute the output string size. */ - len = PyUnicode_GET_LENGTH(pathstr); - /* If there is no separator, this returns -1, so - fname will be 0. */ - fname = rightmost_sep_obj(pathstr, 0, len) + 1; - /* Windows: re-use the last separator character (/ or \\) when - appending the __pycache__ path. */ - if (fname > 0) - lastsep = PyUnicode_READ_CHAR(pathstr, fname -1); - else - lastsep = SEP; - ext = fname - 1; - for(i = fname; i < len; i++) - if (PyUnicode_READ_CHAR(pathstr, i) == '.') - ext = i + 1; - if (ext < fname) - /* No dot in filename; use entire filename */ - ext = len; - - /* result = pathstr[:fname] + "__pycache__" + SEP + - pathstr[fname:ext] + tag + ".py[co]" */ - taglen = strlen(pyc_tag); - result = PyUnicode_New(ext + pycache_len + 1 + taglen + 4, - PyUnicode_MAX_CHAR_VALUE(pathstr)); - if (!result) - return NULL; - kind = PyUnicode_KIND(result); - data = PyUnicode_DATA(result); - PyUnicode_CopyCharacters(result, 0, pathstr, 0, fname); - pos = fname; - for (i = 0; i < pycache_len; i++) - PyUnicode_WRITE(kind, data, pos++, CACHEDIR[i]); - PyUnicode_WRITE(kind, data, pos++, lastsep); - PyUnicode_CopyCharacters(result, pos, pathstr, - fname, ext - fname); - pos += ext - fname; - for (i = 0; pyc_tag[i]; i++) - PyUnicode_WRITE(kind, data, pos++, pyc_tag[i]); - PyUnicode_WRITE(kind, data, pos++, '.'); - PyUnicode_WRITE(kind, data, pos++, 'p'); - PyUnicode_WRITE(kind, data, pos++, 'y'); - PyUnicode_WRITE(kind, data, pos++, debug ? 'c' : 'o'); - return result; -} - /* Given a pathname to a Python byte compiled file, return the path to the source file, if the path matches the PEP 3147 format. This does not check @@ -2991,49 +2925,6 @@ PyDoc_STRVAR(doc_reload, \n\ Reload the module. The module must have been successfully imported before."); -static PyObject * -imp_cache_from_source(PyObject *self, PyObject *args, PyObject *kws) -{ - static char *kwlist[] = {"path", "debug_override", NULL}; - - PyObject *pathname, *cpathname; - PyObject *debug_override = NULL; - int debug = !Py_OptimizeFlag; - - if (!PyArg_ParseTupleAndKeywords( - args, kws, "O&|O", kwlist, - PyUnicode_FSDecoder, &pathname, &debug_override)) - return NULL; - - if (debug_override != NULL && - (debug = PyObject_IsTrue(debug_override)) < 0) { - Py_DECREF(pathname); - return NULL; - } - - if (PyUnicode_READY(pathname) < 0) - return NULL; - - cpathname = make_compiled_pathname(pathname, debug); - Py_DECREF(pathname); - - if (cpathname == NULL) { - PyErr_Format(PyExc_SystemError, "path buffer too short"); - return NULL; - } - return cpathname; -} - -PyDoc_STRVAR(doc_cache_from_source, -"cache_from_source(path, [debug_override]) -> path\n\ -Given the path to a .py file, return the path to its .pyc/.pyo file.\n\ -\n\ -The .py file does not need to exist; this simply returns the path to the\n\ -.pyc/.pyo file calculated as if the .py file were imported. The extension\n\ -will be .pyc unless __debug__ is not defined, then it will be .pyo.\n\ -\n\ -If debug_override is not None, then it must be a boolean and is taken as\n\ -the value of __debug__ instead."); static PyObject * imp_source_from_cache(PyObject *self, PyObject *args, PyObject *kws) @@ -3116,8 +3007,6 @@ static PyMethodDef imp_methods[] = { {"acquire_lock", imp_acquire_lock, METH_NOARGS, doc_acquire_lock}, {"release_lock", imp_release_lock, METH_NOARGS, doc_release_lock}, {"reload", imp_reload, METH_O, doc_reload}, - {"cache_from_source", (PyCFunction)imp_cache_from_source, - METH_VARARGS | METH_KEYWORDS, doc_cache_from_source}, {"source_from_cache", (PyCFunction)imp_source_from_cache, METH_VARARGS | METH_KEYWORDS, doc_source_from_cache}, /* The rest are obsolete */ -- cgit v1.2.1 From c276765b01cb8a531e0204e439053a26691df495 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sat, 21 Apr 2012 18:52:52 -0400 Subject: Issue #13959: Re-implement imp.source_from_cache() in Lib/imp.py. --- Python/import.c | 33 --------------------------------- 1 file changed, 33 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 7298ca51e6..bcf6bd7622 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2926,36 +2926,6 @@ PyDoc_STRVAR(doc_reload, Reload the module. The module must have been successfully imported before."); -static PyObject * -imp_source_from_cache(PyObject *self, PyObject *args, PyObject *kws) -{ - static char *kwlist[] = {"path", NULL}; - PyObject *pathname, *source; - - if (!PyArg_ParseTupleAndKeywords( - args, kws, "O&", kwlist, - PyUnicode_FSDecoder, &pathname)) - return NULL; - - source = make_source_pathname(pathname); - if (source == NULL) { - PyErr_Format(PyExc_ValueError, "Not a PEP 3147 pyc path: %R", - pathname); - Py_DECREF(pathname); - return NULL; - } - Py_DECREF(pathname); - return source; -} - -PyDoc_STRVAR(doc_source_from_cache, -"source_from_cache(path) -> path\n\ -Given the path to a .pyc./.pyo file, return the path to its .py file.\n\ -\n\ -The .pyc/.pyo file does not need to exist; this simply returns the path to\n\ -the .py file calculated to correspond to the .pyc/.pyo file. If path\n\ -does not conform to PEP 3147 format, ValueError will be raised."); - /* Doc strings */ PyDoc_STRVAR(doc_imp, @@ -3007,9 +2977,6 @@ static PyMethodDef imp_methods[] = { {"acquire_lock", imp_acquire_lock, METH_NOARGS, doc_acquire_lock}, {"release_lock", imp_release_lock, METH_NOARGS, doc_release_lock}, {"reload", imp_reload, METH_O, doc_reload}, - {"source_from_cache", (PyCFunction)imp_source_from_cache, - METH_VARARGS | METH_KEYWORDS, doc_source_from_cache}, - /* The rest are obsolete */ {"get_frozen_object", imp_get_frozen_object, METH_VARARGS}, {"is_frozen_package", imp_is_frozen_package, METH_VARARGS}, {"init_builtin", imp_init_builtin, METH_VARARGS}, -- cgit v1.2.1 From 06fad5af9464da87657bf0815c832e8f5d835b43 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sat, 21 Apr 2012 21:09:46 -0400 Subject: Issue #13959: Re-implement imp.find_module() in Lib/imp.py. Thanks to Eric Snow for taking an initial stab at the implementation. --- Python/import.c | 751 -------------------------------------------------------- 1 file changed, 751 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index bcf6bd7622..ab320e523c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1118,8 +1118,6 @@ unchanged: } /* Forward */ -static struct filedescr *find_module(PyObject *, PyObject *, PyObject *, - PyObject **, FILE **, PyObject **); static struct _frozen * find_frozen(PyObject *); @@ -1220,678 +1218,12 @@ PyImport_GetImporter(PyObject *path) { return importer; } -/* Search the path (default sys.path) for a module. Return the - corresponding filedescr struct, and (via return arguments) the - pathname and an open file. Return NULL if the module is not found. */ #ifdef MS_COREDLL extern FILE *_PyWin_FindRegisteredModule(PyObject *, struct filedescr **, PyObject **p_path); #endif -/* Forward */ -static int case_ok(PyObject *, Py_ssize_t, PyObject *); -static int find_init_module(PyObject *); -static struct filedescr importhookdescr = {"", "", IMP_HOOK}; - -/* Get the path of a module: get its importer and call importer.find_module() - hook, or check if the module if a package (if path/__init__.py exists). - - -1: error: a Python error occurred - 0: ignore: an error occurred because of invalid data, but the error is not - important enough to be reported. - 1: get path: module not found, but *buf contains its path - 2: found: *p_fd is the file descriptor (IMP_HOOK or PKG_DIRECTORY) - and *buf is the path */ - -static int -find_module_path(PyObject *fullname, PyObject *name, PyObject *path, - PyObject *path_hooks, PyObject *path_importer_cache, - PyObject **p_path, PyObject **p_loader, struct filedescr **p_fd) -{ - PyObject *path_unicode, *filename = NULL; - Py_ssize_t len, pos; - struct stat statbuf; - static struct filedescr fd_package = {"", "", PKG_DIRECTORY}; - int err, result, addsep; - - if (PyUnicode_Check(path)) { - Py_INCREF(path); - path_unicode = path; - } - else if (PyBytes_Check(path)) { - path_unicode = PyUnicode_DecodeFSDefaultAndSize( - PyBytes_AS_STRING(path), PyBytes_GET_SIZE(path)); - if (path_unicode == NULL) - return -1; - } - else - return 0; - - if (PyUnicode_READY(path_unicode)) - return -1; - - len = PyUnicode_GET_LENGTH(path_unicode); - if (PyUnicode_FindChar(path_unicode, 0, 0, len, 1) != -1) { - result = 0; - goto out; /* path contains '\0' */ - } - - /* sys.path_hooks import hook */ - if (p_loader != NULL) { - _Py_IDENTIFIER(find_module); - PyObject *importer; - - importer = get_path_importer(path_importer_cache, - path_hooks, path); - if (importer == NULL) { - result = -1; - goto out; - } - /* Note: importer is a borrowed reference */ - if (importer != Py_None) { - PyObject *loader; - loader = _PyObject_CallMethodId(importer, - &PyId_find_module, "O", fullname); - if (loader == NULL) { - result = -1; /* error */ - goto out; - } - if (loader != Py_None) { - /* a loader was found */ - *p_loader = loader; - *p_fd = &importhookdescr; - result = 2; - goto out; - } - Py_DECREF(loader); - result = 0; - goto out; - } - } - /* no hook was found, use builtin import */ - - addsep = 0; - if (len > 0 && PyUnicode_READ_CHAR(path_unicode, len-1) != SEP -#ifdef ALTSEP - && PyUnicode_READ_CHAR(path_unicode, len-1) != ALTSEP -#endif - ) - addsep = 1; - filename = PyUnicode_New(len + PyUnicode_GET_LENGTH(name) + addsep, - Py_MAX(PyUnicode_MAX_CHAR_VALUE(path_unicode), - PyUnicode_MAX_CHAR_VALUE(name))); - if (filename == NULL) { - result = -1; - goto out; - } - PyUnicode_CopyCharacters(filename, 0, path_unicode, 0, len); - pos = len; - if (addsep) - PyUnicode_WRITE(PyUnicode_KIND(filename), - PyUnicode_DATA(filename), - pos++, SEP); - PyUnicode_CopyCharacters(filename, pos, name, 0, - PyUnicode_GET_LENGTH(name)); - - /* Check for package import (buf holds a directory name, - and there's an __init__ module in that directory */ -#ifdef HAVE_STAT - err = _Py_stat(filename, &statbuf); - if (err == -2) { - result = -1; - goto out; - } - if (err == 0 && /* it exists */ - S_ISDIR(statbuf.st_mode)) /* it's a directory */ - { - int match; - - match = case_ok(filename, 0, name); - if (match < 0) { - result = -1; - goto out; - } - if (match) { /* case matches */ - if (find_init_module(filename)) { /* and has __init__.py */ - *p_path = filename; - filename = NULL; - *p_fd = &fd_package; - result = 2; - goto out; - } - else { - int err; - err = PyErr_WarnFormat(PyExc_ImportWarning, 1, - "Not importing directory %R: missing __init__.py", - filename); - if (err) { - result = -1; - goto out; - } - } - } - } -#endif - *p_path = filename; - filename = NULL; - result = 1; - out: - Py_DECREF(path_unicode); - Py_XDECREF(filename); - return result; -} - -/* Find a module in search_path_list. For each path, try - find_module_path() or try each _PyImport_Filetab suffix. - - If the module is found, return a file descriptor, write the path in - *p_filename, write the pointer to the file object into *p_fp, and (if - p_loader is not NULL) the loader into *p_loader. - - Otherwise, raise an exception and return NULL. */ - -static struct filedescr* -find_module_path_list(PyObject *fullname, PyObject *name, - PyObject *search_path_list, PyObject *path_hooks, - PyObject *path_importer_cache, - PyObject **p_path, FILE **p_fp, PyObject **p_loader) -{ - Py_ssize_t i, npath; - struct filedescr *fdp = NULL; - char *filemode; - FILE *fp = NULL; - PyObject *prefix, *filename; - int match; - int err; - - npath = PyList_Size(search_path_list); - for (i = 0; i < npath; i++) { - PyObject *path; - int ok; - - path = PyList_GetItem(search_path_list, i); - if (path == NULL) - return NULL; - - prefix = NULL; - ok = find_module_path(fullname, name, path, - path_hooks, path_importer_cache, - &prefix, p_loader, &fdp); - if (ok < 0) - return NULL; - if (ok == 0) - continue; - if (ok == 2) { - *p_path = prefix; - return fdp; - } - - for (fdp = _PyImport_Filetab; fdp->suffix != NULL; fdp++) { - struct stat statbuf; - - filemode = fdp->mode; - if (filemode[0] == 'U') - filemode = "r" PY_STDIOTEXTMODE; - - filename = PyUnicode_FromFormat("%U%s", prefix, fdp->suffix); - if (filename == NULL) { - Py_DECREF(prefix); - return NULL; - } - - if (Py_VerboseFlag > 1) - PySys_FormatStderr("# trying %R\n", filename); - - err = _Py_stat(filename, &statbuf); - if (err == -2) { - Py_DECREF(prefix); - Py_DECREF(filename); - return NULL; - } - if (err != 0 || S_ISDIR(statbuf.st_mode)) { - /* it doesn't exist, or it's a directory */ - Py_DECREF(filename); - continue; - } - - fp = _Py_fopen(filename, filemode); - if (fp == NULL) { - Py_DECREF(filename); - if (PyErr_Occurred()) { - Py_DECREF(prefix); - return NULL; - } - continue; - } - match = case_ok(filename, -(Py_ssize_t)strlen(fdp->suffix), name); - if (match < 0) { - Py_DECREF(prefix); - Py_DECREF(filename); - return NULL; - } - if (match) { - Py_DECREF(prefix); - *p_path = filename; - *p_fp = fp; - return fdp; - } - Py_DECREF(filename); - - fclose(fp); - fp = NULL; - } - Py_DECREF(prefix); - } - PyErr_Format(PyExc_ImportError, - "No module named %R", name); - return NULL; -} - -/* Find a module: - - - try find_module() of each sys.meta_path hook - - try find_frozen() - - try is_builtin() - - try _PyWin_FindRegisteredModule() (Windows only) - - otherwise, call find_module_path_list() with search_path_list (if not - NULL) or sys.path - - fullname can be NULL, but only if p_loader is NULL. - - Return: - - - &fd_builtin (C_BUILTIN) if it is a builtin - - &fd_frozen (PY_FROZEN) if it is frozen - - &fd_package (PKG_DIRECTORY) and write the filename into *p_path - if it is a package - - &importhookdescr (IMP_HOOK) and write the loader into *p_loader if a - importer loader was found - - a file descriptor (PY_SOURCE, PY_COMPILED, C_EXTENSION, PY_RESOURCE or - PY_CODERESOURCE: see _PyImport_Filetab), write the filename into - *p_path and the pointer to the open file into *p_fp - - NULL on error - - By default, *p_path, *p_fp and *p_loader (if set) are set to NULL. - Eg. *p_path is set to NULL for a builtin package. -*/ - -static struct filedescr * -find_module(PyObject *fullname, PyObject *name, PyObject *search_path_list, - PyObject **p_path, FILE **p_fp, PyObject **p_loader) -{ - Py_ssize_t i, npath; - static struct filedescr fd_frozen = {"", "", PY_FROZEN}; - static struct filedescr fd_builtin = {"", "", C_BUILTIN}; - PyObject *path_hooks, *path_importer_cache; - - *p_path = NULL; - *p_fp = NULL; - if (p_loader != NULL) - *p_loader = NULL; - - /* sys.meta_path import hook */ - if (p_loader != NULL) { - _Py_IDENTIFIER(find_module); - PyObject *meta_path; - - meta_path = PySys_GetObject("meta_path"); - if (meta_path == NULL || !PyList_Check(meta_path)) { - PyErr_SetString(PyExc_RuntimeError, - "sys.meta_path must be a list of " - "import hooks"); - return NULL; - } - Py_INCREF(meta_path); /* zap guard */ - npath = PyList_Size(meta_path); - for (i = 0; i < npath; i++) { - PyObject *loader; - PyObject *hook = PyList_GetItem(meta_path, i); - loader = _PyObject_CallMethodId(hook, &PyId_find_module, - "OO", fullname, - search_path_list != NULL ? - search_path_list : Py_None); - if (loader == NULL) { - Py_DECREF(meta_path); - return NULL; /* true error */ - } - if (loader != Py_None) { - /* a loader was found */ - *p_loader = loader; - Py_DECREF(meta_path); - return &importhookdescr; - } - Py_DECREF(loader); - } - Py_DECREF(meta_path); - } - - if (find_frozen(fullname) != NULL) - return &fd_frozen; - - if (search_path_list == NULL) { -#ifdef MS_COREDLL - FILE *fp; - struct filedescr *fdp; -#endif - if (is_builtin(name)) - return &fd_builtin; -#ifdef MS_COREDLL - fp = _PyWin_FindRegisteredModule(name, &fdp, p_path); - if (fp != NULL) { - *p_fp = fp; - return fdp; - } - else if (PyErr_Occurred()) - return NULL; -#endif - search_path_list = PySys_GetObject("path"); - } - - if (search_path_list == NULL || !PyList_Check(search_path_list)) { - PyErr_SetString(PyExc_RuntimeError, - "sys.path must be a list of directory names"); - return NULL; - } - - path_hooks = PySys_GetObject("path_hooks"); - if (path_hooks == NULL || !PyList_Check(path_hooks)) { - PyErr_SetString(PyExc_RuntimeError, - "sys.path_hooks must be a list of " - "import hooks"); - return NULL; - } - path_importer_cache = PySys_GetObject("path_importer_cache"); - if (path_importer_cache == NULL || - !PyDict_Check(path_importer_cache)) { - PyErr_SetString(PyExc_RuntimeError, - "sys.path_importer_cache must be a dict"); - return NULL; - } - - return find_module_path_list(fullname, name, - search_path_list, path_hooks, - path_importer_cache, - p_path, p_fp, p_loader); -} - -/* case_bytes(char* buf, Py_ssize_t len, Py_ssize_t namelen, char* name) - * The arguments here are tricky, best shown by example: - * /a/b/c/d/e/f/g/h/i/j/k/some_long_module_name.py\0 - * ^ ^ ^ ^ - * |--------------------- buf ---------------------| - * |------------------- len ------------------| - * |------ name -------| - * |----- namelen -----| - * buf is the full path, but len only counts up to (& exclusive of) the - * extension. name is the module name, also exclusive of extension. - * - * We've already done a successful stat() or fopen() on buf, so know that - * there's some match, possibly case-insensitive. - * - * case_bytes() is to return 1 if there's a case-sensitive match for - * name, else 0. case_bytes() is also to return 1 if envar PYTHONCASEOK - * exists. - * - * case_bytes() is used to implement case-sensitive import semantics even - * on platforms with case-insensitive filesystems. It's trivial to implement - * for case-sensitive filesystems. It's pretty much a cross-platform - * nightmare for systems with case-insensitive filesystems. - */ - -/* First we may need a pile of platform-specific header files; the sequence - * of #if's here should match the sequence in the body of case_bytes(). - */ -#if defined(MS_WINDOWS) -#include - -#elif defined(DJGPP) -#include - -#elif (defined(__MACH__) && defined(__APPLE__) || defined(__CYGWIN__)) && defined(HAVE_DIRENT_H) -#include -#include - -#elif defined(PYOS_OS2) -#define INCL_DOS -#define INCL_DOSERRORS -#define INCL_NOPMAPI -#include -#endif - -#if defined(DJGPP) \ - || ((defined(__MACH__) && defined(__APPLE__) || defined(__CYGWIN__)) \ - && defined(HAVE_DIRENT_H)) \ - || defined(PYOS_OS2) -# define USE_CASE_OK_BYTES -#endif - - -#ifdef USE_CASE_OK_BYTES -static int -case_bytes(char *buf, Py_ssize_t len, Py_ssize_t namelen, const char *name) -{ -/* Pick a platform-specific implementation; the sequence of #if's here should - * match the sequence just above. - */ - -/* DJGPP */ -#if defined(DJGPP) - struct ffblk ffblk; - int done; - - if (Py_GETENV("PYTHONCASEOK") != NULL) - return 1; - - done = findfirst(buf, &ffblk, FA_ARCH|FA_RDONLY|FA_HIDDEN|FA_DIREC); - if (done) { - PyErr_Format(PyExc_NameError, - "Can't find file for module %.100s\n(filename %.300s)", - name, buf); - return -1; - } - return strncmp(ffblk.ff_name, name, namelen) == 0; - -/* new-fangled macintosh (macosx) or Cygwin */ -#elif (defined(__MACH__) && defined(__APPLE__) || defined(__CYGWIN__)) && defined(HAVE_DIRENT_H) - DIR *dirp; - struct dirent *dp; - char dirname[MAXPATHLEN + 1]; - const int dirlen = len - namelen - 1; /* don't want trailing SEP */ - - if (Py_GETENV("PYTHONCASEOK") != NULL) - return 1; - - /* Copy the dir component into dirname; substitute "." if empty */ - if (dirlen <= 0) { - dirname[0] = '.'; - dirname[1] = '\0'; - } - else { - assert(dirlen <= MAXPATHLEN); - memcpy(dirname, buf, dirlen); - dirname[dirlen] = '\0'; - } - /* Open the directory and search the entries for an exact match. */ - dirp = opendir(dirname); - if (dirp) { - char *nameWithExt = buf + len - namelen; - while ((dp = readdir(dirp)) != NULL) { - const int thislen = -#ifdef _DIRENT_HAVE_D_NAMELEN - dp->d_namlen; -#else - strlen(dp->d_name); -#endif - if (thislen >= namelen && - strcmp(dp->d_name, nameWithExt) == 0) { - (void)closedir(dirp); - return 1; /* Found */ - } - } - (void)closedir(dirp); - } - return 0 ; /* Not found */ - -/* OS/2 */ -#elif defined(PYOS_OS2) - HDIR hdir = 1; - ULONG srchcnt = 1; - FILEFINDBUF3 ffbuf; - APIRET rc; - - if (Py_GETENV("PYTHONCASEOK") != NULL) - return 1; - - rc = DosFindFirst(buf, - &hdir, - FILE_READONLY | FILE_HIDDEN | FILE_SYSTEM | FILE_DIRECTORY, - &ffbuf, sizeof(ffbuf), - &srchcnt, - FIL_STANDARD); - if (rc != NO_ERROR) - return 0; - return strncmp(ffbuf.achName, name, namelen) == 0; - -/* assuming it's a case-sensitive filesystem, so there's nothing to do! */ -#else -# error "USE_CASE_OK_BYTES is not correctly defined" -#endif -} -#endif - -/* - * Check if a filename case matchs the name case. We've already done a - * successful stat() or fopen() on buf, so know that there's some match, - * possibly case-insensitive. - * - * case_ok() is to return 1 if there's a case-sensitive match for name, 0 if it - * the filename doesn't match, or -1 on error. case_ok() is also to return 1 - * if envar PYTHONCASEOK exists. - * - * case_ok() is used to implement case-sensitive import semantics even - * on platforms with case-insensitive filesystems. It's trivial to implement - * for case-sensitive filesystems. It's pretty much a cross-platform - * nightmare for systems with case-insensitive filesystems. - */ - -static int -case_ok(PyObject *filename, Py_ssize_t prefix_delta, PyObject *name) -{ -#ifdef MS_WINDOWS - WIN32_FIND_DATAW data; - HANDLE h; - int cmp; - wchar_t *wfilename, *wname; - Py_ssize_t wname_len; - - if (Py_GETENV("PYTHONCASEOK") != NULL) - return 1; - - wfilename = PyUnicode_AsUnicode(filename); - if (wfilename == NULL) - return -1; - - h = FindFirstFileW(wfilename, &data); - if (h == INVALID_HANDLE_VALUE) { - PyErr_Format(PyExc_NameError, - "Can't find file for module %R\n(filename %R)", - name, filename); - return -1; - } - FindClose(h); - - wname = PyUnicode_AsUnicodeAndSize(name, &wname_len); - if (wname == NULL) - return -1; - - cmp = wcsncmp(data.cFileName, wname, wname_len); - return cmp == 0; -#elif defined(USE_CASE_OK_BYTES) - int match; - PyObject *filebytes, *namebytes; - filebytes = PyUnicode_EncodeFSDefault(filename); - if (filebytes == NULL) - return -1; - namebytes = PyUnicode_EncodeFSDefault(name); - if (namebytes == NULL) { - Py_DECREF(filebytes); - return -1; - } - match = case_bytes( - PyBytes_AS_STRING(filebytes), - PyBytes_GET_SIZE(filebytes) + prefix_delta, - PyBytes_GET_SIZE(namebytes), - PyBytes_AS_STRING(namebytes)); - Py_DECREF(filebytes); - Py_DECREF(namebytes); - return match; -#else - /* assuming it's a case-sensitive filesystem, so there's nothing to do! */ - return 1; - -#endif -} - -#ifdef HAVE_STAT - -/* Helper to look for __init__.py or __init__.py[co] in potential package. - Return 1 if __init__ was found, 0 if not, or -1 on error. */ -static int -find_init_module(PyObject *directory) -{ - struct stat statbuf; - PyObject *filename; - int match; - int err; - - filename = PyUnicode_FromFormat("%U%c__init__.py", directory, SEP); - if (filename == NULL) - return -1; - err = _Py_stat(filename, &statbuf); - if (err == -2) - return -1; - if (err == 0) { - /* 3=len(".py") */ - match = case_ok(filename, -3, initstr); - if (match < 0) { - Py_DECREF(filename); - return -1; - } - if (match) { - Py_DECREF(filename); - return 1; - } - } - Py_DECREF(filename); - - filename = PyUnicode_FromFormat("%U%c__init__.py%c", - directory, SEP, Py_OptimizeFlag ? 'o' : 'c'); - if (filename == NULL) - return -1; - err = _Py_stat(filename, &statbuf); - if (err == -2) { - Py_DECREF(filename); - return -1; - } - if (err == 0) { - /* 4=len(".pyc") */ - match = case_ok(filename, -4, initstr); - if (match < 0) { - Py_DECREF(filename); - return -1; - } - if (match) { - Py_DECREF(filename); - return 1; - } - } - Py_DECREF(filename); - return 0; -} - -#endif /* HAVE_STAT */ - static int init_builtin(PyObject *); /* Forward */ @@ -2688,81 +2020,6 @@ imp_get_suffixes(PyObject *self, PyObject *noargs) return list; } -static PyObject * -call_find_module(PyObject *name, PyObject *path_list) -{ - extern int fclose(FILE *); - PyObject *fob, *ret; - PyObject *pathobj; - struct filedescr *fdp; - FILE *fp; - int fd = -1; - char *found_encoding = NULL; - char *encoding = NULL; - - if (path_list == Py_None) - path_list = NULL; - fdp = find_module(NULL, name, path_list, - &pathobj, &fp, NULL); - if (fdp == NULL) - return NULL; - if (fp != NULL) { - fd = fileno(fp); - if (fd != -1) - fd = dup(fd); - fclose(fp); - fp = NULL; - if (fd == -1) - return PyErr_SetFromErrno(PyExc_OSError); - } - if (fd != -1) { - if (strchr(fdp->mode, 'b') == NULL) { - /* PyTokenizer_FindEncodingFilename() returns PyMem_MALLOC'ed - memory. */ - found_encoding = PyTokenizer_FindEncodingFilename(fd, pathobj); - lseek(fd, 0, 0); /* Reset position */ - if (found_encoding == NULL && PyErr_Occurred()) { - Py_XDECREF(pathobj); - close(fd); - return NULL; - } - encoding = (found_encoding != NULL) ? found_encoding : - (char*)PyUnicode_GetDefaultEncoding(); - } - fob = PyFile_FromFd(fd, NULL, fdp->mode, -1, - (char*)encoding, NULL, NULL, 1); - if (fob == NULL) { - Py_XDECREF(pathobj); - close(fd); - PyMem_FREE(found_encoding); - return NULL; - } - } - else { - fob = Py_None; - Py_INCREF(fob); - } - if (pathobj == NULL) { - Py_INCREF(Py_None); - pathobj = Py_None; - } - ret = Py_BuildValue("NN(ssi)", - fob, pathobj, fdp->suffix, fdp->mode, fdp->type); - PyMem_FREE(found_encoding); - - return ret; -} - -static PyObject * -imp_find_module(PyObject *self, PyObject *args) -{ - PyObject *name, *path_list = NULL; - if (!PyArg_ParseTuple(args, "U|O:find_module", - &name, &path_list)) - return NULL; - return call_find_module(name, path_list); -} - static PyObject * imp_init_builtin(PyObject *self, PyObject *args) { @@ -2931,13 +2188,6 @@ Reload the module. The module must have been successfully imported before."); PyDoc_STRVAR(doc_imp, "(Extremely) low-level import machinery bits as used by importlib and imp."); -PyDoc_STRVAR(doc_find_module, -"find_module(name, [path]) -> (file, filename, (suffix, mode, type))\n\ -Search for a module. If path is omitted or None, search for a\n\ -built-in, frozen or special module and continue search in sys.path.\n\ -The module name cannot contain '.'; to search for a submodule of a\n\ -package, pass the submodule name and the package's __path__."); - PyDoc_STRVAR(doc_get_magic, "get_magic() -> string\n\ Return the magic number for .pyc or .pyo files."); @@ -2969,7 +2219,6 @@ Release the interpreter's import lock.\n\ On platforms without threads, this function does nothing."); static PyMethodDef imp_methods[] = { - {"find_module", imp_find_module, METH_VARARGS, doc_find_module}, {"get_magic", imp_get_magic, METH_NOARGS, doc_get_magic}, {"get_tag", imp_get_tag, METH_NOARGS, doc_get_tag}, {"get_suffixes", imp_get_suffixes, METH_NOARGS, doc_get_suffixes}, -- cgit v1.2.1 From 901bb2ffffe2b1483cdd1a7316df60a4f24cb665 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Tue, 24 Apr 2012 22:03:46 -0400 Subject: Issue #14605: Insert to the front of sys.path_hooks instead of appending. --- Python/import.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index ab320e523c..8cf10e658c 100644 --- a/Python/import.c +++ b/Python/import.c @@ -268,8 +268,8 @@ _PyImportZip_Init(void) "# can't import zipimport.zipimporter\n"); } else { - /* sys.path_hooks.append(zipimporter) */ - err = PyList_Append(path_hooks, zipimporter); + /* sys.path_hooks.insert(0, zipimporter) */ + err = PyList_Insert(path_hooks, 0, zipimporter); Py_DECREF(zipimporter); if (err < 0) { goto error; -- cgit v1.2.1 From 2555301e0aafefea124fe79f268207bc752a4722 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Fri, 27 Apr 2012 13:55:39 +0200 Subject: Check newly created consistency using _PyUnicode_CheckConsistency(str, 1) * In debug mode, fill the string data with invalid characters * Simplify also reference counting in PyCodec_BackslashReplaceErrors() and PyCodec_XMLCharRefReplaceError() --- Python/import.c | 1 + 1 file changed, 1 insertion(+) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 8cf10e658c..103e7de439 100644 --- a/Python/import.c +++ b/Python/import.c @@ -992,6 +992,7 @@ make_source_pathname(PyObject *path) (j = dot0-right)); PyUnicode_WRITE(kind, data, i+j, 'p'); PyUnicode_WRITE(kind, data, i+j+1, 'y'); + assert(_PyUnicode_CheckConsistency(result, 1)); return result; } -- cgit v1.2.1 From 6389a78515e401399bbb9aa29fb76f193c22c101 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 27 Apr 2012 15:30:58 -0400 Subject: Issue #14605: Use None in sys.path_importer_cache to represent no finder instead of using some (now non-existent) implicit finder. --- Python/import.c | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 103e7de439..a47e9881b9 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1186,15 +1186,7 @@ get_path_importer(PyObject *path_importer_cache, PyObject *path_hooks, PyErr_Clear(); } if (importer == NULL) { - importer = PyObject_CallFunctionObjArgs( - (PyObject *)&PyNullImporter_Type, p, NULL - ); - if (importer == NULL) { - if (PyErr_ExceptionMatches(PyExc_ImportError)) { - PyErr_Clear(); - return Py_None; - } - } + return Py_None; } if (importer != NULL) { int err = PyDict_SetItem(path_importer_cache, p, importer); -- cgit v1.2.1 From 4d428cdc3bc818df1b7799d54ff76543d709fa62 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sun, 29 Apr 2012 12:50:03 -0400 Subject: Issue #13959: Re-implement imp.NullImporter in Lib/imp.py. --- Python/import.c | 141 +------------------------------------------------------- 1 file changed, 2 insertions(+), 139 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index a47e9881b9..0580f608fb 100644 --- a/Python/import.c +++ b/Python/import.c @@ -205,9 +205,6 @@ _PyImportHooks_Init(void) PyObject *v, *path_hooks = NULL; int err = 0; - if (PyType_Ready(&PyNullImporter_Type) < 0) - goto error; - /* adding sys.path_hooks and sys.path_importer_cache */ v = PyList_New(0); if (v == NULL) @@ -231,8 +228,7 @@ _PyImportHooks_Init(void) error: PyErr_Print(); Py_FatalError("initializing sys.meta_path, sys.path_hooks, " - "path_importer_cache, or NullImporter failed" - ); + "or path_importer_cache"); } Py_DECREF(path_hooks); } @@ -284,7 +280,7 @@ _PyImportZip_Init(void) error: PyErr_Print(); - Py_FatalError("initializing zipimport or NullImporter failed"); + Py_FatalError("initializing zipimport failed"); } /* Locking primitives to prevent parallel imports of the same module @@ -2244,134 +2240,6 @@ setint(PyObject *d, char *name, int value) return err; } -typedef struct { - PyObject_HEAD -} NullImporter; - -static int -NullImporter_init(NullImporter *self, PyObject *args, PyObject *kwds) -{ -#ifndef MS_WINDOWS - PyObject *path; - struct stat statbuf; - int rv; - - if (!_PyArg_NoKeywords("NullImporter()", kwds)) - return -1; - - if (!PyArg_ParseTuple(args, "O&:NullImporter", - PyUnicode_FSConverter, &path)) - return -1; - - if (PyBytes_GET_SIZE(path) == 0) { - Py_DECREF(path); - PyErr_SetString(PyExc_ImportError, "empty pathname"); - return -1; - } - - rv = stat(PyBytes_AS_STRING(path), &statbuf); - Py_DECREF(path); - if (rv == 0) { - /* it exists */ - if (S_ISDIR(statbuf.st_mode)) { - /* it's a directory */ - PyErr_SetString(PyExc_ImportError, "existing directory"); - return -1; - } - } -#else /* MS_WINDOWS */ - PyObject *pathobj; - DWORD rv; - wchar_t *path; - - if (!_PyArg_NoKeywords("NullImporter()", kwds)) - return -1; - - if (!PyArg_ParseTuple(args, "U:NullImporter", - &pathobj)) - return -1; - - if (PyUnicode_GET_LENGTH(pathobj) == 0) { - PyErr_SetString(PyExc_ImportError, "empty pathname"); - return -1; - } - - path = PyUnicode_AsWideCharString(pathobj, NULL); - if (path == NULL) - return -1; - /* see issue1293 and issue3677: - * stat() on Windows doesn't recognise paths like - * "e:\\shared\\" and "\\\\whiterab-c2znlh\\shared" as dirs. - */ - rv = GetFileAttributesW(path); - PyMem_Free(path); - if (rv != INVALID_FILE_ATTRIBUTES) { - /* it exists */ - if (rv & FILE_ATTRIBUTE_DIRECTORY) { - /* it's a directory */ - PyErr_SetString(PyExc_ImportError, "existing directory"); - return -1; - } - } -#endif - return 0; -} - -static PyObject * -NullImporter_find_module(NullImporter *self, PyObject *args) -{ - Py_RETURN_NONE; -} - -static PyMethodDef NullImporter_methods[] = { - {"find_module", (PyCFunction)NullImporter_find_module, METH_VARARGS, - "Always return None" - }, - {NULL} /* Sentinel */ -}; - - -PyTypeObject PyNullImporter_Type = { - PyVarObject_HEAD_INIT(NULL, 0) - "imp.NullImporter", /*tp_name*/ - sizeof(NullImporter), /*tp_basicsize*/ - 0, /*tp_itemsize*/ - 0, /*tp_dealloc*/ - 0, /*tp_print*/ - 0, /*tp_getattr*/ - 0, /*tp_setattr*/ - 0, /*tp_reserved*/ - 0, /*tp_repr*/ - 0, /*tp_as_number*/ - 0, /*tp_as_sequence*/ - 0, /*tp_as_mapping*/ - 0, /*tp_hash */ - 0, /*tp_call*/ - 0, /*tp_str*/ - 0, /*tp_getattro*/ - 0, /*tp_setattro*/ - 0, /*tp_as_buffer*/ - Py_TPFLAGS_DEFAULT, /*tp_flags*/ - "Null importer object", /* tp_doc */ - 0, /* tp_traverse */ - 0, /* tp_clear */ - 0, /* tp_richcompare */ - 0, /* tp_weaklistoffset */ - 0, /* tp_iter */ - 0, /* tp_iternext */ - NullImporter_methods, /* tp_methods */ - 0, /* tp_members */ - 0, /* tp_getset */ - 0, /* tp_base */ - 0, /* tp_dict */ - 0, /* tp_descr_get */ - 0, /* tp_descr_set */ - 0, /* tp_dictoffset */ - (initproc)NullImporter_init, /* tp_init */ - 0, /* tp_alloc */ - PyType_GenericNew /* tp_new */ -}; - static struct PyModuleDef impmodule = { PyModuleDef_HEAD_INIT, "_imp", @@ -2389,9 +2257,6 @@ PyInit_imp(void) { PyObject *m, *d; - if (PyType_Ready(&PyNullImporter_Type) < 0) - return NULL; - m = PyModule_Create(&impmodule); if (m == NULL) goto failure; @@ -2410,8 +2275,6 @@ PyInit_imp(void) if (setint(d, "PY_CODERESOURCE", PY_CODERESOURCE) < 0) goto failure; if (setint(d, "IMP_HOOK", IMP_HOOK) < 0) goto failure; - Py_INCREF(&PyNullImporter_Type); - PyModule_AddObject(m, "NullImporter", (PyObject *)&PyNullImporter_Type); return m; failure: Py_XDECREF(m); -- cgit v1.2.1 From 50f76bdf6a49c9e80d8c092db2e93fc8c02ecb5c Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sun, 29 Apr 2012 14:38:11 -0400 Subject: Issues #13959, 14647: Re-implement imp.reload() in Lib/imp.py. Thanks to Eric Snow for the patch. --- Python/import.c | 109 +++++++------------------------------------------------- 1 file changed, 12 insertions(+), 97 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 0580f608fb..71485bd1ea 100644 --- a/Python/import.c +++ b/Python/import.c @@ -410,14 +410,6 @@ _PyImport_Fini(void) #endif } -static void -imp_modules_reloading_clear(void) -{ - PyInterpreterState *interp = PyThreadState_Get()->interp; - if (interp->modules_reloading != NULL) - PyDict_Clear(interp->modules_reloading); -} - /* Helper for sys */ PyObject * @@ -575,7 +567,6 @@ PyImport_Cleanup(void) PyDict_Clear(modules); interp->modules = NULL; Py_DECREF(modules); - Py_CLEAR(interp->modules_reloading); } @@ -1783,87 +1774,23 @@ PyImport_ImportModuleLevel(const char *name, PyObject *globals, PyObject *locals PyObject * PyImport_ReloadModule(PyObject *m) { - PyInterpreterState *interp = PyThreadState_Get()->interp; - PyObject *modules_reloading = interp->modules_reloading; + _Py_IDENTIFIER(reload); + PyObject *reloaded_module = NULL; PyObject *modules = PyImport_GetModuleDict(); - PyObject *loader = NULL, *existing_m = NULL; - PyObject *name; - Py_ssize_t subname_start; - PyObject *newm = NULL; - _Py_IDENTIFIER(__loader__); - _Py_IDENTIFIER(load_module); - - if (modules_reloading == NULL) { - Py_FatalError("PyImport_ReloadModule: " - "no modules_reloading dictionary!"); - return NULL; - } - - if (m == NULL || !PyModule_Check(m)) { - PyErr_SetString(PyExc_TypeError, - "reload() argument must be module"); - return NULL; - } - name = PyModule_GetNameObject(m); - if (name == NULL || PyUnicode_READY(name) == -1) - return NULL; - if (m != PyDict_GetItem(modules, name)) { - PyErr_Format(PyExc_ImportError, - "reload(): module %R not in sys.modules", - name); - Py_DECREF(name); - return NULL; - } - existing_m = PyDict_GetItem(modules_reloading, name); - if (existing_m != NULL) { - /* Due to a recursive reload, this module is already - being reloaded. */ - Py_DECREF(name); - Py_INCREF(existing_m); - return existing_m; - } - if (PyDict_SetItem(modules_reloading, name, m) < 0) { - Py_DECREF(name); - return NULL; - } - - subname_start = PyUnicode_FindChar(name, '.', 0, - PyUnicode_GET_LENGTH(name), -1); - if (subname_start != -1) { - PyObject *parentname, *parent; - parentname = PyUnicode_Substring(name, 0, subname_start); - if (parentname == NULL) { - goto error; - } - parent = PyDict_GetItem(modules, parentname); - Py_XDECREF(parent); - if (parent == NULL) { - PyErr_Format(PyExc_ImportError, - "reload(): parent %R not in sys.modules", - parentname); - goto error; + PyObject *imp = PyDict_GetItemString(modules, "imp"); + if (imp == NULL) { + imp = PyImport_ImportModule("imp"); + if (imp == NULL) { + return NULL; } } - - loader = _PyObject_GetAttrId(m, &PyId___loader__); - if (loader == NULL) { - goto error; - } - newm = _PyObject_CallMethodId(loader, &PyId_load_module, "O", name); - Py_DECREF(loader); - if (newm == NULL) { - /* load_module probably removed name from modules because of - * the error. Put back the original module object. We're - * going to return NULL in this case regardless of whether - * replacing name succeeds, so the return value is ignored. - */ - PyDict_SetItem(modules, name, m); + else { + Py_INCREF(imp); } -error: - imp_modules_reloading_clear(); - Py_DECREF(name); - return newm; + reloaded_module = _PyObject_CallMethodId(imp, &PyId_reload, "O", m); + Py_DECREF(imp); + return reloaded_module; } @@ -2160,17 +2087,6 @@ imp_load_dynamic(PyObject *self, PyObject *args) #endif /* HAVE_DYNAMIC_LOADING */ -static PyObject * -imp_reload(PyObject *self, PyObject *v) -{ - return PyImport_ReloadModule(v); -} - -PyDoc_STRVAR(doc_reload, -"reload(module) -> module\n\ -\n\ -Reload the module. The module must have been successfully imported before."); - /* Doc strings */ @@ -2214,7 +2130,6 @@ static PyMethodDef imp_methods[] = { {"lock_held", imp_lock_held, METH_NOARGS, doc_lock_held}, {"acquire_lock", imp_acquire_lock, METH_NOARGS, doc_acquire_lock}, {"release_lock", imp_release_lock, METH_NOARGS, doc_release_lock}, - {"reload", imp_reload, METH_O, doc_reload}, {"get_frozen_object", imp_get_frozen_object, METH_VARARGS}, {"is_frozen_package", imp_is_frozen_package, METH_VARARGS}, {"init_builtin", imp_init_builtin, METH_VARARGS}, -- cgit v1.2.1 From f9e12fdf7205afea929ddb2b8f6e950721c831bf Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 4 May 2012 15:20:40 -0400 Subject: Issue #13959: Re-implement imp.get_suffixes() in Lib/imp.py. This introduces a new function, imp.extension_suffixes(), which is currently undocumented. That is forthcoming once issue #14657 is resolved and how to expose file suffixes is decided. --- Python/import.c | 72 +++++++++++---------------------------------------------- 1 file changed, 13 insertions(+), 59 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 71485bd1ea..8349b776c6 100644 --- a/Python/import.c +++ b/Python/import.c @@ -140,18 +140,6 @@ extern struct _inittab _PyImport_Inittab[]; struct _inittab *PyImport_Inittab = _PyImport_Inittab; -/* these tables define the module suffixes that Python recognizes */ -struct filedescr * _PyImport_Filetab = NULL; - -static const struct filedescr _PyImport_StandardFiletab[] = { - {".py", "U", PY_SOURCE}, -#ifdef MS_WINDOWS - {".pyw", "U", PY_SOURCE}, -#endif - {".pyc", "rb", PY_COMPILED}, - {0, 0} -}; - static PyObject *initstr = NULL; /* Initialize things */ @@ -159,44 +147,9 @@ static PyObject *initstr = NULL; void _PyImport_Init(void) { - const struct filedescr *scan; - struct filedescr *filetab; - int countD = 0; - int countS = 0; - initstr = PyUnicode_InternFromString("__init__"); if (initstr == NULL) Py_FatalError("Can't initialize import variables"); - - /* prepare _PyImport_Filetab: copy entries from - _PyImport_DynLoadFiletab and _PyImport_StandardFiletab. - */ -#ifdef HAVE_DYNAMIC_LOADING - for (scan = _PyImport_DynLoadFiletab; scan->suffix != NULL; ++scan) - ++countD; -#endif - for (scan = _PyImport_StandardFiletab; scan->suffix != NULL; ++scan) - ++countS; - filetab = PyMem_NEW(struct filedescr, countD + countS + 1); - if (filetab == NULL) - Py_FatalError("Can't initialize import file table."); -#ifdef HAVE_DYNAMIC_LOADING - memcpy(filetab, _PyImport_DynLoadFiletab, - countD * sizeof(struct filedescr)); -#endif - memcpy(filetab + countD, _PyImport_StandardFiletab, - countS * sizeof(struct filedescr)); - filetab[countD + countS].suffix = NULL; - - _PyImport_Filetab = filetab; - - if (Py_OptimizeFlag) { - /* Replace ".pyc" with ".pyo" in _PyImport_Filetab */ - for (; filetab->suffix != NULL; filetab++) { - if (strcmp(filetab->suffix, ".pyc") == 0) - filetab->suffix = ".pyo"; - } - } } void @@ -400,8 +353,6 @@ _PyImport_Fini(void) { Py_XDECREF(extensions); extensions = NULL; - PyMem_DEL(_PyImport_Filetab); - _PyImport_Filetab = NULL; #ifdef WITH_THREAD if (import_lock != NULL) { PyThread_free_lock(import_lock); @@ -1911,17 +1862,18 @@ imp_get_tag(PyObject *self, PyObject *noargs) } static PyObject * -imp_get_suffixes(PyObject *self, PyObject *noargs) +imp_extension_suffixes(PyObject *self, PyObject *noargs) { PyObject *list; - struct filedescr *fdp; + const char *suffix; + unsigned int index = 0; list = PyList_New(0); if (list == NULL) return NULL; - for (fdp = _PyImport_Filetab; fdp->suffix != NULL; fdp++) { - PyObject *item = Py_BuildValue("ssi", - fdp->suffix, fdp->mode, fdp->type); +#ifdef HAVE_DYNAMIC_LOADING + while ((suffix = _PyImport_DynLoadFiletab[index])) { + PyObject *item = PyUnicode_FromString(suffix); if (item == NULL) { Py_DECREF(list); return NULL; @@ -1932,7 +1884,9 @@ imp_get_suffixes(PyObject *self, PyObject *noargs) return NULL; } Py_DECREF(item); + index += 1; } +#endif return list; } @@ -2101,10 +2055,9 @@ PyDoc_STRVAR(doc_get_tag, "get_tag() -> string\n\ Return the magic tag for .pyc or .pyo files."); -PyDoc_STRVAR(doc_get_suffixes, -"get_suffixes() -> [(suffix, mode, type), ...]\n\ -Return a list of (suffix, mode, type) tuples describing the files\n\ -that find_module() looks for."); +PyDoc_STRVAR(doc_extension_suffixes, +"extension_suffixes() -> list of strings\n\ +Returns the list of file suffixes used to identify extension modules."); PyDoc_STRVAR(doc_lock_held, "lock_held() -> boolean\n\ @@ -2126,7 +2079,8 @@ On platforms without threads, this function does nothing."); static PyMethodDef imp_methods[] = { {"get_magic", imp_get_magic, METH_NOARGS, doc_get_magic}, {"get_tag", imp_get_tag, METH_NOARGS, doc_get_tag}, - {"get_suffixes", imp_get_suffixes, METH_NOARGS, doc_get_suffixes}, + {"extension_suffixes", imp_extension_suffixes, METH_NOARGS, + doc_extension_suffixes}, {"lock_held", imp_lock_held, METH_NOARGS, doc_lock_held}, {"acquire_lock", imp_acquire_lock, METH_NOARGS, doc_acquire_lock}, {"release_lock", imp_release_lock, METH_NOARGS, doc_release_lock}, -- cgit v1.2.1 From b48b22964182216357bd0b9f07b1d8de65834364 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 4 May 2012 16:04:14 -0400 Subject: Remove dead Windows code which no longer will compile. --- Python/import.c | 6 ------ 1 file changed, 6 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 8349b776c6..4edc3a75f8 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1150,12 +1150,6 @@ PyImport_GetImporter(PyObject *path) { } -#ifdef MS_COREDLL -extern FILE *_PyWin_FindRegisteredModule(PyObject *, struct filedescr **, - PyObject **p_path); -#endif - - static int init_builtin(PyObject *); /* Forward */ /* Initialize a built-in module. -- cgit v1.2.1 From a05192072f95ad8e5fc8b3ecc3717f71902ee208 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 4 May 2012 16:13:30 -0400 Subject: Issue #13959: Move module type constants to Lib/imp.py. --- Python/import.c | 22 ---------------------- 1 file changed, 22 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 4edc3a75f8..4f8229bd0b 100644 --- a/Python/import.c +++ b/Python/import.c @@ -2091,17 +2091,6 @@ static PyMethodDef imp_methods[] = { {NULL, NULL} /* sentinel */ }; -static int -setint(PyObject *d, char *name, int value) -{ - PyObject *v; - int err; - - v = PyLong_FromLong((long)value); - err = PyDict_SetItemString(d, name, v); - Py_XDECREF(v); - return err; -} static struct PyModuleDef impmodule = { PyModuleDef_HEAD_INIT, @@ -2127,17 +2116,6 @@ PyInit_imp(void) if (d == NULL) goto failure; - if (setint(d, "SEARCH_ERROR", SEARCH_ERROR) < 0) goto failure; - if (setint(d, "PY_SOURCE", PY_SOURCE) < 0) goto failure; - if (setint(d, "PY_COMPILED", PY_COMPILED) < 0) goto failure; - if (setint(d, "C_EXTENSION", C_EXTENSION) < 0) goto failure; - if (setint(d, "PY_RESOURCE", PY_RESOURCE) < 0) goto failure; - if (setint(d, "PKG_DIRECTORY", PKG_DIRECTORY) < 0) goto failure; - if (setint(d, "C_BUILTIN", C_BUILTIN) < 0) goto failure; - if (setint(d, "PY_FROZEN", PY_FROZEN) < 0) goto failure; - if (setint(d, "PY_CODERESOURCE", PY_CODERESOURCE) < 0) goto failure; - if (setint(d, "IMP_HOOK", IMP_HOOK) < 0) goto failure; - return m; failure: Py_XDECREF(m); -- cgit v1.2.1 From 315ac927ba98ee268df075c8ec43bb2a23122f5e Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Fri, 4 May 2012 22:40:25 +0200 Subject: Simplify code for load_dynamic() --- Python/import.c | 46 +++------------------------------------------- 1 file changed, 3 insertions(+), 43 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 4f8229bd0b..920d304dbc 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1964,48 +1964,6 @@ imp_is_frozen(PyObject *self, PyObject *args) return PyBool_FromLong((long) (p == NULL ? 0 : p->size)); } -static FILE * -get_file(PyObject *pathname, PyObject *fob, char *mode) -{ - FILE *fp; - if (mode[0] == 'U') - mode = "r" PY_STDIOTEXTMODE; - if (fob == NULL) { - fp = _Py_fopen(pathname, mode); - if (!fp) { - if (!PyErr_Occurred()) - PyErr_SetFromErrno(PyExc_IOError); - return NULL; - } - return fp; - } - else { - int fd = PyObject_AsFileDescriptor(fob); - if (fd == -1) - return NULL; - if (!_PyVerify_fd(fd)) { - PyErr_SetFromErrno(PyExc_IOError); - return NULL; - } - - /* the FILE struct gets a new fd, so that it can be closed - * independently of the file descriptor given - */ - fd = dup(fd); - if (fd == -1) { - PyErr_SetFromErrno(PyExc_IOError); - return NULL; - } - - fp = fdopen(fd, mode); - if (!fp) { - PyErr_SetFromErrno(PyExc_IOError); - return NULL; - } - return fp; - } -} - #ifdef HAVE_DYNAMIC_LOADING static PyObject * @@ -2018,9 +1976,11 @@ imp_load_dynamic(PyObject *self, PyObject *args) &name, PyUnicode_FSDecoder, &pathname, &fob)) return NULL; if (fob != NULL) { - fp = get_file(NULL, fob, "r"); + fp = _Py_fopen(pathname, "r"); if (fp == NULL) { Py_DECREF(pathname); + if (!PyErr_Occurred()) + PyErr_SetFromErrno(PyExc_IOError); return NULL; } } -- cgit v1.2.1 From 416762c05c413705cd100d9fd0afa12db75d17f9 Mon Sep 17 00:00:00 2001 From: Nadeem Vawda Date: Sat, 5 May 2012 12:27:30 +0200 Subject: Fix typo in changeset eb5c5c23ca9b. --- Python/import.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 920d304dbc..ccba1066d2 100644 --- a/Python/import.c +++ b/Python/import.c @@ -181,7 +181,7 @@ _PyImportHooks_Init(void) error: PyErr_Print(); Py_FatalError("initializing sys.meta_path, sys.path_hooks, " - "or path_importer_cache"); + "or path_importer_cache failed"); } Py_DECREF(path_hooks); } -- cgit v1.2.1 From 03507f25cc48d46466ff1923b9a5d5308d54812c Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sun, 6 May 2012 17:15:23 +0200 Subject: Fix too early decrefs. --- Python/import.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index ccba1066d2..fd42a892a1 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1639,7 +1639,6 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, if (level == 0) { final_mod = PyDict_GetItem(interp->modules, front); - Py_DECREF(front); if (final_mod == NULL) { PyErr_Format(PyExc_KeyError, "%R not in sys.modules as expected", front); @@ -1647,6 +1646,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, else { Py_INCREF(final_mod); } + Py_DECREF(front); } else { Py_ssize_t cut_off = PyUnicode_GET_LENGTH(name) - @@ -1660,7 +1660,6 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, } final_mod = PyDict_GetItem(interp->modules, to_return); - Py_DECREF(to_return); if (final_mod == NULL) { PyErr_Format(PyExc_KeyError, "%R not in sys.modules as expected", @@ -1669,6 +1668,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, else { Py_INCREF(final_mod); } + Py_DECREF(to_return); } } else { -- cgit v1.2.1 From b13cec03ded782b4aed581ecb83604d1ebbcefc1 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Mon, 7 May 2012 21:41:59 +0200 Subject: Issue #14583: Fix importlib bug when a package's __init__.py would first import one of its modules then raise an error. --- Python/import.c | 19 +++++++++++-------- 1 file changed, 11 insertions(+), 8 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index fd42a892a1..a8b1aa3fd1 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1633,19 +1633,20 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, goto error_with_unlock; } + if (PyUnicode_GET_LENGTH(PyTuple_GET_ITEM(partition, 1)) == 0) { + /* No dot in module name, simple exit */ + Py_DECREF(partition); + final_mod = mod; + Py_INCREF(mod); + goto exit_with_unlock; + } + front = PyTuple_GET_ITEM(partition, 0); Py_INCREF(front); Py_DECREF(partition); if (level == 0) { - final_mod = PyDict_GetItem(interp->modules, front); - if (final_mod == NULL) { - PyErr_Format(PyExc_KeyError, - "%R not in sys.modules as expected", front); - } - else { - Py_INCREF(final_mod); - } + final_mod = PyObject_CallFunctionObjArgs(builtins_import, front, NULL); Py_DECREF(front); } else { @@ -1682,6 +1683,8 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, fromlist, builtins_import, NULL); } + + exit_with_unlock: error_with_unlock: #ifdef WITH_THREAD if (_PyImport_ReleaseLock() < 0) { -- cgit v1.2.1 From 2c77389854db5e8a9deeaa4210ead02b376c274b Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Thu, 17 May 2012 18:55:59 +0200 Subject: Issue #9260: A finer-grained import lock. Most of the import sequence now uses per-module locks rather than the global import lock, eliminating well-known issues with threads and imports. --- Python/import.c | 88 +++++++++++++++++++++++++++------------------------------ 1 file changed, 41 insertions(+), 47 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index a8b1aa3fd1..ab6ff864a5 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1370,47 +1370,7 @@ PyImport_ImportModule(const char *name) PyObject * PyImport_ImportModuleNoBlock(const char *name) { - PyObject *nameobj, *modules, *result; -#ifdef WITH_THREAD - long me; -#endif - - /* Try to get the module from sys.modules[name] */ - modules = PyImport_GetModuleDict(); - if (modules == NULL) - return NULL; - - nameobj = PyUnicode_FromString(name); - if (nameobj == NULL) - return NULL; - result = PyDict_GetItem(modules, nameobj); - if (result != NULL) { - Py_DECREF(nameobj); - Py_INCREF(result); - return result; - } - PyErr_Clear(); -#ifdef WITH_THREAD - /* check the import lock - * me might be -1 but I ignore the error here, the lock function - * takes care of the problem */ - me = PyThread_get_thread_ident(); - if (import_lock_thread == -1 || import_lock_thread == me) { - /* no thread or me is holding the lock */ - result = PyImport_Import(nameobj); - } - else { - PyErr_Format(PyExc_ImportError, - "Failed to import %R because the import lock" - "is held by another thread.", - nameobj); - result = NULL; - } -#else - result = PyImport_Import(nameobj); -#endif - Py_DECREF(nameobj); - return result; + return PyImport_ImportModule(name); } @@ -1420,11 +1380,13 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, int level) { _Py_IDENTIFIER(__import__); + _Py_IDENTIFIER(__initializing__); _Py_IDENTIFIER(__package__); _Py_IDENTIFIER(__path__); _Py_IDENTIFIER(__name__); _Py_IDENTIFIER(_find_and_load); _Py_IDENTIFIER(_handle_fromlist); + _Py_IDENTIFIER(_lock_unlock_module); _Py_static_string(single_dot, "."); PyObject *abs_name = NULL; PyObject *builtins_import = NULL; @@ -1607,16 +1569,48 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, goto error_with_unlock; } else if (mod != NULL) { + PyObject *value; + int initializing = 0; + Py_INCREF(mod); + /* Only call _bootstrap._lock_unlock_module() if __initializing__ is true. */ + value = _PyObject_GetAttrId(mod, &PyId___initializing__); + if (value == NULL) + PyErr_Clear(); + else { + initializing = PyObject_IsTrue(value); + Py_DECREF(value); + if (initializing == -1) + PyErr_Clear(); + } + if (initializing > 0) { + /* _bootstrap._lock_unlock_module() releases the import lock */ + value = _PyObject_CallMethodObjIdArgs(interp->importlib, + &PyId__lock_unlock_module, abs_name, + NULL); + if (value == NULL) + goto error; + Py_DECREF(value); + } + else { +#ifdef WITH_THREAD + if (_PyImport_ReleaseLock() < 0) { + PyErr_SetString(PyExc_RuntimeError, "not holding the import lock"); + goto error; + } +#endif + } } else { + /* _bootstrap._find_and_load() releases the import lock */ mod = _PyObject_CallMethodObjIdArgs(interp->importlib, &PyId__find_and_load, abs_name, builtins_import, NULL); if (mod == NULL) { - goto error_with_unlock; + goto error; } } + /* From now on we don't hold the import lock anymore. */ if (PyObject_Not(fromlist)) { if (level == 0 || PyUnicode_GET_LENGTH(name) > 0) { @@ -1625,12 +1619,12 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, PyObject *borrowed_dot = _PyUnicode_FromId(&single_dot); if (borrowed_dot == NULL) { - goto error_with_unlock; + goto error; } partition = PyUnicode_Partition(name, borrowed_dot); if (partition == NULL) { - goto error_with_unlock; + goto error; } if (PyUnicode_GET_LENGTH(PyTuple_GET_ITEM(partition, 1)) == 0) { @@ -1638,7 +1632,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, Py_DECREF(partition); final_mod = mod; Py_INCREF(mod); - goto exit_with_unlock; + goto error; } front = PyTuple_GET_ITEM(partition, 0); @@ -1657,7 +1651,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, abs_name_len - cut_off); Py_DECREF(front); if (to_return == NULL) { - goto error_with_unlock; + goto error; } final_mod = PyDict_GetItem(interp->modules, to_return); @@ -1683,8 +1677,8 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, fromlist, builtins_import, NULL); } + goto error; - exit_with_unlock: error_with_unlock: #ifdef WITH_THREAD if (_PyImport_ReleaseLock() < 0) { -- cgit v1.2.1 From e26eba13bcc570d5e912886692d5030db7b74ac3 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 27 May 2012 18:17:07 +1000 Subject: Close #14857: fix regression in references to PEP 3135 implicit __class__ closure variable. Reopens issue #12370, but also updates unittest.mock to workaround that issue --- Python/import.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index ab6ff864a5..98d9dc5d70 100644 --- a/Python/import.c +++ b/Python/import.c @@ -106,6 +106,7 @@ typedef unsigned short mode_t; Python 3.3a0 3200 (__qualname__ added) 3210 (added size modulo 2**32 to the pyc header) Python 3.3a1 3220 (changed PEP 380 implementation) + Python 3.3a4 3230 (revert changes to implicit __class__ closure) */ /* MAGIC must change whenever the bytecode emitted by the compiler may no @@ -118,7 +119,7 @@ typedef unsigned short mode_t; #define STRIFY(name) QUOTE(name) #define MAJOR STRIFY(PY_MAJOR_VERSION) #define MINOR STRIFY(PY_MINOR_VERSION) -#define MAGIC (3220 | ((long)'\r'<<16) | ((long)'\n'<<24)) +#define MAGIC (3230 | ((long)'\r'<<16) | ((long)'\n'<<24)) #define TAG "cpython-" MAJOR MINOR; #define CACHEDIR "__pycache__" /* Current magic word and string tag as globals. */ -- cgit v1.2.1 From e7d1f25f3844ef6eb4a4971bb4c2db50d672e041 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Sat, 16 Jun 2012 03:22:05 +0200 Subject: get_sourcefile(): use PyUnicode_READ() to avoid the creation of a temporary Py_UCS4 buffer --- Python/import.c | 23 +++++++++++------------ 1 file changed, 11 insertions(+), 12 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 98d9dc5d70..701a6e9eb5 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1008,23 +1008,25 @@ static PyObject * get_sourcefile(PyObject *filename) { Py_ssize_t len; - Py_UCS4 *fileuni; PyObject *py; struct stat statbuf; int err; + void *data; + unsigned int kind; len = PyUnicode_GET_LENGTH(filename); if (len == 0) Py_RETURN_NONE; /* don't match *.pyc or *.pyo? */ - fileuni = PyUnicode_AsUCS4Copy(filename); - if (!fileuni) - return NULL; + data = PyUnicode_DATA(filename); + kind = PyUnicode_KIND(filename); if (len < 5 - || fileuni[len-4] != '.' - || (fileuni[len-3] != 'p' && fileuni[len-3] != 'P') - || (fileuni[len-2] != 'y' && fileuni[len-2] != 'Y')) + || PyUnicode_READ(kind, data, len-4) != '.' + || (PyUnicode_READ(kind, data, len-3) != 'p' + && PyUnicode_READ(kind, data, len-3) != 'P') + || (PyUnicode_READ(kind, data, len-2) != 'y' + && PyUnicode_READ(kind, data, len-2) != 'Y')) goto unchanged; /* Start by trying to turn PEP 3147 path into source path. If that @@ -1034,7 +1036,7 @@ get_sourcefile(PyObject *filename) py = make_source_pathname(filename); if (py == NULL) { PyErr_Clear(); - py = PyUnicode_FromKindAndData(PyUnicode_4BYTE_KIND, fileuni, len - 1); + py = PyUnicode_Substring(filename, 0, len - 1); } if (py == NULL) goto error; @@ -1042,17 +1044,14 @@ get_sourcefile(PyObject *filename) err = _Py_stat(py, &statbuf); if (err == -2) goto error; - if (err == 0 && S_ISREG(statbuf.st_mode)) { - PyMem_Free(fileuni); + if (err == 0 && S_ISREG(statbuf.st_mode)) return py; - } Py_DECREF(py); goto unchanged; error: PyErr_Clear(); unchanged: - PyMem_Free(fileuni); Py_INCREF(filename); return filename; } -- cgit v1.2.1 From 80184ef410b1a602b43cd805a6b46aa98a6ca1ae Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 2 Jul 2012 15:13:11 -0400 Subject: Issue #15166: Re-implement imp.get_tag() using sys.implementation. Also eliminates some C code in Python/import.c as well. Patch by Eric Snow with verification by comparing against another patch from Jeff Knupp. --- Python/import.c | 38 ++++++++++++++------------------------ 1 file changed, 14 insertions(+), 24 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 701a6e9eb5..c63f760d07 100644 --- a/Python/import.c +++ b/Python/import.c @@ -112,23 +112,11 @@ typedef unsigned short mode_t; /* MAGIC must change whenever the bytecode emitted by the compiler may no longer be understood by older implementations of the eval loop (usually due to the addition of new opcodes) - TAG must change for each major Python release. The magic number will take - care of any bytecode changes that occur during development. */ -#define QUOTE(arg) #arg -#define STRIFY(name) QUOTE(name) -#define MAJOR STRIFY(PY_MAJOR_VERSION) -#define MINOR STRIFY(PY_MINOR_VERSION) #define MAGIC (3230 | ((long)'\r'<<16) | ((long)'\n'<<24)) -#define TAG "cpython-" MAJOR MINOR; #define CACHEDIR "__pycache__" /* Current magic word and string tag as globals. */ static long pyc_magic = MAGIC; -static const char *pyc_tag = TAG; -#undef QUOTE -#undef STRIFY -#undef MAJOR -#undef MINOR /* See _PyImport_FixupExtensionObject() below */ static PyObject *extensions = NULL; @@ -534,9 +522,22 @@ PyImport_GetMagicNumber(void) const char * PyImport_GetMagicTag(void) { - return pyc_tag; + PyObject *impl, *tag; + const char *raw_tag; + + /* We could also pull it from imp or importlib. */ + impl = PySys_GetObject("implementation"); + if (impl == NULL) + return NULL; + tag = PyObject_GetAttrString(impl, "cache_tag"); + if (tag == NULL) + return NULL; + raw_tag = PyUnicode_DATA(tag); + Py_DECREF(tag); + return raw_tag; } + /* Magic for extension modules (built-in as well as dynamically loaded). To prevent initializing an extension module more than once, we keep a static dictionary 'extensions' keyed by module name @@ -1846,12 +1847,6 @@ imp_get_magic(PyObject *self, PyObject *noargs) return imp_make_magic(pyc_magic); } -static PyObject * -imp_get_tag(PyObject *self, PyObject *noargs) -{ - return PyUnicode_FromString(pyc_tag); -} - static PyObject * imp_extension_suffixes(PyObject *self, PyObject *noargs) { @@ -2002,10 +1997,6 @@ PyDoc_STRVAR(doc_get_magic, "get_magic() -> string\n\ Return the magic number for .pyc or .pyo files."); -PyDoc_STRVAR(doc_get_tag, -"get_tag() -> string\n\ -Return the magic tag for .pyc or .pyo files."); - PyDoc_STRVAR(doc_extension_suffixes, "extension_suffixes() -> list of strings\n\ Returns the list of file suffixes used to identify extension modules."); @@ -2029,7 +2020,6 @@ On platforms without threads, this function does nothing."); static PyMethodDef imp_methods[] = { {"get_magic", imp_get_magic, METH_NOARGS, doc_get_magic}, - {"get_tag", imp_get_tag, METH_NOARGS, doc_get_tag}, {"extension_suffixes", imp_extension_suffixes, METH_NOARGS, doc_extension_suffixes}, {"lock_held", imp_lock_held, METH_NOARGS, doc_lock_held}, -- cgit v1.2.1 From 8768a2b7b880b9483285546dc7946b9c87af11a5 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sun, 8 Jul 2012 12:01:27 +0200 Subject: Issue #15110: Fix the tracebacks generated by "import xxx" to not show the importlib stack frames. --- Python/import.c | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 66 insertions(+) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index c63f760d07..c31819dc81 100644 --- a/Python/import.c +++ b/Python/import.c @@ -8,6 +8,7 @@ #include "errcode.h" #include "marshal.h" #include "code.h" +#include "frameobject.h" #include "osdefs.h" #include "importdl.h" @@ -1375,6 +1376,69 @@ PyImport_ImportModuleNoBlock(const char *name) } +/* Remove importlib frames from the traceback, + * except in Verbose mode. */ +static void +remove_importlib_frames(void) +{ + const char *importlib_filename = ""; + const char *exec_funcname = "_exec_module"; + int always_trim = 0; + int in_importlib; + PyObject *exception, *value, *base_tb, *tb; + PyObject **prev_link, **outer_link; + + /* Synopsis: if it's an ImportError, we trim all importlib chunks + from the traceback. Otherwise, we trim only those chunks which + end with a call to "_exec_module". */ + + PyErr_Fetch(&exception, &value, &base_tb); + if (!exception || Py_VerboseFlag) + goto done; + if (PyType_IsSubtype((PyTypeObject *) exception, + (PyTypeObject *) PyExc_ImportError)) + always_trim = 1; + + prev_link = &base_tb; + tb = base_tb; + in_importlib = 0; + while (tb != NULL) { + PyTracebackObject *traceback = (PyTracebackObject *)tb; + PyObject *next = (PyObject *) traceback->tb_next; + PyFrameObject *frame = traceback->tb_frame; + PyCodeObject *code = frame->f_code; + int now_in_importlib; + + assert(PyTraceBack_Check(tb)); + now_in_importlib = (PyUnicode_CompareWithASCIIString( + code->co_filename, + importlib_filename) == 0); + if (now_in_importlib && !in_importlib) { + /* This is the link to this chunk of importlib tracebacks */ + outer_link = prev_link; + } + in_importlib = now_in_importlib; + + if (in_importlib && + (always_trim || + PyUnicode_CompareWithASCIIString(code->co_name, + exec_funcname) == 0)) { + PyObject *tmp = *outer_link; + *outer_link = next; + Py_XINCREF(next); + Py_DECREF(tmp); + prev_link = outer_link; + } + else { + prev_link = (PyObject **) &traceback->tb_next; + } + tb = next; + } +done: + PyErr_Restore(exception, value, base_tb); +} + + PyObject * PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, PyObject *locals, PyObject *given_fromlist, @@ -1693,6 +1757,8 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, Py_XDECREF(package); Py_XDECREF(globals); Py_XDECREF(fromlist); + if (final_mod == NULL) + remove_importlib_frames(); return final_mod; } -- cgit v1.2.1 From 2218bc0c7a4c4a2df3126cc840b3a2718b0be65e Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 9 Jul 2012 14:22:12 -0400 Subject: Issue #15242: Have PyImport_GetMagicTag() return a const char * defined in sysmodule.c instead of straight out of a Unicode object. Thanks to Amaury Forgeot d'Arc for noticing the bug and Eric Snow for writing the patch. --- Python/import.c | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index c31819dc81..1592045896 100644 --- a/Python/import.c +++ b/Python/import.c @@ -116,7 +116,7 @@ typedef unsigned short mode_t; */ #define MAGIC (3230 | ((long)'\r'<<16) | ((long)'\n'<<24)) #define CACHEDIR "__pycache__" -/* Current magic word and string tag as globals. */ +/* Current magic word as global. */ static long pyc_magic = MAGIC; /* See _PyImport_FixupExtensionObject() below */ @@ -520,22 +520,12 @@ PyImport_GetMagicNumber(void) } +extern const char * _PySys_ImplCacheTag; + const char * PyImport_GetMagicTag(void) { - PyObject *impl, *tag; - const char *raw_tag; - - /* We could also pull it from imp or importlib. */ - impl = PySys_GetObject("implementation"); - if (impl == NULL) - return NULL; - tag = PyObject_GetAttrString(impl, "cache_tag"); - if (tag == NULL) - return NULL; - raw_tag = PyUnicode_DATA(tag); - Py_DECREF(tag); - return raw_tag; + return _PySys_ImplCacheTag; } -- cgit v1.2.1 From 03f5d292413062348e623f4db9067ee65c531eb3 Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Mon, 9 Jul 2012 16:09:00 -0400 Subject: Issue #15167 (as part of #13959): imp.get_magic() is no implemented in Lib/imp.py. --- Python/import.c | 126 +++----------------------------------------------------- 1 file changed, 6 insertions(+), 120 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 1592045896..8dd285f9b0 100644 --- a/Python/import.c +++ b/Python/import.c @@ -27,97 +27,7 @@ typedef unsigned short mode_t; #endif -/* Magic word to reject .pyc files generated by other Python versions. - It should change for each incompatible change to the bytecode. - - The value of CR and LF is incorporated so if you ever read or write - a .pyc file in text mode the magic number will be wrong; also, the - Apple MPW compiler swaps their values, botching string constants. - - The magic numbers must be spaced apart at least 2 values, as the - -U interpeter flag will cause MAGIC+1 being used. They have been - odd numbers for some time now. - - There were a variety of old schemes for setting the magic number. - The current working scheme is to increment the previous value by - 10. - - Starting with the adoption of PEP 3147 in Python 3.2, every bump in magic - number also includes a new "magic tag", i.e. a human readable string used - to represent the magic number in __pycache__ directories. When you change - the magic number, you must also set a new unique magic tag. Generally this - can be named after the Python major version of the magic number bump, but - it can really be anything, as long as it's different than anything else - that's come before. The tags are included in the following table, starting - with Python 3.2a0. - - Known values: - Python 1.5: 20121 - Python 1.5.1: 20121 - Python 1.5.2: 20121 - Python 1.6: 50428 - Python 2.0: 50823 - Python 2.0.1: 50823 - Python 2.1: 60202 - Python 2.1.1: 60202 - Python 2.1.2: 60202 - Python 2.2: 60717 - Python 2.3a0: 62011 - Python 2.3a0: 62021 - Python 2.3a0: 62011 (!) - Python 2.4a0: 62041 - Python 2.4a3: 62051 - Python 2.4b1: 62061 - Python 2.5a0: 62071 - Python 2.5a0: 62081 (ast-branch) - Python 2.5a0: 62091 (with) - Python 2.5a0: 62092 (changed WITH_CLEANUP opcode) - Python 2.5b3: 62101 (fix wrong code: for x, in ...) - Python 2.5b3: 62111 (fix wrong code: x += yield) - Python 2.5c1: 62121 (fix wrong lnotab with for loops and - storing constants that should have been removed) - Python 2.5c2: 62131 (fix wrong code: for x, in ... in listcomp/genexp) - Python 2.6a0: 62151 (peephole optimizations and STORE_MAP opcode) - Python 2.6a1: 62161 (WITH_CLEANUP optimization) - Python 3000: 3000 - 3010 (removed UNARY_CONVERT) - 3020 (added BUILD_SET) - 3030 (added keyword-only parameters) - 3040 (added signature annotations) - 3050 (print becomes a function) - 3060 (PEP 3115 metaclass syntax) - 3061 (string literals become unicode) - 3071 (PEP 3109 raise changes) - 3081 (PEP 3137 make __file__ and __name__ unicode) - 3091 (kill str8 interning) - 3101 (merge from 2.6a0, see 62151) - 3103 (__file__ points to source file) - Python 3.0a4: 3111 (WITH_CLEANUP optimization). - Python 3.0a5: 3131 (lexical exception stacking, including POP_EXCEPT) - Python 3.1a0: 3141 (optimize list, set and dict comprehensions: - change LIST_APPEND and SET_ADD, add MAP_ADD) - Python 3.1a0: 3151 (optimize conditional branches: - introduce POP_JUMP_IF_FALSE and POP_JUMP_IF_TRUE) - Python 3.2a0: 3160 (add SETUP_WITH) - tag: cpython-32 - Python 3.2a1: 3170 (add DUP_TOP_TWO, remove DUP_TOPX and ROT_FOUR) - tag: cpython-32 - Python 3.2a2 3180 (add DELETE_DEREF) - Python 3.3a0 3190 __class__ super closure changed - Python 3.3a0 3200 (__qualname__ added) - 3210 (added size modulo 2**32 to the pyc header) - Python 3.3a1 3220 (changed PEP 380 implementation) - Python 3.3a4 3230 (revert changes to implicit __class__ closure) -*/ - -/* MAGIC must change whenever the bytecode emitted by the compiler may no - longer be understood by older implementations of the eval loop (usually - due to the addition of new opcodes) -*/ -#define MAGIC (3230 | ((long)'\r'<<16) | ((long)'\n'<<24)) #define CACHEDIR "__pycache__" -/* Current magic word as global. */ -static long pyc_magic = MAGIC; /* See _PyImport_FixupExtensionObject() below */ static PyObject *extensions = NULL; @@ -516,7 +426,12 @@ PyImport_Cleanup(void) long PyImport_GetMagicNumber(void) { - return pyc_magic; + PyInterpreterState *interp = PyThreadState_Get()->interp; + PyObject *pyc_magic = PyObject_GetAttrString(interp->importlib, + "_RAW_MAGIC_NUMBER"); + if (pyc_magic == NULL) + return -1; + return PyLong_AsLong(pyc_magic); } @@ -1879,30 +1794,6 @@ PyImport_Import(PyObject *module_name) return r; } - -/* Module 'imp' provides Python access to the primitives used for - importing modules. -*/ - -static PyObject * -imp_make_magic(long magic) -{ - char buf[4]; - - buf[0] = (char) ((magic >> 0) & 0xff); - buf[1] = (char) ((magic >> 8) & 0xff); - buf[2] = (char) ((magic >> 16) & 0xff); - buf[3] = (char) ((magic >> 24) & 0xff); - - return PyBytes_FromStringAndSize(buf, 4); -} - -static PyObject * -imp_get_magic(PyObject *self, PyObject *noargs) -{ - return imp_make_magic(pyc_magic); -} - static PyObject * imp_extension_suffixes(PyObject *self, PyObject *noargs) { @@ -2049,10 +1940,6 @@ imp_load_dynamic(PyObject *self, PyObject *args) PyDoc_STRVAR(doc_imp, "(Extremely) low-level import machinery bits as used by importlib and imp."); -PyDoc_STRVAR(doc_get_magic, -"get_magic() -> string\n\ -Return the magic number for .pyc or .pyo files."); - PyDoc_STRVAR(doc_extension_suffixes, "extension_suffixes() -> list of strings\n\ Returns the list of file suffixes used to identify extension modules."); @@ -2075,7 +1962,6 @@ Release the interpreter's import lock.\n\ On platforms without threads, this function does nothing."); static PyMethodDef imp_methods[] = { - {"get_magic", imp_get_magic, METH_NOARGS, doc_get_magic}, {"extension_suffixes", imp_extension_suffixes, METH_NOARGS, doc_extension_suffixes}, {"lock_held", imp_lock_held, METH_NOARGS, doc_lock_held}, -- cgit v1.2.1 From 0216bfca9a2f81c03b46f83c962f1dcf07aaae15 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Mon, 9 Jul 2012 13:43:53 -0700 Subject: initialize variable for compiler happiness --- Python/import.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 8dd285f9b0..4ae8abfceb 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1289,9 +1289,9 @@ remove_importlib_frames(void) const char *importlib_filename = ""; const char *exec_funcname = "_exec_module"; int always_trim = 0; - int in_importlib; + int in_importlib = 0; PyObject *exception, *value, *base_tb, *tb; - PyObject **prev_link, **outer_link; + PyObject **prev_link, **outer_link = NULL; /* Synopsis: if it's an ImportError, we trim all importlib chunks from the traceback. Otherwise, we trim only those chunks which @@ -1306,7 +1306,6 @@ remove_importlib_frames(void) prev_link = &base_tb; tb = base_tb; - in_importlib = 0; while (tb != NULL) { PyTracebackObject *traceback = (PyTracebackObject *)tb; PyObject *next = (PyObject *) traceback->tb_next; -- cgit v1.2.1 From 4ea09956f8d85aadd13515f026e53a2267533680 Mon Sep 17 00:00:00 2001 From: Benjamin Peterson Date: Mon, 9 Jul 2012 22:21:55 -0700 Subject: fix refleak --- Python/import.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 4ae8abfceb..aeaa9c05c3 100644 --- a/Python/import.c +++ b/Python/import.c @@ -431,7 +431,9 @@ PyImport_GetMagicNumber(void) "_RAW_MAGIC_NUMBER"); if (pyc_magic == NULL) return -1; - return PyLong_AsLong(pyc_magic); + long res = PyLong_AsLong(pyc_magic); + Py_DECREF(pyc_magic); + return res; } -- cgit v1.2.1 From bc5266671e1879adb2d293a7a0179a37db8c092b Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 10 Jul 2012 18:27:54 +0200 Subject: Fix compilation under Windows --- Python/import.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index aeaa9c05c3..26f4772da7 100644 --- a/Python/import.c +++ b/Python/import.c @@ -426,12 +426,13 @@ PyImport_Cleanup(void) long PyImport_GetMagicNumber(void) { + long res; PyInterpreterState *interp = PyThreadState_Get()->interp; PyObject *pyc_magic = PyObject_GetAttrString(interp->importlib, "_RAW_MAGIC_NUMBER"); if (pyc_magic == NULL) return -1; - long res = PyLong_AsLong(pyc_magic); + res = PyLong_AsLong(pyc_magic); Py_DECREF(pyc_magic); return res; } -- cgit v1.2.1 From 405b4e0f944d20bb68e3597b28a734164cbd66ca Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Fri, 13 Jul 2012 13:57:03 -0400 Subject: Issues #15169, #14599: Make PyImport_ExecCodeModuleWithPathnames() use Lib/imp.py for imp.source_from_cache() instead of its own C version. Also change PyImport_ExecCodeModuleObject() to not infer the source path from the bytecode path like PyImport_ExecCodeModuleWithPathnames() does. This makes the function less magical. This also has the side-effect of removing all uses of MAXPATHLEN in Python/import.c which can cause failures on really long filenames. --- Python/import.c | 195 +++++++++----------------------------------------------- 1 file changed, 29 insertions(+), 166 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 26f4772da7..982c389c83 100644 --- a/Python/import.c +++ b/Python/import.c @@ -630,8 +630,6 @@ remove_module(PyObject *name) "sys.modules failed"); } -static PyObject * get_sourcefile(PyObject *filename); -static PyObject *make_source_pathname(PyObject *pathname); /* Execute a code object in a module and return the module object * WITH INCREMENTED REFERENCE COUNT. If an error occurs, name is @@ -668,18 +666,37 @@ PyImport_ExecCodeModuleWithPathnames(char *name, PyObject *co, char *pathname, if (nameobj == NULL) return NULL; - if (pathname != NULL) { - pathobj = PyUnicode_DecodeFSDefault(pathname); - if (pathobj == NULL) - goto error; - } else - pathobj = NULL; if (cpathname != NULL) { cpathobj = PyUnicode_DecodeFSDefault(cpathname); if (cpathobj == NULL) goto error; - } else + } + else cpathobj = NULL; + + if (pathname != NULL) { + pathobj = PyUnicode_DecodeFSDefault(pathname); + if (pathobj == NULL) + goto error; + } + else if (cpathobj != NULL) { + PyInterpreterState *interp = PyThreadState_GET()->interp; + _Py_IDENTIFIER(_get_sourcefile); + + if (interp == NULL) { + Py_FatalError("PyImport_ExecCodeModuleWithPathnames: " + "no interpreter!"); + } + + pathobj = _PyObject_CallMethodObjIdArgs(interp->importlib, + &PyId__get_sourcefile, cpathobj, + NULL); + if (pathobj == NULL) + PyErr_Clear(); + } + else + pathobj = NULL; + m = PyImport_ExecCodeModuleObject(nameobj, co, pathobj, cpathobj); error: Py_DECREF(nameobj); @@ -706,18 +723,13 @@ PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, PyEval_GetBuiltins()) != 0) goto error; } - /* Remember the filename as the __file__ attribute */ if (pathname != NULL) { - v = get_sourcefile(pathname); - if (v == NULL) - PyErr_Clear(); + v = pathname; } - else - v = NULL; - if (v == NULL) { + else { v = ((PyCodeObject *)co)->co_filename; - Py_INCREF(v); } + Py_INCREF(v); if (PyDict_SetItemString(d, "__file__", v) != 0) PyErr_Clear(); /* Not important enough to report */ Py_DECREF(v); @@ -752,100 +764,6 @@ PyImport_ExecCodeModuleObject(PyObject *name, PyObject *co, PyObject *pathname, } -/* Like rightmost_sep, but operate on unicode objects. */ -static Py_ssize_t -rightmost_sep_obj(PyObject* o, Py_ssize_t start, Py_ssize_t end) -{ - Py_ssize_t found, i; - Py_UCS4 c; - for (found = -1, i = start; i < end; i++) { - c = PyUnicode_READ_CHAR(o, i); - if (c == SEP -#ifdef ALTSEP - || c == ALTSEP -#endif - ) - { - found = i; - } - } - return found; -} - - -/* Given a pathname to a Python byte compiled file, return the path to the - source file, if the path matches the PEP 3147 format. This does not check - for any file existence, however, if the pyc file name does not match PEP - 3147 style, NULL is returned. buf must be at least as big as pathname; - the resulting path will always be shorter. - - (...)/__pycache__/foo..pyc -> (...)/foo.py */ - -static PyObject* -make_source_pathname(PyObject *path) -{ - Py_ssize_t left, right, dot0, dot1, len; - Py_ssize_t i, j; - PyObject *result; - int kind; - void *data; - - len = PyUnicode_GET_LENGTH(path); - if (len > MAXPATHLEN) - return NULL; - - /* Look back two slashes from the end. In between these two slashes - must be the string __pycache__ or this is not a PEP 3147 style - path. It's possible for there to be only one slash. - */ - right = rightmost_sep_obj(path, 0, len); - if (right == -1) - return NULL; - left = rightmost_sep_obj(path, 0, right); - if (left == -1) - left = 0; - else - left++; - if (right-left != sizeof(CACHEDIR)-1) - return NULL; - for (i = 0; i < sizeof(CACHEDIR)-1; i++) - if (PyUnicode_READ_CHAR(path, left+i) != CACHEDIR[i]) - return NULL; - - /* Now verify that the path component to the right of the last slash - has two dots in it. - */ - dot0 = PyUnicode_FindChar(path, '.', right+1, len, 1); - if (dot0 < 0) - return NULL; - dot1 = PyUnicode_FindChar(path, '.', dot0+1, len, 1); - if (dot1 < 0) - return NULL; - /* Too many dots? */ - if (PyUnicode_FindChar(path, '.', dot1+1, len, 1) != -1) - return NULL; - - /* This is a PEP 3147 path. Start by copying everything from the - start of pathname up to and including the leftmost slash. Then - copy the file's basename, removing the magic tag and adding a .py - suffix. - */ - result = PyUnicode_New(left + (dot0-right) + 2, - PyUnicode_MAX_CHAR_VALUE(path)); - if (!result) - return NULL; - kind = PyUnicode_KIND(result); - data = PyUnicode_DATA(result); - PyUnicode_CopyCharacters(result, 0, path, 0, (i = left)); - PyUnicode_CopyCharacters(result, left, path, right+1, - (j = dot0-right)); - PyUnicode_WRITE(kind, data, i+j, 'p'); - PyUnicode_WRITE(kind, data, i+j+1, 'y'); - assert(_PyUnicode_CheckConsistency(result, 1)); - return result; -} - - static void update_code_filenames(PyCodeObject *co, PyObject *oldname, PyObject *newname) { @@ -911,61 +829,6 @@ imp_fix_co_filename(PyObject *self, PyObject *args) } -/* Get source file -> unicode or None - * Returns the path to the py file if available, else the given path - */ -static PyObject * -get_sourcefile(PyObject *filename) -{ - Py_ssize_t len; - PyObject *py; - struct stat statbuf; - int err; - void *data; - unsigned int kind; - - len = PyUnicode_GET_LENGTH(filename); - if (len == 0) - Py_RETURN_NONE; - - /* don't match *.pyc or *.pyo? */ - data = PyUnicode_DATA(filename); - kind = PyUnicode_KIND(filename); - if (len < 5 - || PyUnicode_READ(kind, data, len-4) != '.' - || (PyUnicode_READ(kind, data, len-3) != 'p' - && PyUnicode_READ(kind, data, len-3) != 'P') - || (PyUnicode_READ(kind, data, len-2) != 'y' - && PyUnicode_READ(kind, data, len-2) != 'Y')) - goto unchanged; - - /* Start by trying to turn PEP 3147 path into source path. If that - * fails, just chop off the trailing character, i.e. legacy pyc path - * to py. - */ - py = make_source_pathname(filename); - if (py == NULL) { - PyErr_Clear(); - py = PyUnicode_Substring(filename, 0, len - 1); - } - if (py == NULL) - goto error; - - err = _Py_stat(py, &statbuf); - if (err == -2) - goto error; - if (err == 0 && S_ISREG(statbuf.st_mode)) - return py; - Py_DECREF(py); - goto unchanged; - -error: - PyErr_Clear(); -unchanged: - Py_INCREF(filename); - return filename; -} - /* Forward */ static struct _frozen * find_frozen(PyObject *); -- cgit v1.2.1 From 53f3c6a8236f5881b0ec73e2063bd634ef4bffe2 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Sun, 29 Jul 2012 20:30:36 +1000 Subject: Close #15425: Eliminate more importlib related traceback noise --- Python/import.c | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 982c389c83..c4edfacd34 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1154,14 +1154,27 @@ remove_importlib_frames(void) { const char *importlib_filename = ""; const char *exec_funcname = "_exec_module"; + const char *get_code_funcname = "get_code"; + const char *recursive_import = "_recursive_import"; int always_trim = 0; + int trim_get_code = 0; int in_importlib = 0; PyObject *exception, *value, *base_tb, *tb; PyObject **prev_link, **outer_link = NULL; /* Synopsis: if it's an ImportError, we trim all importlib chunks - from the traceback. Otherwise, we trim only those chunks which - end with a call to "_exec_module". */ + from the traceback. If it's a SyntaxError, we trim any chunks that + end with a call to "get_code", We always trim chunks + which end with a call to "_exec_module". */ + + /* Thanks to issue 15425, we also strip any chunk ending with + * _recursive_import. This is used when making a recursive call to the + * full import machinery which means the inner stack gets stripped early + * and the normal heuristics won't fire properly for outer frames. A + * more elegant mechanism would be nice, as this one can misfire if + * builtins.__import__ has been replaced with a custom implementation. + * However, the current approach at least gets the job done. + */ PyErr_Fetch(&exception, &value, &base_tb); if (!exception || Py_VerboseFlag) @@ -1169,6 +1182,9 @@ remove_importlib_frames(void) if (PyType_IsSubtype((PyTypeObject *) exception, (PyTypeObject *) PyExc_ImportError)) always_trim = 1; + if (PyType_IsSubtype((PyTypeObject *) exception, + (PyTypeObject *) PyExc_SyntaxError)) + trim_get_code = 1; prev_link = &base_tb; tb = base_tb; @@ -1191,8 +1207,14 @@ remove_importlib_frames(void) if (in_importlib && (always_trim || - PyUnicode_CompareWithASCIIString(code->co_name, - exec_funcname) == 0)) { + (PyUnicode_CompareWithASCIIString(code->co_name, + exec_funcname) == 0) || + (PyUnicode_CompareWithASCIIString(code->co_name, + recursive_import) == 0) || + (trim_get_code && + PyUnicode_CompareWithASCIIString(code->co_name, + get_code_funcname) == 0) + )) { PyObject *tmp = *outer_link; *outer_link = next; Py_XINCREF(next); -- cgit v1.2.1 From 193694b6c893c5973f233ab06ff26f87fad5a097 Mon Sep 17 00:00:00 2001 From: Nick Coghlan Date: Tue, 31 Jul 2012 21:14:18 +1000 Subject: Close #15486: Simplify the mechanism used to remove importlib frames from tracebacks when they just introduce irrelevant noise --- Python/import.c | 28 +++++----------------------- 1 file changed, 5 insertions(+), 23 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index c4edfacd34..90a89eef4e 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1153,9 +1153,7 @@ static void remove_importlib_frames(void) { const char *importlib_filename = ""; - const char *exec_funcname = "_exec_module"; - const char *get_code_funcname = "get_code"; - const char *recursive_import = "_recursive_import"; + const char *remove_frames = "_call_with_frames_removed"; int always_trim = 0; int trim_get_code = 0; int in_importlib = 0; @@ -1163,18 +1161,8 @@ remove_importlib_frames(void) PyObject **prev_link, **outer_link = NULL; /* Synopsis: if it's an ImportError, we trim all importlib chunks - from the traceback. If it's a SyntaxError, we trim any chunks that - end with a call to "get_code", We always trim chunks - which end with a call to "_exec_module". */ - - /* Thanks to issue 15425, we also strip any chunk ending with - * _recursive_import. This is used when making a recursive call to the - * full import machinery which means the inner stack gets stripped early - * and the normal heuristics won't fire properly for outer frames. A - * more elegant mechanism would be nice, as this one can misfire if - * builtins.__import__ has been replaced with a custom implementation. - * However, the current approach at least gets the job done. - */ + from the traceback. We always trim chunks + which end with a call to "_call_with_frames_removed". */ PyErr_Fetch(&exception, &value, &base_tb); if (!exception || Py_VerboseFlag) @@ -1207,14 +1195,8 @@ remove_importlib_frames(void) if (in_importlib && (always_trim || - (PyUnicode_CompareWithASCIIString(code->co_name, - exec_funcname) == 0) || - (PyUnicode_CompareWithASCIIString(code->co_name, - recursive_import) == 0) || - (trim_get_code && - PyUnicode_CompareWithASCIIString(code->co_name, - get_code_funcname) == 0) - )) { + PyUnicode_CompareWithASCIIString(code->co_name, + remove_frames) == 0)) { PyObject *tmp = *outer_link; *outer_link = next; Py_XINCREF(next); -- cgit v1.2.1 From c60473744b41056df357fd9e5367a7755dbf42b3 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Sat, 4 Aug 2012 23:26:25 +0200 Subject: Remove unused variable `trim_get_code`. --- Python/import.c | 4 ---- 1 file changed, 4 deletions(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 90a89eef4e..6f4b6d0638 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1155,7 +1155,6 @@ remove_importlib_frames(void) const char *importlib_filename = ""; const char *remove_frames = "_call_with_frames_removed"; int always_trim = 0; - int trim_get_code = 0; int in_importlib = 0; PyObject *exception, *value, *base_tb, *tb; PyObject **prev_link, **outer_link = NULL; @@ -1170,9 +1169,6 @@ remove_importlib_frames(void) if (PyType_IsSubtype((PyTypeObject *) exception, (PyTypeObject *) PyExc_ImportError)) always_trim = 1; - if (PyType_IsSubtype((PyTypeObject *) exception, - (PyTypeObject *) PyExc_SyntaxError)) - trim_get_code = 1; prev_link = &base_tb; tb = base_tb; -- cgit v1.2.1 From e95288342d0a23e79fa33ab77fdfcca20280953d Mon Sep 17 00:00:00 2001 From: Brett Cannon Date: Sun, 5 Aug 2012 19:24:57 -0400 Subject: Fix a spelling mistake in a comment. --- Python/import.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 6f4b6d0638..2540f9d7b7 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1243,7 +1243,7 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, } else { /* Only have to care what given_globals is if it will be used - fortsomething. */ + for something. */ if (level > 0 && !PyDict_Check(given_globals)) { PyErr_SetString(PyExc_TypeError, "globals must be a dict"); goto error; -- cgit v1.2.1 From 6fa4b672c0f5e95b9ae16564ad3777ec7b8f6a92 Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 28 Aug 2012 00:24:52 +0200 Subject: Issue #15781: Fix two small race conditions in import's module locking. --- Python/import.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'Python/import.c') diff --git a/Python/import.c b/Python/import.c index 2540f9d7b7..2f71b97bc8 100644 --- a/Python/import.c +++ b/Python/import.c @@ -1408,7 +1408,11 @@ PyImport_ImportModuleLevelObject(PyObject *name, PyObject *given_globals, int initializing = 0; Py_INCREF(mod); - /* Only call _bootstrap._lock_unlock_module() if __initializing__ is true. */ + /* Optimization: only call _bootstrap._lock_unlock_module() if + __initializing__ is true. + NOTE: because of this, __initializing__ must be set *before* + stuffing the new module in sys.modules. + */ value = _PyObject_GetAttrId(mod, &PyId___initializing__); if (value == NULL) PyErr_Clear(); -- cgit v1.2.1