Issue #19700: set __spec__ appropriately in runpy

Note that __spec__.name is not currently guaranteed to be in
sys.modules when the code is running, only __name__ is.

The "running module is in sys.modules" invariant will be
expanded to also cover __spec__.name in a subsequent patch.

1 files changed, 69 insertions, 46 deletions

diff --git a/Lib/runpy.py b/Lib/runpy.py
index 1e0a2be4f0..dc08f4eae9 100644
--- a/Lib/runpy.py
+++ b/Lib/runpy.py
@@ -14,7 +14,9 @@ import os
 import sys
 import importlib.machinery # importlib first so we can test #15386 via -m
 import types
-from pkgutil import read_code, get_loader, get_importer
+from importlib import find_spec
+from importlib.util import spec_from_loader
+from pkgutil import read_code, get_importer
 
 __all__ = [
     "run_module", "run_path",
@@ -58,51 +60,76 @@ class _ModifiedArgv0(object):
         self.value = self._sentinel
         sys.argv[0] = self._saved_value
 
+# TODO: Replace these helpers with importlib._bootstrap._SpecMethods
 def _run_code(code, run_globals, init_globals=None,
-              mod_name=None, mod_fname=None,
-              mod_loader=None, pkg_name=None):
+              mod_name=None, mod_spec=None,
+              pkg_name=None, script_name=None):
     """Helper to run code in nominated namespace"""
     if init_globals is not None:
         run_globals.update(init_globals)
+    if mod_spec is None:
+        loader = None
+        fname = script_name
+        cached = None
+    else:
+        loader = mod_spec.loader
+        fname = mod_spec.origin
+        cached = mod_spec.cached
+        if pkg_name is None:
+            pkg_name = mod_spec.parent
     run_globals.update(__name__ = mod_name,
-                       __file__ = mod_fname,
-                       __cached__ = None,
+                       __file__ = fname,
+                       __cached__ = cached,
                        __doc__ = None,
-                       __loader__ = mod_loader,
-                       __package__ = pkg_name)
+                       __loader__ = loader,
+                       __package__ = pkg_name,
+                       __spec__ = mod_spec)
     exec(code, run_globals)
     return run_globals
 
 def _run_module_code(code, init_globals=None,
-                    mod_name=None, mod_fname=None,
-                    mod_loader=None, pkg_name=None):
+                    mod_name=None, mod_spec=None,
+                    pkg_name=None, script_name=None):
     """Helper to run code in new namespace with sys modified"""
-    with _TempModule(mod_name) as temp_module, _ModifiedArgv0(mod_fname):
+    fname = script_name if mod_spec is None else mod_spec.origin
+    with _TempModule(mod_name) as temp_module, _ModifiedArgv0(fname):
         mod_globals = temp_module.module.__dict__
         _run_code(code, mod_globals, init_globals,
-                  mod_name, mod_fname, mod_loader, pkg_name)
+                  mod_name, mod_spec, pkg_name, script_name)
     # Copy the globals of the temporary module, as they
     # may be cleared when the temporary module goes away
     return mod_globals.copy()
 
 
-# This helper is needed due to a missing component in the PEP 302
-# loader protocol (specifically, "get_filename" is non-standard)
-# Since we can't introduce new features in maintenance releases,
-# support was added to zipimporter under the name '_get_filename'
-def _get_filename(loader, mod_name):
-    for attr in ("get_filename", "_get_filename"):
-        meth = getattr(loader, attr, None)
-        if meth is not None:
-            return os.path.abspath(meth(mod_name))
-    return None
+def _fixed_find_spec(mod_name):
+    # find_spec has the same annoying behaviour as find_loader did (it
+    # fails to work properly for dotted names), so this is a fixed version
+    # ala pkgutil.get_loader
+    if mod_name.startswith('.'):
+        msg = "Relative module name {!r} not supported".format(mod_name)
+        raise ImportError(msg)
+    path = None
+    pkg_name = mod_name.rpartition(".")[0]
+    if pkg_name:
+        pkg = importlib.import_module(pkg_name)
+        path = getattr(pkg, "__path__", None)
+        if path is None:
+            return None
+    try:
+        return importlib.find_spec(mod_name, path)
+    except (ImportError, AttributeError, TypeError, ValueError) as ex:
+        # This hack fixes an impedance mismatch between pkgutil and
+        # importlib, where the latter raises other errors for cases where
+        # pkgutil previously raised ImportError
+        msg = "Error while finding spec for {!r} ({}: {})"
+        raise ImportError(msg.format(mod_name, type(ex), ex)) from ex
 
 # Helper to get the loader, code and filename for a module
 def _get_module_details(mod_name):
-    loader = get_loader(mod_name)
-    if loader is None:
+    spec = _fixed_find_spec(mod_name)
+    if spec is None:
         raise ImportError("No module named %s" % mod_name)
-    if loader.is_package(mod_name):
+    if spec.submodule_search_locations is not None:
         if mod_name == "__main__" or mod_name.endswith(".__main__"):
             raise ImportError("Cannot use package as __main__ module")
         try:
@@ -111,11 +138,14 @@ def _get_module_details(mod_name):
         except ImportError as e:
             raise ImportError(("%s; %r is a package and cannot " +
                                "be directly executed") %(e, mod_name))
+    loader = spec.loader
+    if loader is None:
+        raise ImportError("%r is a namespace package and cannot be executed"
+                                                                 % mod_name)
     code = loader.get_code(mod_name)
     if code is None:
         raise ImportError("No code object available for %s" % mod_name)
-    filename = _get_filename(loader, mod_name)
-    return mod_name, loader, code, filename
+    return mod_name, spec, code
 
 # XXX ncoghlan: Should this be documented and made public?
 # (Current thoughts: don't repeat the mistake that lead to its
@@ -137,9 +167,9 @@ def _run_module_as_main(mod_name, alter_argv=True):
     """
     try:
         if alter_argv or mod_name != "__main__": # i.e. -m switch
-            mod_name, loader, code, fname = _get_module_details(mod_name)
+            mod_name, mod_spec, code = _get_module_details(mod_name)
         else:          # i.e. directory or zipfile execution
-            mod_name, loader, code, fname = _get_main_module_details()
+            mod_name, mod_spec, code = _get_main_module_details()
     except ImportError as exc:
         # Try to provide a good error message
         # for directories, zip files and the -m switch
@@ -152,12 +182,11 @@ def _run_module_as_main(mod_name, alter_argv=True):
             info = "can't find '__main__' module in %r" % sys.argv[0]
         msg = "%s: %s" % (sys.executable, info)
         sys.exit(msg)
-    pkg_name = mod_name.rpartition('.')[0]
     main_globals = sys.modules["__main__"].__dict__
     if alter_argv:
-        sys.argv[0] = fname
+        sys.argv[0] = mod_spec.origin
     return _run_code(code, main_globals, None,
-                     "__main__", fname, loader, pkg_name)
+                     "__main__", mod_spec)
 
 def run_module(mod_name, init_globals=None,
                run_name=None, alter_sys=False):
@@ -165,17 +194,14 @@ def run_module(mod_name, init_globals=None,
 
        Returns the resulting top level namespace dictionary
     """
-    mod_name, loader, code, fname = _get_module_details(mod_name)
+    mod_name, mod_spec, code = _get_module_details(mod_name)
     if run_name is None:
         run_name = mod_name
-    pkg_name = mod_name.rpartition('.')[0]
     if alter_sys:
-        return _run_module_code(code, init_globals, run_name,
-                                fname, loader, pkg_name)
+        return _run_module_code(code, init_globals, run_name, mod_spec)
     else:
         # Leave the sys module alone
-        return _run_code(code, {}, init_globals, run_name,
-                         fname, loader, pkg_name)
+        return _run_code(code, {}, init_globals, run_name, mod_spec)
 
 def _get_main_module_details():
     # Helper that gives a nicer error message when attempting to
@@ -204,10 +230,7 @@ def _get_code_from_file(run_name, fname):
         # That didn't work, so try it as normal source code
         with open(fname, "rb") as f:
             code = compile(f.read(), fname, 'exec')
-            loader = importlib.machinery.SourceFileLoader(run_name, fname)
-    else:
-        loader = importlib.machinery.SourcelessFileLoader(run_name, fname)
-    return code, loader
+    return code, fname
 
 def run_path(path_name, init_globals=None, run_name=None):
     """Execute code located at the specified filesystem location
@@ -231,9 +254,9 @@ def run_path(path_name, init_globals=None, run_name=None):
     if isinstance(importer, type(None)) or is_NullImporter:
         # Not a valid sys.path entry, so run the code directly
         # execfile() doesn't help as we want to allow compiled files
-        code, mod_loader = _get_code_from_file(run_name, path_name)
-        return _run_module_code(code, init_globals, run_name, path_name,
-                                mod_loader, pkg_name)
+        code, fname = _get_code_from_file(run_name, path_name)
+        return _run_module_code(code, init_globals, run_name,
+                                pkg_name=pkg_name, script_name=fname)
     else:
         # Importer is defined for path, so add it to
         # the start of sys.path
@@ -245,12 +268,12 @@ def run_path(path_name, init_globals=None, run_name=None):
             # have no choice and we have to remove it even while we read the
             # code. If we don't do this, a __loader__ attribute in the
             # existing __main__ module may prevent location of the new module.
-            mod_name, loader, code, fname = _get_main_module_details()
+            mod_name, mod_spec, code = _get_main_module_details()
             with _TempModule(run_name) as temp_module, \
                  _ModifiedArgv0(path_name):
                 mod_globals = temp_module.module.__dict__
                 return _run_code(code, mod_globals, init_globals,
-                                    run_name, fname, loader, pkg_name).copy()
+                                    run_name, mod_spec, pkg_name).copy()
         finally:
             try:
                 sys.path.remove(path_name)