diff options
Diffstat (limited to 'Lib/importlib/_bootstrap.py')
| -rw-r--r-- | Lib/importlib/_bootstrap.py | 589 |
1 files changed, 182 insertions, 407 deletions
diff --git a/Lib/importlib/_bootstrap.py b/Lib/importlib/_bootstrap.py index 145be93b74..2fac4999ac 100644 --- a/Lib/importlib/_bootstrap.py +++ b/Lib/importlib/_bootstrap.py @@ -22,7 +22,7 @@ work. One should use importlib as the public-facing version of this module. def _path_join(*args): """Replacement for os.path.join.""" return path_sep.join(x[:-len(path_sep)] if x.endswith(path_sep) else x - for x in args) + for x in args if x) def _path_exists(path): @@ -53,6 +53,8 @@ def _path_isfile(path): # XXX Could also expose Modules/getpath.c:isdir() def _path_isdir(path): """Replacement for os.path.isdir.""" + if not path: + path = _os.getcwd() return _path_is_mode_type(path, 0o040000) @@ -99,6 +101,8 @@ def _wrap(new, old): new.__dict__.update(old.__dict__) +code_type = type(_wrap.__code__) + # Finder/loader utility code ################################################## def set_package(fxn): @@ -138,7 +142,7 @@ def module_for_loader(fxn): the second argument. """ - def decorated(self, fullname): + def decorated(self, fullname, *args, **kwargs): module = sys.modules.get(fullname) is_reload = bool(module) if not is_reload: @@ -148,7 +152,7 @@ def module_for_loader(fxn): module = imp.new_module(fullname) sys.modules[fullname] = module try: - return fxn(self, module) + return fxn(self, module, *args, **kwargs) except: if not is_reload: del sys.modules[fullname] @@ -301,7 +305,59 @@ class FrozenImporter: return imp.is_frozen_package(fullname) -class SourceLoader: +class _LoaderBasics: + + """Base class of common code needed by both SourceLoader and + _SourcelessFileLoader.""" + + def is_package(self, fullname): + """Concrete implementation of InspectLoader.is_package by checking if + the path returned by get_filename has a filename of '__init__.py'.""" + filename = self.get_filename(fullname).rpartition(path_sep)[2] + return filename.rsplit('.', 1)[0] == '__init__' + + def _bytes_from_bytecode(self, fullname, data, source_mtime): + """Return the marshalled bytes from bytecode, verifying the magic + number and timestamp alon the way. + + If source_mtime is None then skip the timestamp check. + + """ + magic = data[:4] + raw_timestamp = data[4:8] + if len(magic) != 4 or magic != imp.get_magic(): + raise ImportError("bad magic number in {}".format(fullname)) + elif len(raw_timestamp) != 4: + raise EOFError("bad timestamp in {}".format(fullname)) + elif source_mtime is not None: + if marshal._r_long(raw_timestamp) != source_mtime: + raise ImportError("bytecode is stale for {}".format(fullname)) + # Can't return the code object as errors from marshal loading need to + # propagate even when source is available. + return data[8:] + + @module_for_loader + def _load_module(self, module, *, sourceless=False): + """Helper for load_module able to handle either source or sourceless + loading.""" + name = module.__name__ + code_object = self.get_code(name) + module.__file__ = self.get_filename(name) + if not sourceless: + module.__cached__ = imp.cache_from_source(module.__file__) + else: + module.__cached__ = module.__file__ + module.__package__ = name + if self.is_package(name): + module.__path__ = [module.__file__.rsplit(path_sep, 1)[0]] + else: + module.__package__ = module.__package__.rpartition('.')[0] + module.__loader__ = self + exec(code_object, module.__dict__) + return module + + +class SourceLoader(_LoaderBasics): def path_mtime(self, path:str) -> int: """Optional method that returns the modification time for the specified @@ -320,11 +376,6 @@ class SourceLoader: """ raise NotImplementedError - def is_package(self, fullname): - """Concrete implementation of InspectLoader.is_package by checking if - the path returned by get_filename has a filename of '__init__.py'.""" - filename = self.get_filename(fullname).rsplit(path_sep, 1)[1] - return filename.rsplit('.', 1)[0] == '__init__' def get_source(self, fullname): """Concrete implementation of InspectLoader.get_source.""" @@ -359,12 +410,18 @@ class SourceLoader: except IOError: pass else: - magic = data[:4] - raw_timestamp = data[4:8] - if (len(magic) == 4 and len(raw_timestamp) == 4 and - magic == imp.get_magic() and - marshal._r_long(raw_timestamp) == source_mtime): - return marshal.loads(data[8:]) + try: + bytes_data = self._bytes_from_bytecode(fullname, data, + source_mtime) + except (ImportError, EOFError): + pass + else: + found = marshal.loads(bytes_data) + if isinstance(found, code_type): + return found + else: + msg = "Non-code object in {}" + raise ImportError(msg.format(bytecode_path)) source_bytes = self.get_data(source_path) code_object = compile(source_bytes, source_path, 'exec', dont_inherit=True) @@ -382,8 +439,7 @@ class SourceLoader: pass return code_object - @module_for_loader - def load_module(self, module): + def load_module(self, fullname): """Concrete implementation of Loader.load_module. Requires ExecutionLoader.get_filename and ResourceLoader.get_data to be @@ -391,31 +447,17 @@ class SourceLoader: get_code uses/writes bytecode. """ - name = module.__name__ - code_object = self.get_code(name) - module.__file__ = self.get_filename(name) - module.__cached__ = imp.cache_from_source(module.__file__) - module.__package__ = name - is_package = self.is_package(name) - if is_package: - module.__path__ = [module.__file__.rsplit(path_sep, 1)[0]] - else: - module.__package__ = module.__package__.rpartition('.')[0] - module.__loader__ = self - exec(code_object, module.__dict__) - return module - + return self._load_module(fullname) -class _SourceFileLoader(SourceLoader): - """Concrete implementation of SourceLoader. +class _FileLoader: - NOT A PUBLIC CLASS! Do not expect any API stability from this class, so DO - NOT SUBCLASS IN YOUR OWN CODE! - - """ + """Base file loader class which implements the loader protocol methods that + require file system usage.""" def __init__(self, fullname, path): + """Cache the module name and the path to the file found by the + finder.""" self._name = fullname self._path = path @@ -424,272 +466,66 @@ class _SourceFileLoader(SourceLoader): """Return the path to the source file as found by the finder.""" return self._path + def get_data(self, path): + """Return the data from path as raw bytes.""" + with _closing(_io.FileIO(path, 'r')) as file: + return file.read() + + +class _SourceFileLoader(_FileLoader, SourceLoader): + + """Concrete implementation of SourceLoader using the file system.""" + def path_mtime(self, path): """Return the modification time for the path.""" return int(_os.stat(path).st_mtime) - def set_data(self, data, path): + def set_data(self, path, data): """Write bytes data to a file.""" try: - with _closing(_io.FileIO(bytecode_path, 'w')) as file: + with _closing(_io.FileIO(path, 'wb')) as file: file.write(data) except IOError as exc: - if exc.errno != errno.EACCES: + if exc.errno == errno.ENOENT: + directory, _, filename = path.rpartition(path_sep) + sub_directories = [] + while not _path_isdir(directory): + directory, _, sub_dir = directory.rpartition(path_sep) + sub_directories.append(sub_dir) + for part in reversed(sub_directories): + directory = _path_join(directory, part) + try: + _os.mkdir(directory) + except IOError as exc: + if exc.errno != errno.EACCES: + raise + else: + return + return self.set_data(path, data) + elif exc.errno != errno.EACCES: raise -class PyLoader: - - """Loader base class for Python source code. - - Subclasses need to implement the methods: - - - source_path - - get_data - - is_package - - """ - - @module_for_loader - def load_module(self, module): - """Initialize the module.""" - name = module.__name__ - code_object = self.get_code(module.__name__) - module.__file__ = self.get_filename(name) - if self.is_package(name): - module.__path__ = [module.__file__.rsplit(path_sep, 1)[0]] - module.__package__ = module.__name__ - if not hasattr(module, '__path__'): - module.__package__ = module.__package__.rpartition('.')[0] - module.__loader__ = self - exec(code_object, module.__dict__) - return module - - def get_filename(self, fullname): - """Return the path to the source file, else raise ImportError.""" - path = self.source_path(fullname) - if path is not None: - return path - else: - raise ImportError("no source path available for " - "{0!r}".format(fullname)) - - def get_code(self, fullname): - """Get a code object from source.""" - source_path = self.source_path(fullname) - if source_path is None: - message = "a source path must exist to load {0}".format(fullname) - raise ImportError(message) - source = self.get_data(source_path) - return compile(source, source_path, 'exec', dont_inherit=True) - - # Never use in implementing import! Imports code within the method. - def get_source(self, fullname): - """Return the source code for a module. - - self.source_path() and self.get_data() are used to implement this - method. - - """ - path = self.source_path(fullname) - if path is None: - return None - try: - source_bytes = self.get_data(path) - except IOError: - return ImportError("source not available through get_data()") - import io - import tokenize - encoding = tokenize.detect_encoding(io.BytesIO(source_bytes).readline) - return source_bytes.decode(encoding[0]) - - -class PyPycLoader(PyLoader): - - """Loader base class for Python source and bytecode. +class _SourcelessFileLoader(_FileLoader, _LoaderBasics): - Requires implementing the methods needed for PyLoader as well as - source_mtime, bytecode_path, and write_bytecode. + """Loader which handles sourceless file imports.""" - """ - - def get_filename(self, fullname): - """Return the source or bytecode file path.""" - path = self.source_path(fullname) - if path is not None: - return path - path = self.bytecode_path(fullname) - if path is not None: - return path - raise ImportError("no source or bytecode path available for " - "{0!r}".format(fullname)) + def load_module(self, fullname): + return self._load_module(fullname, sourceless=True) def get_code(self, fullname): - """Get a code object from source or bytecode.""" - # XXX Care enough to make sure this call does not happen if the magic - # number is bad? - source_timestamp = self.source_mtime(fullname) - # Try to use bytecode if it is available. - bytecode_path = self.bytecode_path(fullname) - if bytecode_path: - data = self.get_data(bytecode_path) - try: - magic = data[:4] - if len(magic) < 4: - raise ImportError("bad magic number in {}".format(fullname)) - raw_timestamp = data[4:8] - if len(raw_timestamp) < 4: - raise EOFError("bad timestamp in {}".format(fullname)) - pyc_timestamp = marshal._r_long(raw_timestamp) - bytecode = data[8:] - # Verify that the magic number is valid. - if imp.get_magic() != magic: - raise ImportError("bad magic number in {}".format(fullname)) - # Verify that the bytecode is not stale (only matters when - # there is source to fall back on. - if source_timestamp: - if pyc_timestamp < source_timestamp: - raise ImportError("bytecode is stale") - except (ImportError, EOFError): - # If source is available give it a shot. - if source_timestamp is not None: - pass - else: - raise - else: - # Bytecode seems fine, so try to use it. - # XXX If the bytecode is ill-formed, would it be beneficial to - # try for using source if available and issue a warning? - return marshal.loads(bytecode) - elif source_timestamp is None: - raise ImportError("no source or bytecode available to create code " - "object for {0!r}".format(fullname)) - # Use the source. - code_object = super().get_code(fullname) - # Generate bytecode and write it out. - if not sys.dont_write_bytecode: - data = bytearray(imp.get_magic()) - data.extend(marshal._w_long(source_timestamp)) - data.extend(marshal.dumps(code_object)) - self.write_bytecode(fullname, data) - return code_object - - -class _PyFileLoader(PyLoader): - - """Load a Python source file.""" - - def __init__(self, name, path, is_pkg): - self._name = name - self._is_pkg = is_pkg - # Figure out the base path based on whether it was source or bytecode - # that was found. - try: - self._base_path = _path_without_ext(path, imp.PY_SOURCE) - except ValueError: - self._base_path = _path_without_ext(path, imp.PY_COMPILED) - - def _find_path(self, ext_type): - """Find a path from the base path and the specified extension type that - exists, returning None if one is not found.""" - for suffix in _suffix_list(ext_type): - path = self._base_path + suffix - if _path_exists(path): - return path + path = self.get_filename(fullname) + data = self.get_data(path) + bytes_data = self._bytes_from_bytecode(fullname, data, None) + found = marshal.loads(bytes_data) + if isinstance(found, code_type): + return found else: - return None - - @_check_name - def source_path(self, fullname): - """Return the path to an existing source file for the module, or None - if one cannot be found.""" - # Not a property so that it is easy to override. - return self._find_path(imp.PY_SOURCE) + raise ImportError("Non-code object in {}".format(path)) - def get_data(self, path): - """Return the data from path as raw bytes.""" - return _io.FileIO(path, 'r').read() # Assuming bytes. - - @_check_name - def is_package(self, fullname): - """Return a boolean based on whether the module is a package. - - Raises ImportError (like get_source) if the loader cannot handle the - package. - - """ - return self._is_pkg - - -class _PyPycFileLoader(PyPycLoader, _PyFileLoader): - - """Load a module from a source or bytecode file.""" - - def _find_path(self, ext_type): - """Return PEP 3147 path if ext_type is PY_COMPILED, otherwise - super()._find_path() is called.""" - if ext_type == imp.PY_COMPILED: - # We don't really care what the extension on self._base_path is, - # as long as it has exactly one dot. - source_path = self._base_path + '.py' - pycache_path = imp.cache_from_source(source_path) - legacy_path = self._base_path + '.pyc' - # The rule is: if the source file exists, then Python always uses - # the __pycache__/foo.<tag>.pyc file. If the source file does not - # exist, then Python uses the legacy path. - pyc_path = (pycache_path - if _path_exists(source_path) - else legacy_path) - return (pyc_path if _path_exists(pyc_path) else None) - return super()._find_path(ext_type) - - @_check_name - def source_mtime(self, name): - """Return the modification time of the source for the specified - module.""" - source_path = self.source_path(name) - if not source_path: - return None - return int(_os.stat(source_path).st_mtime) - - @_check_name - def bytecode_path(self, fullname): - """Return the path to a bytecode file, or None if one does not - exist.""" - # Not a property for easy overriding. - return self._find_path(imp.PY_COMPILED) - - @_check_name - def write_bytecode(self, name, data): - """Write out 'data' for the specified module, returning a boolean - signifying if the write-out actually occurred. - - Raises ImportError (just like get_source) if the specified module - cannot be handled by the loader. - - """ - bytecode_path = self.bytecode_path(name) - if not bytecode_path: - source_path = self.source_path(name) - bytecode_path = imp.cache_from_source(source_path) - # Ensure that the __pycache__ directory exists. We can't use - # os.path.dirname() here. - dirname, sep, basename = bytecode_path.rpartition(path_sep) - try: - _os.mkdir(dirname) - except OSError as error: - if error.errno != errno.EEXIST: - raise - try: - # Assuming bytes. - with _closing(_io.FileIO(bytecode_path, 'w')) as bytecode_file: - bytecode_file.write(data) - return True - except IOError as exc: - if exc.errno == errno.EACCES: - return False - else: - raise + def get_source(self, fullname): + """Return None as there is no source code.""" + return None class _ExtensionFileLoader: @@ -700,7 +536,7 @@ class _ExtensionFileLoader: """ - def __init__(self, name, path, is_pkg): + def __init__(self, name, path): """Initialize the loader. If is_pkg is True then an exception is raised as extension modules @@ -709,8 +545,6 @@ class _ExtensionFileLoader: """ self._name = name self._path = path - if is_pkg: - raise ValueError("extension modules cannot be packages") @_check_name @set_package @@ -808,147 +642,88 @@ class PathFinder: return None -class _ChainedFinder: - - """Finder that sequentially calls other finders.""" - - def __init__(self, *finders): - self._finders = finders - - def find_module(self, fullname, path=None): - for finder in self._finders: - result = finder.find_module(fullname, path) - if result: - return result - else: - return None - - class _FileFinder: - """Base class for file finders. - - Subclasses are expected to define the following attributes: - - * _suffixes - Sequence of file suffixes whose order will be followed. + """File-based finder. - * _possible_package - True if importer should check for packages. - - * _loader - A callable that takes the module name, a file path, and whether - the path points to a package and returns a loader for the module - found at that path. + Constructor takes a list of objects detailing what file extensions their + loader supports along with whether it can be used for a package. """ - def __init__(self, path_entry): - """Initialize an importer for the passed-in sys.path entry (which is - assumed to have already been verified as an existing directory). - - Can be used as an entry on sys.path_hook. - - """ - absolute_path = _path_absolute(path_entry) - if not _path_isdir(absolute_path): - raise ImportError("only directories are supported") - self._path_entry = absolute_path - - def find_module(self, fullname, path=None): + def __init__(self, path, *details): + """Initialize with finder details.""" + packages = [] + modules = [] + for detail in details: + modules.extend((suffix, detail.loader) for suffix in detail.suffixes) + if detail.supports_packages: + packages.extend((suffix, detail.loader) + for suffix in detail.suffixes) + self.packages = packages + self.modules = modules + self.path = path + + def find_module(self, fullname): + """Try to find a loader for the specified module.""" tail_module = fullname.rpartition('.')[2] - package_directory = None - if self._possible_package: - for ext in self._suffixes: - package_directory = _path_join(self._path_entry, tail_module) - init_filename = '__init__' + ext - package_init = _path_join(package_directory, init_filename) - if (_path_isfile(package_init) and - _case_ok(self._path_entry, tail_module) and - _case_ok(package_directory, init_filename)): - return self._loader(fullname, package_init, True) - for ext in self._suffixes: - file_name = tail_module + ext - file_path = _path_join(self._path_entry, file_name) - if (_path_isfile(file_path) and - _case_ok(self._path_entry, file_name)): - return self._loader(fullname, file_path, False) - else: - # Raise a warning if it matches a directory w/o an __init__ file. - if (package_directory is not None and - _path_isdir(package_directory) and - _case_ok(self._path_entry, tail_module)): - _warnings.warn("Not importing directory %s: missing __init__" - % package_directory, ImportWarning) - return None - - -class _PyFileFinder(_FileFinder): - - """Importer for source/bytecode files.""" - - _possible_package = True - _loader = _PyFileLoader - - def __init__(self, path_entry): - # Lack of imp during class creation means _suffixes is set here. - # Make sure that Python source files are listed first! Needed for an - # optimization by the loader. - self._suffixes = _suffix_list(imp.PY_SOURCE) - super().__init__(path_entry) - - -class _PyPycFileFinder(_PyFileFinder): + base_path = _path_join(self.path, tail_module) + if _path_isdir(base_path) and _case_ok(self.path, tail_module): + for suffix, loader in self.packages: + init_filename = '__init__' + suffix + full_path = _path_join(base_path, init_filename) + if (_path_isfile(full_path) and + _case_ok(base_path, init_filename)): + return loader(fullname, full_path) + else: + msg = "Not importing directory {}: missing __init__" + _warnings.warn(msg.format(base_path), ImportWarning) + for suffix, loader in self.modules: + mod_filename = tail_module + suffix + full_path = _path_join(self.path, mod_filename) + if _path_isfile(full_path) and _case_ok(self.path, mod_filename): + return loader(fullname, full_path) + return None - """Finder for source and bytecode files.""" +class _SourceFinderDetails: - _loader = _PyPycFileLoader + loader = _SourceFileLoader + supports_packages = True - def __init__(self, path_entry): - super().__init__(path_entry) - self._suffixes += _suffix_list(imp.PY_COMPILED) + def __init__(self): + self.suffixes = _suffix_list(imp.PY_SOURCE) +class _SourcelessFinderDetails: + loader = _SourcelessFileLoader + supports_packages = True + def __init__(self): + self.suffixes = _suffix_list(imp.PY_COMPILED) -class _ExtensionFileFinder(_FileFinder): - """Importer for extension files.""" +class _ExtensionFinderDetails: - _possible_package = False - _loader = _ExtensionFileLoader + loader = _ExtensionFileLoader + supports_packages = False - def __init__(self, path_entry): - # Assigning to _suffixes here instead of at the class level because - # imp is not imported at the time of class creation. - self._suffixes = _suffix_list(imp.C_EXTENSION) - super().__init__(path_entry) + def __init__(self): + self.suffixes = _suffix_list(imp.C_EXTENSION) # Import itself ############################################################### -def _chained_path_hook(*path_hooks): - """Create a closure which sequentially checks path hooks to see which ones - (if any) can work with a path.""" - def path_hook(entry): - """Check to see if 'entry' matches any of the enclosed path hooks.""" - finders = [] - for hook in path_hooks: - try: - finder = hook(entry) - except ImportError: - continue - else: - finders.append(finder) - if not finders: - raise ImportError("no finder found") - else: - return _ChainedFinder(*finders) - - return path_hook +def _file_path_hook(path): + """If the path is a directory, return a file-based finder.""" + if _path_isdir(path): + return _FileFinder(path, _ExtensionFinderDetails(), + _SourceFinderDetails(), + _SourcelessFinderDetails()) + else: + raise ImportError("only directories are supported") -_DEFAULT_PATH_HOOK = _chained_path_hook(_ExtensionFileFinder, _PyPycFileFinder) +_DEFAULT_PATH_HOOK = _file_path_hook class _DefaultPathFinder(PathFinder): |