# copyright 2003-2013 LOGILAB S.A. (Paris, FRANCE), all rights reserved. # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr # # This file is part of astroid. # # astroid is free software: you can redistribute it and/or modify it # under the terms of the GNU Lesser General Public License as published by the # Free Software Foundation, either version 2.1 of the License, or (at your # option) any later version. # # astroid is distributed in the hope that it will be useful, but # WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or # FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License # for more details. # # You should have received a copy of the GNU Lesser General Public License along # with astroid. If not, see . """astroid manager: avoid multiple astroid build of a same module when possible by providing a class responsible to get astroid representation from various source and using a cache of built modules) """ __docformat__ = "restructuredtext en" import os from os.path import dirname, join, isdir, exists from warnings import warn from logilab.common.configuration import OptionsProviderMixIn from astroid.exceptions import AstroidBuildingException from astroid.modutils import NoSourceFile, is_python_source, \ file_from_modpath, load_module_from_name, modpath_from_file, \ get_module_files, get_source_file, zipimport def astroid_wrapper(func, modname): """wrapper to give to AstroidManager.project_from_files""" print 'parsing %s...' % modname try: return func(modname) except AstroidBuildingException, exc: print exc except Exception, exc: import traceback traceback.print_exc() def _silent_no_wrap(func, modname): """silent wrapper that doesn't do anything; can be used for tests""" return func(modname) def safe_repr(obj): try: return repr(obj) except: return '???' class AstroidManager(OptionsProviderMixIn): """the astroid manager, responsible to build astroid from files or modules. Use the Borg pattern. """ name = 'astroid loader' options = (("ignore", {'type' : "csv", 'metavar' : "", 'dest' : "black_list", "default" : ('CVS',), 'help' : "add (may be a directory) to the black list\ . It should be a base name, not a path. You may set this option multiple times\ ."}), ("project", {'default': "No Name", 'type' : 'string', 'short': 'p', 'metavar' : '', 'help' : 'set the project name.'}), ) brain = {} def __init__(self): self.__dict__ = AstroidManager.brain if not self.__dict__: OptionsProviderMixIn.__init__(self) self.load_defaults() # NOTE: cache entries are added by the [re]builder self.astroid_cache = {} self._mod_file_cache = {} self.transforms = {} def ast_from_file(self, filepath, modname=None, fallback=True, source=False): """given a module name, return the astroid object""" try: filepath = get_source_file(filepath, include_no_ext=True) source = True except NoSourceFile: pass if modname is None: try: modname = '.'.join(modpath_from_file(filepath)) except ImportError: modname = filepath if modname in self.astroid_cache and self.astroid_cache[modname].file == filepath: return self.astroid_cache[modname] if source: from astroid.builder import AstroidBuilder return AstroidBuilder(self).file_build(filepath, modname) elif fallback and modname: return self.ast_from_module_name(modname) raise AstroidBuildingException('unable to get astroid for file %s' % filepath) def ast_from_module_name(self, modname, context_file=None): """given a module name, return the astroid object""" if modname in self.astroid_cache: return self.astroid_cache[modname] if modname == '__main__': from astroid.builder import AstroidBuilder return AstroidBuilder(self).string_build('', modname) old_cwd = os.getcwd() if context_file: os.chdir(dirname(context_file)) try: filepath = self.file_from_module_name(modname, context_file) if filepath is not None and not is_python_source(filepath): module = self.zip_import_data(filepath) if module is not None: return module if filepath is None or not is_python_source(filepath): try: module = load_module_from_name(modname) except Exception, ex: msg = 'Unable to load module %s (%s)' % (modname, ex) raise AstroidBuildingException(msg) return self.ast_from_module(module, modname) return self.ast_from_file(filepath, modname, fallback=False) finally: os.chdir(old_cwd) def zip_import_data(self, filepath): if zipimport is None: return None from astroid.builder import AstroidBuilder builder = AstroidBuilder(self) for ext in ('.zip', '.egg'): try: eggpath, resource = filepath.rsplit(ext + '/', 1) except ValueError: continue try: importer = zipimport.zipimporter(eggpath + ext) zmodname = resource.replace('/', '.') if importer.is_package(resource): zmodname = zmodname + '.__init__' module = builder.string_build(importer.get_source(resource), zmodname, filepath) return module except: continue return None def file_from_module_name(self, modname, contextfile): try: value = self._mod_file_cache[(modname, contextfile)] except KeyError: try: value = file_from_modpath(modname.split('.'), context_file=contextfile) except ImportError, ex: msg = 'Unable to load module %s (%s)' % (modname, ex) value = AstroidBuildingException(msg) self._mod_file_cache[(modname, contextfile)] = value if isinstance(value, AstroidBuildingException): raise value return value def ast_from_module(self, module, modname=None): """given an imported module, return the astroid object""" modname = modname or module.__name__ if modname in self.astroid_cache: return self.astroid_cache[modname] try: # some builtin modules don't have __file__ attribute filepath = module.__file__ if is_python_source(filepath): return self.ast_from_file(filepath, modname) except AttributeError: pass from astroid.builder import AstroidBuilder return AstroidBuilder(self).module_build(module, modname) def ast_from_class(self, klass, modname=None): """get astroid for the given class""" if modname is None: try: modname = klass.__module__ except AttributeError: raise AstroidBuildingException( 'Unable to get module for class %s' % safe_repr(klass)) modastroid = self.ast_from_module_name(modname) return modastroid.getattr(klass.__name__)[0] # XXX def infer_ast_from_something(self, obj, context=None): """infer astroid for the given class""" if hasattr(obj, '__class__') and not isinstance(obj, type): klass = obj.__class__ else: klass = obj try: modname = klass.__module__ except AttributeError: raise AstroidBuildingException( 'Unable to get module for %s' % safe_repr(klass)) except Exception, ex: raise AstroidBuildingException( 'Unexpected error while retrieving module for %s: %s' % (safe_repr(klass), ex)) try: name = klass.__name__ except AttributeError: raise AstroidBuildingException( 'Unable to get name for %s' % safe_repr(klass)) except Exception, ex: raise AstroidBuildingException( 'Unexpected error while retrieving name for %s: %s' % (safe_repr(klass), ex)) # take care, on living object __module__ is regularly wrong :( modastroid = self.ast_from_module_name(modname) if klass is obj: for infered in modastroid.igetattr(name, context): yield infered else: for infered in modastroid.igetattr(name, context): yield infered.instanciate_class() def project_from_files(self, files, func_wrapper=astroid_wrapper, project_name=None, black_list=None): """return a Project from a list of files or modules""" # build the project representation project_name = project_name or self.config.project black_list = black_list or self.config.black_list project = Project(project_name) for something in files: if not exists(something): fpath = file_from_modpath(something.split('.')) elif isdir(something): fpath = join(something, '__init__.py') else: fpath = something astroid = func_wrapper(self.ast_from_file, fpath) if astroid is None: continue # XXX why is first file defining the project.path ? project.path = project.path or astroid.file project.add_module(astroid) base_name = astroid.name # recurse in package except if __init__ was explicitly given if astroid.package and something.find('__init__') == -1: # recurse on others packages / modules if this is a package for fpath in get_module_files(dirname(astroid.file), black_list): astroid = func_wrapper(self.ast_from_file, fpath) if astroid is None or astroid.name == base_name: continue project.add_module(astroid) return project def register_transform(self, node_class, transform, predicate=None): """Register `transform(node)` function to be applied on the given Astroid's `node_class` if `predicate` is None or return a true value when called with the node as argument. The transform function may return a value which is then used to substitute the original node in the tree. """ self.transforms.setdefault(node_class, []).append((transform, predicate)) def unregister_transform(self, node_class, transform, predicate=None): """Unregister the given transform.""" self.transforms[node_class].remove((transform, predicate)) def transform(self, node): """Call matching transforms for the given node if any and return the transformed node. """ cls = node.__class__ if cls not in self.transforms: # no transform registered for this class of node return node transforms = self.transforms[cls] orig_node = node # copy the reference for transform_func, predicate in transforms: if predicate is None or predicate(node): ret = transform_func(node) # if the transformation function returns something, it's # expected to be a replacement for the node if ret is not None: if node is not orig_node: # node has already be modified by some previous # transformation, warn about it warn('node %s substituted multiple times' % node) node = ret return node def cache_module(self, module): """Cache a module if no module with the same name is known yet.""" self.astroid_cache.setdefault(module.name, module) def clear_cache(self): self.astroid_cache.clear() # force bootstrap again, else we may ends up with cache inconsistency # between the manager and CONST_PROXY, making # unittest_lookup.LookupTC.test_builtin_lookup fail depending on the # test order from astroid.raw_building import astroid_bootstrapping astroid_bootstrapping() class Project(object): """a project handle a set of modules / packages""" def __init__(self, name=''): self.name = name self.path = None self.modules = [] self.locals = {} self.__getitem__ = self.locals.__getitem__ self.__iter__ = self.locals.__iter__ self.values = self.locals.values self.keys = self.locals.keys self.items = self.locals.items def add_module(self, node): self.locals[node.name] = node self.modules.append(node) def get_module(self, name): return self.locals[name] def get_children(self): return self.modules def __repr__(self): return '' % (self.name, id(self), len(self.modules))