diff options
| -rw-r--r-- | .gitignore | 2 | ||||
| -rw-r--r-- | README.rst | 16 | ||||
| -rw-r--r-- | isort/finders.py | 325 | ||||
| -rw-r--r-- | isort/isort.py | 120 | ||||
| -rw-r--r-- | isort/utils.py | 30 | ||||
| -rwxr-xr-x | setup.py | 4 | ||||
| -rw-r--r-- | test_isort.py | 80 | ||||
| -rw-r--r-- | tox.ini | 6 |
8 files changed, 463 insertions, 120 deletions
@@ -1,4 +1,5 @@ *.py[cod] +__pycache__ .DS_Store # C extensions *.so @@ -25,6 +26,7 @@ npm-debug.log # Unit test / coverage reports .coverage +.pytest_cache .tox nosetests.xml htmlcov @@ -86,11 +86,23 @@ Installing isort is as simple as: pip install isort -or if you prefer +Install isort with requirements.txt support: .. code-block:: bash - easy_install isort + pip install isort[requirements] + +Install isort with Pipfile support: + +.. code-block:: bash + + pip install isort[pipfile] + +Install isort with both formats support: + +.. code-block:: bash + + pip install isort[requirements,pipfile] Using isort =========== diff --git a/isort/finders.py b/isort/finders.py new file mode 100644 index 00000000..d3a3c074 --- /dev/null +++ b/isort/finders.py @@ -0,0 +1,325 @@ +"""Finders try to find right section for passed module name +""" +from __future__ import absolute_import, division, print_function, unicode_literals + +import inspect +import os +import os.path +import re +import sys +import sysconfig +from fnmatch import fnmatch +from glob import glob + +from .pie_slice import PY2 +from .utils import chdir, exists_case_sensitive + +try: + from pipreqs import pipreqs +except ImportError: + pipreqs = None + +try: + # pip>=10 + from pip._internal.download import PipSession + from pip._internal.req import parse_requirements +except ImportError: + try: + from pip.download import PipSession + from pip.req import parse_requirements + except ImportError: + parse_requirements = None + +try: + from requirementslib import Pipfile +except ImportError: + Pipfile = None + + +KNOWN_SECTION_MAPPING = { + 'STDLIB': 'STANDARD_LIBRARY', + 'FUTURE': 'FUTURE_LIBRARY', + 'FIRSTPARTY': 'FIRST_PARTY', + 'THIRDPARTY': 'THIRD_PARTY', +} + + +class BaseFinder(object): + def __init__(self, config, sections): + self.config = config + self.sections = sections + + +class ForcedSeparateFinder(BaseFinder): + def find(self, module_name): + for forced_separate in self.config['forced_separate']: + # Ensure all forced_separate patterns will match to end of string + path_glob = forced_separate + if not forced_separate.endswith('*'): + path_glob = '%s*' % forced_separate + + if fnmatch(module_name, path_glob) or fnmatch(module_name, '.' + path_glob): + return forced_separate + + +class LocalFinder(BaseFinder): + def find(self, module_name): + if module_name.startswith("."): + return self.sections.LOCALFOLDER + + +class KnownPatternFinder(BaseFinder): + def __init__(self, config, sections): + super(KnownPatternFinder, self).__init__(config, sections) + + self.known_patterns = [] + for placement in reversed(self.sections): + known_placement = KNOWN_SECTION_MAPPING.get(placement, placement) + config_key = 'known_{0}'.format(known_placement.lower()) + known_patterns = self.config.get(config_key, []) + known_patterns = [ + pattern + for known_pattern in known_patterns + for pattern in self._parse_known_pattern(known_pattern) + ] + for known_pattern in known_patterns: + regexp = '^' + known_pattern.replace('*', '.*').replace('?', '.?') + '$' + self.known_patterns.append((re.compile(regexp), placement)) + + @staticmethod + def _is_package(path): + """ + Evaluates if path is a python package + """ + if PY2: + return os.path.exists(os.path.join(path, '__init__.py')) + else: + return os.path.isdir(path) + + def _parse_known_pattern(self, pattern): + """ + Expand pattern if identified as a directory and return found sub packages + """ + if pattern.endswith(os.path.sep): + patterns = [ + filename + for filename in os.listdir(pattern) + if self._is_package(os.path.join(pattern, filename)) + ] + else: + patterns = [pattern] + + return patterns + + def find(self, module_name): + # Try to find most specific placement instruction match (if any) + parts = module_name.split('.') + module_names_to_check = ('.'.join(parts[:first_k]) for first_k in range(len(parts), 0, -1)) + for module_name_to_check in module_names_to_check: + for pattern, placement in self.known_patterns: + if pattern.match(module_name_to_check): + return placement + + +class PathFinder(BaseFinder): + def __init__(self, config, sections): + super(PathFinder, self).__init__(config, sections) + + # Use a copy of sys.path to avoid any unintended modifications + # to it - e.g. `+=` used below will change paths in place and + # if not copied, consequently sys.path, which will grow unbounded + # with duplicates on every call to this method. + self.paths = list(sys.path) + # restore the original import path (i.e. not the path to bin/isort) + self.paths[0] = os.getcwd() + + # virtual env + self.virtual_env = self.config.get('virtual_env') or os.environ.get('VIRTUAL_ENV') + self.virtual_env_src = False + if self.virtual_env: + self.virtual_env_src = '{0}/src/'.format(self.virtual_env) + for path in glob('{0}/lib/python*/site-packages'.format(self.virtual_env)): + if path not in self.paths: + self.paths.append(path) + for path in glob('{0}/src/*'.format(self.virtual_env)): + if os.path.isdir(path): + self.paths.append(path) + + # handle case-insensitive paths on windows + self.stdlib_lib_prefix = os.path.normcase(sysconfig.get_paths()['stdlib']) + + def find(self, module_name): + for prefix in self.paths: + package_path = "/".join((prefix, module_name.split(".")[0])) + is_module = (exists_case_sensitive(package_path + ".py") or + exists_case_sensitive(package_path + ".so")) + is_package = exists_case_sensitive(package_path) and os.path.isdir(package_path) + if is_module or is_package: + if 'site-packages' in prefix: + return self.sections.THIRDPARTY + if 'dist-packages' in prefix: + return self.sections.THIRDPARTY + if self.virtual_env and self.virtual_env_src in prefix: + return self.sections.THIRDPARTY + if os.path.normcase(prefix).startswith(self.stdlib_lib_prefix): + return self.sections.STDLIB + return self.config['default_section'] + + +class ReqsBaseFinder(BaseFinder): + def __init__(self, config, sections, path='.'): + super(ReqsBaseFinder, self).__init__(config, sections) + self.path = path + if self.enabled: + self.mapping = self._load_mapping() + self.names = self._load_names() + + @staticmethod + def _load_mapping(): + """Return list of mappings `package_name -> module_name` + + Example: + django-haystack -> haystack + """ + if not pipreqs: + return + path = os.path.dirname(inspect.getfile(pipreqs)) + path = os.path.join(path, 'mapping') + with open(path, "r") as f: + # pypi_name: import_name + return dict(line.strip().split(":")[::-1] for line in f) + + def _load_names(self): + """Return list of thirdparty modules from requirements + """ + names = [] + for path in self._get_files(): + for name in self._get_names(path): + names.append(self._normalize_name(name)) + return names + + @staticmethod + def _get_parents(path): + prev = '' + while path != prev: + prev = path + yield path + path = os.path.dirname(path) + + def _get_files(self): + """Return paths to all requirements files + """ + path = os.path.abspath(self.path) + if os.path.isfile(path): + path = os.path.dirname(path) + + for path in self._get_parents(path): + for file_path in self._get_files_from_dir(path): + yield file_path + + def _normalize_name(self, name): + """Convert package name to module name + + Examples: + Django -> django + django-haystack -> haystack + Flask-RESTFul -> flask_restful + """ + if self.mapping: + name = self.mapping.get(name, name) + return name.lower().replace('-', '_') + + def find(self, module_name): + # required lib not installed yet + if not self.enabled: + return + + module_name, _sep, _submodules = module_name.partition('.') + module_name = module_name.lower() + if not module_name: + return + + for name in self.names: + if module_name == name: + return self.sections.THIRDPARTY + + +class RequirementsFinder(ReqsBaseFinder): + exts = ('.txt', '.in') + enabled = bool(parse_requirements) + + def _get_files_from_dir(self, path): + """Return paths to requirements files from passed dir. + """ + for fname in os.listdir(path): + if 'requirements' not in fname: + continue + full_path = os.path.join(path, fname) + + # *requirements*/*.{txt,in} + if os.path.isdir(full_path): + for subfile_name in os.listdir(path): + for ext in self.exts: + if subfile_name.endswith(ext): + yield os.path.join(path, subfile_name) + continue + + # *requirements*.{txt,in} + if os.path.isfile(full_path): + for ext in self.exts: + if fname.endswith(ext): + yield full_path + break + + def _get_names(self, path): + """Load required packages from path to requirements file + """ + with chdir(os.path.dirname(path)): + requirements = parse_requirements(path, session=PipSession()) + for req in requirements: + if req.name: + yield req.name + + +class PipfileFinder(ReqsBaseFinder): + enabled = bool(Pipfile) + + def _get_names(self, path): + with chdir(path): + project = Pipfile.load(path) + sections = project.get_sections() + for section in sections.values(): + for name in section: + yield name + + def _get_files_from_dir(self, path): + if 'Pipfile' in os.listdir(path): + yield path + + +class DefaultFinder(BaseFinder): + def find(self, module_name): + return self.config['default_section'] + + +class FindersManager(object): + finders = ( + ForcedSeparateFinder, + LocalFinder, + KnownPatternFinder, + PathFinder, + PipfileFinder, + RequirementsFinder, + DefaultFinder, + ) + + def __init__(self, config, sections, finders=None): + if finders is not None: + self.finders = finders + self.finders = tuple(finder(config, sections) for finder in self.finders) + + def find(self, module_name): + for finder in self.finders: + section = finder.find(module_name) + if section is not None: + return section diff --git a/isort/isort.py b/isort/isort.py index e3594716..30c263b9 100644 --- a/isort/isort.py +++ b/isort/isort.py @@ -32,23 +32,14 @@ import itertools import os import re import sys -import sysconfig from collections import OrderedDict, namedtuple from datetime import datetime from difflib import unified_diff -from fnmatch import fnmatch -from glob import glob from . import settings +from .finders import FindersManager from .natural import nsorted -from .pie_slice import OrderedSet, input, itemsview, PY2 - -KNOWN_SECTION_MAPPING = { - 'STDLIB': 'STANDARD_LIBRARY', - 'FUTURE': 'FUTURE_LIBRARY', - 'FIRSTPARTY': 'FIRST_PARTY', - 'THIRDPARTY': 'THIRD_PARTY', -} +from .pie_slice import OrderedSet, input, itemsview class SortImports(object): @@ -147,19 +138,7 @@ class SortImports(object): for section in itertools.chain(self.sections, self.config['forced_separate']): self.imports[section] = {'straight': OrderedSet(), 'from': OrderedDict()} - self.known_patterns = [] - for placement in reversed(self.sections): - known_placement = KNOWN_SECTION_MAPPING.get(placement, placement) - config_key = 'known_{0}'.format(known_placement.lower()) - known_patterns = self.config.get(config_key, []) - known_patterns = [ - pattern - for known_pattern in known_patterns - for pattern in self._parse_known_pattern(known_pattern) - ] - for known_pattern in known_patterns: - self.known_patterns.append((re.compile('^' + known_pattern.replace('*', '.*').replace('?', '.?') + '$'), - placement)) + self.finder = FindersManager(config=self.config, sections=self.sections) self.index = 0 self.import_index = -1 @@ -222,30 +201,6 @@ class SortImports(object): print("Fixing {0}".format(self.file_path)) output_file.write(self.output) - def _is_package(self, path): - """ - Evaluates if path is a python package - """ - if PY2: - return os.path.exists(os.path.join(path, '__init__.py')) - else: - return os.path.isdir(path) - - def _parse_known_pattern(self, pattern): - """ - Expand pattern if identified as a directory and return found sub packages - """ - if pattern.endswith(os.path.sep): - patterns = [ - filename - for filename in os.listdir(pattern) - if self._is_package(os.path.join(pattern, filename)) - ] - else: - patterns = [pattern] - - return patterns - def _show_diff(self, file_contents): for line in unified_diff( file_contents.splitlines(1), @@ -272,59 +227,7 @@ class SortImports(object): if it can't determine - it assumes it is project code """ - for forced_separate in self.config['forced_separate']: - # Ensure all forced_separate patterns will match to end of string - path_glob = forced_separate - if not forced_separate.endswith('*'): - path_glob = '%s*' % forced_separate - - if fnmatch(module_name, path_glob) or fnmatch(module_name, '.' + path_glob): - return forced_separate - - if module_name.startswith("."): - return self.sections.LOCALFOLDER - - # Try to find most specific placement instruction match (if any) - parts = module_name.split('.') - module_names_to_check = ['.'.join(parts[:first_k]) for first_k in range(len(parts), 0, -1)] - for module_name_to_check in module_names_to_check: - for pattern, placement in self.known_patterns: - if pattern.match(module_name_to_check): - return placement - - # Use a copy of sys.path to avoid any unintended modifications - # to it - e.g. `+=` used below will change paths in place and - # if not copied, consequently sys.path, which will grow unbounded - # with duplicates on every call to this method. - paths = list(sys.path) - # restore the original import path (i.e. not the path to bin/isort) - paths[0] = os.getcwd() - virtual_env = self.config.get('virtual_env') or os.environ.get('VIRTUAL_ENV') - virtual_env_src = False - if virtual_env: - paths += [path for path in glob('{0}/lib/python*/site-packages'.format(virtual_env)) - if path not in paths] - paths += [path for path in glob('{0}/src/*'.format(virtual_env)) if os.path.isdir(path)] - virtual_env_src = '{0}/src/'.format(virtual_env) - - # handle case-insensitive paths on windows - stdlib_lib_prefix = os.path.normcase(sysconfig.get_paths()['stdlib']) - - for prefix in paths: - package_path = "/".join((prefix, module_name.split(".")[0])) - is_module = (exists_case_sensitive(package_path + ".py") or - exists_case_sensitive(package_path + ".so")) - is_package = exists_case_sensitive(package_path) and os.path.isdir(package_path) - if is_module or is_package: - if ('site-packages' in prefix or 'dist-packages' in prefix or - (virtual_env and virtual_env_src in prefix)): - return self.sections.THIRDPARTY - elif os.path.normcase(prefix).startswith(stdlib_lib_prefix): - return self.sections.STDLIB - else: - return self.config['default_section'] - - return self.config['default_section'] + return self.finder.find(module_name) def _get_line(self): """Returns the current line from the file while incrementing the index.""" @@ -1065,18 +968,3 @@ def coding_check(fname, default='utf-8'): break return coding - - -def exists_case_sensitive(path): - """ - Returns if the given path exists and also matches the case on Windows. - - When finding files that can be imported, it is important for the cases to match because while - file os.path.exists("module.py") and os.path.exists("MODULE.py") both return True on Windows, Python - can only import using the case of the real file. - """ - result = os.path.exists(path) - if (sys.platform.startswith('win') or sys.platform == 'darwin') and result: - directory, basename = os.path.split(path) - result = basename in os.listdir(directory) - return result diff --git a/isort/utils.py b/isort/utils.py new file mode 100644 index 00000000..9206c88e --- /dev/null +++ b/isort/utils.py @@ -0,0 +1,30 @@ +import os +import sys +from contextlib import contextmanager + + +def exists_case_sensitive(path): + """ + Returns if the given path exists and also matches the case on Windows. + + When finding files that can be imported, it is important for the cases to match because while + file os.path.exists("module.py") and os.path.exists("MODULE.py") both return True on Windows, Python + can only import using the case of the real file. + """ + result = os.path.exists(path) + if (sys.platform.startswith('win') or sys.platform == 'darwin') and result: + directory, basename = os.path.split(path) + result = basename in os.listdir(directory) + return result + + +@contextmanager +def chdir(path): + """Context manager for changing dir and restoring previous workdir after exit. + """ + curdir = os.getcwd() + os.chdir(path) + try: + yield + finally: + os.chdir(curdir) @@ -54,6 +54,10 @@ setup(name='isort', 'pylama.linter': ['isort = isort.pylama_isort:Linter'], }, packages=['isort'], + extras_require={ + 'requirements': ['pip', 'pipreqs'], + 'pipfile': ['pipreqs', 'requirementslib'], + }, install_requires=['futures; python_version < "3.2"'], python_requires=">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*", cmdclass={'test': PyTest}, diff --git a/test_isort.py b/test_isort.py index 0bffe9f1..909d83f2 100644 --- a/test_isort.py +++ b/test_isort.py @@ -31,7 +31,9 @@ import shutil import sys import tempfile -from isort.isort import SortImports, exists_case_sensitive +from isort import finders +from isort.isort import SortImports +from isort.utils import exists_case_sensitive from isort.main import is_python_file from isort.pie_slice import PY2 from isort.settings import WrapModes @@ -2465,3 +2467,79 @@ def test_new_lines_are_preserved(): with io.open(n_newline.name, newline='') as n_newline_file: n_newline_contents = n_newline_file.read() assert n_newline_contents == 'import os\nimport sys\n' + + +def test_requirements_finder(tmpdir): + subdir = tmpdir.mkdir('subdir').join("lol.txt") + subdir.write("flask") + req_file = tmpdir.join('requirements.txt') + req_file.write( + "Django==1.11\n" + "-e git+https://github.com/orsinium/deal.git#egg=deal\n" + ) + si = SortImports(file_contents="") + for path in (str(tmpdir), str(subdir)): + finder = finders.RequirementsFinder( + config=si.config, + sections=si.sections, + path=path + ) + + files = list(finder._get_files()) + assert len(files) == 1 # file finding + assert files[0].endswith('requirements.txt') # file finding + assert list(finder._get_names(str(req_file))) == ['Django', 'deal'] # file parsing + + assert finder.find("django") == si.sections.THIRDPARTY # package in reqs + assert finder.find("flask") is None # package not in reqs + assert finder.find("deal") == si.sections.THIRDPARTY # vcs + + assert len(finder.mapping) > 100 + assert finder._normalize_name('deal') == 'deal' + assert finder._normalize_name('Django') == 'django' # lowercase + assert finder._normalize_name('django_haystack') == 'haystack' # mapping + assert finder._normalize_name('Flask-RESTful') == 'flask_restful' # conver `-`to `_` + + req_file.remove() + + +PIPFILE = """ +[[source]] +url = "https://pypi.org/simple" +verify_ssl = true +name = "pypi" + +[requires] +python_version = "3.5" + +[packages] +Django = "~=1.11" +deal = {editable = true, git = "https://github.com/orsinium/deal.git"} + +[dev-packages] +""" + + +def test_pipfile_finder(tmpdir): + pipfile = tmpdir.join('Pipfile') + pipfile.write(PIPFILE) + si = SortImports(file_contents="") + finder = finders.PipfileFinder( + config=si.config, + sections=si.sections, + path=str(tmpdir) + ) + + assert set(finder._get_names(str(tmpdir))) == {'Django', 'deal'} # file parsing + + assert finder.find("django") == si.sections.THIRDPARTY # package in reqs + assert finder.find("flask") is None # package not in reqs + assert finder.find("deal") == si.sections.THIRDPARTY # vcs + + assert len(finder.mapping) > 100 + assert finder._normalize_name('deal') == 'deal' + assert finder._normalize_name('Django') == 'django' # lowercase + assert finder._normalize_name('django_haystack') == 'haystack' # mapping + assert finder._normalize_name('Flask-RESTful') == 'flask_restful' # conver `-`to `_` + + pipfile.remove() @@ -5,7 +5,11 @@ envlist = py{27,34,35,36,37,py} [testenv] -deps = pytest +deps = + pytest + pip + pipreqs + requirementslib commands = py.test test_isort.py {posargs} [testenv:isort-check] |