diff options
| author | Jason R. Coombs <jaraco@jaraco.com> | 2017-09-03 19:57:54 -0400 |
|---|---|---|
| committer | Jason R. Coombs <jaraco@jaraco.com> | 2017-09-03 20:01:45 -0400 |
| commit | dcb24ad15465c266a3f258471766fbbe8fc8a42e (patch) | |
| tree | 13123440610d78e398476a8ce1e8cc3d9f9ec72e /pkg_resources | |
| parent | f14930e66601b462699c44384c482cd966f53b8f (diff) | |
| parent | 1b192005562d5cf0de30c02154c58fd1dca577c8 (diff) | |
| download | python-setuptools-git-dcb24ad15465c266a3f258471766fbbe8fc8a42e.tar.gz | |
Merge branch 'master' into drop-py26
Diffstat (limited to 'pkg_resources')
| -rw-r--r-- | pkg_resources/__init__.py | 183 | ||||
| -rw-r--r-- | pkg_resources/py31compat.py | 22 | ||||
| -rw-r--r-- | pkg_resources/tests/test_find_distributions.py | 65 | ||||
| -rw-r--r-- | pkg_resources/tests/test_markers.py | 5 | ||||
| -rw-r--r-- | pkg_resources/tests/test_resources.py | 2 | ||||
| -rw-r--r-- | pkg_resources/tests/test_working_set.py | 478 |
6 files changed, 688 insertions, 67 deletions
diff --git a/pkg_resources/__init__.py b/pkg_resources/__init__.py index d8000b00..497448de 100644 --- a/pkg_resources/__init__.py +++ b/pkg_resources/__init__.py @@ -34,9 +34,11 @@ import platform import collections import plistlib import email.parser +import errno import tempfile import textwrap import itertools +import inspect from pkgutil import get_importer try: @@ -67,6 +69,7 @@ try: except ImportError: importlib_machinery = None +from . import py31compat from pkg_resources.extern import appdirs from pkg_resources.extern import packaging __import__('pkg_resources.extern.packaging.version') @@ -74,9 +77,15 @@ __import__('pkg_resources.extern.packaging.specifiers') __import__('pkg_resources.extern.packaging.requirements') __import__('pkg_resources.extern.packaging.markers') + if (3, 0) < sys.version_info < (3, 3): raise RuntimeError("Python 3.3 or later is required") +if six.PY2: + # Those builtin exceptions are only defined in Python 3 + PermissionError = None + NotADirectoryError = None + # declare some globals that will be defined later to # satisfy the linters. require = None @@ -786,7 +795,7 @@ class WorkingSet(object): self._added_new(dist) def resolve(self, requirements, env=None, installer=None, - replace_conflicting=False): + replace_conflicting=False, extras=None): """List all distributions needed to (recursively) meet `requirements` `requirements` must be a sequence of ``Requirement`` objects. `env`, @@ -802,6 +811,12 @@ class WorkingSet(object): the wrong version. Otherwise, if an `installer` is supplied it will be invoked to obtain the correct version of the requirement and activate it. + + `extras` is a list of the extras to be used with these requirements. + This is important because extra requirements may look like `my_req; + extra = "my_extra"`, which would otherwise be interpreted as a purely + optional requirement. Instead, we want to be able to assert that these + requirements are truly required. """ # set up the stack @@ -825,7 +840,7 @@ class WorkingSet(object): # Ignore cyclic or redundant dependencies continue - if not req_extras.markers_pass(req): + if not req_extras.markers_pass(req, extras): continue dist = best.get(req.key) @@ -843,7 +858,10 @@ class WorkingSet(object): # distribution env = Environment([]) ws = WorkingSet([]) - dist = best[req.key] = env.best_match(req, ws, installer) + dist = best[req.key] = env.best_match( + req, ws, installer, + replace_conflicting=replace_conflicting + ) if dist is None: requirers = required_by.get(req, None) raise DistributionNotFound(req, requirers) @@ -1004,7 +1022,7 @@ class _ReqExtras(dict): Map each requirement to the extras that demanded it. """ - def markers_pass(self, req): + def markers_pass(self, req, extras=None): """ Evaluate markers for req against each extra that demanded it. @@ -1014,7 +1032,7 @@ class _ReqExtras(dict): """ extra_evals = ( req.marker.evaluate({'extra': extra}) - for extra in self.get(req, ()) + (None,) + for extra in self.get(req, ()) + (extras or (None,)) ) return not req.marker or any(extra_evals) @@ -1095,7 +1113,7 @@ class Environment(object): dists.append(dist) dists.sort(key=operator.attrgetter('hashcmp'), reverse=True) - def best_match(self, req, working_set, installer=None): + def best_match(self, req, working_set, installer=None, replace_conflicting=False): """Find distribution best matching `req` and usable on `working_set` This calls the ``find(req)`` method of the `working_set` to see if a @@ -1108,7 +1126,12 @@ class Environment(object): calling the environment's ``obtain(req, installer)`` method will be returned. """ - dist = working_set.find(req) + try: + dist = working_set.find(req) + except VersionConflict: + if not replace_conflicting: + raise + dist = None if dist is not None: return dist for dist in self[req.key]: @@ -1544,7 +1567,7 @@ class EggProvider(NullProvider): path = self.module_path old = None while path != old: - if _is_unpacked_egg(path): + if _is_egg_path(path): self.egg_name = os.path.basename(path) self.egg_info = os.path.join(path, 'EGG-INFO') self.egg_root = path @@ -1927,10 +1950,16 @@ def find_eggs_in_zip(importer, path_item, only=False): # don't yield nested distros return for subitem in metadata.resource_listdir('/'): - if _is_unpacked_egg(subitem): + if _is_egg_path(subitem): subpath = os.path.join(path_item, subitem) for dist in find_eggs_in_zip(zipimport.zipimporter(subpath), subpath): yield dist + elif subitem.lower().endswith('.dist-info'): + subpath = os.path.join(path_item, subitem) + submeta = EggMetadata(zipimport.zipimporter(subpath)) + submeta.egg_info = subpath + yield Distribution.from_location(path_item, subitem, submeta) + register_finder(zipimport.zipimporter, find_eggs_in_zip) @@ -1973,46 +2002,57 @@ def find_on_path(importer, path_item, only=False): """Yield distributions accessible on a sys.path directory""" path_item = _normalize_cached(path_item) - if os.path.isdir(path_item) and os.access(path_item, os.R_OK): - if _is_unpacked_egg(path_item): - yield Distribution.from_filename( - path_item, metadata=PathMetadata( - path_item, os.path.join(path_item, 'EGG-INFO') - ) + if _is_unpacked_egg(path_item): + yield Distribution.from_filename( + path_item, metadata=PathMetadata( + path_item, os.path.join(path_item, 'EGG-INFO') ) - else: - # scan for .egg and .egg-info in directory - path_item_entries = _by_version_descending(os.listdir(path_item)) - for entry in path_item_entries: - lower = entry.lower() - if lower.endswith('.egg-info') or lower.endswith('.dist-info'): - fullpath = os.path.join(path_item, entry) - if os.path.isdir(fullpath): - # egg-info directory, allow getting metadata - if len(os.listdir(fullpath)) == 0: - # Empty egg directory, skip. - continue - metadata = PathMetadata(path_item, fullpath) - else: - metadata = FileMetadata(fullpath) - yield Distribution.from_location( - path_item, entry, metadata, precedence=DEVELOP_DIST - ) - elif not only and _is_unpacked_egg(entry): - dists = find_distributions(os.path.join(path_item, entry)) - for dist in dists: - yield dist - elif not only and lower.endswith('.egg-link'): - with open(os.path.join(path_item, entry)) as entry_file: - entry_lines = entry_file.readlines() - for line in entry_lines: - if not line.strip(): - continue - path = os.path.join(path_item, line.rstrip()) - dists = find_distributions(path) - for item in dists: - yield item - break + ) + else: + try: + entries = os.listdir(path_item) + except (PermissionError, NotADirectoryError): + return + except OSError as e: + # Ignore the directory if does not exist, not a directory or we + # don't have permissions + if (e.errno in (errno.ENOTDIR, errno.EACCES, errno.ENOENT) + # Python 2 on Windows needs to be handled this way :( + or hasattr(e, "winerror") and e.winerror == 267): + return + raise + # scan for .egg and .egg-info in directory + path_item_entries = _by_version_descending(entries) + for entry in path_item_entries: + lower = entry.lower() + if lower.endswith('.egg-info') or lower.endswith('.dist-info'): + fullpath = os.path.join(path_item, entry) + if os.path.isdir(fullpath): + # egg-info directory, allow getting metadata + if len(os.listdir(fullpath)) == 0: + # Empty egg directory, skip. + continue + metadata = PathMetadata(path_item, fullpath) + else: + metadata = FileMetadata(fullpath) + yield Distribution.from_location( + path_item, entry, metadata, precedence=DEVELOP_DIST + ) + elif not only and _is_egg_path(entry): + dists = find_distributions(os.path.join(path_item, entry)) + for dist in dists: + yield dist + elif not only and lower.endswith('.egg-link'): + with open(os.path.join(path_item, entry)) as entry_file: + entry_lines = entry_file.readlines() + for line in entry_lines: + if not line.strip(): + continue + path = os.path.join(path_item, line.rstrip()) + dists = find_distributions(path) + for item in dists: + yield item + break register_finder(pkgutil.ImpImporter, find_on_path) @@ -2093,6 +2133,10 @@ def _rebuild_mod_path(orig_path, package_name, module): parts = path_parts[:-module_parts] return safe_sys_path_index(_normalize_cached(os.sep.join(parts))) + if not isinstance(orig_path, list): + # Is this behavior useful when module.__path__ is not a list? + return + orig_path.sort(key=position_in_sys_path) module.__path__[:] = [_normalize_cached(p) for p in orig_path] @@ -2182,12 +2226,22 @@ def _normalize_cached(filename, _cache={}): return result +def _is_egg_path(path): + """ + Determine if given path appears to be an egg. + """ + return ( + path.lower().endswith('.egg') + ) + + def _is_unpacked_egg(path): """ Determine if given path appears to be an unpacked egg. """ return ( - path.lower().endswith('.egg') + _is_egg_path(path) and + os.path.isfile(os.path.join(path, 'EGG-INFO', 'PKG-INFO')) ) @@ -2279,8 +2333,14 @@ class EntryPoint(object): def require(self, env=None, installer=None): if self.extras and not self.dist: raise UnknownExtra("Can't require() without a distribution", self) + + # Get the requirements for this entry point with all its extras and + # then resolve them. We have to pass `extras` along when resolving so + # that the working set knows what extras we want. Otherwise, for + # dist-info distributions, the working set will assume that the + # requirements for that extra are purely optional and skip over them. reqs = self.dist.requires(self.extras) - items = working_set.resolve(reqs, env, installer) + items = working_set.resolve(reqs, env, installer, extras=self.extras) list(map(working_set.add, items)) pattern = re.compile( @@ -2895,20 +2955,20 @@ class Requirement(packaging.requirements.Requirement): return req -def _get_mro(cls): - """Get an mro for a type or classic class""" - if not isinstance(cls, type): - - class cls(cls, object): - pass - - return cls.__mro__[1:] - return cls.__mro__ +def _always_object(classes): + """ + Ensure object appears in the mro even + for old-style classes. + """ + if object not in classes: + return classes + (object,) + return classes def _find_adapter(registry, ob): """Return an adapter factory for `ob` from `registry`""" - for t in _get_mro(getattr(ob, '__class__', type(ob))): + types = _always_object(inspect.getmro(getattr(ob, '__class__', type(ob)))) + for t in types: if t in registry: return registry[t] @@ -2916,8 +2976,7 @@ def _find_adapter(registry, ob): def ensure_directory(path): """Ensure that the parent directory of `path` exists""" dirname = os.path.dirname(path) - if not os.path.isdir(dirname): - os.makedirs(dirname) + py31compat.makedirs(dirname, exist_ok=True) def _bypass_ensure_directory(path): diff --git a/pkg_resources/py31compat.py b/pkg_resources/py31compat.py new file mode 100644 index 00000000..331a51bb --- /dev/null +++ b/pkg_resources/py31compat.py @@ -0,0 +1,22 @@ +import os +import errno +import sys + + +def _makedirs_31(path, exist_ok=False): + try: + os.makedirs(path) + except OSError as exc: + if not exist_ok or exc.errno != errno.EEXIST: + raise + + +# rely on compatibility behavior until mode considerations +# and exists_ok considerations are disentangled. +# See https://github.com/pypa/setuptools/pull/1083#issuecomment-315168663 +needs_makedirs = ( + sys.version_info < (3, 2, 5) or + (3, 3) <= sys.version_info < (3, 3, 6) or + (3, 4) <= sys.version_info < (3, 4, 1) +) +makedirs = _makedirs_31 if needs_makedirs else os.makedirs diff --git a/pkg_resources/tests/test_find_distributions.py b/pkg_resources/tests/test_find_distributions.py new file mode 100644 index 00000000..97999b33 --- /dev/null +++ b/pkg_resources/tests/test_find_distributions.py @@ -0,0 +1,65 @@ +import subprocess +import sys + +import pytest +import pkg_resources + +SETUP_TEMPLATE = """ +import setuptools +setuptools.setup( + name="my-test-package", + version="1.0", + zip_safe=True, +) +""".lstrip() + +class TestFindDistributions: + + @pytest.fixture + def target_dir(self, tmpdir): + target_dir = tmpdir.mkdir('target') + # place a .egg named directory in the target that is not an egg: + target_dir.mkdir('not.an.egg') + return str(target_dir) + + @pytest.fixture + def project_dir(self, tmpdir): + project_dir = tmpdir.mkdir('my-test-package') + (project_dir / "setup.py").write(SETUP_TEMPLATE) + return str(project_dir) + + def test_non_egg_dir_named_egg(self, target_dir): + dists = pkg_resources.find_distributions(target_dir) + assert not list(dists) + + def test_standalone_egg_directory(self, project_dir, target_dir): + # install this distro as an unpacked egg: + args = [ + sys.executable, + '-c', 'from setuptools.command.easy_install import main; main()', + '-mNx', + '-d', target_dir, + '--always-unzip', + project_dir, + ] + subprocess.check_call(args) + dists = pkg_resources.find_distributions(target_dir) + assert [dist.project_name for dist in dists] == ['my-test-package'] + dists = pkg_resources.find_distributions(target_dir, only=True) + assert not list(dists) + + def test_zipped_egg(self, project_dir, target_dir): + # install this distro as an unpacked egg: + args = [ + sys.executable, + '-c', 'from setuptools.command.easy_install import main; main()', + '-mNx', + '-d', target_dir, + '--zip-ok', + project_dir, + ] + subprocess.check_call(args) + dists = pkg_resources.find_distributions(target_dir) + assert [dist.project_name for dist in dists] == ['my-test-package'] + dists = pkg_resources.find_distributions(target_dir, only=True) + assert not list(dists) diff --git a/pkg_resources/tests/test_markers.py b/pkg_resources/tests/test_markers.py index 78810b6e..15a3b499 100644 --- a/pkg_resources/tests/test_markers.py +++ b/pkg_resources/tests/test_markers.py @@ -1,7 +1,4 @@ -try: - import unittest.mock as mock -except ImportError: - import mock +import mock from pkg_resources import evaluate_marker diff --git a/pkg_resources/tests/test_resources.py b/pkg_resources/tests/test_resources.py index 8223963c..dcd2f42c 100644 --- a/pkg_resources/tests/test_resources.py +++ b/pkg_resources/tests/test_resources.py @@ -585,7 +585,7 @@ class TestParsing: [Requirement('Twis-Ted>=1.2-1')] ) assert ( - list(parse_requirements('Twisted >=1.2, \ # more\n<2.0')) + list(parse_requirements('Twisted >=1.2, \\ # more\n<2.0')) == [Requirement('Twisted>=1.2,<2.0')] ) diff --git a/pkg_resources/tests/test_working_set.py b/pkg_resources/tests/test_working_set.py new file mode 100644 index 00000000..422a7283 --- /dev/null +++ b/pkg_resources/tests/test_working_set.py @@ -0,0 +1,478 @@ +import inspect +import re +import textwrap + +import pytest + +import pkg_resources + +from .test_resources import Metadata + + +def strip_comments(s): + return '\n'.join( + l for l in s.split('\n') + if l.strip() and not l.strip().startswith('#') + ) + +def parse_distributions(s): + ''' + Parse a series of distribution specs of the form: + {project_name}-{version} + [optional, indented requirements specification] + + Example: + + foo-0.2 + bar-1.0 + foo>=3.0 + [feature] + baz + + yield 2 distributions: + - project_name=foo, version=0.2 + - project_name=bar, version=1.0, requires=['foo>=3.0', 'baz; extra=="feature"'] + ''' + s = s.strip() + for spec in re.split('\n(?=[^\s])', s): + if not spec: + continue + fields = spec.split('\n', 1) + assert 1 <= len(fields) <= 2 + name, version = fields.pop(0).split('-') + if fields: + requires = textwrap.dedent(fields.pop(0)) + metadata=Metadata(('requires.txt', requires)) + else: + metadata = None + dist = pkg_resources.Distribution(project_name=name, + version=version, + metadata=metadata) + yield dist + + +class FakeInstaller(object): + + def __init__(self, installable_dists): + self._installable_dists = installable_dists + + def __call__(self, req): + return next(iter(filter(lambda dist: dist in req, + self._installable_dists)), None) + + +def parametrize_test_working_set_resolve(*test_list): + idlist = [] + argvalues = [] + for test in test_list: + ( + name, + installed_dists, + installable_dists, + requirements, + expected1, expected2 + ) = [ + strip_comments(s.lstrip()) for s in + textwrap.dedent(test).lstrip().split('\n\n', 5) + ] + installed_dists = list(parse_distributions(installed_dists)) + installable_dists = list(parse_distributions(installable_dists)) + requirements = list(pkg_resources.parse_requirements(requirements)) + for id_, replace_conflicting, expected in ( + (name, False, expected1), + (name + '_replace_conflicting', True, expected2), + ): + idlist.append(id_) + expected = strip_comments(expected.strip()) + if re.match('\w+$', expected): + expected = getattr(pkg_resources, expected) + assert issubclass(expected, Exception) + else: + expected = list(parse_distributions(expected)) + argvalues.append(pytest.param(installed_dists, installable_dists, + requirements, replace_conflicting, + expected)) + return pytest.mark.parametrize('installed_dists,installable_dists,' + 'requirements,replace_conflicting,' + 'resolved_dists_or_exception', + argvalues, ids=idlist) + + +@parametrize_test_working_set_resolve( + ''' + # id + noop + + # installed + + # installable + + # wanted + + # resolved + + # resolved [replace conflicting] + ''', + + ''' + # id + already_installed + + # installed + foo-3.0 + + # installable + + # wanted + foo>=2.1,!=3.1,<4 + + # resolved + foo-3.0 + + # resolved [replace conflicting] + foo-3.0 + ''', + + ''' + # id + installable_not_installed + + # installed + + # installable + foo-3.0 + foo-4.0 + + # wanted + foo>=2.1,!=3.1,<4 + + # resolved + foo-3.0 + + # resolved [replace conflicting] + foo-3.0 + ''', + + ''' + # id + not_installable + + # installed + + # installable + + # wanted + foo>=2.1,!=3.1,<4 + + # resolved + DistributionNotFound + + # resolved [replace conflicting] + DistributionNotFound + ''', + + ''' + # id + no_matching_version + + # installed + + # installable + foo-3.1 + + # wanted + foo>=2.1,!=3.1,<4 + + # resolved + DistributionNotFound + + # resolved [replace conflicting] + DistributionNotFound + ''', + + ''' + # id + installable_with_installed_conflict + + # installed + foo-3.1 + + # installable + foo-3.5 + + # wanted + foo>=2.1,!=3.1,<4 + + # resolved + VersionConflict + + # resolved [replace conflicting] + foo-3.5 + ''', + + ''' + # id + not_installable_with_installed_conflict + + # installed + foo-3.1 + + # installable + + # wanted + foo>=2.1,!=3.1,<4 + + # resolved + VersionConflict + + # resolved [replace conflicting] + DistributionNotFound + ''', + + ''' + # id + installed_with_installed_require + + # installed + foo-3.9 + baz-0.1 + foo>=2.1,!=3.1,<4 + + # installable + + # wanted + baz + + # resolved + foo-3.9 + baz-0.1 + + # resolved [replace conflicting] + foo-3.9 + baz-0.1 + ''', + + ''' + # id + installed_with_conflicting_installed_require + + # installed + foo-5 + baz-0.1 + foo>=2.1,!=3.1,<4 + + # installable + + # wanted + baz + + # resolved + VersionConflict + + # resolved [replace conflicting] + DistributionNotFound + ''', + + ''' + # id + installed_with_installable_conflicting_require + + # installed + foo-5 + baz-0.1 + foo>=2.1,!=3.1,<4 + + # installable + foo-2.9 + + # wanted + baz + + # resolved + VersionConflict + + # resolved [replace conflicting] + baz-0.1 + foo-2.9 + ''', + + ''' + # id + installed_with_installable_require + + # installed + baz-0.1 + foo>=2.1,!=3.1,<4 + + # installable + foo-3.9 + + # wanted + baz + + # resolved + foo-3.9 + baz-0.1 + + # resolved [replace conflicting] + foo-3.9 + baz-0.1 + ''', + + ''' + # id + installable_with_installed_require + + # installed + foo-3.9 + + # installable + baz-0.1 + foo>=2.1,!=3.1,<4 + + # wanted + baz + + # resolved + foo-3.9 + baz-0.1 + + # resolved [replace conflicting] + foo-3.9 + baz-0.1 + ''', + + ''' + # id + installable_with_installable_require + + # installed + + # installable + foo-3.9 + baz-0.1 + foo>=2.1,!=3.1,<4 + + # wanted + baz + + # resolved + foo-3.9 + baz-0.1 + + # resolved [replace conflicting] + foo-3.9 + baz-0.1 + ''', + + ''' + # id + installable_with_conflicting_installable_require + + # installed + foo-5 + + # installable + foo-2.9 + baz-0.1 + foo>=2.1,!=3.1,<4 + + # wanted + baz + + # resolved + VersionConflict + + # resolved [replace conflicting] + baz-0.1 + foo-2.9 + ''', + + ''' + # id + conflicting_installables + + # installed + + # installable + foo-2.9 + foo-5.0 + + # wanted + foo>=2.1,!=3.1,<4 + foo>=4 + + # resolved + VersionConflict + + # resolved [replace conflicting] + VersionConflict + ''', + + ''' + # id + installables_with_conflicting_requires + + # installed + + # installable + foo-2.9 + dep==1.0 + baz-5.0 + dep==2.0 + dep-1.0 + dep-2.0 + + # wanted + foo + baz + + # resolved + VersionConflict + + # resolved [replace conflicting] + VersionConflict + ''', + + ''' + # id + installables_with_conflicting_nested_requires + + # installed + + # installable + foo-2.9 + dep1 + dep1-1.0 + subdep<1.0 + baz-5.0 + dep2 + dep2-1.0 + subdep>1.0 + subdep-0.9 + subdep-1.1 + + # wanted + foo + baz + + # resolved + VersionConflict + + # resolved [replace conflicting] + VersionConflict + ''', +) +def test_working_set_resolve(installed_dists, installable_dists, requirements, + replace_conflicting, resolved_dists_or_exception): + ws = pkg_resources.WorkingSet([]) + list(map(ws.add, installed_dists)) + resolve_call = lambda: ws.resolve( + requirements, installer=FakeInstaller(installable_dists), + replace_conflicting=replace_conflicting, + ) + if inspect.isclass(resolved_dists_or_exception): + with pytest.raises(resolved_dists_or_exception): + resolve_call() + else: + assert sorted(resolve_call()) == sorted(resolved_dists_or_exception) |
