From 3fcd24b6bec780a56664ea5bb74a4e8eaadf3014 Mon Sep 17 00:00:00 2001 From: "phillip.eby" Date: Wed, 15 May 2013 21:59:07 +0000 Subject: Snapshot pre-merger changes, mostly SSL support and a few bugfixes git-svn-id: http://svn.python.org/projects/sandbox/trunk/setuptools@88994 6015fed2-1504-0410-9fe1-9d1591cc4771 --- pkg_resources.py | 152 +++++++++++++++++++++++++++++++++---- setup.py | 34 ++++----- setuptools.txt | 8 +- setuptools/command/easy_install.py | 2 +- setuptools/dist.py | 8 +- setuptools/package_index.py | 30 ++++---- setuptools/sandbox.py | 2 +- setuptools/tests/api_tests.txt | 91 ++++++++++++++++++++++ setuptools/tests/test_resources.py | 8 +- 9 files changed, 272 insertions(+), 63 deletions(-) diff --git a/pkg_resources.py b/pkg_resources.py index 4fae479..efc4475 100644 --- a/pkg_resources.py +++ b/pkg_resources.py @@ -144,7 +144,7 @@ __all__ = [ # Parsing functions and string utilities 'parse_requirements', 'parse_version', 'safe_name', 'safe_version', 'get_platform', 'compatible_platforms', 'yield_lines', 'split_sections', - 'safe_extra', 'to_filename', + 'safe_extra', 'to_filename', 'invalid_marker', 'evaluate_marker', # filesystem utilities 'ensure_directory', 'normalize_path', @@ -1146,6 +1146,129 @@ def to_filename(name): +_marker_names = { + 'os': ['name'], 'sys': ['platform'], + 'platform': ['version','machine','python_implementation'], + 'python_version': [], 'python_full_version': [], 'extra':[], +} + +_marker_values = { + 'os_name': lambda: os.name, + 'sys_platform': lambda: sys.platform, + 'python_full_version': lambda: sys.version.split()[0], + 'python_version': lambda:'%s.%s' % (sys.version_info[0], sys.version_info[1]), + 'platform_version': lambda: _platinfo('version'), + 'platform_machine': lambda: _platinfo('machine'), + 'python_implementation': lambda: _platinfo('python_implementation') or _pyimp(), +} + +def _platinfo(attr): + try: + import platform + except ImportError: + return '' + return getattr(platform, attr, lambda:'')() + +def _pyimp(): + if sys.platform=='cli': + return 'IronPython' + elif sys.platform.startswith('java'): + return 'Jython' + elif '__pypy__' in sys.builtin_module_names: + return 'PyPy' + else: + return 'CPython' + +def invalid_marker(text): + """Validate text as a PEP 426 environment marker; return exception or False""" + try: + evaluate_marker(text) + except SyntaxError: + return sys.exc_info()[1] + return False + +def evaluate_marker(text, extra=None, _ops={}): + """Evaluate a PEP 426 environment marker; SyntaxError if marker is invalid""" + + if not _ops: + + from token import NAME, STRING + import token, symbol, operator + + def and_test(nodelist): + # MUST NOT short-circuit evaluation, or invalid syntax can be skipped! + return reduce(operator.and_, [interpret(nodelist[i]) for i in range(1,len(nodelist),2)]) + + def test(nodelist): + # MUST NOT short-circuit evaluation, or invalid syntax can be skipped! + return reduce(operator.or_, [interpret(nodelist[i]) for i in range(1,len(nodelist),2)]) + + def atom(nodelist): + t = nodelist[1][0] + if t == token.LPAR: + if nodelist[2][0] == token.RPAR: + raise SyntaxError("Empty parentheses") + return interpret(nodelist[2]) + raise SyntaxError("Language feature not supported in environment markers") + + def comparison(nodelist): + if len(nodelist)>4: + raise SyntaxError("Chained comparison not allowed in environment markers") + comp = nodelist[2][1] + cop = comp[1] + if comp[0] == NAME: + if len(nodelist[2]) == 3: + if cop == 'not': + cop = 'not in' + else: + cop = 'is not' + try: + cop = _ops[cop] + except KeyError: + raise SyntaxError(repr(cop)+" operator not allowed in environment markers") + return cop(evaluate(nodelist[1]), evaluate(nodelist[3])) + + _ops.update({ + symbol.test: test, symbol.and_test: and_test, symbol.atom: atom, + symbol.comparison: comparison, 'not in': lambda x,y: x not in y, + 'in': lambda x,y: x in y, '==': operator.eq, '!=': operator.ne, + }) + if hasattr(symbol,'or_test'): + _ops[symbol.or_test] = test + + def interpret(nodelist): + while len(nodelist)==2: nodelist = nodelist[1] + try: + op = _ops[nodelist[0]] + except KeyError: + raise SyntaxError("Comparison or logical expression expected") + raise SyntaxError("Language feature not supported in environment markers: "+symbol.sym_name[nodelist[0]]) + return op(nodelist) + + def evaluate(nodelist): + while len(nodelist)==2: nodelist = nodelist[1] + kind = nodelist[0] + name = nodelist[1] + #while len(name)==2: name = name[1] + if kind==NAME: + try: + op = _marker_values[name] + except KeyError: + raise SyntaxError("Unknown name %r" % name) + return op() + if kind==STRING: + s = nodelist[1] + if s[:1] not in "'\"" or s.startswith('"""') or s.startswith("'''") \ + or '\\' in s: + raise SyntaxError( + "Only plain strings allowed in environment markers") + return s[1:-1] + raise SyntaxError("Language feature not supported in environment markers") + + import parser + return interpret(parser.expr(text).totuple(1)[1]) + + class NullProvider: """Try to implement resources and metadata for arbitrary PEP 302 loaders""" @@ -1843,7 +1966,6 @@ def parse_version(s): parts.pop() parts.append(part) return tuple(parts) - class EntryPoint(object): """Object representing an advertised importable object""" @@ -2057,7 +2179,14 @@ class Distribution(object): dm = self.__dep_map = {None: []} for name in 'requires.txt', 'depends.txt': for extra,reqs in split_sections(self._get_metadata(name)): - if extra: extra = safe_extra(extra) + if extra: + if ':' in extra: + extra, marker = extra.split(':',1) + if invalid_marker(marker): + reqs=[] # XXX warn + elif not evaluate_marker(marker): + reqs=[] + extra = safe_extra(extra) or None dm.setdefault(extra,[]).extend(parse_requirements(reqs)) return dm _dep_map = property(_dep_map) @@ -2081,6 +2210,8 @@ class Distribution(object): for line in self.get_metadata_lines(name): yield line + + def activate(self,path=None): """Ensure distribution is importable on `path` (default=sys.path)""" if path is None: path = sys.path @@ -2119,6 +2250,9 @@ class Distribution(object): raise AttributeError,attr return getattr(self._provider, attr) + + + #@classmethod def from_filename(cls,filename,metadata=None, **kw): return cls.from_location( @@ -2160,18 +2294,6 @@ class Distribution(object): - - - - - - - - - - - - def insert_on(self, path, loc = None): """Insert self.location in path before its nearest parent directory""" diff --git a/setup.py b/setup.py index bf310a5..72f9a5f 100755 --- a/setup.py +++ b/setup.py @@ -91,33 +91,33 @@ setup( Topic :: System :: Archiving :: Packaging Topic :: System :: Systems Administration Topic :: Utilities""".splitlines() if f.strip()], + extras_require = { + "ssl:sys_platform=='win32'": "wincertstore==0.1", + "ssl:sys_platform=='win32' and python_version in '2.3, 2.4'": "ctypes==1.0.2", + "ssl:python_version in '2.3, 2.4, 2.5'":"ssl==1.16", + "certs": "certifi==0.0.8", + }, + dependency_links = [ + 'http://pypi.python.org/packages/source/c/certifi/certifi-0.0.8.tar.gz#md5=dc5f5e7f0b5fc08d27654b17daa6ecec', + 'http://pypi.python.org/packages/source/s/ssl/ssl-1.16.tar.gz#md5=fb12d335d56f3c8c7c1fefc1c06c4bfb', + 'http://pypi.python.org/packages/source/w/wincertstore/wincertstore-0.1.zip#md5=2f9accbebe8f7b4c06ac7aa83879b81c', + 'http://sourceforge.net/projects/ctypes/files/ctypes/1.0.2/ctypes-1.0.2.win32-py2.3.exe/download#md5=9afe4b75240a8808a24df7a76b6081e3', + 'http://sourceforge.net/projects/ctypes/files/ctypes/1.0.2/ctypes-1.0.2.win32-py2.4.exe/download#md5=9092a0ad5a3d79fa2d980f1ddc5e9dbc', + 'http://peak.telecommunity.com/dist/ssl-1.16-py2.3-win32.egg#md5=658f74b3eb6f32050e8531bb73de8e74', + 'http://peak.telecommunity.com/dist/ssl-1.16-py2.4-win32.egg#md5=3cfa2c526dc66e318e8520b6f1aadce5', + 'http://peak.telecommunity.com/dist/ssl-1.16-py2.5-win32.egg#md5=85ad1cda806d639743121c0bbcb5f39b', + ], scripts = [], # uncomment for testing # setup_requires = ['setuptools>=0.6a0'], + # tests_require = "setuptools[ssl]", ) - - - - - - - - - - - - - - - - - diff --git a/setuptools.txt b/setuptools.txt index efe2c68..98b974a 100755 --- a/setuptools.txt +++ b/setuptools.txt @@ -217,11 +217,7 @@ For the most part, setuptools' interpretation of version numbers is intuitive, but here are a few tips that will keep you out of trouble in the corner cases: * Don't use ``-`` or any other character than ``.`` as a separator, unless you - really want a post-release. Remember that ``2.1-rc2`` means you've - *already* released ``2.1``, whereas ``2.1rc2`` and ``2.1.c2`` are candidates - you're putting out *before* ``2.1``. If you accidentally distribute copies - of a post-release that you meant to be a pre-release, the only safe fix is to - bump your main release number (e.g. to ``2.1.1``) and re-release the project. + really want a post-release. * Don't stick adjoining pre-release tags together without a dot or number between them. Version ``1.9adev`` is the ``adev`` prerelease of ``1.9``, @@ -239,7 +235,7 @@ but here are a few tips that will keep you out of trouble in the corner cases: >>> parse_version('1.9.a.dev') == parse_version('1.9a0dev') True >>> parse_version('2.1-rc2') < parse_version('2.1') - False + True >>> parse_version('0.6a9dev-r41475') < parse_version('0.6a9') True diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py index b5db0e8..920af02 100755 --- a/setuptools/command/easy_install.py +++ b/setuptools/command/easy_install.py @@ -156,7 +156,7 @@ class easy_install(Command): else: self.all_site_dirs.append(normalize_path(d)) if not self.editable: self.check_site_dir() - self.index_url = self.index_url or "http://pypi.python.org/simple" + self.index_url = self.index_url or "https://pypi.python.org/simple" self.shadow_path = self.all_site_dirs[:] for path_item in self.install_dir, normalize_path(self.script_dir): if path_item not in self.shadow_path: diff --git a/setuptools/dist.py b/setuptools/dist.py index 482c6bf..e7f5454 100644 --- a/setuptools/dist.py +++ b/setuptools/dist.py @@ -47,7 +47,6 @@ def assert_string_list(dist, attr, value): raise DistutilsSetupError( "%r must be a list of strings (got %r)" % (attr,value) ) - def check_nsp(dist, attr, value): """Verify that namespace packages are valid""" assert_string_list(dist,attr,value) @@ -69,6 +68,10 @@ def check_extras(dist, attr, value): """Verify that extras_require mapping is valid""" try: for k,v in value.items(): + if ':' in k: + k,m = k.split(':',1) + if pkg_resources.invalid_marker(m): + raise DistutilsSetupError("Invalid environment marker: "+m) list(pkg_resources.parse_requirements(v)) except (TypeError,ValueError,AttributeError): raise DistutilsSetupError( @@ -77,9 +80,6 @@ def check_extras(dist, attr, value): "requirement specifiers." ) - - - def assert_bool(dist, attr, value): """Verify that value is True, False, 0, or 1""" if bool(value) != value: diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 9a9c5d6..aab2bb2 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1,6 +1,6 @@ """PyPI and direct package downloading""" import sys, os.path, re, urlparse, urllib2, shutil, random, socket, cStringIO -import httplib, urllib +import httplib, urllib; from setuptools import ssl_support from pkg_resources import * from distutils import log from distutils.errors import DistutilsError @@ -145,12 +145,11 @@ user_agent = "Python-urllib/%s setuptools/%s" % ( urllib2.__version__, require('setuptools')[0].version ) - class PackageIndex(Environment): """A distribution index that scans web pages for download URLs""" - def __init__(self, index_url="http://pypi.python.org/simple", hosts=('*',), - *args, **kw + def __init__(self, index_url="https://pypi.python.org/simple", hosts=('*',), + ca_bundle=None, verify_ssl=True, *args, **kw ): Environment.__init__(self,*args,**kw) self.index_url = index_url + "/"[:not index_url.endswith('/')] @@ -159,8 +158,9 @@ class PackageIndex(Environment): self.package_pages = {} self.allows = re.compile('|'.join(map(translate,hosts))).match self.to_scan = [] - - + if verify_ssl and ssl_support.is_available and (ca_bundle or ssl_support.find_ca_bundle()): + self.opener = ssl_support.opener_for(ca_bundle) + else: self.opener = urllib2.urlopen def process_url(self, url, retrieve=False): """Evaluate a URL as a possible download, and maybe retrieve it""" @@ -575,12 +575,13 @@ class PackageIndex(Environment): def open_url(self, url, warning=None): if url.startswith('file:'): return local_open(url) try: - return open_with_auth(url) - except urllib2.HTTPError, v: - return v - except urllib2.URLError, v: - reason = v.reason - except httplib.HTTPException, v: + return open_with_auth(url, self.opener) + except urllib2.HTTPError: + return sys.exc_info()[1] + except urllib2.URLError: + reason = sys.exc_info()[1].reason + except httplib.HTTPException: + v = sys.exc_info()[1] reason = "%s: %s" % (v.__doc__ or v.__class__.__name__, v) if warning: self.warn(warning, reason) @@ -612,7 +613,6 @@ class PackageIndex(Environment): self.url_ok(url, True) # raises error if not allowed return self._attempt_download(url, filename) - def scan_url(self, url): self.process_url(url, True) @@ -736,7 +736,7 @@ def htmldecode(text): -def open_with_auth(url): +def open_with_auth(url, opener=urllib2.urlopen): """Open a urllib2 request, handling HTTP authentication""" scheme, netloc, path, params, query, frag = urlparse.urlparse(url) @@ -755,7 +755,7 @@ def open_with_auth(url): request = urllib2.Request(url) request.add_header('User-Agent', user_agent) - fp = urllib2.urlopen(request) + fp = opener(request) if auth: # Put authentication info back into request URL if same host, diff --git a/setuptools/sandbox.py b/setuptools/sandbox.py index 4c5e712..8b889a7 100755 --- a/setuptools/sandbox.py +++ b/setuptools/sandbox.py @@ -206,7 +206,7 @@ class DirectorySandbox(AbstractSandbox): def tmpnam(self): self._violation("tmpnam") def _ok(self,path): - if hasattr(_os,'devnull') and path==_os.devnull: return True + if hasattr(os,'devnull') and path==os.devnull: return True active = self._active try: self._active = False diff --git a/setuptools/tests/api_tests.txt b/setuptools/tests/api_tests.txt index 823d815..e35dad1 100755 --- a/setuptools/tests/api_tests.txt +++ b/setuptools/tests/api_tests.txt @@ -328,3 +328,94 @@ setuptools is provided as well:: >>> cp("darwin-8.2.0-Power_Macintosh", "macosx-10.3-ppc") False + +Environment Markers +------------------- + + >>> from pkg_resources import invalid_marker as im, evaluate_marker as em + >>> import os + + >>> print(im("sys_platform")) + Comparison or logical expression expected + + >>> print(im("sys_platform==")) + unexpected EOF while parsing (line 1) + + >>> print(im("sys_platform=='win32'")) + False + + >>> print(im("sys=='x'")) + Unknown name 'sys' + + >>> print(im("(extra)")) + Comparison or logical expression expected + + >>> print(im("(extra")) + unexpected EOF while parsing (line 1) + + >>> print(im("os.open('foo')=='y'")) + Language feature not supported in environment markers + + >>> print(im("'x'=='y' and os.open('foo')=='y'")) # no short-circuit! + Language feature not supported in environment markers + + >>> print(im("'x'=='x' or os.open('foo')=='y'")) # no short-circuit! + Language feature not supported in environment markers + + >>> print(im("'x' < 'y'")) + '<' operator not allowed in environment markers + + >>> print(im("'x' < 'y' < 'z'")) + Chained comparison not allowed in environment markers + + >>> print(im("r'x'=='x'")) + Only plain strings allowed in environment markers + + >>> print(im("'''x'''=='x'")) + Only plain strings allowed in environment markers + + >>> print(im('"""x"""=="x"')) + Only plain strings allowed in environment markers + + >>> print(im(r"'x\n'=='x'")) + Only plain strings allowed in environment markers + + >>> print(im("os.open=='y'")) + Language feature not supported in environment markers + + >>> em('"x"=="x"') + True + + >>> em('"x"=="y"') + False + + >>> em('"x"=="y" and "x"=="x"') + False + + >>> em('"x"=="y" or "x"=="x"') + True + + >>> em('"x"=="y" and "x"=="q" or "z"=="z"') + True + + >>> em('"x"=="y" and ("x"=="q" or "z"=="z")') + False + + >>> em('"x"=="y" and "z"=="z" or "x"=="q"') + False + + >>> em('"x"=="x" and "z"=="z" or "x"=="q"') + True + + >>> em("sys_platform=='win32'") == (sys.platform=='win32') + True + + >>> em("'x' in 'yx'") + True + + >>> em("'yx' in 'x'") + False + + + + diff --git a/setuptools/tests/test_resources.py b/setuptools/tests/test_resources.py index 1c010e7..ad2d02f 100644 --- a/setuptools/tests/test_resources.py +++ b/setuptools/tests/test_resources.py @@ -507,7 +507,7 @@ class ScriptHeaderTests(TestCase): def test_get_script_header_jython_workaround(self): platform = sys.platform sys.platform = 'java1.5.0_13' - stdout = sys.stdout + stdout, stderr = sys.stdout, sys.stderr try: # A mock sys.executable that uses a shebang line (this file) exe = os.path.normpath(os.path.splitext(__file__)[0] + '.py') @@ -517,17 +517,17 @@ class ScriptHeaderTests(TestCase): # Ensure we generate what is basically a broken shebang line # when there's options, with a warning emitted - sys.stdout = StringIO.StringIO() + sys.stdout = sys.stderr = StringIO.StringIO() self.assertEqual(get_script_header('#!/usr/bin/python -x', executable=exe), '#!%s -x\n' % exe) self.assert_('Unable to adapt shebang line' in sys.stdout.getvalue()) - sys.stdout = StringIO.StringIO() + sys.stdout = sys.stderr = StringIO.StringIO() self.assertEqual(get_script_header('#!/usr/bin/python', executable=self.non_ascii_exe), '#!%s -x\n' % self.non_ascii_exe) self.assert_('Unable to adapt shebang line' in sys.stdout.getvalue()) finally: sys.platform = platform - sys.stdout = stdout + sys.stdout, sys.stderr = stdout, stderr -- cgit v1.2.1