Snapshot pre-merger changes, mostly SSL support and a few bugfixes

git-svn-id: http://svn.python.org/projects/sandbox/trunk/setuptools@88994 6015fed2-1504-0410-9fe1-9d1591cc4771

9 files changed, 272 insertions, 63 deletions

diff --git a/pkg_resources.py b/pkg_resources.py
index 4fae479..efc4475 100644
--- a/pkg_resources.py
+++ b/pkg_resources.py
@@ -144,7 +144,7 @@ __all__ = [
     # Parsing functions and string utilities
     'parse_requirements', 'parse_version', 'safe_name', 'safe_version',
     'get_platform', 'compatible_platforms', 'yield_lines', 'split_sections',
-    'safe_extra', 'to_filename',
+    'safe_extra', 'to_filename', 'invalid_marker', 'evaluate_marker',
 
     # filesystem utilities
     'ensure_directory', 'normalize_path',
@@ -1146,6 +1146,129 @@ def to_filename(name):
 
 
 
+_marker_names = {
+    'os': ['name'], 'sys': ['platform'],
+    'platform': ['version','machine','python_implementation'],
+    'python_version': [], 'python_full_version': [], 'extra':[],
+}
+
+_marker_values = {
+    'os_name': lambda: os.name,
+    'sys_platform': lambda: sys.platform,
+    'python_full_version': lambda: sys.version.split()[0],
+    'python_version': lambda:'%s.%s' % (sys.version_info[0], sys.version_info[1]),
+    'platform_version': lambda: _platinfo('version'),
+    'platform_machine': lambda: _platinfo('machine'),
+    'python_implementation': lambda: _platinfo('python_implementation') or _pyimp(),
+}
+
+def _platinfo(attr):
+    try:
+        import platform
+    except ImportError:
+        return ''
+    return getattr(platform, attr, lambda:'')()
+
+def _pyimp():
+    if sys.platform=='cli':
+        return 'IronPython'
+    elif sys.platform.startswith('java'):
+        return 'Jython'
+    elif '__pypy__' in sys.builtin_module_names:
+        return 'PyPy'
+    else:
+        return 'CPython'                
+
+def invalid_marker(text):
+    """Validate text as a PEP 426 environment marker; return exception or False"""
+    try:
+        evaluate_marker(text)
+    except SyntaxError:
+        return sys.exc_info()[1]            
+    return False
+
+def evaluate_marker(text, extra=None, _ops={}):
+    """Evaluate a PEP 426 environment marker; SyntaxError if marker is invalid"""
+
+    if not _ops:
+
+        from token import NAME, STRING
+        import token, symbol, operator
+        
+        def and_test(nodelist):
+            # MUST NOT short-circuit evaluation, or invalid syntax can be skipped!
+            return reduce(operator.and_, [interpret(nodelist[i]) for i in range(1,len(nodelist),2)])
+        
+        def test(nodelist):
+            # MUST NOT short-circuit evaluation, or invalid syntax can be skipped!
+            return reduce(operator.or_, [interpret(nodelist[i]) for i in range(1,len(nodelist),2)])
+        
+        def atom(nodelist):
+            t = nodelist[1][0]
+            if t == token.LPAR:
+                if nodelist[2][0] == token.RPAR:
+                    raise SyntaxError("Empty parentheses")
+                return interpret(nodelist[2])
+            raise SyntaxError("Language feature not supported in environment markers")
+        
+        def comparison(nodelist):
+            if len(nodelist)>4:
+                raise SyntaxError("Chained comparison not allowed in environment markers")
+            comp = nodelist[2][1]
+            cop = comp[1]
+            if comp[0] == NAME:
+                if len(nodelist[2]) == 3:
+                    if cop == 'not':
+                        cop = 'not in'
+                    else:
+                        cop = 'is not'
+            try:
+                cop = _ops[cop]
+            except KeyError:
+                raise SyntaxError(repr(cop)+" operator not allowed in environment markers")
+            return cop(evaluate(nodelist[1]), evaluate(nodelist[3]))
+        
+        _ops.update({
+            symbol.test: test, symbol.and_test: and_test, symbol.atom: atom,
+            symbol.comparison: comparison, 'not in': lambda x,y: x not in y,
+            'in': lambda x,y: x in y, '==': operator.eq, '!=': operator.ne,
+        })
+        if hasattr(symbol,'or_test'):
+            _ops[symbol.or_test] = test
+
+    def interpret(nodelist):
+        while len(nodelist)==2: nodelist = nodelist[1]
+        try:
+            op = _ops[nodelist[0]]
+        except KeyError:
+            raise SyntaxError("Comparison or logical expression expected")
+            raise SyntaxError("Language feature not supported in environment markers: "+symbol.sym_name[nodelist[0]])
+        return op(nodelist)
+        
+    def evaluate(nodelist):
+        while len(nodelist)==2: nodelist = nodelist[1]
+        kind = nodelist[0]
+        name = nodelist[1]
+        #while len(name)==2: name = name[1]
+        if kind==NAME:
+            try:
+                op = _marker_values[name]
+            except KeyError:
+                raise SyntaxError("Unknown name %r" % name)
+            return op()
+        if kind==STRING:
+            s = nodelist[1]
+            if s[:1] not in "'\"" or s.startswith('"""') or s.startswith("'''") \
+            or '\\' in s:
+                raise SyntaxError(
+                    "Only plain strings allowed in environment markers")
+            return s[1:-1]
+        raise SyntaxError("Language feature not supported in environment markers")
+
+    import parser        
+    return interpret(parser.expr(text).totuple(1)[1])
+
+
 class NullProvider:
     """Try to implement resources and metadata for arbitrary PEP 302 loaders"""
 
@@ -1843,7 +1966,6 @@ def parse_version(s):
                 parts.pop()
         parts.append(part)
     return tuple(parts)
-
 class EntryPoint(object):
     """Object representing an advertised importable object"""
 
@@ -2057,7 +2179,14 @@ class Distribution(object):
             dm = self.__dep_map = {None: []}
             for name in 'requires.txt', 'depends.txt':
                 for extra,reqs in split_sections(self._get_metadata(name)):
-                    if extra: extra = safe_extra(extra)
+                    if extra:
+                        if ':' in extra:
+                            extra, marker = extra.split(':',1)
+                            if invalid_marker(marker):
+                                reqs=[] # XXX warn
+                            elif not evaluate_marker(marker):
+                                reqs=[]
+                        extra = safe_extra(extra) or None                                                            
                     dm.setdefault(extra,[]).extend(parse_requirements(reqs))
             return dm
     _dep_map = property(_dep_map)
@@ -2081,6 +2210,8 @@ class Distribution(object):
             for line in self.get_metadata_lines(name):
                 yield line
 
+
+
     def activate(self,path=None):
         """Ensure distribution is importable on `path` (default=sys.path)"""
         if path is None: path = sys.path
@@ -2119,6 +2250,9 @@ class Distribution(object):
             raise AttributeError,attr
         return getattr(self._provider, attr)
 
+
+
+
     #@classmethod
     def from_filename(cls,filename,metadata=None, **kw):
         return cls.from_location(
@@ -2160,18 +2294,6 @@ class Distribution(object):
 
 
 
-
-
-
-
-
-
-
-
-
-
-
-
     def insert_on(self, path, loc = None):
         """Insert self.location in path before its nearest parent directory"""
 
diff --git a/setup.py b/setup.py
index bf310a5..72f9a5f 100755
--- a/setup.py
+++ b/setup.py
@@ -91,10 +91,27 @@ setup(
     Topic :: System :: Archiving :: Packaging
     Topic :: System :: Systems Administration
     Topic :: Utilities""".splitlines() if f.strip()],
+    extras_require = {
+        "ssl:sys_platform=='win32'": "wincertstore==0.1",
+        "ssl:sys_platform=='win32' and python_version in '2.3, 2.4'": "ctypes==1.0.2",
+        "ssl:python_version in '2.3, 2.4, 2.5'":"ssl==1.16",
+        "certs": "certifi==0.0.8",
+    },
+    dependency_links = [
+        'http://pypi.python.org/packages/source/c/certifi/certifi-0.0.8.tar.gz#md5=dc5f5e7f0b5fc08d27654b17daa6ecec',
+        'http://pypi.python.org/packages/source/s/ssl/ssl-1.16.tar.gz#md5=fb12d335d56f3c8c7c1fefc1c06c4bfb',
+        'http://pypi.python.org/packages/source/w/wincertstore/wincertstore-0.1.zip#md5=2f9accbebe8f7b4c06ac7aa83879b81c',
+        'http://sourceforge.net/projects/ctypes/files/ctypes/1.0.2/ctypes-1.0.2.win32-py2.3.exe/download#md5=9afe4b75240a8808a24df7a76b6081e3',
+        'http://sourceforge.net/projects/ctypes/files/ctypes/1.0.2/ctypes-1.0.2.win32-py2.4.exe/download#md5=9092a0ad5a3d79fa2d980f1ddc5e9dbc',
+        'http://peak.telecommunity.com/dist/ssl-1.16-py2.3-win32.egg#md5=658f74b3eb6f32050e8531bb73de8e74',
+        'http://peak.telecommunity.com/dist/ssl-1.16-py2.4-win32.egg#md5=3cfa2c526dc66e318e8520b6f1aadce5',
+        'http://peak.telecommunity.com/dist/ssl-1.16-py2.5-win32.egg#md5=85ad1cda806d639743121c0bbcb5f39b',
+    ],
     scripts = [],
 
     # uncomment for testing
     # setup_requires = ['setuptools>=0.6a0'],
+    # tests_require = "setuptools[ssl]",
 )
 
 
@@ -104,20 +121,3 @@ setup(
 
 
 
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
-
diff --git a/setuptools.txt b/setuptools.txt
index efe2c68..98b974a 100755
--- a/setuptools.txt
+++ b/setuptools.txt
@@ -217,11 +217,7 @@ For the most part, setuptools' interpretation of version numbers is intuitive,
 but here are a few tips that will keep you out of trouble in the corner cases:
 
 * Don't use ``-`` or any other character than ``.`` as a separator, unless you
-  really want a post-release.  Remember that ``2.1-rc2`` means you've
-  *already* released ``2.1``, whereas ``2.1rc2`` and ``2.1.c2`` are candidates
-  you're putting out *before* ``2.1``.  If you accidentally distribute copies
-  of a post-release that you meant to be a pre-release, the only safe fix is to
-  bump your main release number (e.g. to ``2.1.1``) and re-release the project.
+  really want a post-release.
 
 * Don't stick adjoining pre-release tags together without a dot or number
   between them.  Version ``1.9adev`` is the ``adev`` prerelease of ``1.9``,
@@ -239,7 +235,7 @@ but here are a few tips that will keep you out of trouble in the corner cases:
     >>> parse_version('1.9.a.dev') == parse_version('1.9a0dev')
     True
     >>> parse_version('2.1-rc2') < parse_version('2.1')
-    False
+    True
     >>> parse_version('0.6a9dev-r41475') < parse_version('0.6a9')
     True
 
diff --git a/setuptools/command/easy_install.py b/setuptools/command/easy_install.py
index b5db0e8..920af02 100755
--- a/setuptools/command/easy_install.py
+++ b/setuptools/command/easy_install.py
@@ -156,7 +156,7 @@ class easy_install(Command):
                 else:
                     self.all_site_dirs.append(normalize_path(d))
         if not self.editable: self.check_site_dir()
-        self.index_url = self.index_url or "http://pypi.python.org/simple"
+        self.index_url = self.index_url or "https://pypi.python.org/simple"
         self.shadow_path = self.all_site_dirs[:]
         for path_item in self.install_dir, normalize_path(self.script_dir):
             if path_item not in self.shadow_path:
diff --git a/setuptools/dist.py b/setuptools/dist.py
index 482c6bf..e7f5454 100644
--- a/setuptools/dist.py
+++ b/setuptools/dist.py
@@ -47,7 +47,6 @@ def assert_string_list(dist, attr, value):
         raise DistutilsSetupError(
             "%r must be a list of strings (got %r)" % (attr,value)
         )
-
 def check_nsp(dist, attr, value):
     """Verify that namespace packages are valid"""
     assert_string_list(dist,attr,value)
@@ -69,6 +68,10 @@ def check_extras(dist, attr, value):
     """Verify that extras_require mapping is valid"""
     try:
         for k,v in value.items():
+            if ':' in k:
+                k,m = k.split(':',1)
+                if pkg_resources.invalid_marker(m):
+                    raise DistutilsSetupError("Invalid environment marker: "+m)
             list(pkg_resources.parse_requirements(v))
     except (TypeError,ValueError,AttributeError):
         raise DistutilsSetupError(
@@ -77,9 +80,6 @@ def check_extras(dist, attr, value):
             "requirement specifiers."
         )
 
-
-
-
 def assert_bool(dist, attr, value):
     """Verify that value is True, False, 0, or 1"""
     if bool(value) != value:
diff --git a/setuptools/package_index.py b/setuptools/package_index.py
index 9a9c5d6..aab2bb2 100755
--- a/setuptools/package_index.py
+++ b/setuptools/package_index.py
@@ -1,6 +1,6 @@
 """PyPI and direct package downloading"""
 import sys, os.path, re, urlparse, urllib2, shutil, random, socket, cStringIO
-import httplib, urllib
+import httplib, urllib; from setuptools import ssl_support
 from pkg_resources import *
 from distutils import log
 from distutils.errors import DistutilsError
@@ -145,12 +145,11 @@ user_agent = "Python-urllib/%s setuptools/%s" % (
     urllib2.__version__, require('setuptools')[0].version
 )
 
-
 class PackageIndex(Environment):
     """A distribution index that scans web pages for download URLs"""
 
-    def __init__(self, index_url="http://pypi.python.org/simple", hosts=('*',),
-        *args, **kw
+    def __init__(self, index_url="https://pypi.python.org/simple", hosts=('*',),
+        ca_bundle=None, verify_ssl=True, *args, **kw
     ):
         Environment.__init__(self,*args,**kw)
         self.index_url = index_url + "/"[:not index_url.endswith('/')]
@@ -159,8 +158,9 @@ class PackageIndex(Environment):
         self.package_pages = {}
         self.allows = re.compile('|'.join(map(translate,hosts))).match
         self.to_scan = []
-
-
+        if verify_ssl and ssl_support.is_available and (ca_bundle or ssl_support.find_ca_bundle()):
+            self.opener = ssl_support.opener_for(ca_bundle)
+        else: self.opener = urllib2.urlopen
 
     def process_url(self, url, retrieve=False):
         """Evaluate a URL as a possible download, and maybe retrieve it"""
@@ -575,12 +575,13 @@ class PackageIndex(Environment):
     def open_url(self, url, warning=None):
         if url.startswith('file:'): return local_open(url)
         try:
-            return open_with_auth(url)
-        except urllib2.HTTPError, v:
-            return v
-        except urllib2.URLError, v:
-            reason = v.reason
-        except httplib.HTTPException, v: 
+            return open_with_auth(url, self.opener)
+        except urllib2.HTTPError:
+            return sys.exc_info()[1]
+        except urllib2.URLError:
+            reason = sys.exc_info()[1].reason
+        except httplib.HTTPException:
+            v = sys.exc_info()[1]
             reason = "%s: %s" % (v.__doc__ or v.__class__.__name__, v)
         if warning:
             self.warn(warning, reason)
@@ -612,7 +613,6 @@ class PackageIndex(Environment):
             self.url_ok(url, True)   # raises error if not allowed
             return self._attempt_download(url, filename)
 
-
     def scan_url(self, url):
         self.process_url(url, True)
 
@@ -736,7 +736,7 @@ def htmldecode(text):
 
 
 
-def open_with_auth(url):
+def open_with_auth(url, opener=urllib2.urlopen):
     """Open a urllib2 request, handling HTTP authentication"""
 
     scheme, netloc, path, params, query, frag = urlparse.urlparse(url)
@@ -755,7 +755,7 @@ def open_with_auth(url):
         request = urllib2.Request(url)
 
     request.add_header('User-Agent', user_agent)
-    fp = urllib2.urlopen(request)
+    fp = opener(request)
 
     if auth:
         # Put authentication info back into request URL if same host,
diff --git a/setuptools/sandbox.py b/setuptools/sandbox.py
index 4c5e712..8b889a7 100755
--- a/setuptools/sandbox.py
+++ b/setuptools/sandbox.py
@@ -206,7 +206,7 @@ class DirectorySandbox(AbstractSandbox):
     def tmpnam(self): self._violation("tmpnam")
 
     def _ok(self,path):
-        if hasattr(_os,'devnull') and path==_os.devnull: return True
+        if hasattr(os,'devnull') and path==os.devnull: return True
         active = self._active
         try:
             self._active = False
diff --git a/setuptools/tests/api_tests.txt b/setuptools/tests/api_tests.txt
index 823d815..e35dad1 100755
--- a/setuptools/tests/api_tests.txt
+++ b/setuptools/tests/api_tests.txt
@@ -328,3 +328,94 @@ setuptools is provided as well::
     >>> cp("darwin-8.2.0-Power_Macintosh", "macosx-10.3-ppc")
     False
 
+
+Environment Markers
+-------------------
+
+    >>> from pkg_resources import invalid_marker as im, evaluate_marker as em
+    >>> import os
+
+    >>> print(im("sys_platform"))
+    Comparison or logical expression expected
+
+    >>> print(im("sys_platform=="))
+    unexpected EOF while parsing (line 1)
+    
+    >>> print(im("sys_platform=='win32'"))
+    False
+
+    >>> print(im("sys=='x'"))
+    Unknown name 'sys'
+
+    >>> print(im("(extra)"))
+    Comparison or logical expression expected
+
+    >>> print(im("(extra"))
+    unexpected EOF while parsing (line 1)
+
+    >>> print(im("os.open('foo')=='y'"))
+    Language feature not supported in environment markers
+    
+    >>> print(im("'x'=='y' and os.open('foo')=='y'"))   # no short-circuit!
+    Language feature not supported in environment markers
+
+    >>> print(im("'x'=='x' or os.open('foo')=='y'"))   # no short-circuit!
+    Language feature not supported in environment markers
+
+    >>> print(im("'x' < 'y'"))
+    '<' operator not allowed in environment markers
+    
+    >>> print(im("'x' < 'y' < 'z'"))
+    Chained comparison not allowed in environment markers
+
+    >>> print(im("r'x'=='x'"))
+    Only plain strings allowed in environment markers
+    
+    >>> print(im("'''x'''=='x'"))
+    Only plain strings allowed in environment markers
+    
+    >>> print(im('"""x"""=="x"'))
+    Only plain strings allowed in environment markers
+    
+    >>> print(im(r"'x\n'=='x'"))
+    Only plain strings allowed in environment markers
+
+    >>> print(im("os.open=='y'"))
+    Language feature not supported in environment markers
+    
+    >>> em('"x"=="x"')
+    True
+
+    >>> em('"x"=="y"')
+    False
+
+    >>> em('"x"=="y" and "x"=="x"')
+    False
+
+    >>> em('"x"=="y" or "x"=="x"')
+    True
+
+    >>> em('"x"=="y" and "x"=="q" or "z"=="z"')
+    True
+
+    >>> em('"x"=="y" and ("x"=="q" or "z"=="z")')
+    False
+
+    >>> em('"x"=="y" and "z"=="z" or "x"=="q"')
+    False
+
+    >>> em('"x"=="x" and "z"=="z" or "x"=="q"')
+    True
+
+    >>> em("sys_platform=='win32'") == (sys.platform=='win32')
+    True
+
+    >>> em("'x' in 'yx'")
+    True
+
+    >>> em("'yx' in 'x'")
+    False
+
+
+
+
diff --git a/setuptools/tests/test_resources.py b/setuptools/tests/test_resources.py
index 1c010e7..ad2d02f 100644
--- a/setuptools/tests/test_resources.py
+++ b/setuptools/tests/test_resources.py
@@ -507,7 +507,7 @@ class ScriptHeaderTests(TestCase):
     def test_get_script_header_jython_workaround(self):
         platform = sys.platform
         sys.platform = 'java1.5.0_13'
-        stdout = sys.stdout
+        stdout, stderr = sys.stdout, sys.stderr
         try:
             # A mock sys.executable that uses a shebang line (this file)
             exe = os.path.normpath(os.path.splitext(__file__)[0] + '.py')
@@ -517,17 +517,17 @@ class ScriptHeaderTests(TestCase):
 
             # Ensure we generate what is basically a broken shebang line
             # when there's options, with a warning emitted
-            sys.stdout = StringIO.StringIO()
+            sys.stdout = sys.stderr = StringIO.StringIO()
             self.assertEqual(get_script_header('#!/usr/bin/python -x',
                                                executable=exe),
                              '#!%s  -x\n' % exe)
             self.assert_('Unable to adapt shebang line' in sys.stdout.getvalue())
-            sys.stdout = StringIO.StringIO()
+            sys.stdout = sys.stderr = StringIO.StringIO()
             self.assertEqual(get_script_header('#!/usr/bin/python',
                                                executable=self.non_ascii_exe),
                              '#!%s -x\n' % self.non_ascii_exe)
             self.assert_('Unable to adapt shebang line' in sys.stdout.getvalue())
         finally:
             sys.platform = platform
-            sys.stdout = stdout
+            sys.stdout, sys.stderr = stdout, stderr