Merged in philip_thiem/setuptools (pull request #27)

Fixed tests for skipping when svn not present.

7 files changed, 200 insertions, 182 deletions

diff --git a/CHANGES.txt b/CHANGES.txt
index f04f5b91..90e2aaa8 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -6,6 +6,7 @@ CHANGES
 2.0
 ---
 
+* Issue #121: Exempt lib2to3 pickled grammars from DirectorySandbox.
 * Issue #41: Dropped support for Python 2.4 and Python 2.5. Clients requiring
   setuptools for those versions of Python should use setuptools 1.x.
 * Removed ``setuptools.command.easy_install.HAS_USER_SITE``. Clients
diff --git a/ez_setup.py b/ez_setup.py
index 2aa994b3..371000f9 100644
--- a/ez_setup.py
+++ b/ez_setup.py
@@ -30,7 +30,7 @@ try:
 except ImportError:
     USER_SITE = None
 
-DEFAULT_VERSION = "2.0"
+DEFAULT_VERSION = "2.1"
 DEFAULT_URL = "https://pypi.python.org/packages/source/s/setuptools/"
 
 def _python_cmd(*args):
diff --git a/pkg_resources.py b/pkg_resources.py
index 36a7b306..c6228545 100644
--- a/pkg_resources.py
+++ b/pkg_resources.py
@@ -24,6 +24,10 @@ import warnings
 import stat
 import functools
 import pkgutil
+import token
+import symbol
+import operator
+import platform
 from pkgutil import get_importer
 
 try:
@@ -1103,143 +1107,133 @@ def to_filename(name):
     """
     return name.replace('-','_')
 
-_marker_names = {
-    'os': ['name'], 'sys': ['platform'],
-    'platform': ['version','machine','python_implementation'],
-    'python_version': [], 'python_full_version': [], 'extra':[],
-}
-
-_marker_values = {
-    'os_name': lambda: os.name,
-    'sys_platform': lambda: sys.platform,
-    'python_full_version': lambda: sys.version.split()[0],
-    'python_version': lambda:'%s.%s' % (sys.version_info[0], sys.version_info[1]),
-    'platform_version': lambda: _platinfo('version'),
-    'platform_machine': lambda: _platinfo('machine'),
-    'python_implementation': lambda: _platinfo('python_implementation') or _pyimp(),
-}
-
-def _platinfo(attr):
-    try:
-        import platform
-    except ImportError:
-        return ''
-    return getattr(platform, attr, lambda:'')()
-
-def _pyimp():
-    if sys.platform=='cli':
-        return 'IronPython'
-    elif sys.platform.startswith('java'):
-        return 'Jython'
-    elif '__pypy__' in sys.builtin_module_names:
-        return 'PyPy'
-    else:
-        return 'CPython'
 
-def normalize_exception(exc):
-    """
-    Given a SyntaxError from a marker evaluation, normalize the error message:
-     - Remove indications of filename and line number.
-     - Replace platform-specific error messages with standard error messages.
-    """
-    subs = {
-        'unexpected EOF while parsing': 'invalid syntax',
-        'parenthesis is never closed': 'invalid syntax',
+class MarkerEvaluation(object):
+    values = {
+        'os_name': lambda: os.name,
+        'sys_platform': lambda: sys.platform,
+        'python_full_version': lambda: sys.version.split()[0],
+        'python_version': lambda:'%s.%s' % (sys.version_info[0], sys.version_info[1]),
+        'platform_version': platform.version,
+        'platform_machine': platform.machine,
+        'python_implementation': platform.python_implementation,
     }
-    exc.filename = None
-    exc.lineno = None
-    exc.msg = subs.get(exc.msg, exc.msg)
-    return exc
 
+    @classmethod
+    def is_invalid_marker(cls, text):
+        """
+        Validate text as a PEP 426 environment marker; return an exception
+        if invalid or False otherwise.
+        """
+        try:
+            cls.evaluate_marker(text)
+        except SyntaxError:
+            return cls.normalize_exception(sys.exc_info()[1])
+        return False
 
-def invalid_marker(text):
-    """Validate text as a PEP 426 environment marker; return exception or False"""
-    try:
-        evaluate_marker(text)
-    except SyntaxError:
-        return normalize_exception(sys.exc_info()[1])
-    return False
+    @staticmethod
+    def normalize_exception(exc):
+        """
+        Given a SyntaxError from a marker evaluation, normalize the error message:
+         - Remove indications of filename and line number.
+         - Replace platform-specific error messages with standard error messages.
+        """
+        subs = {
+            'unexpected EOF while parsing': 'invalid syntax',
+            'parenthesis is never closed': 'invalid syntax',
+        }
+        exc.filename = None
+        exc.lineno = None
+        exc.msg = subs.get(exc.msg, exc.msg)
+        return exc
+
+    @classmethod
+    def and_test(cls, nodelist):
+        # MUST NOT short-circuit evaluation, or invalid syntax can be skipped!
+        return functools.reduce(operator.and_, [cls.interpret(nodelist[i]) for i in range(1,len(nodelist),2)])
+
+    @classmethod
+    def test(cls, nodelist):
+        # MUST NOT short-circuit evaluation, or invalid syntax can be skipped!
+        return functools.reduce(operator.or_, [cls.interpret(nodelist[i]) for i in range(1,len(nodelist),2)])
+
+    @classmethod
+    def atom(cls, nodelist):
+        t = nodelist[1][0]
+        if t == token.LPAR:
+            if nodelist[2][0] == token.RPAR:
+                raise SyntaxError("Empty parentheses")
+            return cls.interpret(nodelist[2])
+        raise SyntaxError("Language feature not supported in environment markers")
 
-def evaluate_marker(text, extra=None, _ops={}):
-    """
-    Evaluate a PEP 426 environment marker on CPython 2.4+.
-    Return a boolean indicating the marker result in this environment.
-    Raise SyntaxError if marker is invalid.
+    @classmethod
+    def comparison(cls, nodelist):
+        if len(nodelist)>4:
+            raise SyntaxError("Chained comparison not allowed in environment markers")
+        comp = nodelist[2][1]
+        cop = comp[1]
+        if comp[0] == token.NAME:
+            if len(nodelist[2]) == 3:
+                if cop == 'not':
+                    cop = 'not in'
+                else:
+                    cop = 'is not'
+        try:
+            cop = cls.get_op(cop)
+        except KeyError:
+            raise SyntaxError(repr(cop)+" operator not allowed in environment markers")
+        return cop(cls.evaluate(nodelist[1]), cls.evaluate(nodelist[3]))
+
+    @classmethod
+    def get_op(cls, op):
+        ops = {
+            symbol.test: cls.test,
+            symbol.and_test: cls.and_test,
+            symbol.atom: cls.atom,
+            symbol.comparison: cls.comparison,
+            'not in': lambda x, y: x not in y,
+            'in': lambda x, y: x in y,
+            '==': operator.eq,
+            '!=': operator.ne,
+        }
+        if hasattr(symbol, 'or_test'):
+            ops[symbol.or_test] = cls.test
+        return ops[op]
+
+    @classmethod
+    def evaluate_marker(cls, text, extra=None):
+        """
+        Evaluate a PEP 426 environment marker on CPython 2.4+.
+        Return a boolean indicating the marker result in this environment.
+        Raise SyntaxError if marker is invalid.
 
-    This implementation uses the 'parser' module, which is not implemented on
-    Jython and has been superseded by the 'ast' module in Python 2.6 and
-    later.
-    """
+        This implementation uses the 'parser' module, which is not implemented on
+        Jython and has been superseded by the 'ast' module in Python 2.6 and
+        later.
+        """
+        return cls.interpret(parser.expr(text).totuple(1)[1])
 
-    if not _ops:
-
-        from token import NAME, STRING
-        import token
-        import symbol
-        import operator
-
-        def and_test(nodelist):
-            # MUST NOT short-circuit evaluation, or invalid syntax can be skipped!
-            return functools.reduce(operator.and_, [interpret(nodelist[i]) for i in range(1,len(nodelist),2)])
-
-        def test(nodelist):
-            # MUST NOT short-circuit evaluation, or invalid syntax can be skipped!
-            return functools.reduce(operator.or_, [interpret(nodelist[i]) for i in range(1,len(nodelist),2)])
-
-        def atom(nodelist):
-            t = nodelist[1][0]
-            if t == token.LPAR:
-                if nodelist[2][0] == token.RPAR:
-                    raise SyntaxError("Empty parentheses")
-                return interpret(nodelist[2])
-            raise SyntaxError("Language feature not supported in environment markers")
-
-        def comparison(nodelist):
-            if len(nodelist)>4:
-                raise SyntaxError("Chained comparison not allowed in environment markers")
-            comp = nodelist[2][1]
-            cop = comp[1]
-            if comp[0] == NAME:
-                if len(nodelist[2]) == 3:
-                    if cop == 'not':
-                        cop = 'not in'
-                    else:
-                        cop = 'is not'
-            try:
-                cop = _ops[cop]
-            except KeyError:
-                raise SyntaxError(repr(cop)+" operator not allowed in environment markers")
-            return cop(evaluate(nodelist[1]), evaluate(nodelist[3]))
-
-        _ops.update({
-            symbol.test: test, symbol.and_test: and_test, symbol.atom: atom,
-            symbol.comparison: comparison, 'not in': lambda x,y: x not in y,
-            'in': lambda x,y: x in y, '==': operator.eq, '!=': operator.ne,
-        })
-        if hasattr(symbol,'or_test'):
-            _ops[symbol.or_test] = test
-
-    def interpret(nodelist):
+    @classmethod
+    def interpret(cls, nodelist):
         while len(nodelist)==2: nodelist = nodelist[1]
         try:
-            op = _ops[nodelist[0]]
+            op = cls.get_op(nodelist[0])
         except KeyError:
             raise SyntaxError("Comparison or logical expression expected")
-            raise SyntaxError("Language feature not supported in environment markers: "+symbol.sym_name[nodelist[0]])
         return op(nodelist)
 
-    def evaluate(nodelist):
+    @classmethod
+    def evaluate(cls, nodelist):
         while len(nodelist)==2: nodelist = nodelist[1]
         kind = nodelist[0]
         name = nodelist[1]
-        #while len(name)==2: name = name[1]
-        if kind==NAME:
+        if kind==token.NAME:
             try:
-                op = _marker_values[name]
+                op = cls.values[name]
             except KeyError:
                 raise SyntaxError("Unknown name %r" % name)
             return op()
-        if kind==STRING:
+        if kind==token.STRING:
             s = nodelist[1]
             if s[:1] not in "'\"" or s.startswith('"""') or s.startswith("'''") \
                     or '\\' in s:
@@ -1248,8 +1242,6 @@ def evaluate_marker(text, extra=None, _ops={}):
             return s[1:-1]
         raise SyntaxError("Language feature not supported in environment markers")
 
-    return interpret(parser.expr(text).totuple(1)[1])
-
 def _markerlib_evaluate(text):
     """
     Evaluate a PEP 426 environment marker using markerlib.
@@ -1270,7 +1262,11 @@ def _markerlib_evaluate(text):
         raise SyntaxError(e.args[0])
     return result
 
-if 'parser' not in globals():
+invalid_marker = MarkerEvaluation.is_invalid_marker
+
+if 'parser' in globals():
+    evaluate_marker = MarkerEvaluation.evaluate_marker
+else:
     # fallback to less-complete _markerlib implementation if 'parser' module
     #  is not available.
     evaluate_marker = _markerlib_evaluate
diff --git a/setuptools.egg-info/entry_points.txt b/setuptools.egg-info/entry_points.txt
index d423a67d..23b577f6 100644
--- a/setuptools.egg-info/entry_points.txt
+++ b/setuptools.egg-info/entry_points.txt
@@ -1,62 +1,62 @@
-[console_scripts]
-easy_install = setuptools.command.easy_install:main
-easy_install-3.3 = setuptools.command.easy_install:main
-
-[setuptools.installation]
-eggsecutable = setuptools.command.easy_install:bootstrap
-
-[setuptools.file_finders]
-svn_cvs = setuptools.command.sdist:_default_revctrl
-
 [distutils.setup_keywords]
-use_2to3_fixers = setuptools.dist:assert_string_list
-package_data = setuptools.dist:check_package_data
-use_2to3 = setuptools.dist:assert_bool
-namespace_packages = setuptools.dist:check_nsp
-test_suite = setuptools.dist:check_test_suite
-use_2to3_exclude_fixers = setuptools.dist:assert_string_list
-packages = setuptools.dist:check_packages
-exclude_package_data = setuptools.dist:check_package_data
-dependency_links = setuptools.dist:assert_string_list
-eager_resources = setuptools.dist:assert_string_list
+convert_2to3_doctests = setuptools.dist:assert_string_list
 install_requires = setuptools.dist:check_requirements
+exclude_package_data = setuptools.dist:check_package_data
 include_package_data = setuptools.dist:assert_bool
-convert_2to3_doctests = setuptools.dist:assert_string_list
-entry_points = setuptools.dist:check_entry_points
 extras_require = setuptools.dist:check_extras
-test_loader = setuptools.dist:check_importable
+entry_points = setuptools.dist:check_entry_points
+dependency_links = setuptools.dist:assert_string_list
+use_2to3 = setuptools.dist:assert_bool
+use_2to3_fixers = setuptools.dist:assert_string_list
+use_2to3_exclude_fixers = setuptools.dist:assert_string_list
+packages = setuptools.dist:check_packages
+package_data = setuptools.dist:check_package_data
 tests_require = setuptools.dist:check_requirements
+test_suite = setuptools.dist:check_test_suite
+namespace_packages = setuptools.dist:check_nsp
 zip_safe = setuptools.dist:assert_bool
+test_loader = setuptools.dist:check_importable
+eager_resources = setuptools.dist:assert_string_list
+
+[setuptools.installation]
+eggsecutable = setuptools.command.easy_install:bootstrap
+
+[egg_info.writers]
+eager_resources.txt = setuptools.command.egg_info:overwrite_arg
+dependency_links.txt = setuptools.command.egg_info:overwrite_arg
+depends.txt = setuptools.command.egg_info:warn_depends_obsolete
+namespace_packages.txt = setuptools.command.egg_info:overwrite_arg
+requires.txt = setuptools.command.egg_info:write_requirements
+top_level.txt = setuptools.command.egg_info:write_toplevel_names
+PKG-INFO = setuptools.command.egg_info:write_pkg_info
+entry_points.txt = setuptools.command.egg_info:write_entries
 
 [distutils.commands]
-bdist_wininst = setuptools.command.bdist_wininst:bdist_wininst
+install_egg_info = setuptools.command.install_egg_info:install_egg_info
+egg_info = setuptools.command.egg_info:egg_info
+saveopts = setuptools.command.saveopts:saveopts
+sdist = setuptools.command.sdist:sdist
+rotate = setuptools.command.rotate:rotate
+bdist_rpm = setuptools.command.bdist_rpm:bdist_rpm
+upload_docs = setuptools.command.upload_docs:upload_docs
+build_py = setuptools.command.build_py:build_py
+setopt = setuptools.command.setopt:setopt
+develop = setuptools.command.develop:develop
+install_lib = setuptools.command.install_lib:install_lib
 test = setuptools.command.test:test
-install = setuptools.command.install:install
+install_scripts = setuptools.command.install_scripts:install_scripts
 register = setuptools.command.register:register
-develop = setuptools.command.develop:develop
-bdist_egg = setuptools.command.bdist_egg:bdist_egg
-egg_info = setuptools.command.egg_info:egg_info
 build_ext = setuptools.command.build_ext:build_ext
-setopt = setuptools.command.setopt:setopt
 easy_install = setuptools.command.easy_install:easy_install
-upload_docs = setuptools.command.upload_docs:upload_docs
-bdist_rpm = setuptools.command.bdist_rpm:bdist_rpm
-install_lib = setuptools.command.install_lib:install_lib
-rotate = setuptools.command.rotate:rotate
-sdist = setuptools.command.sdist:sdist
 alias = setuptools.command.alias:alias
-saveopts = setuptools.command.saveopts:saveopts
-build_py = setuptools.command.build_py:build_py
-install_egg_info = setuptools.command.install_egg_info:install_egg_info
-install_scripts = setuptools.command.install_scripts:install_scripts
+install = setuptools.command.install:install
+bdist_egg = setuptools.command.bdist_egg:bdist_egg
+bdist_wininst = setuptools.command.bdist_wininst:bdist_wininst
 
-[egg_info.writers]
-PKG-INFO = setuptools.command.egg_info:write_pkg_info
-top_level.txt = setuptools.command.egg_info:write_toplevel_names
-namespace_packages.txt = setuptools.command.egg_info:overwrite_arg
-eager_resources.txt = setuptools.command.egg_info:overwrite_arg
-depends.txt = setuptools.command.egg_info:warn_depends_obsolete
-dependency_links.txt = setuptools.command.egg_info:overwrite_arg
-requires.txt = setuptools.command.egg_info:write_requirements
-entry_points.txt = setuptools.command.egg_info:write_entries
+[setuptools.file_finders]
+svn_cvs = setuptools.command.sdist:_default_revctrl
+
+[console_scripts]
+easy_install = setuptools.command.easy_install:main
+easy_install-3.3 = setuptools.command.easy_install:main
 
diff --git a/setuptools.egg-info/requires.txt b/setuptools.egg-info/requires.txt
index 9a6bf437..4fd464d8 100644
--- a/setuptools.egg-info/requires.txt
+++ b/setuptools.egg-info/requires.txt
@@ -1,7 +1,7 @@
 
 
-[certs]
-certifi==0.0.8
-
 [ssl:sys_platform=='win32']
-wincertstore==0.1
\ No newline at end of file
+wincertstore==0.1
+
+[certs]
+certifi==0.0.8
\ No newline at end of file
diff --git a/setuptools/sandbox.py b/setuptools/sandbox.py
index 6dd1ca07..042c5958 100755
--- a/setuptools/sandbox.py
+++ b/setuptools/sandbox.py
@@ -3,6 +3,8 @@ import sys
 import tempfile
 import operator
 import functools
+import itertools
+import re
 
 import pkg_resources
 
@@ -197,10 +199,19 @@ class DirectorySandbox(AbstractSandbox):
         "utime", "lchown", "chroot", "mkfifo", "mknod", "tempnam",
     ])
 
+    _exception_patterns = [
+        # Allow lib2to3 to attempt to save a pickled grammar object (#121)
+        '.*lib2to3.*\.pickle$',
+    ]
+    "exempt writing to paths that match the pattern"
+
     def __init__(self, sandbox, exceptions=_EXCEPTIONS):
         self._sandbox = os.path.normcase(os.path.realpath(sandbox))
         self._prefix = os.path.join(self._sandbox,'')
-        self._exceptions = [os.path.normcase(os.path.realpath(path)) for path in exceptions]
+        self._exceptions = [
+            os.path.normcase(os.path.realpath(path))
+            for path in exceptions
+        ]
         AbstractSandbox.__init__(self)
 
     def _violation(self, operation, *args, **kw):
@@ -220,28 +231,38 @@ class DirectorySandbox(AbstractSandbox):
     def tmpnam(self):
         self._violation("tmpnam")
 
-    def _ok(self,path):
+    def _ok(self, path):
         active = self._active
         try:
             self._active = False
             realpath = os.path.normcase(os.path.realpath(path))
-            if (self._exempted(realpath) or realpath == self._sandbox
-                    or realpath.startswith(self._prefix)):
-                return True
+            return (
+                self._exempted(realpath)
+                or realpath == self._sandbox
+                or realpath.startswith(self._prefix)
+            )
         finally:
             self._active = active
 
     def _exempted(self, filepath):
-        exception_matches = map(filepath.startswith, self._exceptions)
-        return True in exception_matches
+        start_matches = (
+            filepath.startswith(exception)
+            for exception in self._exceptions
+        )
+        pattern_matches = (
+            re.match(pattern, filepath)
+            for pattern in self._exception_patterns
+        )
+        candidates = itertools.chain(start_matches, pattern_matches)
+        return any(candidates)
 
-    def _remap_input(self,operation,path,*args,**kw):
+    def _remap_input(self, operation, path, *args, **kw):
         """Called for path inputs"""
         if operation in self.write_ops and not self._ok(path):
             self._violation(operation, os.path.realpath(path), *args, **kw)
         return path
 
-    def _remap_pair(self,operation,src,dst,*args,**kw):
+    def _remap_pair(self, operation, src, dst, *args, **kw):
         """Called for path pairs like rename, link, and symlink operations"""
         if not self._ok(src) or not self._ok(dst):
             self._violation(operation, src, dst, *args, **kw)
diff --git a/setuptools/version.py b/setuptools/version.py
index 3b3dacb9..d980f276 100644
--- a/setuptools/version.py
+++ b/setuptools/version.py
@@ -1 +1 @@
-__version__ = '2.0'
+__version__ = '2.1'