Merged in moben/setuptools (pull request #79)

make order of lines in top_level.txt deterministic

8 files changed, 180 insertions, 57 deletions

diff --git a/CHANGES.txt b/CHANGES.txt
index 83b05bbe..4b20333d 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -3,6 +3,40 @@ CHANGES
 =======
 
 ---
+6.0
+---
+
+* Issue #100: When building a distribution, Setuptools will no longer match
+  default files using platform-dependent case sensitivity, but rather will
+  only match the files if their case matches exactly. As a result, on Windows
+  and other case-insensitive file systems, files with names such as
+  'readme.txt' or 'README.TXT' will be omitted from the distribution and a
+  warning will be issued indicating that 'README.txt' was not found. Other
+  filenames affected are:
+
+    - README.rst
+    - README
+    - setup.cfg
+    - setup.py (or the script name)
+    - test/test*.py
+
+  Any users producing distributions with filenames that match those above
+  case-insensitively, but not case-sensitively, should rename those files in
+  their repository for better portability.
+* Pull Request #72: When using ``single_version_externally_managed``, the
+  exclusion list now includes Python 3.2 ``__pycache__`` entries.
+* Pull Request #76 and Pull Request #78: lines in top_level.txt are now
+  ordered deterministically.
+
+---
+5.8
+---
+
+* Issue #237: ``pkg_resources`` now uses explicit detection of Python 2 vs.
+  Python 3, supporting environments where builtins have been patched to make
+  Python 3 look more like Python 2.
+
+---
 5.7
 ---
 
diff --git a/ez_setup.py b/ez_setup.py
index 74c8224a..0fd1c872 100644
--- a/ez_setup.py
+++ b/ez_setup.py
@@ -36,7 +36,7 @@ try:
 except ImportError:
     USER_SITE = None
 
-DEFAULT_VERSION = "5.8"
+DEFAULT_VERSION = "5.9"
 DEFAULT_URL = "https://pypi.python.org/packages/source/s/setuptools/"
 
 def _python_cmd(*args):
diff --git a/pkg_resources.py b/pkg_resources.py
index ee2c553b..517298c9 100644
--- a/pkg_resources.py
+++ b/pkg_resources.py
@@ -16,6 +16,7 @@ method.
 
 import sys
 import os
+import io
 import time
 import re
 import imp
@@ -35,28 +36,19 @@ import email.parser
 import tempfile
 from pkgutil import get_importer
 
-try:
-    from urlparse import urlparse, urlunparse
-except ImportError:
+PY3 = sys.version_info > (3,)
+PY2 = not PY3
+
+if PY3:
     from urllib.parse import urlparse, urlunparse
 
-try:
-    frozenset
-except NameError:
-    from sets import ImmutableSet as frozenset
-try:
-    basestring
-    next = lambda o: o.next()
-    from cStringIO import StringIO as BytesIO
-except NameError:
-    basestring = str
-    from io import BytesIO
-    def execfile(fn, globs=None, locs=None):
-        if globs is None:
-            globs = globals()
-        if locs is None:
-            locs = globs
-        exec(compile(open(fn).read(), fn, 'exec'), globs, locs)
+if PY2:
+    from urlparse import urlparse, urlunparse
+
+if PY3:
+    string_types = str,
+else:
+    string_types = str, eval('unicode')
 
 # capture these to bypass sandboxing
 from os import utime
@@ -81,15 +73,6 @@ try:
 except ImportError:
     pass
 
-def _bypass_ensure_directory(name, mode=0o777):
-    # Sandbox-bypassing version of ensure_directory()
-    if not WRITE_SUPPORT:
-        raise IOError('"os.mkdir" not supported on this platform.')
-    dirname, filename = split(name)
-    if dirname and filename and not isdir(dirname):
-        _bypass_ensure_directory(dirname)
-        mkdir(dirname, mode)
-
 
 _state_vars = {}
 
@@ -343,7 +326,7 @@ run_main = run_script
 
 def get_distribution(dist):
     """Return a current distribution object for a Requirement or string"""
-    if isinstance(dist, basestring):
+    if isinstance(dist, string_types):
         dist = Requirement.parse(dist)
     if isinstance(dist, Requirement):
         dist = get_provider(dist)
@@ -1387,7 +1370,7 @@ class NullProvider:
         return self._fn(self.module_path, resource_name)
 
     def get_resource_stream(self, manager, resource_name):
-        return BytesIO(self.get_resource_string(manager, resource_name))
+        return io.BytesIO(self.get_resource_string(manager, resource_name))
 
     def get_resource_string(self, manager, resource_name):
         return self._get(self._fn(self.module_path, resource_name))
@@ -1435,7 +1418,9 @@ class NullProvider:
         script_filename = self._fn(self.egg_info, script)
         namespace['__file__'] = script_filename
         if os.path.exists(script_filename):
-            execfile(script_filename, namespace, namespace)
+            source = open(script_filename).read()
+            code = compile(source, script_filename, 'exec')
+            exec(code, namespace, namespace)
         else:
             from linecache import cache
             cache[script_filename] = (
@@ -2063,8 +2048,8 @@ def _set_parent_ns(packageName):
 
 
 def yield_lines(strs):
-    """Yield non-empty/non-comment lines of a ``basestring`` or sequence"""
-    if isinstance(strs, basestring):
+    """Yield non-empty/non-comment lines of a string or sequence"""
+    if isinstance(strs, string_types):
         for s in strs.splitlines():
             s = s.strip()
             # skip blank lines/comments
@@ -2656,8 +2641,7 @@ def issue_warning(*args,**kw):
 def parse_requirements(strs):
     """Yield ``Requirement`` objects for each specification in `strs`
 
-    `strs` must be an instance of ``basestring``, or a (possibly-nested)
-    iterable thereof.
+    `strs` must be a string, or a (possibly-nested) iterable thereof.
     """
     # create a steppable iterator, so we can handle \-continuations
     lines = iter(yield_lines(strs))
@@ -2758,7 +2742,7 @@ class Requirement:
             # only get if we need it
             if self.index:
                 item = item.parsed_version
-        elif isinstance(item, basestring):
+        elif isinstance(item, string_types):
             item = parse_version(item)
         last = None
         # -1, 0, 1
@@ -2824,6 +2808,17 @@ def ensure_directory(path):
     if not os.path.isdir(dirname):
         os.makedirs(dirname)
 
+
+def _bypass_ensure_directory(path, mode=0o777):
+    """Sandbox-bypassing version of ensure_directory()"""
+    if not WRITE_SUPPORT:
+        raise IOError('"os.mkdir" not supported on this platform.')
+    dirname, filename = split(path)
+    if dirname and filename and not isdir(dirname):
+        _bypass_ensure_directory(dirname)
+        mkdir(dirname, mode)
+
+
 def split_sections(s):
     """Split a string or iterable thereof into (section, content) pairs
 
diff --git a/setuptools/command/install_lib.py b/setuptools/command/install_lib.py
index d7e117f0..3cd16a8f 100644
--- a/setuptools/command/install_lib.py
+++ b/setuptools/command/install_lib.py
@@ -1,6 +1,7 @@
-import distutils.command.install_lib as orig
 import os
-
+import imp
+from itertools import product, starmap
+import distutils.command.install_lib as orig
 
 class install_lib(orig.install_lib):
     """Don't add compiled flags to filenames of non-Python files"""
@@ -13,19 +14,71 @@ class install_lib(orig.install_lib):
             self.byte_compile(outfiles)
 
     def get_exclusions(self):
-        exclude = {}
-        nsp = self.distribution.namespace_packages
-        svem = (nsp and self.get_finalized_command('install')
-                .single_version_externally_managed)
-        if svem:
-            for pkg in nsp:
-                parts = pkg.split('.')
-                while parts:
-                    pkgdir = os.path.join(self.install_dir, *parts)
-                    for f in '__init__.py', '__init__.pyc', '__init__.pyo':
-                        exclude[os.path.join(pkgdir, f)] = 1
-                    parts.pop()
-        return exclude
+        """
+        Return a collections.Sized collections.Container of paths to be
+        excluded for single_version_externally_managed installations.
+        """
+        all_packages = (
+            pkg
+            for ns_pkg in self._get_SVEM_NSPs()
+            for pkg in self._all_packages(ns_pkg)
+        )
+
+        excl_specs = product(all_packages, self._gen_exclusion_paths())
+        return set(starmap(self._exclude_pkg_path, excl_specs))
+
+    def _exclude_pkg_path(self, pkg, exclusion_path):
+        """
+        Given a package name and exclusion path within that package,
+        compute the full exclusion path.
+        """
+        parts = pkg.split('.') + [exclusion_path]
+        return os.path.join(self.install_dir, *parts)
+
+    @staticmethod
+    def _all_packages(pkg_name):
+        """
+        >>> list(install_lib._all_packages('foo.bar.baz'))
+        ['foo.bar.baz', 'foo.bar', 'foo']
+        """
+        while pkg_name:
+            yield pkg_name
+            pkg_name, sep, child = pkg_name.partition('.')
+
+    def _get_SVEM_NSPs(self):
+        """
+        Get namespace packages (list) but only for
+        single_version_externally_managed installations and empty otherwise.
+        """
+        # TODO: is it necessary to short-circuit here? i.e. what's the cost
+        # if get_finalized_command is called even when namespace_packages is
+        # False?
+        if not self.distribution.namespace_packages:
+            return []
+
+        install_cmd = self.get_finalized_command('install')
+        svem = install_cmd.single_version_externally_managed
+
+        return self.distribution.namespace_packages if svem else []
+
+    @staticmethod
+    def _gen_exclusion_paths():
+        """
+        Generate file paths to be excluded for namespace packages (bytecode
+        cache files).
+        """
+        # always exclude the package module itself
+        yield '__init__.py'
+
+        yield '__init__.pyc'
+        yield '__init__.pyo'
+
+        if not hasattr(imp, 'get_tag'):
+            return
+
+        base = os.path.join('__pycache__', '__init__.' + imp.get_tag())
+        yield base + '.pyc'
+        yield base + '.pyo'
 
     def copy_tree(
             self, infile, outfile,
diff --git a/setuptools/command/sdist.py b/setuptools/command/sdist.py
index 2aa1ee20..a77c39f2 100755
--- a/setuptools/command/sdist.py
+++ b/setuptools/command/sdist.py
@@ -8,6 +8,8 @@ import sys
 
 from setuptools import svn_utils
 from setuptools.compat import PY3
+from setuptools.utils import cs_path_exists
+
 import pkg_resources
 
 READMES = ('README', 'README.rst', 'README.txt')
@@ -146,7 +148,7 @@ class sdist(orig.sdist):
                 alts = fn
                 got_it = 0
                 for fn in alts:
-                    if os.path.exists(fn):
+                    if cs_path_exists(fn):
                         got_it = 1
                         self.filelist.append(fn)
                         break
@@ -155,14 +157,14 @@ class sdist(orig.sdist):
                     self.warn("standard file not found: should have one of " +
                               ', '.join(alts))
             else:
-                if os.path.exists(fn):
+                if cs_path_exists(fn):
                     self.filelist.append(fn)
                 else:
                     self.warn("standard file '%s' not found" % fn)
 
         optional = ['test/test*.py', 'setup.cfg']
         for pattern in optional:
-            files = list(filter(os.path.isfile, glob(pattern)))
+            files = list(filter(cs_path_exists, glob(pattern)))
             if files:
                 self.filelist.extend(files)
 
diff --git a/setuptools/tests/test_sdist.py b/setuptools/tests/test_sdist.py
index 5b3862e9..5f8a190f 100644
--- a/setuptools/tests/test_sdist.py
+++ b/setuptools/tests/test_sdist.py
@@ -86,6 +86,7 @@ class TestSdistTest(unittest.TestCase):
         f = open(os.path.join(self.temp_dir, 'setup.py'), 'w')
         f.write(SETUP_PY)
         f.close()
+
         # Set up the rest of the test package
         test_pkg = os.path.join(self.temp_dir, 'sdist_test')
         os.mkdir(test_pkg)
@@ -121,6 +122,33 @@ class TestSdistTest(unittest.TestCase):
         self.assertTrue(os.path.join('sdist_test', 'b.txt') in manifest)
         self.assertTrue(os.path.join('sdist_test', 'c.rst') not in manifest)
 
+
+    def test_defaults_case_sensitivity(self):
+        """
+            Make sure default files (README.*, etc.) are added in a case-sensitive
+            way to avoid problems with packages built on Windows.
+        """
+
+        open(os.path.join(self.temp_dir, 'readme.rst'), 'w').close()
+        open(os.path.join(self.temp_dir, 'SETUP.cfg'), 'w').close()
+
+        dist = Distribution(SETUP_ATTRS)
+        # the extension deliberately capitalized for this test
+        # to make sure the actual filename (not capitalized) gets added
+        # to the manifest
+        dist.script_name = 'setup.PY'
+        cmd = sdist(dist)
+        cmd.ensure_finalized()
+
+        with quiet():
+            cmd.run()
+
+        # lowercase all names so we can test in a case-insensitive way to make sure the files are not included
+        manifest = map(lambda x: x.lower(), cmd.filelist.files)
+        self.assertFalse('readme.rst' in manifest, manifest)
+        self.assertFalse('setup.py' in manifest, manifest)
+        self.assertFalse('setup.cfg' in manifest, manifest)
+
     def test_manifest_is_written_with_utf8_encoding(self):
         # Test for #303.
         dist = Distribution(SETUP_ATTRS)
diff --git a/setuptools/utils.py b/setuptools/utils.py
new file mode 100644
index 00000000..91e4b87f
--- /dev/null
+++ b/setuptools/utils.py
@@ -0,0 +1,11 @@
+import os
+import os.path
+
+
+def cs_path_exists(fspath):
+    if not os.path.exists(fspath): 
+        return False
+    # make absolute so we always have a directory
+    abspath = os.path.abspath(fspath)
+    directory, filename = os.path.split(abspath)
+    return filename in os.listdir(directory)
\ No newline at end of file
diff --git a/setuptools/version.py b/setuptools/version.py
index 868f2d33..7244139e 100644
--- a/setuptools/version.py
+++ b/setuptools/version.py
@@ -1 +1 @@
-__version__ = '5.8'
+__version__ = '5.9'