From 8f64fbe5e5a016bd88f19d108e126be7c23e757a Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Sun, 12 Jun 2005 03:44:07 +0000 Subject: Move package index/downloading stuff to setuptools.package_index module. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041051 --- setuptools/package_index.py | 369 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 369 insertions(+) create mode 100755 setuptools/package_index.py (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py new file mode 100755 index 00000000..9e48de35 --- /dev/null +++ b/setuptools/package_index.py @@ -0,0 +1,369 @@ +"""PyPI and direct package downloading""" + +import sys, os.path, re, urlparse, urllib2 +from pkg_resources import * + +HREF = re.compile(r"""href\s*=\s*['"]?([^'"> ]+)""", re.I) +URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):',re.I).match +EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split() + +__all__ = [ + 'PackageIndex', 'distros_for_url', +] + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +def distros_for_url(url, metadata=None): + """Yield egg or source distribution objects that might be found at a URL""" + + path = urlparse.urlparse(url)[2] + base = urllib2.unquote(path.split('/')[-1]) + + if base.endswith('.egg'): + dist = Distribution.from_filename(base, metadata) + dist.path = url + yield dist + return # only one, unambiguous interpretation + + for ext in EXTENSIONS: + if base.endswith(ext): + base = base[:-len(ext)] + break + else: + return # no extension matched + + # Generate alternative interpretations of a source distro name + # Because some packages are ambiguous as to name/versions split + # e.g. "adns-python-1.1.0", "egenix-mx-commercial", etc. + # So, we generate each possible interepretation (e.g. "adns, python-1.1.0" + # "adns-python, 1.1.0", and "adns-python-1.1.0, no version"). In practice, + # the spurious interpretations should be ignored, because in the event + # there's also an "adns" package, the spurious "python-1.1.0" version will + # compare lower than any numeric version number, and is therefore unlikely + # to match a request for it. It's still a potential problem, though, and + # in the long run PyPI and the distutils should go for "safe" names and + # versions in distribution archive names (sdist and bdist). + + parts = base.split('-') + for p in range(1,len(parts)+1): + yield Distribution( + url, metadata, '-'.join(parts[:p]), '-'.join(parts[p:]), + distro_type = SOURCE_DIST + ) + + + + +class PackageIndex(AvailableDistributions): + """A distribution index that scans web pages for download URLs""" + + def __init__(self,index_url="http://www.python.org/pypi",*args,**kw): + AvailableDistributions.__init__(self,*args,**kw) + self.index_url = index_url + "/"[:not index_url.endswith('/')] + self.scanned_urls = {} + self.fetched_urls = {} + self.package_pages = {} + + def scan_url(self, url): + self.process_url(url, True) + + def process_url(self, url, retrieve=False): + if url in self.scanned_urls and not retrieve: + return + + self.scanned_urls[url] = True + dists = list(distros_for_url(url)) + map(self.add, dists) + + if dists or not retrieve or url in self.fetched_urls: + # don't need the actual page + return + + f = self.open_url(url) + self.fetched_urls[url] = self.fetched_urls[f.url] = True + if 'html' not in f.headers['content-type'].lower(): + f.close() # not html, we can't process it + return + + base = f.url # handle redirects + page = f.read() + f.close() + if url.startswith(self.index_url): + self.process_index(url, page) + else: + for match in HREF.finditer(page): + link = urlparse.urljoin(base, match.group(1)) + self.process_url(link) + + def find_packages(self,requirement): + self.scan_url(self.index_url + requirement.distname) + if not self.package_pages.get(requirement.key): + # We couldn't find the target package, so search the index page too + self.scan_url(self.index_url) + for url in self.package_pages.get(requirement.key,()): + # scan each page that might be related to the desired package + self.scan_url(url) + + def process_index(self,url,page): + def scan(link): + if link.startswith(self.index_url): + parts = map( + urllib2.unquote, link[len(self.index_url):].split('/') + ) + if len(parts)==2: + # it's a package page, sanitize and index it + pkg = safe_name(parts[0]) + ver = safe_version(parts[1]) + self.package_pages.setdefault(pkg.lower(),{})[link] = True + if url==self.index_url or 'Index of Packages' in page: + # process an index page into the package-page index + for match in HREF.finditer(page): + scan( urlparse.urljoin(url, match.group(1)) ) + else: + scan(url) # ensure this page is in the page index + # process individual package page + for tag in ("Home Page", "Download URL"): + pos = page.find(tag) + if pos!=-1: + match = HREF.search(page,pos) + if match: + # Process the found URL + self.scan_url(urlparse.urljoin(url, match.group(1))) + + def obtain(self,requirement): + self.find_packages(requirement) + for dist in self.get(requirement.key, ()): + if dist in requirement: + return dist + + def download(self, spec, tmpdir): + """Locate and/or download `spec`, returning a local filename + + `spec` may be a ``Requirement`` object, or a string containing a URL, + an existing local filename, or a package/version requirement spec + (i.e. the string form of a ``Requirement`` object). + + If necessary, the requirement is searched for in the package index. + If the download is successful, the return value is a local file path, + and it is a subpath of `tmpdir` if the distribution had to be + downloaded. If no matching distribution is found, return ``None``. + Various errors may be raised if a problem occurs during downloading. + """ + + if not isinstance(spec,Requirement): + scheme = URL_SCHEME(spec) + if scheme: + # It's a url, download it to tmpdir + return self._download_url(scheme.group(1), spec, tmpdir) + + elif os.path.exists(spec): + # Existing file or directory, just return it + return spec + else: + try: + spec = Requirement.parse(spec) + except ValueError: + raise RuntimeError( + "Not a URL, existing file, or requirement spec: %r" % + (spec,) + ) + + # process a Requirement + dist = self.best_match(spec,[]) + if dist is not None: + return self.download(dist.path, tmpdir) + + return None + + + + dl_blocksize = 8192 + + def _download_to(self, url, filename): + # Download the file + fp, tfp = None, None + try: + fp = self.open_url(url) + if isinstance(fp, urllib2.HTTPError): + raise RuntimeError( + "Can't download %s: %s %s" % (url, fp.code,fp.msg) + ) + + headers = fp.info() + blocknum = 0 + bs = self.dl_blocksize + size = -1 + + if "content-length" in headers: + size = int(headers["Content-Length"]) + self.reporthook(url, filename, blocknum, bs, size) + + tfp = open(filename,'wb') + while True: + block = fp.read(bs) + if block: + tfp.write(block) + blocknum += 1 + self.reporthook(url, filename, blocknum, bs, size) + else: + break + return headers + + finally: + if fp: fp.close() + if tfp: tfp.close() + + def reporthook(self, url, filename, blocknum, blksize, size): + pass # no-op + + + + def open_url(self, url): + try: + return urllib2.urlopen(url) + except urllib2.HTTPError, v: + return v + except urllib2.URLError, v: + raise RuntimeError("Download error: %s" % v.reason) + + + def _download_url(self, scheme, url, tmpdir): + + # Determine download filename + # + name = filter(None,urlparse.urlparse(url)[2].split('/')) + if name: + name = name[-1] + while '..' in name: + name = name.replace('..','.').replace('\\','_') + else: + name = "__downloaded__" # default if URL has no path contents + + filename = os.path.join(tmpdir,name) + + # Download the file + # + if scheme=='svn' or scheme.startswith('svn+'): + return self._download_svn(url, filename) + else: + headers = self._download_to(url, filename) + if 'html' in headers['content-type'].lower(): + return self._download_html(url, headers, filename, tmpdir) + else: + return filename + + + + + + + + + def _download_html(self, url, headers, filename, tmpdir): + # Check for a sourceforge URL + sf_url = url.startswith('http://prdownloads.') + file = open(filename) + for line in file: + if line.strip(): + # Check for a subversion index page + if re.search(r'Revision \d+:', line): + # it's a subversion index page: + file.close() + os.unlink(filename) + return self._download_svn(url, filename) + # Check for a SourceForge header + elif sf_url: + if re.search(r'^<HTML><HEAD>', line, re.I): + continue # skip first line + elif re.search(r'<TITLE>Select a Mirror for File:',line): + # Sourceforge mirror page + page = file.read() + file.close() + os.unlink(filename) + return self._download_sourceforge(url, page, tmpdir) + break # not an index page + file.close() + raise RuntimeError("Unexpected HTML page found at "+url) + + + def _download_svn(self, url, filename): + os.system("svn checkout -q %s %s" % (url, filename)) + return filename + + + + + + + + + + + + def _download_sourceforge(self, source_url, sf_page, tmpdir): + """Download package from randomly-selected SourceForge mirror""" + + mirror_regex = re.compile(r'HREF=(/.*?\?use_mirror=[^>]*)') + urls = [m.group(1) for m in mirror_regex.finditer(sf_page)] + if not urls: + raise RuntimeError( + "URL looks like a Sourceforge mirror page, but no URLs found" + ) + + import random + url = urlparse.urljoin(source_url, random.choice(urls)) + f = self.open_url(url) + match = re.search( + r'<META HTTP-EQUIV="refresh" content=".*?URL=(.*?)"', + f.read() + ) + f.close() + + if match: + download_url = match.group(1) + scheme = URL_SCHEME(download_url) + return self._download_url(scheme.group(1), download_url, tmpdir) + else: + raise RuntimeError( + 'No META HTTP-EQUIV="refresh" found in Sourceforge page at %s' + % url + ) + + + + + + + + + + + + + -- cgit v1.2.1 From 5c31d27554b5e5785759248373e509db0815c130 Mon Sep 17 00:00:00 2001 From: PJ Eby <distutils-sig@python.org> Date: Tue, 14 Jun 2005 01:26:26 +0000 Subject: Add lots of progress messages, so people know what the package search is doing. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041056 --- setuptools/package_index.py | 115 ++++++++++++++++++++++++++++++-------------- 1 file changed, 78 insertions(+), 37 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 9e48de35..04c1879e 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -90,21 +90,21 @@ class PackageIndex(AvailableDistributions): self.fetched_urls = {} self.package_pages = {} - def scan_url(self, url): - self.process_url(url, True) - def process_url(self, url, retrieve=False): if url in self.scanned_urls and not retrieve: return self.scanned_urls[url] = True dists = list(distros_for_url(url)) - map(self.add, dists) + if dists: self.debug("Found link: %s", url) - if dists or not retrieve or url in self.fetched_urls: + if dists or not retrieve or url in self.fetched_urls: + for dist in dists: + self.add(dist) # don't need the actual page return + self.info("Reading %s", url) f = self.open_url(url) self.fetched_urls[url] = self.fetched_urls[f.url] = True if 'html' not in f.headers['content-type'].lower(): @@ -121,17 +121,11 @@ class PackageIndex(AvailableDistributions): link = urlparse.urljoin(base, match.group(1)) self.process_url(link) - def find_packages(self,requirement): - self.scan_url(self.index_url + requirement.distname) - if not self.package_pages.get(requirement.key): - # We couldn't find the target package, so search the index page too - self.scan_url(self.index_url) - for url in self.package_pages.get(requirement.key,()): - # scan each page that might be related to the desired package - self.scan_url(url) - def process_index(self,url,page): + """Process the contents of a PyPI page""" + def scan(link): + # Process a URL to see if it's for a package page if link.startswith(self.index_url): parts = map( urllib2.unquote, link[len(self.index_url):].split('/') @@ -141,10 +135,12 @@ class PackageIndex(AvailableDistributions): pkg = safe_name(parts[0]) ver = safe_version(parts[1]) self.package_pages.setdefault(pkg.lower(),{})[link] = True + if url==self.index_url or 'Index of Packages' in page: # process an index page into the package-page index for match in HREF.finditer(page): scan( urlparse.urljoin(url, match.group(1)) ) + else: scan(url) # ensure this page is in the page index # process individual package page @@ -156,11 +152,56 @@ class PackageIndex(AvailableDistributions): # Process the found URL self.scan_url(urlparse.urljoin(url, match.group(1))) + + + + + + + + + + + def find_packages(self,requirement): + self.scan_url(self.index_url + requirement.distname+'/') + if not self.package_pages.get(requirement.key): + # We couldn't find the target package, so search the index page too + self.warn( + "Couldn't find index page for %r (maybe misspelled?)", + requirement.distname + ) + if self.index_url not in self.fetched_urls: + self.warn( + "Scanning index of all packages (this may take a while)" + ) + self.scan_url(self.index_url) + + for url in self.package_pages.get(requirement.key,()): + # scan each page that might be related to the desired package + self.scan_url(url) + def obtain(self,requirement): self.find_packages(requirement) for dist in self.get(requirement.key, ()): if dist in requirement: return dist + self.debug("%s does not match %s", requirement, dist) + + + + + + + + + + + + + + + + def download(self, spec, tmpdir): """Locate and/or download `spec`, returning a local filename @@ -193,19 +234,21 @@ class PackageIndex(AvailableDistributions): "Not a URL, existing file, or requirement spec: %r" % (spec,) ) - # process a Requirement + self.info("Searching for %s", spec) dist = self.best_match(spec,[]) if dist is not None: + self.info("Best match: %s", dist) return self.download(dist.path, tmpdir) + self.warn("No local packages or download links found for %s", spec) return None - - dl_blocksize = 8192 def _download_to(self, url, filename): + self.info("Downloading %s", url) + # Download the file fp, tfp = None, None try: @@ -242,8 +285,6 @@ class PackageIndex(AvailableDistributions): def reporthook(self, url, filename, blocknum, blksize, size): pass # no-op - - def open_url(self, url): try: return urllib2.urlopen(url) @@ -278,8 +319,8 @@ class PackageIndex(AvailableDistributions): else: return filename - - + def scan_url(self, url): + self.process_url(url, True) @@ -313,22 +354,24 @@ class PackageIndex(AvailableDistributions): def _download_svn(self, url, filename): + self.info("Doing subversion checkout from %s to %s", url, filename) os.system("svn checkout -q %s %s" % (url, filename)) return filename + def debug(self, msg, *args): + pass #print msg % args # XXX - - - - - - - - + def info(self, msg, *args): + print msg % args # XXX + + def warn(self, msg, *args): + print msg % args # XXX def _download_sourceforge(self, source_url, sf_page, tmpdir): """Download package from randomly-selected SourceForge mirror""" + self.debug("Processing SourceForge mirror page") + mirror_regex = re.compile(r'HREF=(/.*?\?use_mirror=[^>]*)') urls = [m.group(1) for m in mirror_regex.finditer(sf_page)] if not urls: @@ -338,6 +381,12 @@ class PackageIndex(AvailableDistributions): import random url = urlparse.urljoin(source_url, random.choice(urls)) + + self.info( + "Requesting redirect to (randomly selected) %r mirror", + url.split('=',1)[-1] + ) + f = self.open_url(url) match = re.search( r' Date: Tue, 14 Jun 2005 12:46:35 +0000 Subject: Support downloading packages that were uploaded to PyPI (by scanning all links on package pages, not just the homepage/download links). --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041062 --- setuptools/package_index.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 04c1879e..0645dc3e 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -116,10 +116,10 @@ class PackageIndex(AvailableDistributions): f.close() if url.startswith(self.index_url): self.process_index(url, page) - else: - for match in HREF.finditer(page): - link = urlparse.urljoin(base, match.group(1)) - self.process_url(link) + + for match in HREF.finditer(page): + link = urlparse.urljoin(base, match.group(1)) + self.process_url(link) def process_index(self,url,page): """Process the contents of a PyPI page""" -- cgit v1.2.1 From 8a1ba5d6cdaa0318c4f3fc5de1ae4d092d58003e Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Tue, 14 Jun 2005 15:30:32 +0000 Subject: Add support for quiet/verbose/dry-run/optimize flags. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041064 --- setuptools/package_index.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 0645dc3e..be40df9c 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -2,6 +2,7 @@ import sys, os.path, re, urlparse, urllib2 from pkg_resources import * +from distutils import log HREF = re.compile(r"""href\s*=\s*['"]?([^'"> ]+)""", re.I) URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):',re.I).match @@ -29,7 +30,6 @@ __all__ = [ - @@ -359,13 +359,13 @@ class PackageIndex(AvailableDistributions): return filename def debug(self, msg, *args): - pass #print msg % args # XXX + log.debug(msg, *args) def info(self, msg, *args): - print msg % args # XXX + log.info(msg, *args) def warn(self, msg, *args): - print msg % args # XXX + log.warn(msg, *args) def _download_sourceforge(self, source_url, sf_page, tmpdir): """Download package from randomly-selected SourceForge mirror""" -- cgit v1.2.1 From 645693ba88a2bd2beacbd50d57a5608960190a74 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Wed, 15 Jun 2005 02:23:48 +0000 Subject: Add support for installing from .win32.exe's created by distutils (by converting them to eggs). Bump version to 0.5a1. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041070 --- setuptools/package_index.py | 75 +++++++++++++++++++++++++++++++++++---------- 1 file changed, 58 insertions(+), 17 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index be40df9c..d9d2fc00 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -9,13 +9,25 @@ URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):',re.I).match EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split() __all__ = [ - 'PackageIndex', 'distros_for_url', + 'PackageIndex', 'distros_for_url', 'parse_bdist_wininst', + 'interpret_distro_name', ] +def parse_bdist_wininst(name): + """Return (base,pyversion) or (None,None) for possible .exe name""" + lower = name.lower() + base, py_ver = None, None + if lower.endswith('.exe'): + if lower.endswith('.win32.exe'): + base = name[:-10] + elif lower[-16:].startswith('.win32-py'): + py_ver = base[-7:-4] + base = name[:-16] + return base,py_ver @@ -27,8 +39,32 @@ __all__ = [ +def distros_for_url(url, metadata=None): + """Yield egg or source distribution objects that might be found at a URL""" + path = urlparse.urlparse(url)[2] + base = urllib2.unquote(path.split('/')[-1]) + + if base.endswith('.egg'): + dist = Distribution.from_filename(base, metadata) + dist.path = url + return [dist] # only one, unambiguous interpretation + if base.endswith('.exe'): + win_base, py_ver = parse_bdist_wininst(name) + if win_base is not None: + return interpret_distro_name( + url, win_base, metadata, py_ver, BINARY_DIST, "win32" + ) + + # Try source distro extensions (.zip, .tgz, etc.) + # + for ext in EXTENSIONS: + if base.endswith(ext): + base = base[:-len(ext)] + return interpret_distro_name(url, base, metadata) + + return [] # no extension matched @@ -39,24 +75,14 @@ __all__ = [ -def distros_for_url(url, metadata=None): - """Yield egg or source distribution objects that might be found at a URL""" - path = urlparse.urlparse(url)[2] - base = urllib2.unquote(path.split('/')[-1]) - if base.endswith('.egg'): - dist = Distribution.from_filename(base, metadata) - dist.path = url - yield dist - return # only one, unambiguous interpretation - for ext in EXTENSIONS: - if base.endswith(ext): - base = base[:-len(ext)] - break - else: - return # no extension matched + + +def interpret_distro_name(url, base, metadata, + py_version=None, distro_type=SOURCE_DIST, platform=None +): # Generate alternative interpretations of a source distro name # Because some packages are ambiguous as to name/versions split @@ -74,12 +100,27 @@ def distros_for_url(url, metadata=None): for p in range(1,len(parts)+1): yield Distribution( url, metadata, '-'.join(parts[:p]), '-'.join(parts[p:]), - distro_type = SOURCE_DIST + py_version=py_version, distro_type = distro_type, + platform = platform ) + + + + + + + + + + + + + + class PackageIndex(AvailableDistributions): """A distribution index that scans web pages for download URLs""" -- cgit v1.2.1 From 0fd80962ecc527dbb57ec6472559cdb90ecfe582 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Wed, 15 Jun 2005 02:39:10 +0000 Subject: Fix stupid typos. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041071 --- setuptools/package_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index d9d2fc00..9ed9a8a8 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -24,7 +24,7 @@ def parse_bdist_wininst(name): if lower.endswith('.win32.exe'): base = name[:-10] elif lower[-16:].startswith('.win32-py'): - py_ver = base[-7:-4] + py_ver = name[-7:-4] base = name[:-16] return base,py_ver @@ -51,7 +51,7 @@ def distros_for_url(url, metadata=None): return [dist] # only one, unambiguous interpretation if base.endswith('.exe'): - win_base, py_ver = parse_bdist_wininst(name) + win_base, py_ver = parse_bdist_wininst(base) if win_base is not None: return interpret_distro_name( url, win_base, metadata, py_ver, BINARY_DIST, "win32" -- cgit v1.2.1 From 5ed7f988bca676d52388b7d0db6e540bfd1476f7 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Sat, 25 Jun 2005 19:33:06 +0000 Subject: 0.5a3 bugfix release --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041072 --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 9ed9a8a8..a4f81882 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -23,7 +23,7 @@ def parse_bdist_wininst(name): if lower.endswith('.exe'): if lower.endswith('.win32.exe'): base = name[:-10] - elif lower[-16:].startswith('.win32-py'): + elif lower.startswith('.win32-py',-16): py_ver = name[-7:-4] base = name[:-16] -- cgit v1.2.1 From 643acd6ad1eb4aeebac199c91af001181c7786f3 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Mon, 27 Jun 2005 00:31:03 +0000 Subject: EasyInstall/setuptools 0.5a4: significant new features, including automatic installation of dependencies, the ability to specify dependencies in a setup script, and several new options to control EasyInstall's behavior. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041073 --- setuptools/package_index.py | 69 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 55 insertions(+), 14 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index a4f81882..78962eee 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -203,7 +203,7 @@ class PackageIndex(AvailableDistributions): - def find_packages(self,requirement): + def find_packages(self, requirement): self.scan_url(self.index_url + requirement.distname+'/') if not self.package_pages.get(requirement.key): # We couldn't find the target package, so search the index page too @@ -221,13 +221,13 @@ class PackageIndex(AvailableDistributions): # scan each page that might be related to the desired package self.scan_url(url) - def obtain(self,requirement): + def obtain(self, requirement, installer=None): self.find_packages(requirement) for dist in self.get(requirement.key, ()): if dist in requirement: return dist self.debug("%s does not match %s", requirement, dist) - + return super(PackageIndex, self).obtain(requirement,installer) @@ -245,19 +245,20 @@ class PackageIndex(AvailableDistributions): def download(self, spec, tmpdir): - """Locate and/or download `spec`, returning a local filename + """Locate and/or download `spec` to `tmpdir`, returning a local path `spec` may be a ``Requirement`` object, or a string containing a URL, - an existing local filename, or a package/version requirement spec + an existing local filename, or a project/version requirement spec (i.e. the string form of a ``Requirement`` object). - If necessary, the requirement is searched for in the package index. - If the download is successful, the return value is a local file path, - and it is a subpath of `tmpdir` if the distribution had to be - downloaded. If no matching distribution is found, return ``None``. - Various errors may be raised if a problem occurs during downloading. + If `spec` is a ``Requirement`` object or a string containing a + project/version requirement spec, this method is equivalent to + the ``fetch()`` method. If `spec` is a local, existing file or + directory name, it is simply returned unchanged. If `spec` is a URL, + it is downloaded to a subpath of `tmpdir`, and the local filename is + returned. Various errors may be raised if a problem occurs during + downloading. """ - if not isinstance(spec,Requirement): scheme = URL_SCHEME(spec) if scheme: @@ -275,16 +276,56 @@ class PackageIndex(AvailableDistributions): "Not a URL, existing file, or requirement spec: %r" % (spec,) ) + + return self.fetch(spec, tmpdir, force_scan) + + + + + + + + def fetch(self, requirement, tmpdir, force_scan=False): + """Obtain a file suitable for fulfilling `requirement` + + `requirement` must be a ``pkg_resources.Requirement`` instance. + If necessary, or if the `force_scan` flag is set, the requirement is + searched for in the (online) package index as well as the locally + installed packages. If a distribution matching `requirement` is found, + the return value is the same as if you had called the ``download()`` + method with the matching distribution's URL. If no matching + distribution is found, returns ``None``. + """ + # process a Requirement - self.info("Searching for %s", spec) - dist = self.best_match(spec,[]) + self.info("Searching for %s", requirement) + + if force_scan: + self.find_packages(requirement) + + dist = self.best_match(requirement, []) # XXX + if dist is not None: self.info("Best match: %s", dist) return self.download(dist.path, tmpdir) - self.warn("No local packages or download links found for %s", spec) + self.warn( + "No local packages or download links found for %s", requirement + ) return None + + + + + + + + + + + + dl_blocksize = 8192 def _download_to(self, url, filename): -- cgit v1.2.1 From c1112aa5fb7da9996e5a0ebe8689e5cf9ca72cee Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Thu, 7 Jul 2005 16:28:43 +0000 Subject: Add upload support to setuptools, and make default downloads of setuptools come from PyPI/python.org rather than from telecommunity.com. Bump to version 0.5a7. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041090 --- setuptools/package_index.py | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 78962eee..0a8ccddc 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -44,7 +44,9 @@ def distros_for_url(url, metadata=None): path = urlparse.urlparse(url)[2] base = urllib2.unquote(path.split('/')[-1]) - + if base.endswith('.egg.zip'): + base = base[:-4] # strip the .zip + if base.endswith('.egg'): dist = Distribution.from_filename(base, metadata) dist.path = url @@ -58,10 +60,10 @@ def distros_for_url(url, metadata=None): ) # Try source distro extensions (.zip, .tgz, etc.) - # + # for ext in EXTENSIONS: if base.endswith(ext): - base = base[:-len(ext)] + base = base[:-len(ext)] return interpret_distro_name(url, base, metadata) return [] # no extension matched @@ -78,8 +80,6 @@ def distros_for_url(url, metadata=None): - - def interpret_distro_name(url, base, metadata, py_version=None, distro_type=SOURCE_DIST, platform=None ): @@ -139,7 +139,7 @@ class PackageIndex(AvailableDistributions): dists = list(distros_for_url(url)) if dists: self.debug("Found link: %s", url) - if dists or not retrieve or url in self.fetched_urls: + if dists or not retrieve or url in self.fetched_urls: for dist in dists: self.add(dist) # don't need the actual page @@ -164,7 +164,7 @@ class PackageIndex(AvailableDistributions): def process_index(self,url,page): """Process the contents of a PyPI page""" - + def scan(link): # Process a URL to see if it's for a package page if link.startswith(self.index_url): @@ -226,7 +226,7 @@ class PackageIndex(AvailableDistributions): for dist in self.get(requirement.key, ()): if dist in requirement: return dist - self.debug("%s does not match %s", requirement, dist) + self.debug("%s does not match %s", requirement, dist) return super(PackageIndex, self).obtain(requirement,installer) @@ -388,6 +388,9 @@ class PackageIndex(AvailableDistributions): else: name = "__downloaded__" # default if URL has no path contents + if name.endswith('.egg.zip'): + name = name[:-4] # strip the extra .zip before download + filename = os.path.join(tmpdir,name) # Download the file @@ -405,9 +408,6 @@ class PackageIndex(AvailableDistributions): self.process_url(url, True) - - - def _download_html(self, url, headers, filename, tmpdir): # Check for a sourceforge URL sf_url = url.startswith('http://prdownloads.') @@ -445,7 +445,7 @@ class PackageIndex(AvailableDistributions): def info(self, msg, *args): log.info(msg, *args) - + def warn(self, msg, *args): log.warn(msg, *args) -- cgit v1.2.1 From 451377d0e877fc610d1bdf8181ba70a90e4c14cc Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Sun, 10 Jul 2005 04:49:31 +0000 Subject: Detect and handle conflicts with "unmanaged" packages when installing packages managed by EasyInstall. Also, add an option to exclude source files from .egg distributions. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041109 --- setuptools/package_index.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 0a8ccddc..21d37cf9 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -3,6 +3,7 @@ import sys, os.path, re, urlparse, urllib2 from pkg_resources import * from distutils import log +from distutils.errors import DistutilsError HREF = re.compile(r"""href\s*=\s*['"]?([^'"> ]+)""", re.I) URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):',re.I).match @@ -38,7 +39,6 @@ def parse_bdist_wininst(name): - def distros_for_url(url, metadata=None): """Yield egg or source distribution objects that might be found at a URL""" @@ -272,7 +272,7 @@ class PackageIndex(AvailableDistributions): try: spec = Requirement.parse(spec) except ValueError: - raise RuntimeError( + raise DistutilsError( "Not a URL, existing file, or requirement spec: %r" % (spec,) ) @@ -336,7 +336,7 @@ class PackageIndex(AvailableDistributions): try: fp = self.open_url(url) if isinstance(fp, urllib2.HTTPError): - raise RuntimeError( + raise DistutilsError( "Can't download %s: %s %s" % (url, fp.code,fp.msg) ) @@ -373,7 +373,7 @@ class PackageIndex(AvailableDistributions): except urllib2.HTTPError, v: return v except urllib2.URLError, v: - raise RuntimeError("Download error: %s" % v.reason) + raise DistutilsError("Download error: %s" % v.reason) def _download_url(self, scheme, url, tmpdir): @@ -432,7 +432,7 @@ class PackageIndex(AvailableDistributions): return self._download_sourceforge(url, page, tmpdir) break # not an index page file.close() - raise RuntimeError("Unexpected HTML page found at "+url) + raise DistutilsError("Unexpected HTML page found at "+url) def _download_svn(self, url, filename): @@ -457,7 +457,7 @@ class PackageIndex(AvailableDistributions): mirror_regex = re.compile(r'HREF=(/.*?\?use_mirror=[^>]*)') urls = [m.group(1) for m in mirror_regex.finditer(sf_page)] if not urls: - raise RuntimeError( + raise DistutilsError( "URL looks like a Sourceforge mirror page, but no URLs found" ) @@ -481,7 +481,7 @@ class PackageIndex(AvailableDistributions): scheme = URL_SCHEME(download_url) return self._download_url(scheme.group(1), download_url, tmpdir) else: - raise RuntimeError( + raise DistutilsError( 'No META HTTP-EQUIV="refresh" found in Sourceforge page at %s' % url ) -- cgit v1.2.1 From 3efcdec07fa19acd77cb3a5a38ef5052b509681a Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Tue, 12 Jul 2005 05:31:36 +0000 Subject: Fix bugs and implement features reported/requested by folks on the Distutils-SIG. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041117 --- setuptools/package_index.py | 83 +++++++++++++++++++++++++++++++++------------ 1 file changed, 62 insertions(+), 21 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 21d37cf9..5412fd1f 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -44,27 +44,32 @@ def distros_for_url(url, metadata=None): path = urlparse.urlparse(url)[2] base = urllib2.unquote(path.split('/')[-1]) - if base.endswith('.egg.zip'): - base = base[:-4] # strip the .zip + return distros_for_filename(url, base, metadata) - if base.endswith('.egg'): - dist = Distribution.from_filename(base, metadata) - dist.path = url + +def distros_for_filename(url_or_path, basename, metadata=None): + """Yield egg or source distribution objects based on basename""" + if basename.endswith('.egg.zip'): + basename = basename[:-4] # strip the .zip + + if basename.endswith('.egg'): + dist = Distribution.from_filename(basename, metadata) + dist.path = url_or_path return [dist] # only one, unambiguous interpretation - if base.endswith('.exe'): - win_base, py_ver = parse_bdist_wininst(base) + if basename.endswith('.exe'): + win_base, py_ver = parse_bdist_wininst(basename) if win_base is not None: return interpret_distro_name( - url, win_base, metadata, py_ver, BINARY_DIST, "win32" + url_or_path, win_base, metadata, py_ver, BINARY_DIST, "win32" ) # Try source distro extensions (.zip, .tgz, etc.) # for ext in EXTENSIONS: - if base.endswith(ext): - base = base[:-len(ext)] - return interpret_distro_name(url, base, metadata) + if basename.endswith(ext): + basename = basename[:-len(ext)] + return interpret_distro_name(url_or_path, basename, metadata) return [] # no extension matched @@ -75,12 +80,7 @@ def distros_for_url(url, metadata=None): - - - - - -def interpret_distro_name(url, base, metadata, +def interpret_distro_name(url_or_path, basename, metadata, py_version=None, distro_type=SOURCE_DIST, platform=None ): @@ -96,10 +96,10 @@ def interpret_distro_name(url, base, metadata, # in the long run PyPI and the distutils should go for "safe" names and # versions in distribution archive names (sdist and bdist). - parts = base.split('-') + parts = basename.split('-') for p in range(1,len(parts)+1): yield Distribution( - url, metadata, '-'.join(parts[:p]), '-'.join(parts[p:]), + url_or_path, metadata, '-'.join(parts[:p]), '-'.join(parts[p:]), py_version=py_version, distro_type = distro_type, platform = platform ) @@ -132,12 +132,35 @@ class PackageIndex(AvailableDistributions): self.package_pages = {} def process_url(self, url, retrieve=False): + """Evaluate a URL as a possible download, and maybe retrieve it""" + if url in self.scanned_urls and not retrieve: return self.scanned_urls[url] = True - dists = list(distros_for_url(url)) - if dists: self.debug("Found link: %s", url) + + if not URL_SCHEME(url): + # process filenames or directories + if os.path.isfile(url): + dists = list( + distros_for_filename( + os.path.realpath(url), os.path.basename(url) + ) + ) + elif os.path.isdir(url): + url = os.path.realpath(url) + for item in os.listdir(url): + self.process_url(os.path.join(url,item)) + return + else: + self.warn("Not found: %s", url) + return + else: + dists = list(distros_for_url(url)) + + if dists: + self.debug("Found link: %s", url) + if dists or not retrieve or url in self.fetched_urls: for dist in dists: @@ -148,6 +171,7 @@ class PackageIndex(AvailableDistributions): self.info("Reading %s", url) f = self.open_url(url) self.fetched_urls[url] = self.fetched_urls[f.url] = True + if 'html' not in f.headers['content-type'].lower(): f.close() # not html, we can't process it return @@ -162,6 +186,23 @@ class PackageIndex(AvailableDistributions): link = urlparse.urljoin(base, match.group(1)) self.process_url(link) + + + + + + + + + + + + + + + + + def process_index(self,url,page): """Process the contents of a PyPI page""" -- cgit v1.2.1 From 30f1c5ad93e21ec007d371313e2a27e4d0efb661 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Sun, 17 Jul 2005 04:42:42 +0000 Subject: Renamings for consistent terminology; distributions and requirements now both have 'project_name' attributes, instead of one having 'name' and the other 'distname'. Requirements no longer have 'options', they have 'extras'. This is the beginning of the terminology/architecture refactoring described at: http://mail.python.org/pipermail/distutils-sig/2005-June/004652.html --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041132 --- setuptools/package_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 5412fd1f..f553c2da 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -245,12 +245,12 @@ class PackageIndex(AvailableDistributions): def find_packages(self, requirement): - self.scan_url(self.index_url + requirement.distname+'/') + self.scan_url(self.index_url + requirement.project_name+'/') if not self.package_pages.get(requirement.key): # We couldn't find the target package, so search the index page too self.warn( "Couldn't find index page for %r (maybe misspelled?)", - requirement.distname + requirement.project_name ) if self.index_url not in self.fetched_urls: self.warn( -- cgit v1.2.1 From 61a0e7109e42e844dcda2637fa3bbf5d1f897938 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Sun, 17 Jul 2005 19:54:38 +0000 Subject: The ``path`` attribute of ``Distribution`` objects is now ``location``, because it isn't necessarily a filesystem path (and hasn't been for some time now). ``Distribution`` objects now have an ``as_requirement()`` method that returns a ``Requirement`` for the distribution's project name and version. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041134 --- setuptools/package_index.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index f553c2da..4c2d7616 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -52,10 +52,8 @@ def distros_for_filename(url_or_path, basename, metadata=None): if basename.endswith('.egg.zip'): basename = basename[:-4] # strip the .zip - if basename.endswith('.egg'): - dist = Distribution.from_filename(basename, metadata) - dist.path = url_or_path - return [dist] # only one, unambiguous interpretation + if basename.endswith('.egg'): # only one, unambiguous interpretation + return [Distribution.from_location(url_or_path, basename, metadata)] if basename.endswith('.exe'): win_base, py_ver = parse_bdist_wininst(basename) @@ -80,6 +78,8 @@ def distros_for_filename(url_or_path, basename, metadata=None): + + def interpret_distro_name(url_or_path, basename, metadata, py_version=None, distro_type=SOURCE_DIST, platform=None ): -- cgit v1.2.1 From c7214855992b9657d6a490f46764ffaf4089c1cb Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Mon, 18 Jul 2005 01:39:45 +0000 Subject: Massive API refactoring; see setuptools.txt changelog for details. Also, add ``#egg=project-version`` link support, and docs on how to make your package available for EasyInstall to find. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041135 --- setuptools/package_index.py | 80 ++++++++++++++++++++++----------------------- 1 file changed, 40 insertions(+), 40 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 4c2d7616..7c4fa49d 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -5,7 +5,8 @@ from pkg_resources import * from distutils import log from distutils.errors import DistutilsError -HREF = re.compile(r"""href\s*=\s*['"]?([^'"> ]+)""", re.I) +EGG_FRAGMENT = re.compile('^egg=(\\w+(-\\w+)?)$') +HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I) URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):',re.I).match EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split() @@ -38,28 +39,34 @@ def parse_bdist_wininst(name): - def distros_for_url(url, metadata=None): """Yield egg or source distribution objects that might be found at a URL""" - path = urlparse.urlparse(url)[2] + scheme, server, path, parameters, query, fragment = urlparse.urlparse(url) base = urllib2.unquote(path.split('/')[-1]) - return distros_for_filename(url, base, metadata) + dists = distros_for_location(url, base, metadata) + if fragment and not dists: + match = EGG_FRAGMENT.match(fragment) + if match: + return interpret_distro_name( + url, match.group(1), metadata, precedence = CHECKOUT_DIST + ) + return dists -def distros_for_filename(url_or_path, basename, metadata=None): +def distros_for_location(location, basename, metadata=None): """Yield egg or source distribution objects based on basename""" if basename.endswith('.egg.zip'): basename = basename[:-4] # strip the .zip if basename.endswith('.egg'): # only one, unambiguous interpretation - return [Distribution.from_location(url_or_path, basename, metadata)] + return [Distribution.from_location(location, basename, metadata)] if basename.endswith('.exe'): win_base, py_ver = parse_bdist_wininst(basename) if win_base is not None: return interpret_distro_name( - url_or_path, win_base, metadata, py_ver, BINARY_DIST, "win32" + location, win_base, metadata, py_ver, BINARY_DIST, "win32" ) # Try source distro extensions (.zip, .tgz, etc.) @@ -67,22 +74,28 @@ def distros_for_filename(url_or_path, basename, metadata=None): for ext in EXTENSIONS: if basename.endswith(ext): basename = basename[:-len(ext)] - return interpret_distro_name(url_or_path, basename, metadata) + return interpret_distro_name(location, basename, metadata) return [] # no extension matched +def distros_for_filename(filename, metadata=None): + """Yield possible egg or source distribution objects based on a filename""" + return distros_for_location( + normalize_path(filename), os.path.basename(filename), metadata + ) - - - - - -def interpret_distro_name(url_or_path, basename, metadata, - py_version=None, distro_type=SOURCE_DIST, platform=None +def interpret_distro_name(location, basename, metadata, + py_version=None, precedence=SOURCE_DIST, platform=None ): + """Generate alternative interpretations of a source distro name + + Note: if `location` is a filesystem filename, you should call + ``pkg_resources.normalize_path()`` on it before passing it to this + routine! + """ # Generate alternative interpretations of a source distro name # Because some packages are ambiguous as to name/versions split @@ -99,8 +112,8 @@ def interpret_distro_name(url_or_path, basename, metadata, parts = basename.split('-') for p in range(1,len(parts)+1): yield Distribution( - url_or_path, metadata, '-'.join(parts[:p]), '-'.join(parts[p:]), - py_version=py_version, distro_type = distro_type, + location, metadata, '-'.join(parts[:p]), '-'.join(parts[p:]), + py_version=py_version, precedence = precedence, platform = platform ) @@ -108,19 +121,6 @@ def interpret_distro_name(url_or_path, basename, metadata, - - - - - - - - - - - - - class PackageIndex(AvailableDistributions): """A distribution index that scans web pages for download URLs""" @@ -142,11 +142,7 @@ class PackageIndex(AvailableDistributions): if not URL_SCHEME(url): # process filenames or directories if os.path.isfile(url): - dists = list( - distros_for_filename( - os.path.realpath(url), os.path.basename(url) - ) - ) + dists = list(distros_for_filename(url)) elif os.path.isdir(url): url = os.path.realpath(url) for item in os.listdir(url): @@ -160,8 +156,6 @@ class PackageIndex(AvailableDistributions): if dists: self.debug("Found link: %s", url) - - if dists or not retrieve or url in self.fetched_urls: for dist in dists: self.add(dist) @@ -199,6 +193,12 @@ class PackageIndex(AvailableDistributions): + + + + + + @@ -344,11 +344,11 @@ class PackageIndex(AvailableDistributions): if force_scan: self.find_packages(requirement) - dist = self.best_match(requirement, []) # XXX + dist = self.best_match(requirement, WorkingSet([])) # XXX if dist is not None: self.info("Best match: %s", dist) - return self.download(dist.path, tmpdir) + return self.download(dist.location, tmpdir) self.warn( "No local packages or download links found for %s", requirement @@ -475,8 +475,8 @@ class PackageIndex(AvailableDistributions): file.close() raise DistutilsError("Unexpected HTML page found at "+url) - def _download_svn(self, url, filename): + url = url.split('#',1)[0] # remove any fragment for svn's sake self.info("Doing subversion checkout from %s to %s", url, filename) os.system("svn checkout -q %s %s" % (url, filename)) return filename -- cgit v1.2.1 From baaa492f8f3100a24d723a7bb6c1d4c13295d5fb Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Sun, 24 Jul 2005 02:41:44 +0000 Subject: Implement --editable option, which allows you to just download and extract (or check out from Subversion) one or more source distributions, without actually building or installing them (or their dependencies). --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041147 --- setuptools/package_index.py | 48 ++++++++++++++++++++++----------------------- 1 file changed, 24 insertions(+), 24 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 7c4fa49d..d5013795 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -318,7 +318,7 @@ class PackageIndex(AvailableDistributions): (spec,) ) - return self.fetch(spec, tmpdir, force_scan) + return self.fetch(spec, tmpdir) @@ -326,7 +326,7 @@ class PackageIndex(AvailableDistributions): - def fetch(self, requirement, tmpdir, force_scan=False): + def fetch(self, requirement, tmpdir, force_scan=False, source=False): """Obtain a file suitable for fulfilling `requirement` `requirement` must be a ``pkg_resources.Requirement`` instance. @@ -336,35 +336,35 @@ class PackageIndex(AvailableDistributions): the return value is the same as if you had called the ``download()`` method with the matching distribution's URL. If no matching distribution is found, returns ``None``. - """ + If the `source` flag is set, only source distributions and source + checkout links will be considered. + """ # process a Requirement self.info("Searching for %s", requirement) + def find(req): + for dist in self.get(req.key, ()): + if dist in req and (dist.precedence<=SOURCE_DIST or not source): + self.info("Best match: %s", dist) + return self.download(dist.location, tmpdir) + if force_scan: self.find_packages(requirement) + dist = find(requirement) + else: + dist = find(requirement) + if dist is None: + self.find_packages(requirement) + dist = find(requirement) - dist = self.best_match(requirement, WorkingSet([])) # XXX - - if dist is not None: - self.info("Best match: %s", dist) - return self.download(dist.location, tmpdir) - - self.warn( - "No local packages or download links found for %s", requirement - ) - return None - - - - - - - - - - - + if dist is None: + self.warn( + "No local packages or download links found for %s%s", + (source and "a source distribution of " or ""), + requirement, + ) + return dist dl_blocksize = 8192 -- cgit v1.2.1 From 57f7bd7e62e6b0ab53bcb33f91abe3ca9917e782 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Sun, 7 Aug 2005 01:03:36 +0000 Subject: Renamed AvailableDistributions -> Environment. Add sketch of pkg_resources manual outline. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041184 --- setuptools/package_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index d5013795..6ef185f0 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -121,11 +121,11 @@ def interpret_distro_name(location, basename, metadata, -class PackageIndex(AvailableDistributions): +class PackageIndex(Environment): """A distribution index that scans web pages for download URLs""" def __init__(self,index_url="http://www.python.org/pypi",*args,**kw): - AvailableDistributions.__init__(self,*args,**kw) + Environment.__init__(self,*args,**kw) self.index_url = index_url + "/"[:not index_url.endswith('/')] self.scanned_urls = {} self.fetched_urls = {} -- cgit v1.2.1 From 8cc0d5c27e393605ac2c729143b8730aa973a128 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Thu, 11 Aug 2005 00:37:37 +0000 Subject: Fix bugs reported by Ian Bicking, Walter Doerwald, and Vincenzo Di Massa. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041189 --- setuptools/package_index.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 6ef185f0..f2731a1e 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -7,6 +7,8 @@ from distutils.errors import DistutilsError EGG_FRAGMENT = re.compile('^egg=(\\w+(-\\w+)?)$') HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I) +# this is here to fix emacs' cruddy broken syntax highlighting: ' + URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):',re.I).match EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split() @@ -37,8 +39,6 @@ def parse_bdist_wininst(name): - - def distros_for_url(url, metadata=None): """Yield egg or source distribution objects that might be found at a URL""" @@ -371,10 +371,10 @@ class PackageIndex(Environment): def _download_to(self, url, filename): self.info("Downloading %s", url) - # Download the file fp, tfp = None, None try: + url = url.split('#', 1)[0] fp = self.open_url(url) if isinstance(fp, urllib2.HTTPError): raise DistutilsError( -- cgit v1.2.1 From 15233b3d39a81fb7164465f65b182eee65983b56 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Sun, 14 Aug 2005 01:45:38 +0000 Subject: Document the "Environment" class, and simplify its API. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041194 --- setuptools/package_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index f2731a1e..95b57178 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -264,7 +264,7 @@ class PackageIndex(Environment): def obtain(self, requirement, installer=None): self.find_packages(requirement) - for dist in self.get(requirement.key, ()): + for dist in self[requirement.key]: if dist in requirement: return dist self.debug("%s does not match %s", requirement, dist) @@ -344,7 +344,7 @@ class PackageIndex(Environment): self.info("Searching for %s", requirement) def find(req): - for dist in self.get(req.key, ()): + for dist in self[req.key]: if dist in req and (dist.precedence<=SOURCE_DIST or not source): self.info("Best match: %s", dist) return self.download(dist.location, tmpdir) -- cgit v1.2.1 From 26eee2978579ed2bff77b37257b77e1e67ac162b Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Mon, 22 Aug 2005 01:13:14 +0000 Subject: Implemented md5 validation for PyPI and for URLs with a "#md5=..." anchor. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041219 --- setuptools/package_index.py | 60 ++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 30 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 95b57178..b3c94797 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -4,10 +4,16 @@ import sys, os.path, re, urlparse, urllib2 from pkg_resources import * from distutils import log from distutils.errors import DistutilsError +from md5 import md5 EGG_FRAGMENT = re.compile('^egg=(\\w+(-\\w+)?)$') HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I) -# this is here to fix emacs' cruddy broken syntax highlighting: ' +# this is here to fix emacs' cruddy broken syntax highlighting + +PYPI_MD5 = re.compile( + '([^<]+)\n\s+\\(md5\\)' +) URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):',re.I).match EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split() @@ -33,18 +39,12 @@ def parse_bdist_wininst(name): return base,py_ver - - - - - - def distros_for_url(url, metadata=None): """Yield egg or source distribution objects that might be found at a URL""" scheme, server, path, parameters, query, fragment = urlparse.urlparse(url) base = urllib2.unquote(path.split('/')[-1]) - dists = distros_for_location(url, base, metadata) + dists = distros_for_location(url, base, metadata) if fragment and not dists: match = EGG_FRAGMENT.match(fragment) if match: @@ -59,7 +59,7 @@ def distros_for_location(location, basename, metadata=None): if basename.endswith('.egg.zip'): basename = basename[:-4] # strip the .zip - if basename.endswith('.egg'): # only one, unambiguous interpretation + if basename.endswith('.egg'): # only one, unambiguous interpretation return [Distribution.from_location(location, basename, metadata)] if basename.endswith('.exe'): @@ -174,7 +174,7 @@ class PackageIndex(Environment): page = f.read() f.close() if url.startswith(self.index_url): - self.process_index(url, page) + page = self.process_index(url, page) for match in HREF.finditer(page): link = urlparse.urljoin(base, match.group(1)) @@ -234,9 +234,9 @@ class PackageIndex(Environment): # Process the found URL self.scan_url(urlparse.urljoin(url, match.group(1))) - - - + return PYPI_MD5.sub( + lambda m: '%s' % m.group(1,3,2), page + ) @@ -270,16 +270,16 @@ class PackageIndex(Environment): self.debug("%s does not match %s", requirement, dist) return super(PackageIndex, self).obtain(requirement,installer) - - - - - - - - - - + def check_md5(self, cs, info, filename, tfp): + if re.match('md5=[0-9a-f]{32}$', info): + self.debug("Validating md5 checksum for %s", filename) + if cs.hexdigest()<>info[4:]: + tfp.close() + os.unlink(filename) + raise DistutilsError( + "MD5 validation failed for "+os.path.basename(filename)+ + "; possible download problem?" + ) @@ -348,7 +348,7 @@ class PackageIndex(Environment): if dist in req and (dist.precedence<=SOURCE_DIST or not source): self.info("Best match: %s", dist) return self.download(dist.location, tmpdir) - + if force_scan: self.find_packages(requirement) dist = find(requirement) @@ -372,35 +372,35 @@ class PackageIndex(Environment): def _download_to(self, url, filename): self.info("Downloading %s", url) # Download the file - fp, tfp = None, None + fp, tfp, info = None, None, None try: - url = url.split('#', 1)[0] + if '#' in url: + url, info = url.split('#', 1) fp = self.open_url(url) if isinstance(fp, urllib2.HTTPError): raise DistutilsError( "Can't download %s: %s %s" % (url, fp.code,fp.msg) ) - + cs = md5() headers = fp.info() blocknum = 0 bs = self.dl_blocksize size = -1 - if "content-length" in headers: size = int(headers["Content-Length"]) self.reporthook(url, filename, blocknum, bs, size) - tfp = open(filename,'wb') while True: block = fp.read(bs) if block: + cs.update(block) tfp.write(block) blocknum += 1 self.reporthook(url, filename, blocknum, bs, size) else: break + if info: self.check_md5(cs, info, filename, tfp) return headers - finally: if fp: fp.close() if tfp: tfp.close() -- cgit v1.2.1 From 6a36f2af98bd1065e37856dcc1bdb7e2e133b1ec Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Wed, 14 Sep 2005 03:28:10 +0000 Subject: Correctly handle URL fragments in --find-links. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041241 --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index b3c94797..da6cf8cd 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -44,6 +44,7 @@ def distros_for_url(url, metadata=None): scheme, server, path, parameters, query, fragment = urlparse.urlparse(url) base = urllib2.unquote(path.split('/')[-1]) + if '#' in base: base, fragment = base.split('#',1) dists = distros_for_location(url, base, metadata) if fragment and not dists: match = EGG_FRAGMENT.match(fragment) @@ -79,7 +80,6 @@ def distros_for_location(location, basename, metadata=None): return [] # no extension matched - def distros_for_filename(filename, metadata=None): """Yield possible egg or source distribution objects based on a filename""" return distros_for_location( -- cgit v1.2.1 From 0ae86e5c56ef3fb2b6ddc1e5ac95b36452ba5104 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Sat, 24 Sep 2005 19:44:27 +0000 Subject: Fix a bug parsing #egg links reported by Ben Bangert on the distutils-sig. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041251 --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index da6cf8cd..f5b322b0 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -6,7 +6,7 @@ from distutils import log from distutils.errors import DistutilsError from md5 import md5 -EGG_FRAGMENT = re.compile('^egg=(\\w+(-\\w+)?)$') +EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.]+)$') HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I) # this is here to fix emacs' cruddy broken syntax highlighting -- cgit v1.2.1 From 272fed819ba188ac5d6c0ceb9de6d7a8cb543a18 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Wed, 19 Oct 2005 03:00:35 +0000 Subject: Added "--allow-hosts" option to restrict downloading and spidering to a specified list of server glob patterns. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041266 --- setuptools/package_index.py | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index f5b322b0..76d53d11 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -5,11 +5,11 @@ from pkg_resources import * from distutils import log from distutils.errors import DistutilsError from md5 import md5 +from fnmatch import translate EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.]+)$') HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I) # this is here to fix emacs' cruddy broken syntax highlighting - PYPI_MD5 = re.compile( '([^<]+)\n\s+\\(md5\\)' @@ -124,25 +124,25 @@ def interpret_distro_name(location, basename, metadata, class PackageIndex(Environment): """A distribution index that scans web pages for download URLs""" - def __init__(self,index_url="http://www.python.org/pypi",*args,**kw): + def __init__(self,index_url="http://www.python.org/pypi",hosts=('*',),*args,**kw): Environment.__init__(self,*args,**kw) self.index_url = index_url + "/"[:not index_url.endswith('/')] self.scanned_urls = {} self.fetched_urls = {} self.package_pages = {} + self.allows = re.compile('|'.join(map(translate,hosts))).match def process_url(self, url, retrieve=False): """Evaluate a URL as a possible download, and maybe retrieve it""" - if url in self.scanned_urls and not retrieve: return self.scanned_urls[url] = True - if not URL_SCHEME(url): # process filenames or directories if os.path.isfile(url): - dists = list(distros_for_filename(url)) + map(self.add, distros_for_filename(url)) + return # no need to retrieve anything elif os.path.isdir(url): url = os.path.realpath(url) for item in os.listdir(url): @@ -153,13 +153,16 @@ class PackageIndex(Environment): return else: dists = list(distros_for_url(url)) + if dists: + if not self.url_ok(url): + return + self.debug("Found link: %s", url) - if dists: - self.debug("Found link: %s", url) if dists or not retrieve or url in self.fetched_urls: - for dist in dists: - self.add(dist) - # don't need the actual page + map(self.add, dists) + return # don't need the actual page + + if not self.url_ok(url): return self.info("Reading %s", url) @@ -181,17 +184,14 @@ class PackageIndex(Environment): self.process_url(link) - - - - - - - - - - - + def url_ok(self, url, fatal=False): + if self.allows(urlparse.urlparse(url)[1]): + return True + msg = "\nLink to % s ***BLOCKED*** by --allow-hosts\n" + if fatal: + raise DistutilsError(msg % url) + else: + self.warn(msg, url) @@ -368,8 +368,8 @@ class PackageIndex(Environment): dl_blocksize = 8192 - def _download_to(self, url, filename): + self.url_ok(url,True) # raises error if not allowed self.info("Downloading %s", url) # Download the file fp, tfp, info = None, None, None -- cgit v1.2.1 From 12022171055a008a3dbd9e9eb38e249452414a5a Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Fri, 4 Nov 2005 03:18:44 +0000 Subject: Reduce the number of redundant host blocking warnings by not retrying the same previously-blocked URLs. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041395 --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 76d53d11..1665bd37 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -163,6 +163,7 @@ class PackageIndex(Environment): return # don't need the actual page if not self.url_ok(url): + self.fetched_urls[url] = True return self.info("Reading %s", url) @@ -202,7 +203,6 @@ class PackageIndex(Environment): - def process_index(self,url,page): """Process the contents of a PyPI page""" -- cgit v1.2.1 From 1a44c5da3c8179423740bab0dc6c3d96e56c8263 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Wed, 16 Nov 2005 18:43:05 +0000 Subject: Update for SourceForge's changed mirror page formats --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041458 --- setuptools/package_index.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 1665bd37..199996e8 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -463,16 +463,14 @@ class PackageIndex(Environment): return self._download_svn(url, filename) # Check for a SourceForge header elif sf_url: - if re.search(r'^', line, re.I): - continue # skip first line - elif re.search(r'Select a Mirror for File:',line): - # Sourceforge mirror page - page = file.read() - file.close() + page = ''.join(list(file)) + if '?use_mirror=' in page: + file.close(); os.unlink(filename) return self._download_sourceforge(url, page, tmpdir) break # not an index page file.close() + os.unlink(filename) raise DistutilsError("Unexpected HTML page found at "+url) def _download_svn(self, url, filename): @@ -490,12 +488,14 @@ class PackageIndex(Environment): def warn(self, msg, *args): log.warn(msg, *args) + + def _download_sourceforge(self, source_url, sf_page, tmpdir): """Download package from randomly-selected SourceForge mirror""" self.debug("Processing SourceForge mirror page") - mirror_regex = re.compile(r'HREF=(/.*?\?use_mirror=[^>]*)') + mirror_regex = re.compile(r'HREF="?(/.*?\?use_mirror=[^">]*)', re.I) urls = [m.group(1) for m in mirror_regex.finditer(sf_page)] if not urls: raise DistutilsError( -- cgit v1.2.1 From ff0c5cde2b4bd5ca705ede09784584113785c5c3 Mon Sep 17 00:00:00 2001 From: PJ Eby <distutils-sig@python.org> Date: Wed, 16 Nov 2005 19:18:08 +0000 Subject: 0.6a8 final. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041459 --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 199996e8..df39b299 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -465,7 +465,7 @@ class PackageIndex(Environment): elif sf_url: page = ''.join(list(file)) if '?use_mirror=' in page: - file.close(); + file.close() os.unlink(filename) return self._download_sourceforge(url, page, tmpdir) break # not an index page -- cgit v1.2.1 From f95da2a33da64374abfa0d5c45dd3e5d01d73bd3 Mon Sep 17 00:00:00 2001 From: PJ Eby <distutils-sig@python.org> Date: Sat, 7 Jan 2006 00:19:58 +0000 Subject: PyPI searches now use the exact spelling of requirements specified on the command line or in a project's ``install_requires``. Previously, a normalized form of the name was used, which could lead to unnecessary full-index searches when a project's name had an underscore (``_``) in it. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041942 --- setuptools/package_index.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index df39b299..964e3c1c 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -245,12 +245,16 @@ class PackageIndex(Environment): def find_packages(self, requirement): - self.scan_url(self.index_url + requirement.project_name+'/') + self.scan_url(self.index_url + requirement.unsafe_name+'/') + if not self.package_pages.get(requirement.key): + # Fall back to safe version of the name + self.scan_url(self.index_url + requirement.project_name+'/') + if not self.package_pages.get(requirement.key): # We couldn't find the target package, so search the index page too self.warn( "Couldn't find index page for %r (maybe misspelled?)", - requirement.project_name + requirement.unsafe_name ) if self.index_url not in self.fetched_urls: self.warn( @@ -281,10 +285,6 @@ class PackageIndex(Environment): "; possible download problem?" ) - - - - def download(self, spec, tmpdir): """Locate and/or download `spec` to `tmpdir`, returning a local path -- cgit v1.2.1 From abed75c8f1a0b6a449b5411caf3d9581fabae3df Mon Sep 17 00:00:00 2001 From: PJ Eby <distutils-sig@python.org> Date: Tue, 10 Jan 2006 04:00:54 +0000 Subject: EasyInstall can now download bare ``.py`` files and wrap them in an egg, as long as you include an ``#egg=name-version`` suffix on the URL, or if the ``.py`` file is listed as the "Download URL" on the project's PyPI page. This allows third parties to "package" trivial Python modules just by linking to them (e.g. from within their own PyPI page or download links page). --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041995 --- setuptools/package_index.py | 114 ++++++++++++++++++++++++++++++++------------ 1 file changed, 83 insertions(+), 31 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 964e3c1c..869fa7b0 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1,6 +1,6 @@ """PyPI and direct package downloading""" -import sys, os.path, re, urlparse, urllib2 +import sys, os.path, re, urlparse, urllib2, shutil from pkg_resources import * from distutils import log from distutils.errors import DistutilsError @@ -39,12 +39,15 @@ def parse_bdist_wininst(name): return base,py_ver -def distros_for_url(url, metadata=None): - """Yield egg or source distribution objects that might be found at a URL""" - +def egg_info_for_url(url): scheme, server, path, parameters, query, fragment = urlparse.urlparse(url) base = urllib2.unquote(path.split('/')[-1]) if '#' in base: base, fragment = base.split('#',1) + return base,fragment + +def distros_for_url(url, metadata=None): + """Yield egg or source distribution objects that might be found at a URL""" + base, fragment = egg_info_for_url(url) dists = distros_for_location(url, base, metadata) if fragment and not dists: match = EGG_FRAGMENT.match(fragment) @@ -54,12 +57,10 @@ def distros_for_url(url, metadata=None): ) return dists - def distros_for_location(location, basename, metadata=None): """Yield egg or source distribution objects based on basename""" if basename.endswith('.egg.zip'): basename = basename[:-4] # strip the .zip - if basename.endswith('.egg'): # only one, unambiguous interpretation return [Distribution.from_location(location, basename, metadata)] @@ -76,7 +77,6 @@ def distros_for_location(location, basename, metadata=None): if basename.endswith(ext): basename = basename[:-len(ext)] return interpret_distro_name(location, basename, metadata) - return [] # no extension matched @@ -205,7 +205,6 @@ class PackageIndex(Environment): def process_index(self,url,page): """Process the contents of a PyPI page""" - def scan(link): # Process a URL to see if it's for a package page if link.startswith(self.index_url): @@ -217,14 +216,15 @@ class PackageIndex(Environment): pkg = safe_name(parts[0]) ver = safe_version(parts[1]) self.package_pages.setdefault(pkg.lower(),{})[link] = True + return to_filename(pkg), to_filename(ver) + return None, None if url==self.index_url or 'Index of Packages' in page: # process an index page into the package-page index for match in HREF.finditer(page): scan( urlparse.urljoin(url, match.group(1)) ) - else: - scan(url) # ensure this page is in the page index + pkg,ver = scan(url) # ensure this page is in the page index # process individual package page for tag in ("Home Page", "Download URL"): pos = page.find(tag) @@ -232,35 +232,44 @@ class PackageIndex(Environment): match = HREF.search(page,pos) if match: # Process the found URL - self.scan_url(urlparse.urljoin(url, match.group(1))) - + new_url = urlparse.urljoin(url, match.group(1)) + base, frag = egg_info_for_url(new_url) + if base.endswith('.py') and not frag: + if pkg and ver: + new_url+='#egg=%s-%s' % (pkg,ver) + else: + self.need_version_info(url) + self.scan_url(new_url) return PYPI_MD5.sub( lambda m: '%s' % m.group(1,3,2), page ) + def need_version_info(self, url): + self.scan_all( + "Page at %s links to .py file(s) without version info; an index " + "scan is required.", url + ) - - - - + def scan_all(self, msg, *args): + if self.index_url not in self.fetched_urls: + if msg: self.warn(msg,*args) + self.warn( + "Scanning index of all packages (this may take a while)" + ) + self.scan_url(self.index_url) def find_packages(self, requirement): self.scan_url(self.index_url + requirement.unsafe_name+'/') if not self.package_pages.get(requirement.key): # Fall back to safe version of the name self.scan_url(self.index_url + requirement.project_name+'/') - if not self.package_pages.get(requirement.key): # We couldn't find the target package, so search the index page too self.warn( "Couldn't find index page for %r (maybe misspelled?)", requirement.unsafe_name ) - if self.index_url not in self.fetched_urls: - self.warn( - "Scanning index of all packages (this may take a while)" - ) - self.scan_url(self.index_url) + self.scan_all() for url in self.package_pages.get(requirement.key,()): # scan each page that might be related to the desired package @@ -274,6 +283,8 @@ class PackageIndex(Environment): self.debug("%s does not match %s", requirement, dist) return super(PackageIndex, self).obtain(requirement,installer) + + def check_md5(self, cs, info, filename, tfp): if re.match('md5=[0-9a-f]{32}$', info): self.debug("Validating md5 checksum for %s", filename) @@ -290,7 +301,10 @@ class PackageIndex(Environment): `spec` may be a ``Requirement`` object, or a string containing a URL, an existing local filename, or a project/version requirement spec - (i.e. the string form of a ``Requirement`` object). + (i.e. the string form of a ``Requirement`` object). If it is the URL + of a .py file with an unambiguous ``#egg=name-version`` tag (i.e., one + that escapes ``-`` as ``_`` throughout), a trivial ``setup.py`` is + automatically created alongside the downloaded file. If `spec` is a ``Requirement`` object or a string containing a project/version requirement spec, this method is equivalent to @@ -304,8 +318,11 @@ class PackageIndex(Environment): scheme = URL_SCHEME(spec) if scheme: # It's a url, download it to tmpdir - return self._download_url(scheme.group(1), spec, tmpdir) - + found = self._download_url(scheme.group(1), spec, tmpdir) + base, fragment = egg_info_for_url(spec) + if base.endswith('.py'): + found = self.gen_setup(found,fragment,tmpdir) + return found elif os.path.exists(spec): # Existing file or directory, just return it return spec @@ -317,15 +334,9 @@ class PackageIndex(Environment): "Not a URL, existing file, or requirement spec: %r" % (spec,) ) - return self.fetch(spec, tmpdir) - - - - - def fetch(self, requirement, tmpdir, force_scan=False, source=False): """Obtain a file suitable for fulfilling `requirement` @@ -367,6 +378,47 @@ class PackageIndex(Environment): return dist + def gen_setup(self, filename, fragment, tmpdir): + match = EGG_FRAGMENT.match(fragment); #import pdb; pdb.set_trace() + dists = match and [d for d in + interpret_distro_name(filename, match.group(1), None) if d.version + ] or [] + + if len(dists)==1: # unambiguous ``#egg`` fragment + basename = os.path.basename(filename) + + # Make sure the file has been downloaded to the temp dir. + if os.path.dirname(filename) != tmpdir: + dst = os.path.join(tmpdir, basename) + from setuptools.command.easy_install import samefile + if not samefile(filename, dst): + shutil.copy2(filename, dst) + filename=dst + + file = open(os.path.join(tmpdir, 'setup.py'), 'w') + file.write( + "from setuptools import setup\n" + "setup(name=%r, version=%r, py_modules=[%r])\n" + % ( + dists[0].project_name, dists[0].version, + os.path.splitext(basename)[0] + ) + ) + file.close() + return filename + + elif match: + raise DistutilsError( + "Can't unambiguously interpret project/version identifier %r; " + "any dashes in the name or version should be escaped using " + "underscores. %r" % (fragment,dists) + ) + else: + raise DistutilsError( + "Can't process plain .py files without an '#egg=name-version'" + " suffix to enable automatic setup script generation." + ) + dl_blocksize = 8192 def _download_to(self, url, filename): self.url_ok(url,True) # raises error if not allowed -- cgit v1.2.1 From 889ba71dfffd93fd5d0203512f1302ca1269ca55 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Tue, 17 Jan 2006 18:44:46 +0000 Subject: More sourceforge changes. :( --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4042086 --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 869fa7b0..b7c71f23 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -564,7 +564,7 @@ class PackageIndex(Environment): f = self.open_url(url) match = re.search( - r' Date: Tue, 17 Jan 2006 18:47:56 +0000 Subject: Fix editing error (reported by Ian Bicking). --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4042087 --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index b7c71f23..35cd04a6 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -250,7 +250,7 @@ class PackageIndex(Environment): "scan is required.", url ) - def scan_all(self, msg, *args): + def scan_all(self, msg=None, *args): if self.index_url not in self.fetched_urls: if msg: self.warn(msg,*args) self.warn( -- cgit v1.2.1 From 48bda8b1882e8d785af53081a73b9f5f3ee2d992 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Tue, 17 Jan 2006 19:56:59 +0000 Subject: Scrape-proof Sourceforge mirror processing! --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4042088 --- setuptools/package_index.py | 100 ++++++++++++++++++++++++++------------------ 1 file changed, 60 insertions(+), 40 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 35cd04a6..c48968f1 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -134,9 +134,9 @@ class PackageIndex(Environment): def process_url(self, url, retrieve=False): """Evaluate a URL as a possible download, and maybe retrieve it""" + url = fix_sf_url(url) if url in self.scanned_urls and not retrieve: return - self.scanned_urls[url] = True if not URL_SCHEME(url): # process filenames or directories @@ -296,6 +296,36 @@ class PackageIndex(Environment): "; possible download problem?" ) + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + def download(self, spec, tmpdir): """Locate and/or download `spec` to `tmpdir`, returning a local path @@ -502,8 +532,6 @@ class PackageIndex(Environment): def _download_html(self, url, headers, filename, tmpdir): - # Check for a sourceforge URL - sf_url = url.startswith('http://prdownloads.') file = open(filename) for line in file: if line.strip(): @@ -513,13 +541,6 @@ class PackageIndex(Environment): file.close() os.unlink(filename) return self._download_svn(url, filename) - # Check for a SourceForge header - elif sf_url: - page = ''.join(list(file)) - if '?use_mirror=' in page: - file.close() - os.unlink(filename) - return self._download_sourceforge(url, page, tmpdir) break # not an index page file.close() os.unlink(filename) @@ -539,45 +560,44 @@ class PackageIndex(Environment): def warn(self, msg, *args): log.warn(msg, *args) + + +def fix_sf_url(url): + scheme, server, path, param, query, frag = urlparse.urlparse(url) + if server!='prdownloads.sourceforge.net': + return url + return urlparse.urlunparse( + (scheme, 'dl.sourceforge.net', 'sourceforge'+path, param, '', frag) + ) + + + + + + + + + + + + + + + + + + + + - def _download_sourceforge(self, source_url, sf_page, tmpdir): - """Download package from randomly-selected SourceForge mirror""" - self.debug("Processing SourceForge mirror page") - mirror_regex = re.compile(r'HREF="?(/.*?\?use_mirror=[^">]*)', re.I) - urls = [m.group(1) for m in mirror_regex.finditer(sf_page)] - if not urls: - raise DistutilsError( - "URL looks like a Sourceforge mirror page, but no URLs found" - ) - import random - url = urlparse.urljoin(source_url, random.choice(urls)) - self.info( - "Requesting redirect to (randomly selected) %r mirror", - url.split('=',1)[-1] - ) - f = self.open_url(url) - match = re.search( - r'(?i) Date: Mon, 23 Jan 2006 16:29:16 +0000 Subject: Randomly select a SourceForge mirror IP for each download, to work around too-aggressive DNS caches on some platforms, that could otherwise result in a stuck bad IP. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4042156 --- setuptools/package_index.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index c48968f1..2813631e 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1,6 +1,6 @@ """PyPI and direct package downloading""" -import sys, os.path, re, urlparse, urllib2, shutil +import sys, os.path, re, urlparse, urllib2, shutil, random, socket from pkg_resources import * from distutils import log from distutils.errors import DistutilsError @@ -562,18 +562,28 @@ class PackageIndex(Environment): log.warn(msg, *args) + + + + + + + + + + def fix_sf_url(url): scheme, server, path, param, query, frag = urlparse.urlparse(url) if server!='prdownloads.sourceforge.net': return url return urlparse.urlunparse( - (scheme, 'dl.sourceforge.net', 'sourceforge'+path, param, '', frag) + (scheme, get_sf_ip(), 'sourceforge'+path, param, '', frag) ) - - - - +def get_sf_ip(_mirrors=[]): + if not _mirrors: + _mirrors[:] = socket.gethostbyname_ex('dl.sourceforge.net')[-1] + return random.choice(_mirrors) -- cgit v1.2.1 From ca9ccbf6e73daf553bc41d5abd02f8ebf0d44b45 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Tue, 7 Feb 2006 16:43:41 +0000 Subject: The ``--always-copy`` option now skips "system" and "development" eggs since they can't be reliably copied. Note that this may cause EasyInstall to choose an older version of a package than what you expected, or it may cause downloading and installation of a fresh version of what's already installed. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4042260 --- setuptools/package_index.py | 69 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 55 insertions(+), 14 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 2813631e..669692b6 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -337,12 +337,12 @@ class PackageIndex(Environment): automatically created alongside the downloaded file. If `spec` is a ``Requirement`` object or a string containing a - project/version requirement spec, this method is equivalent to - the ``fetch()`` method. If `spec` is a local, existing file or - directory name, it is simply returned unchanged. If `spec` is a URL, - it is downloaded to a subpath of `tmpdir`, and the local filename is - returned. Various errors may be raised if a problem occurs during - downloading. + project/version requirement spec, this method returns the location of + a matching distribution (possibly after downloading it to `tmpdir`). + If `spec` is a locally existing file or directory name, it is simply + returned unchanged. If `spec` is a URL, it is downloaded to a subpath + of `tmpdir`, and the local filename is returned. Various errors may be + raised if a problem occurs during downloading. """ if not isinstance(spec,Requirement): scheme = URL_SCHEME(spec) @@ -364,31 +364,49 @@ class PackageIndex(Environment): "Not a URL, existing file, or requirement spec: %r" % (spec,) ) - return self.fetch(spec, tmpdir) + return getattr(self.fetch_distribution(spec, tmpdir),'location',None) - def fetch(self, requirement, tmpdir, force_scan=False, source=False): - """Obtain a file suitable for fulfilling `requirement` + def fetch_distribution(self, + requirement, tmpdir, force_scan=False, source=False, develop_ok=False + ): + """Obtain a distribution suitable for fulfilling `requirement` `requirement` must be a ``pkg_resources.Requirement`` instance. If necessary, or if the `force_scan` flag is set, the requirement is searched for in the (online) package index as well as the locally installed packages. If a distribution matching `requirement` is found, - the return value is the same as if you had called the ``download()`` - method with the matching distribution's URL. If no matching - distribution is found, returns ``None``. + the returned distribution's ``location`` is the value you would have + gotten from calling the ``download()`` method with the matching + distribution's URL or filename. If no matching distribution is found, + ``None`` is returned. If the `source` flag is set, only source distributions and source - checkout links will be considered. + checkout links will be considered. Unless the `develop_ok` flag is + set, development and system eggs (i.e., those using the ``.egg-info`` + format) will be ignored. """ + # process a Requirement self.info("Searching for %s", requirement) + skipped = {} def find(req): + # Find a matching distribution; may be called more than once + for dist in self[req.key]: + + if dist.precedence==DEVELOP_DIST and not develop_ok: + if dist not in skipped: + self.warn("Skipping development or system egg: %s",dist) + skipped[dist] = 1 + continue + if dist in req and (dist.precedence<=SOURCE_DIST or not source): self.info("Best match: %s", dist) - return self.download(dist.location, tmpdir) + return dist.clone( + location=self.download(dist.location, tmpdir) + ) if force_scan: self.find_packages(requirement) @@ -407,6 +425,29 @@ class PackageIndex(Environment): ) return dist + def fetch(self, requirement, tmpdir, force_scan=False, source=False): + """Obtain a file suitable for fulfilling `requirement` + + DEPRECATED; use the ``fetch_distribution()`` method now instead. For + backward compatibility, this routine is identical but returns the + ``location`` of the downloaded distribution instead of a distribution + object. + """ + dist = self.fetch_dist(requirement,tmpdir,force_scan,source) + if dist is not None: + return dist.location + return None + + + + + + + + + + + def gen_setup(self, filename, fragment, tmpdir): match = EGG_FRAGMENT.match(fragment); #import pdb; pdb.set_trace() -- cgit v1.2.1 From 81bd937426b4f47094ae50157b12f82093a8f3ef Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Wed, 8 Feb 2006 05:46:54 +0000 Subject: The ``--find-links`` option previously scanned all supplied URLs and directories as early as possible, but now only directories and direct archive links are scanned immediately. URLs are not retrieved unless a package search was already going to go online due to a package not being available locally, or due to the use of the ``--update`` or ``-U`` option. Also, fixed the ``develop`` command ignoring ``--find-links``. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4042262 --- setuptools/package_index.py | 98 ++++++++++++++++++++++----------------------- 1 file changed, 49 insertions(+), 49 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 669692b6..c02f3b4e 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -131,6 +131,7 @@ class PackageIndex(Environment): self.fetched_urls = {} self.package_pages = {} self.allows = re.compile('|'.join(map(translate,hosts))).match + self.to_scan = [] def process_url(self, url, retrieve=False): """Evaluate a URL as a possible download, and maybe retrieve it""" @@ -139,18 +140,8 @@ class PackageIndex(Environment): return self.scanned_urls[url] = True if not URL_SCHEME(url): - # process filenames or directories - if os.path.isfile(url): - map(self.add, distros_for_filename(url)) - return # no need to retrieve anything - elif os.path.isdir(url): - url = os.path.realpath(url) - for item in os.listdir(url): - self.process_url(os.path.join(url,item)) - return - else: - self.warn("Not found: %s", url) - return + self.process_filename(url) + return else: dists = list(distros_for_url(url)) if dists: @@ -170,6 +161,7 @@ class PackageIndex(Environment): f = self.open_url(url) self.fetched_urls[url] = self.fetched_urls[f.url] = True + if 'html' not in f.headers['content-type'].lower(): f.close() # not html, we can't process it return @@ -184,6 +176,21 @@ class PackageIndex(Environment): link = urlparse.urljoin(base, match.group(1)) self.process_url(link) + def process_filename(self, fn, nested=False): + # process filenames or directories + if not os.path.exists(fn): + self.warn("Not found: %s", url) + return + + if os.path.isdir(fn): + path = os.path.realpath(fn) + for item in os.listdir(path): + self.process_filename(os.path.join(path,item), True) + + dists = distros_for_filename(fn) + if dists: + self.debug("Found: %s", fn) + map(self.add, dists) def url_ok(self, url, fatal=False): if self.allows(urlparse.urlparse(url)[1]): @@ -196,13 +203,6 @@ class PackageIndex(Environment): - - - - - - - def process_index(self,url,page): """Process the contents of a PyPI page""" def scan(link): @@ -260,9 +260,11 @@ class PackageIndex(Environment): def find_packages(self, requirement): self.scan_url(self.index_url + requirement.unsafe_name+'/') + if not self.package_pages.get(requirement.key): # Fall back to safe version of the name self.scan_url(self.index_url + requirement.project_name+'/') + if not self.package_pages.get(requirement.key): # We couldn't find the target package, so search the index page too self.warn( @@ -276,15 +278,13 @@ class PackageIndex(Environment): self.scan_url(url) def obtain(self, requirement, installer=None): - self.find_packages(requirement) + self.prescan(); self.find_packages(requirement) for dist in self[requirement.key]: if dist in requirement: return dist self.debug("%s does not match %s", requirement, dist) return super(PackageIndex, self).obtain(requirement,installer) - - def check_md5(self, cs, info, filename, tfp): if re.match('md5=[0-9a-f]{32}$', info): self.debug("Validating md5 checksum for %s", filename) @@ -296,26 +296,26 @@ class PackageIndex(Environment): "; possible download problem?" ) + def add_find_links(self, urls): + """Add `urls` to the list that will be prescanned for searches""" + for url in urls: + if ( + self.to_scan is None # if we have already "gone online" + or not URL_SCHEME(url) # or it's a local file/directory + or url.startswith('file:') + or list(distros_for_url(url)) # or a direct package link + ): + # then go ahead and process it now + self.scan_url(url) + else: + # otherwise, defer retrieval till later + self.to_scan.append(url) - - - - - - - - - - - - - - - - - - - + def prescan(self): + """Scan urls scheduled for prescanning (e.g. --find-links)""" + if self.to_scan: + map(self.scan_url, self.to_scan) + self.to_scan = None # from now on, go ahead and process immediately @@ -409,13 +409,17 @@ class PackageIndex(Environment): ) if force_scan: + self.prescan() self.find_packages(requirement) + + dist = find(requirement) + if dist is None and self.to_scan is not None: + self.prescan() dist = find(requirement) - else: + + if dist is None and not force_scan: + self.find_packages(requirement) dist = find(requirement) - if dist is None: - self.find_packages(requirement) - dist = find(requirement) if dist is None: self.warn( @@ -445,10 +449,6 @@ class PackageIndex(Environment): - - - - def gen_setup(self, filename, fragment, tmpdir): match = EGG_FRAGMENT.match(fragment); #import pdb; pdb.set_trace() dists = match and [d for d in -- cgit v1.2.1 From dc66bc4eb7cd84ba849eea2c652f795bc243265f Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Fri, 3 Mar 2006 23:51:00 +0000 Subject: Don't recurse into subdirectories when scanning --find-links --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4042822 --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index c02f3b4e..ad996e57 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -182,7 +182,7 @@ class PackageIndex(Environment): self.warn("Not found: %s", url) return - if os.path.isdir(fn): + if os.path.isdir(fn) and not nested: path = os.path.realpath(fn) for item in os.listdir(path): self.process_filename(os.path.join(path,item), True) -- cgit v1.2.1 From d5309081b92d1a6c1cc5565fb84a9e4c367e43f9 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Fri, 17 Mar 2006 18:05:54 +0000 Subject: Fix a problem with fetch() method backward compatibility. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4043122 --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index ad996e57..3d858e77 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -437,7 +437,7 @@ class PackageIndex(Environment): ``location`` of the downloaded distribution instead of a distribution object. """ - dist = self.fetch_dist(requirement,tmpdir,force_scan,source) + dist = self.fetch_distribution(requirement,tmpdir,force_scan,source) if dist is not None: return dist.location return None -- cgit v1.2.1 From 176f17c6eb0527a4f251a5ef7ea7fc0e37382ce9 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Tue, 28 Mar 2006 23:06:58 +0000 Subject: Fall back to a reasonable default Sourceforge address if the machine is unable to obtain the mirror IP list via DNS. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4043413 --- setuptools/package_index.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 3d858e77..c0dc92a8 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -623,7 +623,11 @@ def fix_sf_url(url): def get_sf_ip(_mirrors=[]): if not _mirrors: - _mirrors[:] = socket.gethostbyname_ex('dl.sourceforge.net')[-1] + try: + _mirrors[:] = socket.gethostbyname_ex('dl.sourceforge.net')[-1] + except socket.error: + # DNS-bl0ck1n9 f1r3w4llz sUx0rs! + _mirrors[:] = ['dl.sourceforge.net'] return random.choice(_mirrors) @@ -641,10 +645,6 @@ def get_sf_ip(_mirrors=[]): - - - - -- cgit v1.2.1 From c51417f82157df15a40d6fb3b1ba13a851a68763 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Sun, 9 Apr 2006 20:25:49 +0000 Subject: Added automatic retry for Sourceforge mirrors. The new download process is to first just try dl.sourceforge.net, then randomly select mirror IPs and remove ones that fail, until something works. The removed IPs stay removed for the remainder of the run. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4045219 --- setuptools/package_index.py | 59 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 50 insertions(+), 9 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index c0dc92a8..3d66a7c5 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -530,6 +530,47 @@ class PackageIndex(Environment): def reporthook(self, url, filename, blocknum, blksize, size): pass # no-op + + def retry_sf_download(self, url, filename): + try: + return self._download_to(url, filename) + except: + scheme, server, path, param, query, frag = urlparse.urlparse(url) + if server!='dl.sourceforge.net': + raise + + mirror = get_sf_ip() + + while _sf_mirrors: + self.warn("Download failed: %s", sys.exc_info()[1]) + url = urlparse.urlunparse((scheme, mirror, path, param, '', frag)) + try: + return self._download_to(url, filename) + except: + _sf_mirrors.remove(mirror) # don't retry the same mirror + mirror = get_sf_ip() + + raise # fail if no mirror works + + + + + + + + + + + + + + + + + + + + def open_url(self, url): try: @@ -562,7 +603,7 @@ class PackageIndex(Environment): if scheme=='svn' or scheme.startswith('svn+'): return self._download_svn(url, filename) else: - headers = self._download_to(url, filename) + headers = self.retry_sf_download(url, filename) if 'html' in headers['content-type'].lower(): return self._download_html(url, headers, filename, tmpdir) else: @@ -618,19 +659,19 @@ def fix_sf_url(url): if server!='prdownloads.sourceforge.net': return url return urlparse.urlunparse( - (scheme, get_sf_ip(), 'sourceforge'+path, param, '', frag) + (scheme, 'dl.sourceforge.net', 'sourceforge'+path, param, '', frag) ) -def get_sf_ip(_mirrors=[]): - if not _mirrors: +_sf_mirrors = [] + +def get_sf_ip(): + if not _sf_mirrors: try: - _mirrors[:] = socket.gethostbyname_ex('dl.sourceforge.net')[-1] + _sf_mirrors[:] = socket.gethostbyname_ex('dl.sourceforge.net')[-1] except socket.error: # DNS-bl0ck1n9 f1r3w4llz sUx0rs! - _mirrors[:] = ['dl.sourceforge.net'] - return random.choice(_mirrors) - - + _sf_mirrors[:] = ['dl.sourceforge.net'] + return random.choice(_sf_mirrors) -- cgit v1.2.1 From e3207bd63bcf365a1f91b7c3e75a4b3354435501 Mon Sep 17 00:00:00 2001 From: Dirley Rodrigues Date: Mon, 4 Feb 2013 11:30:58 -0200 Subject: Improve external links finder to not yield duplicate links. --HG-- branch : distribute extra : rebase_source : 78e932fca32ee0ee1f50794cf998f4e7db78131b --- setuptools/package_index.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 0ee21e3b..4393c83a 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -139,20 +139,26 @@ REL = re.compile("""<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>""", re.I) def find_external_links(url, page): """Find rel="homepage" and rel="download" links in `page`, yielding URLs""" + seen = set() for match in REL.finditer(page): tag, rel = match.groups() rels = map(str.strip, rel.lower().split(',')) if 'homepage' in rels or 'download' in rels: for match in HREF.finditer(tag): - yield urlparse.urljoin(url, htmldecode(match.group(1))) + url = urlparse.urljoin(url, htmldecode(match.group(1))) + if not url in seen: + yield url for tag in ("Home Page", "Download URL"): pos = page.find(tag) if pos!=-1: match = HREF.search(page,pos) if match: - yield urlparse.urljoin(url, htmldecode(match.group(1))) + url = urlparse.urljoin(url, htmldecode(match.group(1))) + if not url in seen: + yield url + user_agent = "Python-urllib/%s distribute/%s" % ( sys.version[:3], require('distribute')[0].version -- cgit v1.2.1 From 116420fe62842f18f2d37de46c2177028231755a Mon Sep 17 00:00:00 2001 From: Dirley Rodrigues Date: Mon, 4 Feb 2013 11:39:28 -0200 Subject: avoid naming problems --HG-- branch : distribute extra : rebase_source : 29eeb99013055b8d27cad7f7e8898d06a865b188 --- setuptools/package_index.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 4393c83a..984feef4 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -139,25 +139,25 @@ REL = re.compile("""<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>""", re.I) def find_external_links(url, page): """Find rel="homepage" and rel="download" links in `page`, yielding URLs""" - seen = set() + seen_links = set() for match in REL.finditer(page): tag, rel = match.groups() rels = map(str.strip, rel.lower().split(',')) if 'homepage' in rels or 'download' in rels: for match in HREF.finditer(tag): - url = urlparse.urljoin(url, htmldecode(match.group(1))) - if not url in seen: - yield url + link = urlparse.urljoin(url, htmldecode(match.group(1))) + if not link in seen_links: + yield link for tag in ("Home Page", "Download URL"): pos = page.find(tag) if pos!=-1: match = HREF.search(page,pos) if match: - url = urlparse.urljoin(url, htmldecode(match.group(1))) - if not url in seen: - yield url + link = urlparse.urljoin(url, htmldecode(match.group(1))) + if not link in seen_links: + yield link user_agent = "Python-urllib/%s distribute/%s" % ( -- cgit v1.2.1 From d7ba7ce3c4ce2427d78cb2393bd062aa3a8496b4 Mon Sep 17 00:00:00 2001 From: Dirley Rodrigues Date: Mon, 4 Feb 2013 11:54:53 -0200 Subject: actually filter the links --HG-- branch : distribute extra : rebase_source : cb6e3497e1f8594181f10110cbc833bd6c81f89e --- setuptools/package_index.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 984feef4..8974a647 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -148,6 +148,7 @@ def find_external_links(url, page): for match in HREF.finditer(tag): link = urlparse.urljoin(url, htmldecode(match.group(1))) if not link in seen_links: + seen_links.add(link) yield link for tag in ("Home Page", "Download URL"): @@ -157,6 +158,7 @@ def find_external_links(url, page): if match: link = urlparse.urljoin(url, htmldecode(match.group(1))) if not link in seen_links: + seen_links.add(link) yield link -- cgit v1.2.1 From b8327d7f646141415beb30acd39ce8840ebc708b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 24 May 2013 10:36:51 -0400 Subject: Add unique_everseen from Python 2.7 docs --HG-- branch : distribute --- setuptools/package_index.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 8974a647..e542f586 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1,5 +1,6 @@ """PyPI and direct package downloading""" import sys, os.path, re, urlparse, urllib, urllib2, shutil, random, socket, cStringIO +import itertools import base64 import httplib from pkg_resources import * @@ -134,6 +135,24 @@ def interpret_distro_name(location, basename, metadata, platform = platform ) +# From Python 2.7 docs +def unique_everseen(iterable, key=None): + "List unique elements, preserving order. Remember all elements ever seen." + # unique_everseen('AAAABBBCCDAABBB') --> A B C D + # unique_everseen('ABBCcAD', str.lower) --> A B C D + seen = set() + seen_add = seen.add + if key is None: + for element in itertools.ifilterfalse(seen.__contains__, iterable): + seen_add(element) + yield element + else: + for element in iterable: + k = key(element) + if k not in seen: + seen_add(k) + yield element + REL = re.compile("""<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>""", re.I) # this line is here to fix emacs' cruddy broken syntax highlighting -- cgit v1.2.1 From 3b9a57a0c80ee11995fbe937e7dbeca3d83ec10a Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 24 May 2013 11:02:05 -0400 Subject: Use a wrapper to ensure unique values on find_external_links. Factors out uniqueness test into a re-usable decorator and simplifies the body of find_external_links. --HG-- branch : distribute --- setuptools/package_index.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index e542f586..0a3f9e05 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -12,6 +12,8 @@ except ImportError: from md5 import md5 from fnmatch import translate +from .py24compat import wraps + EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.]+)$') HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I) # this is here to fix emacs' cruddy broken syntax highlighting @@ -153,32 +155,36 @@ def unique_everseen(iterable, key=None): seen_add(k) yield element +def unique_values(func): + """ + Wrap a function returning an iterable such that the resulting iterable + only ever yields unique items. + """ + @wraps(func) + def wrapper(*args, **kwargs): + return unique_everseen(func(*args, **kwargs)) + return wrapper + REL = re.compile("""<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>""", re.I) # this line is here to fix emacs' cruddy broken syntax highlighting +@unique_values def find_external_links(url, page): """Find rel="homepage" and rel="download" links in `page`, yielding URLs""" - seen_links = set() for match in REL.finditer(page): tag, rel = match.groups() rels = map(str.strip, rel.lower().split(',')) if 'homepage' in rels or 'download' in rels: for match in HREF.finditer(tag): - link = urlparse.urljoin(url, htmldecode(match.group(1))) - if not link in seen_links: - seen_links.add(link) - yield link + yield urlparse.urljoin(url, htmldecode(match.group(1))) for tag in ("Home Page", "Download URL"): pos = page.find(tag) if pos!=-1: match = HREF.search(page,pos) if match: - link = urlparse.urljoin(url, htmldecode(match.group(1))) - if not link in seen_links: - seen_links.add(link) - yield link + yield urlparse.urljoin(url, htmldecode(match.group(1))) user_agent = "Python-urllib/%s distribute/%s" % ( -- cgit v1.2.1 From f471fd20e76f794f04c69beca6095b98804bb971 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Wed, 19 Apr 2006 00:21:57 +0000 Subject: Backport support for file:// directory URLs in --find-links to 0.6 branch. --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4045556 --- setuptools/package_index.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 3d66a7c5..054220ba 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -179,7 +179,7 @@ class PackageIndex(Environment): def process_filename(self, fn, nested=False): # process filenames or directories if not os.path.exists(fn): - self.warn("Not found: %s", url) + self.warn("Not found: %s", fn) return if os.path.isdir(fn) and not nested: @@ -260,7 +260,7 @@ class PackageIndex(Environment): def find_packages(self, requirement): self.scan_url(self.index_url + requirement.unsafe_name+'/') - + if not self.package_pages.get(requirement.key): # Fall back to safe version of the name self.scan_url(self.index_url + requirement.project_name+'/') @@ -450,7 +450,7 @@ class PackageIndex(Environment): def gen_setup(self, filename, fragment, tmpdir): - match = EGG_FRAGMENT.match(fragment); #import pdb; pdb.set_trace() + match = EGG_FRAGMENT.match(fragment) dists = match and [d for d in interpret_distro_name(filename, match.group(1), None) if d.version ] or [] @@ -489,7 +489,7 @@ class PackageIndex(Environment): "Can't process plain .py files without an '#egg=name-version'" " suffix to enable automatic setup script generation." ) - + dl_blocksize = 8192 def _download_to(self, url, filename): self.url_ok(url,True) # raises error if not allowed @@ -582,7 +582,6 @@ class PackageIndex(Environment): def _download_url(self, scheme, url, tmpdir): - # Determine download filename # name = filter(None,urlparse.urlparse(url)[2].split('/')) @@ -602,6 +601,8 @@ class PackageIndex(Environment): # if scheme=='svn' or scheme.startswith('svn+'): return self._download_svn(url, filename) + elif scheme=='file': + return urllib2.url2pathname(urlparse.urlparse(url)[2]) else: headers = self.retry_sf_download(url, filename) if 'html' in headers['content-type'].lower(): @@ -612,7 +613,6 @@ class PackageIndex(Environment): def scan_url(self, url): self.process_url(url, True) - def _download_html(self, url, headers, filename, tmpdir): file = open(filename) for line in file: @@ -694,4 +694,4 @@ def get_sf_ip(): - +# this line is a kludge to keep the trailing blank lines for pje's editor -- cgit v1.2.1 From bfff594d486ab12b76b9957776227a18b4abc6c5 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Fri, 21 Apr 2006 17:08:23 +0000 Subject: Ignore bdist_dumb distributions when looking at download URLs --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4045627 --- setuptools/package_index.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 054220ba..e0db7273 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -96,7 +96,6 @@ def interpret_distro_name(location, basename, metadata, ``pkg_resources.normalize_path()`` on it before passing it to this routine! """ - # Generate alternative interpretations of a source distro name # Because some packages are ambiguous as to name/versions split # e.g. "adns-python-1.1.0", "egenix-mx-commercial", etc. @@ -110,6 +109,11 @@ def interpret_distro_name(location, basename, metadata, # versions in distribution archive names (sdist and bdist). parts = basename.split('-') + if not py_version: + for i,p in enumerate(parts[2:]): + if len(p)==5 and p.startswith('py2.'): + return # It's a bdist_dumb, not an sdist -- bail out + for p in range(1,len(parts)+1): yield Distribution( location, metadata, '-'.join(parts[:p]), '-'.join(parts[p:]), @@ -117,10 +121,6 @@ def interpret_distro_name(location, basename, metadata, platform = platform ) - - - - class PackageIndex(Environment): """A distribution index that scans web pages for download URLs""" @@ -534,6 +534,8 @@ class PackageIndex(Environment): def retry_sf_download(self, url, filename): try: return self._download_to(url, filename) + except (KeyboardInterrupt,SystemExit): + raise except: scheme, server, path, param, query, frag = urlparse.urlparse(url) if server!='dl.sourceforge.net': @@ -569,8 +571,6 @@ class PackageIndex(Environment): - - def open_url(self, url): try: -- cgit v1.2.1 From 27874986d6539405fc971e9cf70563a096ee5b83 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Fri, 12 May 2006 22:17:31 +0000 Subject: Better ambiguity management: accept #egg name/version even if processing what appears to be a correctly-named distutils file, and ignore .egg files with no '-', since valid Python .egg files always have a version number (but Scheme eggs often don't). --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4045984 --- setuptools/package_index.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index e0db7273..2760dc8d 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -48,20 +48,21 @@ def egg_info_for_url(url): def distros_for_url(url, metadata=None): """Yield egg or source distribution objects that might be found at a URL""" base, fragment = egg_info_for_url(url) - dists = distros_for_location(url, base, metadata) - if fragment and not dists: + for dist in distros_for_location(url, base, metadata): yield dist + if fragment: match = EGG_FRAGMENT.match(fragment) if match: - return interpret_distro_name( + for dist in interpret_distro_name( url, match.group(1), metadata, precedence = CHECKOUT_DIST - ) - return dists + ): + yield dist def distros_for_location(location, basename, metadata=None): """Yield egg or source distribution objects based on basename""" if basename.endswith('.egg.zip'): basename = basename[:-4] # strip the .zip - if basename.endswith('.egg'): # only one, unambiguous interpretation + if basename.endswith('.egg') and '-' in basename: + # only one, unambiguous interpretation return [Distribution.from_location(location, basename, metadata)] if basename.endswith('.exe'): @@ -79,7 +80,6 @@ def distros_for_location(location, basename, metadata=None): return interpret_distro_name(location, basename, metadata) return [] # no extension matched - def distros_for_filename(filename, metadata=None): """Yield possible egg or source distribution objects based on a filename""" return distros_for_location( -- cgit v1.2.1 From cf2cb97ab9dcb6b4245e0b4784f916a1c50c5156 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Tue, 20 Jun 2006 21:21:08 +0000 Subject: Fix ``ftp://`` directory listing URLs from causing a crash when used in the "Home page" or "Download URL" slots on PyPI. (merged from the trunk) --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4047051 --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 2760dc8d..d13bfc0f 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -162,7 +162,7 @@ class PackageIndex(Environment): self.fetched_urls[url] = self.fetched_urls[f.url] = True - if 'html' not in f.headers['content-type'].lower(): + if 'html' not in f.headers.get('content-type', '').lower(): f.close() # not html, we can't process it return -- cgit v1.2.1 From ac7808e631535975e29298f7b41c38a6de3b26e3 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Mon, 10 Jul 2006 21:55:25 +0000 Subject: Fix not recognizing HTML 404 pages from package indexes. (backport from trunk) --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4050552 --- setuptools/package_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index d13bfc0f..4c81580b 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -169,7 +169,7 @@ class PackageIndex(Environment): base = f.url # handle redirects page = f.read() f.close() - if url.startswith(self.index_url): + if url.startswith(self.index_url) and getattr(f,'code',None)!=404: page = self.process_index(url, page) for match in HREF.finditer(page): @@ -253,7 +253,7 @@ class PackageIndex(Environment): def scan_all(self, msg=None, *args): if self.index_url not in self.fetched_urls: if msg: self.warn(msg,*args) - self.warn( + self.info( "Scanning index of all packages (this may take a while)" ) self.scan_url(self.index_url) -- cgit v1.2.1 From 55742ce120b051dbc90e12c468da81dbe902cfea Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Mon, 10 Jul 2006 23:03:20 +0000 Subject: Allow ``file://`` URLs to be used as a package index. URLs that refer to directories will use an internally-generated directory listing if there is no ``index.html`` file in the directory. (backport from trunk) --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4050555 --- setuptools/package_index.py | 56 ++++++++++++++++++++++----------------------- 1 file changed, 28 insertions(+), 28 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 4c81580b..efdd02f3 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1,6 +1,6 @@ """PyPI and direct package downloading""" -import sys, os.path, re, urlparse, urllib2, shutil, random, socket +import sys, os.path, re, urlparse, urllib2, shutil, random, socket, cStringIO from pkg_resources import * from distutils import log from distutils.errors import DistutilsError @@ -573,6 +573,8 @@ class PackageIndex(Environment): def open_url(self, url): + if url.startswith('file:'): + return local_open(url) try: return urllib2.urlopen(url) except urllib2.HTTPError, v: @@ -610,6 +612,7 @@ class PackageIndex(Environment): else: return filename + def scan_url(self, url): self.process_url(url, True) @@ -643,17 +646,6 @@ class PackageIndex(Environment): def warn(self, msg, *args): log.warn(msg, *args) - - - - - - - - - - - def fix_sf_url(url): scheme, server, path, param, query, frag = urlparse.urlparse(url) if server!='prdownloads.sourceforge.net': @@ -674,22 +666,30 @@ def get_sf_ip(): return random.choice(_sf_mirrors) - - - - - - - - - - - - - - - - +def local_open(url): + """Read a local path, with special support for directories""" + scheme, server, path, param, query, frag = urlparse.urlparse(url) + filename = urllib2.url2pathname(path) + if os.path.isfile(filename): + return urllib2.urlopen(url) + elif path.endswith('/') and os.path.isdir(filename): + files = [] + for f in os.listdir(filename): + if f=='index.html': + body = open(os.path.join(filename,f),'rb').read() + break + elif os.path.isdir(os.path.join(filename,f)): + f+='/' + files.append("%s" % (f,f)) + else: + body = ("%s" % url) + \ + "%s" % '\n'.join(files) + status, message = 200, "OK" + else: + status, message, body = 404, "Path not found", "Not found" + + return urllib2.HTTPError(url, status, message, + {'content-type':'text/html'}, cStringIO.StringIO(body)) -- cgit v1.2.1 From 6f78e2f6d9e8aff2697c7262b8489ba9e635f17d Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Tue, 11 Jul 2006 00:14:36 +0000 Subject: Reduce screenscraping required for a package index. Homepage and download URLs can now be marked with 'rel="download"' and 'rel="homepage"' respectively, and the 'Index of Packages' string is no longer required. Since PyPI doesn't yet support rel="" attributes, the old ""-matching code remains, as does the MD5 scraping. (backport from trunk) --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4050557 --- setuptools/package_index.py | 91 ++++++++++++++++++++++++++++++++------------- 1 file changed, 66 insertions(+), 25 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index efdd02f3..5bd08f8a 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -120,6 +120,47 @@ def interpret_distro_name(location, basename, metadata, py_version=py_version, precedence = precedence, platform = platform ) + +REL = re.compile("""<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>""", re.I) +# this line is here to fix emacs' cruddy broken syntax highlighting + +def find_external_links(url, page): + """Find rel="homepage" and rel="download" links in `page`, yielding URLs""" + + for match in REL.finditer(page): + tag, rel = match.groups() + rels = map(str.strip, rel.lower().split(',')) + if 'homepage' in rels or 'download' in rels: + for match in HREF.finditer(tag): + yield urlparse.urljoin(url, match.group(1)) + + for tag in ("Home Page", "Download URL"): + pos = page.find(tag) + if pos!=-1: + match = HREF.search(page,pos) + if match: + yield urlparse.urljoin(url, match.group(1)) + + + + + + + + + + + + + + + + + + + + + class PackageIndex(Environment): """A distribution index that scans web pages for download URLs""" @@ -211,7 +252,7 @@ class PackageIndex(Environment): parts = map( urllib2.unquote, link[len(self.index_url):].split('/') ) - if len(parts)==2: + if len(parts)==2 and '#' not in parts[1]: # it's a package page, sanitize and index it pkg = safe_name(parts[0]) ver = safe_version(parts[1]) @@ -219,30 +260,30 @@ class PackageIndex(Environment): return to_filename(pkg), to_filename(ver) return None, None - if url==self.index_url or 'Index of Packages' in page: - # process an index page into the package-page index - for match in HREF.finditer(page): - scan( urlparse.urljoin(url, match.group(1)) ) - else: - pkg,ver = scan(url) # ensure this page is in the page index + # process an index page into the package-page index + for match in HREF.finditer(page): + scan( urlparse.urljoin(url, match.group(1)) ) + + pkg, ver = scan(url) # ensure this page is in the page index + if pkg: # process individual package page - for tag in ("Home Page", "Download URL"): - pos = page.find(tag) - if pos!=-1: - match = HREF.search(page,pos) - if match: - # Process the found URL - new_url = urlparse.urljoin(url, match.group(1)) - base, frag = egg_info_for_url(new_url) - if base.endswith('.py') and not frag: - if pkg and ver: - new_url+='#egg=%s-%s' % (pkg,ver) - else: - self.need_version_info(url) - self.scan_url(new_url) - return PYPI_MD5.sub( - lambda m: '%s' % m.group(1,3,2), page - ) + for new_url in find_external_links(url, page): + # Process the found URL + base, frag = egg_info_for_url(new_url) + if base.endswith('.py') and not frag: + if ver: + new_url+='#egg=%s-%s' % (pkg,ver) + else: + self.need_version_info(url) + self.scan_url(new_url) + + return PYPI_MD5.sub( + lambda m: '%s' % m.group(1,3,2), page + ) + else: + return "" # no sense double-scanning non-package pages + + def need_version_info(self, url): self.scan_all( @@ -273,7 +314,7 @@ class PackageIndex(Environment): ) self.scan_all() - for url in self.package_pages.get(requirement.key,()): + for url in list(self.package_pages.get(requirement.key,())): # scan each page that might be related to the desired package self.scan_url(url) -- cgit v1.2.1 From 95637d99856e04680612f139a57f4b112f491852 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Tue, 11 Jul 2006 18:26:54 +0000 Subject: Suppressed warning message about possibly-misspelled project name, if an egg or link for that project name has already been seen. (backport from trunk) --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4050581 --- setuptools/package_index.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 5bd08f8a..525f0e73 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -308,11 +308,7 @@ class PackageIndex(Environment): if not self.package_pages.get(requirement.key): # We couldn't find the target package, so search the index page too - self.warn( - "Couldn't find index page for %r (maybe misspelled?)", - requirement.unsafe_name - ) - self.scan_all() + self.not_found_in_index(requirement) for url in list(self.package_pages.get(requirement.key,())): # scan each page that might be related to the desired package @@ -326,6 +322,10 @@ class PackageIndex(Environment): self.debug("%s does not match %s", requirement, dist) return super(PackageIndex, self).obtain(requirement,installer) + + + + def check_md5(self, cs, info, filename, tfp): if re.match('md5=[0-9a-f]{32}$', info): self.debug("Validating md5 checksum for %s", filename) @@ -358,14 +358,14 @@ class PackageIndex(Environment): map(self.scan_url, self.to_scan) self.to_scan = None # from now on, go ahead and process immediately - - - - - - - - + def not_found_in_index(self, requirement): + if self[requirement.key]: # we've seen at least one distro + meth, msg = self.info, "Couldn't retrieve index page for %r" + else: # no distros seen for this name, might be misspelled + meth, msg = (self.warn, + "Couldn't find index page for %r (maybe misspelled?)") + meth(msg, requirement.unsafe_name) + self.scan_all() def download(self, spec, tmpdir): """Locate and/or download `spec` to `tmpdir`, returning a local path -- cgit v1.2.1 From bad362fe5671f2b1567b2f779f57b76edac7f55e Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Thu, 20 Jul 2006 20:41:01 +0000 Subject: EasyInstall now includes setuptools version information in the ``User-Agent`` string sent to websites it visits. (backport from trunk) --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4050733 --- setuptools/package_index.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 525f0e73..5bccbb8e 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -141,9 +141,9 @@ def find_external_links(url, page): if match: yield urlparse.urljoin(url, match.group(1)) - - - +user_agent = "Python-urllib/%s setuptools/%s" % ( + urllib2.__version__, require('setuptools')[0].version +) @@ -617,13 +617,14 @@ class PackageIndex(Environment): if url.startswith('file:'): return local_open(url) try: - return urllib2.urlopen(url) + request = urllib2.Request(url) + request.add_header('User-Agent', user_agent) + return urllib2.urlopen(request) except urllib2.HTTPError, v: return v except urllib2.URLError, v: raise DistutilsError("Download error: %s" % v.reason) - def _download_url(self, scheme, url, tmpdir): # Determine download filename # @@ -653,7 +654,6 @@ class PackageIndex(Environment): else: return filename - def scan_url(self, url): self.process_url(url, True) -- cgit v1.2.1 From 9dea600914272888a5c78e0de581000f854b0d6e Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Thu, 20 Jul 2006 20:47:41 +0000 Subject: Backport PyPI regex change. --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4050735 --- setuptools/package_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 5bccbb8e..837eff19 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -11,8 +11,8 @@ EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.]+)$') HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I) # this is here to fix emacs' cruddy broken syntax highlighting PYPI_MD5 = re.compile( - '([^<]+)\n\s+\\(md5\\)' + '([^<]+)\n\s+\\(md5\\)' ) URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):',re.I).match -- cgit v1.2.1 From 2c5978d3bb4f91f7f837e974c4abdc8248ed8d3f Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Mon, 25 Sep 2006 21:31:59 +0000 Subject: Fixed SF downloads aborting when a SF mirror returns an HTML page when it should've returned a 404. Fall back to ``sf-mirrors.telecommunity.com`` round-robin address for SF mirrors if ``dl.sourceforge.net`` doesn't work. (backport from trunk) --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4052004 --- setuptools/package_index.py | 44 ++++++++++++++++++++++---------------------- 1 file changed, 22 insertions(+), 22 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 837eff19..78e89daf 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -533,7 +533,6 @@ class PackageIndex(Environment): dl_blocksize = 8192 def _download_to(self, url, filename): - self.url_ok(url,True) # raises error if not allowed self.info("Downloading %s", url) # Download the file fp, tfp, info = None, None, None @@ -572,9 +571,18 @@ class PackageIndex(Environment): def reporthook(self, url, filename, blocknum, blksize, size): pass # no-op - def retry_sf_download(self, url, filename): + + def _attempt_download(self, url, filename): + headers = self._download_to(url, filename) + if 'html' in headers['content-type'].lower(): + return self._download_html(url, headers, filename) + else: + return filename + + def _retry_sf_download(self, url, filename): + self.url_ok(url, True) # raises error if not allowed try: - return self._download_to(url, filename) + return self._attempt_download(url, filename) except (KeyboardInterrupt,SystemExit): raise except: @@ -588,7 +596,9 @@ class PackageIndex(Environment): self.warn("Download failed: %s", sys.exc_info()[1]) url = urlparse.urlunparse((scheme, mirror, path, param, '', frag)) try: - return self._download_to(url, filename) + return self._attempt_download(url, filename) + except (KeyboardInterrupt,SystemExit): + raise except: _sf_mirrors.remove(mirror) # don't retry the same mirror mirror = get_sf_ip() @@ -603,16 +613,6 @@ class PackageIndex(Environment): - - - - - - - - - - def open_url(self, url): if url.startswith('file:'): return local_open(url) @@ -648,16 +648,16 @@ class PackageIndex(Environment): elif scheme=='file': return urllib2.url2pathname(urlparse.urlparse(url)[2]) else: - headers = self.retry_sf_download(url, filename) - if 'html' in headers['content-type'].lower(): - return self._download_html(url, headers, filename, tmpdir) - else: - return filename + return self._retry_sf_download(url, filename) + + + + def scan_url(self, url): self.process_url(url, True) - def _download_html(self, url, headers, filename, tmpdir): + def _download_html(self, url, headers, filename): file = open(filename) for line in file: if line.strip(): @@ -700,7 +700,8 @@ _sf_mirrors = [] def get_sf_ip(): if not _sf_mirrors: try: - _sf_mirrors[:] = socket.gethostbyname_ex('dl.sourceforge.net')[-1] + _sf_mirrors[:] = socket.gethostbyname_ex( + 'sf-mirrors.telecommunity.com')[-1] except socket.error: # DNS-bl0ck1n9 f1r3w4llz sUx0rs! _sf_mirrors[:] = ['dl.sourceforge.net'] @@ -734,5 +735,4 @@ def local_open(url): - # this line is a kludge to keep the trailing blank lines for pje's editor -- cgit v1.2.1 From c542669d4d7a6a2ee3a797a14362eef2a69ed089 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Wed, 27 Sep 2006 01:58:22 +0000 Subject: Allow explicit selection of Sourceforge mirror(s) with ``--sf-mirror``, and further refine download/retry algorithm. (backport from trunk) --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4052013 --- setuptools/package_index.py | 105 ++++++++++++++++++++++++++++++-------------- 1 file changed, 73 insertions(+), 32 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 78e89daf..ae816609 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -14,7 +14,7 @@ PYPI_MD5 = re.compile( '([^<]+)\n\s+\\(md5\\)' ) - +SF_DOWNLOAD = 'dl.sourceforge.net' URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):',re.I).match EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split() @@ -165,7 +165,9 @@ user_agent = "Python-urllib/%s setuptools/%s" % ( class PackageIndex(Environment): """A distribution index that scans web pages for download URLs""" - def __init__(self,index_url="http://www.python.org/pypi",hosts=('*',),*args,**kw): + def __init__(self, index_url="http://www.python.org/pypi", hosts=('*',), + sf_mirrors=None, *args, **kw + ): Environment.__init__(self,*args,**kw) self.index_url = index_url + "/"[:not index_url.endswith('/')] self.scanned_urls = {} @@ -173,6 +175,33 @@ class PackageIndex(Environment): self.package_pages = {} self.allows = re.compile('|'.join(map(translate,hosts))).match self.to_scan = [] + if sf_mirrors: + if isinstance(sf_mirrors,str): + self.sf_mirrors = map(str.strip, sf_mirrors.split(',')) + else: + self.sf_mirrors = map(str.strip, sf_mirrors) + else: + self.sf_mirrors = () + + + def _get_mirrors(self): + mirrors = [] + for mirror in self.sf_mirrors: + if mirror: + if '.' not in mirror: + mirror += '.dl.sourceforge.net' + mirrors.append(mirror) + + if not mirrors: + try: + mirrors.extend( + socket.gethostbyname_ex('sf-mirrors.telecommunity.com')[-1] + ) + except socket.error: + # DNS-bl0ck1n9 f1r3w4llz sUx0rs! + mirrors[:] = [SF_DOWNLOAD] + + return mirrors def process_url(self, url, retrieve=False): """Evaluate a URL as a possible download, and maybe retrieve it""" @@ -202,7 +231,6 @@ class PackageIndex(Environment): f = self.open_url(url) self.fetched_urls[url] = self.fetched_urls[f.url] = True - if 'html' not in f.headers.get('content-type', '').lower(): f.close() # not html, we can't process it return @@ -212,7 +240,6 @@ class PackageIndex(Environment): f.close() if url.startswith(self.index_url) and getattr(f,'code',None)!=404: page = self.process_index(url, page) - for match in HREF.finditer(page): link = urlparse.urljoin(base, match.group(1)) self.process_url(link) @@ -244,6 +271,20 @@ class PackageIndex(Environment): + + + + + + + + + + + + + + def process_index(self,url,page): """Process the contents of a PyPI page""" def scan(link): @@ -581,27 +622,27 @@ class PackageIndex(Environment): def _retry_sf_download(self, url, filename): self.url_ok(url, True) # raises error if not allowed - try: - return self._attempt_download(url, filename) - except (KeyboardInterrupt,SystemExit): - raise - except: - scheme, server, path, param, query, frag = urlparse.urlparse(url) - if server!='dl.sourceforge.net': - raise + scheme, server, path, param, query, frag = urlparse.urlparse(url) + + if server == SF_DOWNLOAD: + mirrors = self._get_mirrors() + query = '' + else: + mirrors = [server] - mirror = get_sf_ip() + while mirrors or server != SF_DOWNLOAD: + mirror = random.choice(mirrors) + url = urlparse.urlunparse((scheme,mirror,path,param,query,frag)) - while _sf_mirrors: - self.warn("Download failed: %s", sys.exc_info()[1]) - url = urlparse.urlunparse((scheme, mirror, path, param, '', frag)) try: return self._attempt_download(url, filename) except (KeyboardInterrupt,SystemExit): raise except: - _sf_mirrors.remove(mirror) # don't retry the same mirror - mirror = get_sf_ip() + if server != SF_DOWNLOAD: + raise + self.warn("Download failed: %s", sys.exc_info()[1]) + mirrors.remove(mirror) raise # fail if no mirror works @@ -692,22 +733,9 @@ def fix_sf_url(url): if server!='prdownloads.sourceforge.net': return url return urlparse.urlunparse( - (scheme, 'dl.sourceforge.net', 'sourceforge'+path, param, '', frag) + (scheme, SF_DOWNLOAD, 'sourceforge'+path, param, '', frag) ) -_sf_mirrors = [] - -def get_sf_ip(): - if not _sf_mirrors: - try: - _sf_mirrors[:] = socket.gethostbyname_ex( - 'sf-mirrors.telecommunity.com')[-1] - except socket.error: - # DNS-bl0ck1n9 f1r3w4llz sUx0rs! - _sf_mirrors[:] = ['dl.sourceforge.net'] - return random.choice(_sf_mirrors) - - def local_open(url): """Read a local path, with special support for directories""" scheme, server, path, param, query, frag = urlparse.urlparse(url) @@ -735,4 +763,17 @@ def local_open(url): + + + + + + + + + + + + + # this line is a kludge to keep the trailing blank lines for pje's editor -- cgit v1.2.1 From a2c732458d913a8eac68ad2c19af09167e0ccbfa Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Fri, 29 Dec 2006 00:34:24 +0000 Subject: Removed all special support for Sourceforge mirrors, as Sourceforge's mirror system now works well for non-browser downloads. (backport from trunk) --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4053178 --- setuptools/package_index.py | 116 +++++++------------------------------------- 1 file changed, 17 insertions(+), 99 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index ae816609..7e290d0c 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -14,7 +14,7 @@ PYPI_MD5 = re.compile( '([^<]+)\n\s+\\(md5\\)' ) -SF_DOWNLOAD = 'dl.sourceforge.net' + URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):',re.I).match EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split() @@ -146,27 +146,11 @@ user_agent = "Python-urllib/%s setuptools/%s" % ( ) - - - - - - - - - - - - - - - - class PackageIndex(Environment): """A distribution index that scans web pages for download URLs""" def __init__(self, index_url="http://www.python.org/pypi", hosts=('*',), - sf_mirrors=None, *args, **kw + *args, **kw ): Environment.__init__(self,*args,**kw) self.index_url = index_url + "/"[:not index_url.endswith('/')] @@ -175,39 +159,14 @@ class PackageIndex(Environment): self.package_pages = {} self.allows = re.compile('|'.join(map(translate,hosts))).match self.to_scan = [] - if sf_mirrors: - if isinstance(sf_mirrors,str): - self.sf_mirrors = map(str.strip, sf_mirrors.split(',')) - else: - self.sf_mirrors = map(str.strip, sf_mirrors) - else: - self.sf_mirrors = () - - def _get_mirrors(self): - mirrors = [] - for mirror in self.sf_mirrors: - if mirror: - if '.' not in mirror: - mirror += '.dl.sourceforge.net' - mirrors.append(mirror) - if not mirrors: - try: - mirrors.extend( - socket.gethostbyname_ex('sf-mirrors.telecommunity.com')[-1] - ) - except socket.error: - # DNS-bl0ck1n9 f1r3w4llz sUx0rs! - mirrors[:] = [SF_DOWNLOAD] - - return mirrors def process_url(self, url, retrieve=False): """Evaluate a URL as a possible download, and maybe retrieve it""" - url = fix_sf_url(url) if url in self.scanned_urls and not retrieve: return + self.scanned_urls[url] = True if not URL_SCHEME(url): self.process_filename(url) @@ -613,47 +572,6 @@ class PackageIndex(Environment): pass # no-op - def _attempt_download(self, url, filename): - headers = self._download_to(url, filename) - if 'html' in headers['content-type'].lower(): - return self._download_html(url, headers, filename) - else: - return filename - - def _retry_sf_download(self, url, filename): - self.url_ok(url, True) # raises error if not allowed - scheme, server, path, param, query, frag = urlparse.urlparse(url) - - if server == SF_DOWNLOAD: - mirrors = self._get_mirrors() - query = '' - else: - mirrors = [server] - - while mirrors or server != SF_DOWNLOAD: - mirror = random.choice(mirrors) - url = urlparse.urlunparse((scheme,mirror,path,param,query,frag)) - - try: - return self._attempt_download(url, filename) - except (KeyboardInterrupt,SystemExit): - raise - except: - if server != SF_DOWNLOAD: - raise - self.warn("Download failed: %s", sys.exc_info()[1]) - mirrors.remove(mirror) - - raise # fail if no mirror works - - - - - - - - - def open_url(self, url): if url.startswith('file:'): return local_open(url) @@ -689,15 +607,19 @@ class PackageIndex(Environment): elif scheme=='file': return urllib2.url2pathname(urlparse.urlparse(url)[2]) else: - return self._retry_sf_download(url, filename) - - - - + self.url_ok(url, True) # raises error if not allowed + return self._attempt_download(url, filename) def scan_url(self, url): self.process_url(url, True) + def _attempt_download(self, url, filename): + headers = self._download_to(url, filename) + if 'html' in headers['content-type'].lower(): + return self._download_html(url, headers, filename) + else: + return filename + def _download_html(self, url, headers, filename): file = open(filename) for line in file: @@ -728,13 +650,12 @@ class PackageIndex(Environment): def warn(self, msg, *args): log.warn(msg, *args) + + + + def fix_sf_url(url): - scheme, server, path, param, query, frag = urlparse.urlparse(url) - if server!='prdownloads.sourceforge.net': - return url - return urlparse.urlunparse( - (scheme, SF_DOWNLOAD, 'sourceforge'+path, param, '', frag) - ) + return url # backward compatibility def local_open(url): """Read a local path, with special support for directories""" @@ -773,7 +694,4 @@ def local_open(url): - - - # this line is a kludge to keep the trailing blank lines for pje's editor -- cgit v1.2.1 From 5e2fb1dcd218f4dab91a0beb3c5ce5fe7b89dd81 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Sat, 30 Dec 2006 16:05:41 +0000 Subject: Add Basic Auth support for http URLs with embedded credentials. If an authenticated page contains links to the same protocol and host, those links should inherit the same credentials. (backport from trunk) --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4053203 --- setuptools/package_index.py | 47 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 7e290d0c..f6bc45f5 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -576,9 +576,7 @@ class PackageIndex(Environment): if url.startswith('file:'): return local_open(url) try: - request = urllib2.Request(url) - request.add_header('User-Agent', user_agent) - return urllib2.urlopen(request) + return open_with_auth(url) except urllib2.HTTPError, v: return v except urllib2.URLError, v: @@ -613,6 +611,8 @@ class PackageIndex(Environment): def scan_url(self, url): self.process_url(url, True) + + def _attempt_download(self, url, filename): headers = self._download_to(url, filename) if 'html' in headers['content-type'].lower(): @@ -654,6 +654,47 @@ class PackageIndex(Environment): +def open_with_auth(url): + """Open a urllib2 request, handling HTTP authentication""" + + scheme, netloc, path, params, query, frag = urlparse.urlparse(url) + + if scheme in ('http', 'https'): + auth, host = urllib2.splituser(netloc) + else: + auth = None + + if auth: + auth = "Basic " + urllib2.unquote(auth).encode('base64').strip() + new_url = urlparse.urlunparse((scheme,host,path,params,query,frag)) + request = urllib2.Request(new_url) + request.add_header("Authorization", auth) + else: + request = urllib2.Request(url) + + request.add_header('User-Agent', user_agent) + fp = urllib2.urlopen(request) + + if auth: + # Put authentication info back into request URL if same host, + # so that links found on the page will work + s2, h2, path2, param2, query2, frag2 = urlparse.urlparse(fp.url) + if s2==scheme and h2==host: + fp.url = urlparse.urlunparse((s2,netloc,path2,param2,query2,frag2)) + + return fp + + + + + + + + + + + + def fix_sf_url(url): return url # backward compatibility -- cgit v1.2.1 From 2dd7ed9ccf64f5136c0e6fc8f9bd8216fde53cf2 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Wed, 24 Jan 2007 15:42:26 +0000 Subject: EasyInstall no longer aborts the installation process if a URL it wants to retrieve can't be downloaded, unless the URL is an actual package download. Instead, it issues a warning and tries to keep going. (backport from trunk) --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4053542 --- setuptools/package_index.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index f6bc45f5..d3f3f561 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -166,7 +166,6 @@ class PackageIndex(Environment): """Evaluate a URL as a possible download, and maybe retrieve it""" if url in self.scanned_urls and not retrieve: return - self.scanned_urls[url] = True if not URL_SCHEME(url): self.process_filename(url) @@ -187,7 +186,8 @@ class PackageIndex(Environment): return self.info("Reading %s", url) - f = self.open_url(url) + f = self.open_url(url, "Download error: %s -- Some packages may not be found!") + if f is None: return self.fetched_urls[url] = self.fetched_urls[f.url] = True if 'html' not in f.headers.get('content-type', '').lower(): @@ -572,7 +572,7 @@ class PackageIndex(Environment): pass # no-op - def open_url(self, url): + def open_url(self, url, warning=None): if url.startswith('file:'): return local_open(url) try: @@ -580,7 +580,8 @@ class PackageIndex(Environment): except urllib2.HTTPError, v: return v except urllib2.URLError, v: - raise DistutilsError("Download error: %s" % v.reason) + if warning: self.warn(warning, v.reason) + else: raise DistutilsError("Download error: %s" % v.reason) def _download_url(self, scheme, url, tmpdir): # Determine download filename @@ -612,7 +613,6 @@ class PackageIndex(Environment): self.process_url(url, True) - def _attempt_download(self, url, filename): headers = self._download_to(url, filename) if 'html' in headers['content-type'].lower(): -- cgit v1.2.1 From fb98a95e9badb23765ea520fd5bd9e30d0f1fe4a Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Fri, 23 Feb 2007 20:29:58 +0000 Subject: Added ``--local-snapshots-ok`` flag, to allow building eggs from projects installed using ``setup.py develop``. (backport from trunk) --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4053877 --- setuptools/package_index.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index d3f3f561..e4d7e6b9 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -228,20 +228,20 @@ class PackageIndex(Environment): else: self.warn(msg, url) - - - - - - - - - - - - - - + def scan_egg_links(self, search_path): + for item in search_path: + if os.path.isdir(item): + for entry in os.listdir(item): + if entry.endswith('.egg-link'): + self.scan_egg_link(item, entry) + + def scan_egg_link(self, path, entry): + lines = filter(None, map(str.strip, file(os.path.join(path, entry)))) + if len(lines)==2: + for dist in find_distributions(os.path.join(path, lines[0])): + dist.location = os.path.join(path, *lines) + dist.precedence = SOURCE_DIST + self.add(dist) def process_index(self,url,page): -- cgit v1.2.1 From 89111e6143f3a9bb510433f529d4281681b7c66e Mon Sep 17 00:00:00 2001 From: Jim Fulton Date: Fri, 9 Mar 2007 15:56:51 +0000 Subject: Changed setuptools.package_index.PackageIndex.open_url to include the url in the exception. --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4054241 --- setuptools/package_index.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index e4d7e6b9..e4f96f0b 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -581,7 +581,9 @@ class PackageIndex(Environment): return v except urllib2.URLError, v: if warning: self.warn(warning, v.reason) - else: raise DistutilsError("Download error: %s" % v.reason) + else: + raise DistutilsError("Download error for %s: %s" + % (url, v.reason)) def _download_url(self, scheme, url, tmpdir): # Determine download filename -- cgit v1.2.1 From b364978eee6eaf2e03999ab0590a16278a03b13e Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Thu, 31 May 2007 17:30:55 +0000 Subject: Backport fixes and doc updates; prep for 0.6c6 release --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4055712 --- setuptools/package_index.py | 48 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 44 insertions(+), 4 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index e4f96f0b..3da253a5 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -132,14 +132,14 @@ def find_external_links(url, page): rels = map(str.strip, rel.lower().split(',')) if 'homepage' in rels or 'download' in rels: for match in HREF.finditer(tag): - yield urlparse.urljoin(url, match.group(1)) + yield urlparse.urljoin(url, htmldecode(match.group(1))) for tag in ("Home Page", "Download URL"): pos = page.find(tag) if pos!=-1: match = HREF.search(page,pos) if match: - yield urlparse.urljoin(url, match.group(1)) + yield urlparse.urljoin(url, htmldecode(match.group(1))) user_agent = "Python-urllib/%s setuptools/%s" % ( urllib2.__version__, require('setuptools')[0].version @@ -200,7 +200,7 @@ class PackageIndex(Environment): if url.startswith(self.index_url) and getattr(f,'code',None)!=404: page = self.process_index(url, page) for match in HREF.finditer(page): - link = urlparse.urljoin(base, match.group(1)) + link = urlparse.urljoin(base, htmldecode(match.group(1))) self.process_url(link) def process_filename(self, fn, nested=False): @@ -262,7 +262,7 @@ class PackageIndex(Environment): # process an index page into the package-page index for match in HREF.finditer(page): - scan( urlparse.urljoin(url, match.group(1)) ) + scan( urlparse.urljoin(url, htmldecode(match.group(1))) ) pkg, ver = scan(url) # ensure this page is in the page index if pkg: @@ -611,6 +611,8 @@ class PackageIndex(Environment): self.url_ok(url, True) # raises error if not allowed return self._attempt_download(url, filename) + + def scan_url(self, url): self.process_url(url, True) @@ -652,6 +654,44 @@ class PackageIndex(Environment): def warn(self, msg, *args): log.warn(msg, *args) +# This pattern matches a character entity reference (a decimal numeric +# references, a hexadecimal numeric reference, or a named reference). +entity_sub = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub + +def uchr(c): + if not isinstance(c, int): + return c + if c>255: return unichr(c) + return chr(c) + +def decode_entity(match): + what = match.group(1) + if what.startswith('#x'): + what = int(what[2:], 16) + elif what.startswith('#'): + what = int(what[1:]) + else: + from htmlentitydefs import name2codepoint + what = name2codepoint.get(what, match.group(0)) + return uchr(what) + +def htmldecode(text): + """Decode HTML entities in the given text.""" + return entity_sub(decode_entity, text) + + + + + + + + + + + + + + -- cgit v1.2.1 From 996980750968d02c921bcfd33562969e97573ed9 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Fri, 24 Aug 2007 03:38:27 +0000 Subject: ``ftp:`` download URLs now work correctly. (backport from trunk) --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4057371 --- setuptools/package_index.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 3da253a5..d558b648 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -619,7 +619,7 @@ class PackageIndex(Environment): def _attempt_download(self, url, filename): headers = self._download_to(url, filename) - if 'html' in headers['content-type'].lower(): + if 'html' in headers.get('content-type','').lower(): return self._download_html(url, headers, filename) else: return filename @@ -695,7 +695,6 @@ def htmldecode(text): - def open_with_auth(url): """Open a urllib2 request, handling HTTP authentication""" -- cgit v1.2.1 From a44ba4da6f1bbdc0290905c139becf9b728cb0f6 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Tue, 4 Sep 2007 15:56:34 +0000 Subject: The default ``--index-url`` is now ``http://pypi.python.org/simple``, to use the Python Package Index's new simpler (and faster!) REST API. (backport from trunk) --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4057965 --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index d558b648..fed36a88 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -149,7 +149,7 @@ user_agent = "Python-urllib/%s setuptools/%s" % ( class PackageIndex(Environment): """A distribution index that scans web pages for download URLs""" - def __init__(self, index_url="http://www.python.org/pypi", hosts=('*',), + def __init__(self, index_url="http://pypi.python.org/simple", hosts=('*',), *args, **kw ): Environment.__init__(self,*args,**kw) -- cgit v1.2.1 From 3430324f7fcd38e9d1a8c045812e80376292788e Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Thu, 21 Aug 2008 17:56:23 +0000 Subject: Fix for http://bugs.python.org/setuptools/issue7 (backport from trunk) --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4065943 --- setuptools/package_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index fed36a88..84f1fcea 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -220,7 +220,8 @@ class PackageIndex(Environment): map(self.add, dists) def url_ok(self, url, fatal=False): - if self.allows(urlparse.urlparse(url)[1]): + s = URL_SCHEME(url) + if (s and s.group(1).lower()=='file') or self.allows(urlparse.urlparse(url)[1]): return True msg = "\nLink to % s ***BLOCKED*** by --allow-hosts\n" if fatal: @@ -243,7 +244,6 @@ class PackageIndex(Environment): dist.precedence = SOURCE_DIST self.add(dist) - def process_index(self,url,page): """Process the contents of a PyPI page""" def scan(link): -- cgit v1.2.1 From 2d3007539b9ed75f8f5d5a35284640a03235f422 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Thu, 21 Aug 2008 18:00:54 +0000 Subject: Fix for http://bugs.python.org/setuptools/issue5 (backport from trunk) --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4065945 --- setuptools/package_index.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 84f1fcea..055291a1 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1,10 +1,12 @@ """PyPI and direct package downloading""" - import sys, os.path, re, urlparse, urllib2, shutil, random, socket, cStringIO from pkg_resources import * from distutils import log from distutils.errors import DistutilsError -from md5 import md5 +try: + from hashlib import md5 +except ImportError: + from md5 import md5 from fnmatch import translate EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.]+)$') @@ -14,7 +16,6 @@ PYPI_MD5 = re.compile( '([^<]+)\n\s+\\(md5\\)' ) - URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):',re.I).match EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split() @@ -23,7 +24,6 @@ __all__ = [ 'interpret_distro_name', ] - def parse_bdist_wininst(name): """Return (base,pyversion) or (None,None) for possible .exe name""" -- cgit v1.2.1 From d76df70dd1dffb2591c6a26ecc2a20958ee03073 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Thu, 21 Aug 2008 19:10:15 +0000 Subject: Fix for http://bugs.python.org/setuptools/issue29 (backport from trunk) --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4065953 --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 055291a1..da3fed12 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -629,7 +629,7 @@ class PackageIndex(Environment): for line in file: if line.strip(): # Check for a subversion index page - if re.search(r'Revision \d+:', line): + if re.search(r'<title>([^- ]+ - )?Revision \d+:', line): # it's a subversion index page: file.close() os.unlink(filename) -- cgit v1.2.1 From 334f5b69c08cc7b3f60f963a689987412db2e635 Mon Sep 17 00:00:00 2001 From: Hanno Schlichting <hanno@hannosch.eu> Date: Thu, 16 Jul 2009 16:57:09 +0200 Subject: Apply patch from pjenvey. Closes #3. --HG-- branch : distribute extra : rebase_source : 3a61d0692c74559b140c179dcc5f4ac4905bb982 --- setuptools/package_index.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index da3fed12..8321eece 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -749,7 +749,9 @@ def local_open(url): files = [] for f in os.listdir(filename): if f=='index.html': - body = open(os.path.join(filename,f),'rb').read() + fp = open(os.path.join(filename,f),'rb') + body = fp.read() + fp.close() break elif os.path.isdir(os.path.join(filename,f)): f+='/' -- cgit v1.2.1 From e6684119501416733ff9c824f5e3d90cc8b75b0f Mon Sep 17 00:00:00 2001 From: tarek <none@none> Date: Tue, 21 Jul 2009 09:59:32 +0200 Subject: The User-Agent is also changed to Distribute --HG-- branch : distribute extra : rebase_source : 7a440abb08d1202a09ea2a0e46cac2568df2c759 --- setuptools/package_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 8321eece..e601cc15 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -141,8 +141,8 @@ def find_external_links(url, page): if match: yield urlparse.urljoin(url, htmldecode(match.group(1))) -user_agent = "Python-urllib/%s setuptools/%s" % ( - urllib2.__version__, require('setuptools')[0].version +user_agent = "Python-urllib/%s distribute/%s" % ( + urllib2.__version__, require('distribute')[0].version ) -- cgit v1.2.1 From 0ae5113b9bc5824a09b0e2c4e317084a4402cc0c Mon Sep 17 00:00:00 2001 From: tarek <none@none> Date: Wed, 26 Aug 2009 08:42:44 +0200 Subject: fixed #16 and #18: BadStatusLine and ValueError in package_index.urlopen --HG-- branch : distribute extra : rebase_source : 6159cf23c0dc4effd40b525066266eefd292b96e --- setuptools/package_index.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index e601cc15..fef62942 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1,5 +1,6 @@ """PyPI and direct package downloading""" import sys, os.path, re, urlparse, urllib2, shutil, random, socket, cStringIO +import httplib from pkg_resources import * from distutils import log from distutils.errors import DistutilsError @@ -577,13 +578,27 @@ class PackageIndex(Environment): return local_open(url) try: return open_with_auth(url) + except ValueError, v: + msg = ' '.join([str(arg) for arg in v.args]) + if warning: + self.warn(warning, msg) + else: + raise DistutilsError('%s %s' % (url, msg)) except urllib2.HTTPError, v: return v except urllib2.URLError, v: - if warning: self.warn(warning, v.reason) + if warning: + self.warn(warning, v.reason) else: raise DistutilsError("Download error for %s: %s" % (url, v.reason)) + except httplib.BadStatusLine, v: + if warning: + self.warn(warning, v.line) + else: + raise DistutilsError('%s returned a bad status line. ' + 'The server might be down, %s' % \ + (url, v.line)) def _download_url(self, scheme, url, tmpdir): # Determine download filename -- cgit v1.2.1 From ae8f6226eacfce9c5519f562c750b9db7aa98bdf Mon Sep 17 00:00:00 2001 From: tarek <none@none> Date: Fri, 28 Aug 2009 11:01:48 +0200 Subject: fixed #20 - catching invalid URL error from httplib --HG-- branch : distribute extra : rebase_source : 6c6a92f65f6ac2d6d071ce525d0ef3e41167d59e --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index fef62942..220cdec7 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -578,7 +578,7 @@ class PackageIndex(Environment): return local_open(url) try: return open_with_auth(url) - except ValueError, v: + except (ValueError, httplib.InvalidURL), v: msg = ' '.join([str(arg) for arg in v.args]) if warning: self.warn(warning, msg) -- cgit v1.2.1 From 6f3378c098a481588f5ec5813060376351e5a576 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= <martin@v.loewis.de> Date: Fri, 11 Sep 2009 23:49:34 +0200 Subject: Shortcut User-agent computation, as 2to3 won't update urllib2.__version__ correctly. --HG-- branch : distribute extra : rebase_source : 9035aa6fb13225181f7ce22429efa91c771247a6 --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 220cdec7..48494aff 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -143,7 +143,7 @@ def find_external_links(url, page): yield urlparse.urljoin(url, htmldecode(match.group(1))) user_agent = "Python-urllib/%s distribute/%s" % ( - urllib2.__version__, require('distribute')[0].version + sys.version[:3], require('distribute')[0].version ) -- cgit v1.2.1 From bd900046b3457bfd98ac36dfbdaac63d7acf0a67 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= <martin@v.loewis.de> Date: Sun, 13 Sep 2009 15:33:43 +0200 Subject: Convert HTML pages to text. --HG-- branch : distribute extra : rebase_source : 9ab9cc8b84da2be50c520cd6f23efb15b8744bd9 --- setuptools/package_index.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 48494aff..b6283dbc 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -197,6 +197,8 @@ class PackageIndex(Environment): base = f.url # handle redirects page = f.read() + charset = f.headers.get_param('charset') or 'latin-1' + page = page.decode(charset, "ignore") f.close() if url.startswith(self.index_url) and getattr(f,'code',None)!=404: page = self.process_index(url, page) -- cgit v1.2.1 From 55413ad562eddc5ffc235bde5471fdf010421f9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Martin=20v=2E=20L=C3=B6wis?= <martin@v.loewis.de> Date: Sun, 13 Sep 2009 21:30:44 +0200 Subject: Only decode HTML pages in 3.x. --HG-- branch : distribute extra : rebase_source : 58a286fe297947272398fc9ddd8f50594f54d4ae --- setuptools/package_index.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index b6283dbc..084370d5 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -197,8 +197,9 @@ class PackageIndex(Environment): base = f.url # handle redirects page = f.read() - charset = f.headers.get_param('charset') or 'latin-1' - page = page.decode(charset, "ignore") + if sys.version_info >= (3,): + charset = f.headers.get_param('charset') or 'latin-1' + page = page.decode(charset, "ignore") f.close() if url.startswith(self.index_url) and getattr(f,'code',None)!=404: page = self.process_index(url, page) -- cgit v1.2.1 From 3f06b14ae21d70149f4f404f2d9d82f3842c23dc Mon Sep 17 00:00:00 2001 From: agronholm <none@none> Date: Sun, 27 Sep 2009 01:12:15 +0300 Subject: Changed file() calls to open() calls --HG-- branch : distribute extra : rebase_source : a2567f3f28d896dd0abbbed8a2626cc4ecb3e44e --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 084370d5..98799de5 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -241,7 +241,7 @@ class PackageIndex(Environment): self.scan_egg_link(item, entry) def scan_egg_link(self, path, entry): - lines = filter(None, map(str.strip, file(os.path.join(path, entry)))) + lines = filter(None, map(str.strip, open(os.path.join(path, entry)))) if len(lines)==2: for dist in find_distributions(os.path.join(path, lines[0])): dist.location = os.path.join(path, *lines) -- cgit v1.2.1 From 9294929b0028f551a54dd48cc3325581933b3c5f Mon Sep 17 00:00:00 2001 From: PJ Eby <distutils-sig@python.org> Date: Mon, 12 Oct 2009 20:00:02 +0000 Subject: Major updates and fixes include: * Fix for the Python 2.6.3 build_ext API change * Support for the most recent Sourceforge download link insanity * Support for SVN 1.6 * Stop crashing on certain types of HTTP error * Stop re-trying URLs that already failed retrieval once * Fixes for various dependency management problems such as looping builds, re-downloading packages already present on sys.path (but not in a registered "site" directory), and randomly preferring local -f packages over local installed packages * Prevent lots of spurious "already imported from another path" warnings (e.g. when pkg_resources is imported late) * Ensure C libraries (as opposed to extensions) are also built when doing bdist_egg Other changes: * Misc. documentation fixes * Improved Jython support * Fewer warnings under Python 2.6+ * Warn when 'packages' uses paths instead of package names (because it causes other problems, like spurious "already imported" warnings) * Stop using /usr/bin/sw_vers on Mac OS (replaced w/'platform' module calls) Note: This is NOT a merge from Distribute; upon review, many of the tracker-submitted patches used as a basis for forking were incorrect, incomplete, introduced new bugs, or were not addressing the root causes. (E.g., one of the changes in this patch fixes three superficially unrelated issues in the setuptools bug tracker.) Careful review will be required if you want to merge this work back into Distribute. --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4075385 --- setuptools/package_index.py | 62 ++++++++++++++++++++++----------------------- 1 file changed, 31 insertions(+), 31 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index da3fed12..70b75a6f 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1,5 +1,6 @@ """PyPI and direct package downloading""" import sys, os.path, re, urlparse, urllib2, shutil, random, socket, cStringIO +import httplib from pkg_resources import * from distutils import log from distutils.errors import DistutilsError @@ -8,7 +9,6 @@ try: except ImportError: from md5 import md5 from fnmatch import translate - EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.]+)$') HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I) # this is here to fix emacs' cruddy broken syntax highlighting @@ -42,6 +42,8 @@ def parse_bdist_wininst(name): def egg_info_for_url(url): scheme, server, path, parameters, query, fragment = urlparse.urlparse(url) base = urllib2.unquote(path.split('/')[-1]) + if server=='sourceforge.net' and base=='download': # XXX Yuck + base = urllib2.unquote(path.split('/')[-2]) if '#' in base: base, fragment = base.split('#',1) return base,fragment @@ -64,14 +66,12 @@ def distros_for_location(location, basename, metadata=None): if basename.endswith('.egg') and '-' in basename: # only one, unambiguous interpretation return [Distribution.from_location(location, basename, metadata)] - if basename.endswith('.exe'): win_base, py_ver = parse_bdist_wininst(basename) if win_base is not None: return interpret_distro_name( location, win_base, metadata, py_ver, BINARY_DIST, "win32" ) - # Try source distro extensions (.zip, .tgz, etc.) # for ext in EXTENSIONS: @@ -186,10 +186,10 @@ class PackageIndex(Environment): return self.info("Reading %s", url) + self.fetched_urls[url] = True # prevent multiple fetch attempts f = self.open_url(url, "Download error: %s -- Some packages may not be found!") if f is None: return - self.fetched_urls[url] = self.fetched_urls[f.url] = True - + self.fetched_urls[f.url] = True if 'html' not in f.headers.get('content-type', '').lower(): f.close() # not html, we can't process it return @@ -329,7 +329,7 @@ class PackageIndex(Environment): def check_md5(self, cs, info, filename, tfp): if re.match('md5=[0-9a-f]{32}$', info): self.debug("Validating md5 checksum for %s", filename) - if cs.hexdigest()<>info[4:]: + if cs.hexdigest()!=info[4:]: tfp.close() os.unlink(filename) raise DistutilsError( @@ -409,7 +409,8 @@ class PackageIndex(Environment): def fetch_distribution(self, - requirement, tmpdir, force_scan=False, source=False, develop_ok=False + requirement, tmpdir, force_scan=False, source=False, develop_ok=False, + local_index=None, ): """Obtain a distribution suitable for fulfilling `requirement` @@ -427,15 +428,15 @@ class PackageIndex(Environment): set, development and system eggs (i.e., those using the ``.egg-info`` format) will be ignored. """ - # process a Requirement self.info("Searching for %s", requirement) skipped = {} + dist = None - def find(req): + def find(env, req): # Find a matching distribution; may be called more than once - for dist in self[req.key]: + for dist in env[req.key]: if dist.precedence==DEVELOP_DIST and not develop_ok: if dist not in skipped: @@ -444,23 +445,25 @@ class PackageIndex(Environment): continue if dist in req and (dist.precedence<=SOURCE_DIST or not source): - self.info("Best match: %s", dist) - return dist.clone( - location=self.download(dist.location, tmpdir) - ) + return dist + + if force_scan: self.prescan() self.find_packages(requirement) + dist = find(self, requirement) + + if local_index is not None: + dist = dist or find(local_index, requirement) - dist = find(requirement) if dist is None and self.to_scan is not None: self.prescan() - dist = find(requirement) + dist = find(self, requirement) if dist is None and not force_scan: self.find_packages(requirement) - dist = find(requirement) + dist = find(self, requirement) if dist is None: self.warn( @@ -468,7 +471,9 @@ class PackageIndex(Environment): (source and "a source distribution of " or ""), requirement, ) - return dist + self.info("Best match: %s", dist) + return dist.clone(location=self.download(dist.location, tmpdir)) + def fetch(self, requirement, tmpdir, force_scan=False, source=False): """Obtain a file suitable for fulfilling `requirement` @@ -485,11 +490,6 @@ class PackageIndex(Environment): - - - - - def gen_setup(self, filename, fragment, tmpdir): match = EGG_FRAGMENT.match(fragment) dists = match and [d for d in @@ -573,17 +573,19 @@ class PackageIndex(Environment): def open_url(self, url, warning=None): - if url.startswith('file:'): - return local_open(url) + if url.startswith('file:'): return local_open(url) try: return open_with_auth(url) except urllib2.HTTPError, v: return v except urllib2.URLError, v: - if warning: self.warn(warning, v.reason) - else: - raise DistutilsError("Download error for %s: %s" - % (url, v.reason)) + reason = v.reason + except httplib.HTTPException, v: + reason = "%s: %s" % (v.__doc__ or v.__class__.__name__, v) + if warning: + self.warn(warning, reason) + else: + raise DistutilsError("Download error for %s: %s" % (url, reason)) def _download_url(self, scheme, url, tmpdir): # Determine download filename @@ -611,8 +613,6 @@ class PackageIndex(Environment): self.url_ok(url, True) # raises error if not allowed return self._attempt_download(url, filename) - - def scan_url(self, url): self.process_url(url, True) -- cgit v1.2.1 From 9b02ba04ef7690724330d3d8ff35cd38284d3b07 Mon Sep 17 00:00:00 2001 From: Hanno Schlichting <hanno@hannosch.eu> Date: Sat, 24 Oct 2009 01:24:05 +0200 Subject: Issue 21: Allow PackageIndex.open_url to gracefully handle all cases of a httplib.HTTPException instead of just InvalidURL and BadStatusLine. --HG-- branch : distribute extra : rebase_source : 24986ae1074b564fbd8c34a227265afd3b90ebce --- setuptools/package_index.py | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 98799de5..ee980214 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -602,6 +602,12 @@ class PackageIndex(Environment): raise DistutilsError('%s returned a bad status line. ' 'The server might be down, %s' % \ (url, v.line)) + except httplib.HTTPException, v: + if warning: + self.warn(warning, v) + else: + raise DistutilsError("Download error for %s: %s" + % (url, v)) def _download_url(self, scheme, url, tmpdir): # Determine download filename -- cgit v1.2.1 From 03cee8b587cf41af1a4ad1071cb6aa7b97c4f182 Mon Sep 17 00:00:00 2001 From: PJ Eby <distutils-sig@python.org> Date: Wed, 28 Oct 2009 17:12:57 +0000 Subject: Fix for issue 88 --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4075926 --- setuptools/package_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 70b75a6f..1a4fabde 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1,6 +1,6 @@ """PyPI and direct package downloading""" import sys, os.path, re, urlparse, urllib2, shutil, random, socket, cStringIO -import httplib +import httplib, urllib from pkg_resources import * from distutils import log from distutils.errors import DistutilsError @@ -701,7 +701,7 @@ def open_with_auth(url): scheme, netloc, path, params, query, frag = urlparse.urlparse(url) if scheme in ('http', 'https'): - auth, host = urllib2.splituser(netloc) + auth, host = urllib.splituser(netloc) else: auth = None -- cgit v1.2.1 From 4478cdc28c1550f64b7918e3d88af08d0c28a0cb Mon Sep 17 00:00:00 2001 From: PJ Eby <distutils-sig@python.org> Date: Thu, 5 Nov 2009 16:00:25 +0000 Subject: Backport to 0.6 --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4076123 --- setuptools/package_index.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 1a4fabde..78b07f4b 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -471,8 +471,9 @@ class PackageIndex(Environment): (source and "a source distribution of " or ""), requirement, ) - self.info("Best match: %s", dist) - return dist.clone(location=self.download(dist.location, tmpdir)) + else: + self.info("Best match: %s", dist) + return dist.clone(location=self.download(dist.location, tmpdir)) def fetch(self, requirement, tmpdir, force_scan=False, source=False): @@ -489,7 +490,6 @@ class PackageIndex(Environment): return None - def gen_setup(self, filename, fragment, tmpdir): match = EGG_FRAGMENT.match(fragment) dists = match and [d for d in -- cgit v1.2.1 From b3423443dc7a7b2d524c76959d6dd67013e2165c Mon Sep 17 00:00:00 2001 From: tarek <none@none> Date: Sat, 19 Dec 2009 22:30:13 +0100 Subject: Introduced a socket timeout of 15 seconds on url openings fixes #48 --HG-- branch : distribute extra : rebase_source : d4e3831771aa049e71d64f22e315352fa8906dd1 --- setuptools/package_index.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index ee980214..924c15e1 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -25,6 +25,8 @@ __all__ = [ 'interpret_distro_name', ] +_SOCKET_TIMEOUT = 15 + def parse_bdist_wininst(name): """Return (base,pyversion) or (None,None) for possible .exe name""" @@ -717,6 +719,17 @@ def htmldecode(text): +def socket_timeout(timeout=15): + def _socket_timeout(func): + def _socket_timeout(*args, **kwargs): + old_timeout = socket.getdefaulttimeout() + socket.setdefaulttimeout(timeout) + try: + return func(*args, **kwargs) + finally: + socket.setdefaulttimeout(old_timeout) + return _socket_timeout + return _socket_timeout def open_with_auth(url): @@ -749,6 +762,8 @@ def open_with_auth(url): return fp +# adding a timeout to avoid freezing package_index +open_with_auth = socket_timeout(_SOCKET_TIMEOUT)(open_with_auth) -- cgit v1.2.1 From 16dc2a11fb1b33955ad55ab00f186fcc909e35f6 Mon Sep 17 00:00:00 2001 From: PJ Eby <distutils-sig@python.org> Date: Mon, 1 Feb 2010 16:42:04 +0000 Subject: Backport SF download fix --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4077904 --- setuptools/package_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 78b07f4b..32498d0f 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -590,9 +590,8 @@ class PackageIndex(Environment): def _download_url(self, scheme, url, tmpdir): # Determine download filename # - name = filter(None,urlparse.urlparse(url)[2].split('/')) + name, fragment = egg_info_for_url(url) if name: - name = name[-1] while '..' in name: name = name.replace('..','.').replace('\\','_') else: @@ -613,6 +612,7 @@ class PackageIndex(Environment): self.url_ok(url, True) # raises error if not allowed return self._attempt_download(url, filename) + def scan_url(self, url): self.process_url(url, True) -- cgit v1.2.1 From a33917aed3c98e2537386bd904658f0b4f963d1c Mon Sep 17 00:00:00 2001 From: Tarek Ziade <tarek@ziade.org> Date: Wed, 19 May 2010 12:09:56 +0200 Subject: malformed urls in 2.7 are catched now - fixes #160 --HG-- branch : distribute extra : rebase_source : de334e49e876c8ea88f738e03995a461ea669879 --- setuptools/package_index.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 924c15e1..1c50d86f 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -268,7 +268,10 @@ class PackageIndex(Environment): # process an index page into the package-page index for match in HREF.finditer(page): - scan( urlparse.urljoin(url, htmldecode(match.group(1))) ) + try: + scan( urlparse.urljoin(url, htmldecode(match.group(1))) ) + except ValueError: + pass pkg, ver = scan(url) # ensure this page is in the page index if pkg: -- cgit v1.2.1 From f3411291c4ec89df2f9fe18263a5509fb4caaddf Mon Sep 17 00:00:00 2001 From: Christophe Combelles <ccomb@free.fr> Date: Wed, 19 May 2010 23:11:22 +0200 Subject: fixed issue 163 : don't include md5 when comparing two distributions, and scan index links before external page links. --HG-- branch : distribute extra : rebase_source : d190057280e7cb27317eb4aa40e75f1c851ed6e5 --- setuptools/package_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 1c50d86f..ba43cfbf 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -203,11 +203,11 @@ class PackageIndex(Environment): charset = f.headers.get_param('charset') or 'latin-1' page = page.decode(charset, "ignore") f.close() - if url.startswith(self.index_url) and getattr(f,'code',None)!=404: - page = self.process_index(url, page) for match in HREF.finditer(page): link = urlparse.urljoin(base, htmldecode(match.group(1))) self.process_url(link) + if url.startswith(self.index_url) and getattr(f,'code',None)!=404: + page = self.process_index(url, page) def process_filename(self, fn, nested=False): # process filenames or directories -- cgit v1.2.1 From 72c44b3258e02b56ac106f878a64eca726a6f01f Mon Sep 17 00:00:00 2001 From: David Cournapeau <david@silveregg.co.jp> Date: Thu, 20 May 2010 19:31:44 +0900 Subject: BUG: Fix #142 - easy_install ignore locally installed packages. Backport from setuptools 0.6c10. --HG-- branch : distribute extra : rebase_source : d06cbdae906a725410d4993d9e1a631e2acac345 --- setuptools/package_index.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 1c50d86f..67e9f6ae 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -418,7 +418,8 @@ class PackageIndex(Environment): def fetch_distribution(self, - requirement, tmpdir, force_scan=False, source=False, develop_ok=False + requirement, tmpdir, force_scan=False, source=False, develop_ok=False, + local_index=None ): """Obtain a distribution suitable for fulfilling `requirement` @@ -440,11 +441,14 @@ class PackageIndex(Environment): # process a Requirement self.info("Searching for %s", requirement) skipped = {} + dist = None - def find(req): + def find(req, env=None): + if env is None: + env = self # Find a matching distribution; may be called more than once - for dist in self[req.key]: + for dist in env[req.key]: if dist.precedence==DEVELOP_DIST and not develop_ok: if dist not in skipped: @@ -461,8 +465,11 @@ class PackageIndex(Environment): if force_scan: self.prescan() self.find_packages(requirement) + dist = find(requirement) + + if local_index is not None: + dist = dist or find(requirement, local_index) - dist = find(requirement) if dist is None and self.to_scan is not None: self.prescan() dist = find(requirement) -- cgit v1.2.1 From fadd3a199f2c99ef0fb5d58e78016274cad6cc20 Mon Sep 17 00:00:00 2001 From: Lennart Regebro <regebro@gmail.com> Date: Sat, 24 Jul 2010 12:55:44 +0100 Subject: Importing url2pathname from the correct place, so 2to3 can do it's job. --HG-- branch : distribute extra : rebase_source : 43178821d2b4776dbfdf5ad9493bca8dc401b048 --- setuptools/package_index.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 1d467f78..1f9b6bd8 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1,5 +1,5 @@ """PyPI and direct package downloading""" -import sys, os.path, re, urlparse, urllib2, shutil, random, socket, cStringIO +import sys, os.path, re, urlparse, urllib, urllib2, shutil, random, socket, cStringIO import httplib from pkg_resources import * from distutils import log @@ -642,7 +642,7 @@ class PackageIndex(Environment): if scheme=='svn' or scheme.startswith('svn+'): return self._download_svn(url, filename) elif scheme=='file': - return urllib2.url2pathname(urlparse.urlparse(url)[2]) + return urllib.url2pathname(urlparse.urlparse(url)[2]) else: self.url_ok(url, True) # raises error if not allowed return self._attempt_download(url, filename) @@ -791,7 +791,7 @@ def fix_sf_url(url): def local_open(url): """Read a local path, with special support for directories""" scheme, server, path, param, query, frag = urlparse.urlparse(url) - filename = urllib2.url2pathname(path) + filename = urllib.url2pathname(path) if os.path.isfile(filename): return urllib2.urlopen(url) elif path.endswith('/') and os.path.isdir(filename): -- cgit v1.2.1 From 35b06106c2c50e0d0709f2e75be4bc8fe56aef34 Mon Sep 17 00:00:00 2001 From: Lennart Regebro <regebro@gmail.com> Date: Sat, 24 Jul 2010 14:56:42 +0100 Subject: Only do the decoding if there is something to decode. --HG-- branch : distribute extra : rebase_source : 6510b361ce8885f536a3babfc66a7ebdfaa25b2d --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 1f9b6bd8..4ff96303 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -199,7 +199,7 @@ class PackageIndex(Environment): base = f.url # handle redirects page = f.read() - if sys.version_info >= (3,): + if sys.version_info >= (3,) and not isinstance(f, urllib2.HTTPError): charset = f.headers.get_param('charset') or 'latin-1' page = page.decode(charset, "ignore") f.close() -- cgit v1.2.1 From d722bd3324c1d22541a8e8f924fa490fd7c1046f Mon Sep 17 00:00:00 2001 From: Lennart Regebro <regebro@gmail.com> Date: Mon, 22 Nov 2010 18:12:18 +0100 Subject: We need to make sure that the result is always a str, even if the result is an error response. Otherwise you get an error when trying to pattern match in line 206. --HG-- branch : distribute extra : rebase_source : dc5fe8b1365544fc763414b67227cc78dc1f8524 --- setuptools/package_index.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 4ff96303..459cae2c 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -199,8 +199,12 @@ class PackageIndex(Environment): base = f.url # handle redirects page = f.read() - if sys.version_info >= (3,) and not isinstance(f, urllib2.HTTPError): - charset = f.headers.get_param('charset') or 'latin-1' + if not isinstance(page, str): # We are in Python 3 and got bytes. We want str. + if isinstance(f, urllib2.HTTPError): + # Errors have no charset, assume latin1: + charset = 'latin-1' + else: + charset = f.headers.get_param('charset') or 'latin-1' page = page.decode(charset, "ignore") f.close() for match in HREF.finditer(page): -- cgit v1.2.1 From 9f3c9810de9de1358d4f62ebd74bdf2c866c7d73 Mon Sep 17 00:00:00 2001 From: PJ Eby <distutils-sig@python.org> Date: Wed, 23 Mar 2011 20:38:12 +0000 Subject: Backport. --HG-- branch : setuptools-0.6 extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/branches/setuptools-0.6%4088793 --- setuptools/package_index.py | 45 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 43 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 32498d0f..9a9c5d62 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -550,7 +550,7 @@ class PackageIndex(Environment): bs = self.dl_blocksize size = -1 if "content-length" in headers: - size = int(headers["Content-Length"]) + size = max(map(int,headers.getheaders("Content-Length"))) self.reporthook(url, filename, blocknum, bs, size) tfp = open(filename,'wb') while True: @@ -639,10 +639,39 @@ class PackageIndex(Environment): os.unlink(filename) raise DistutilsError("Unexpected HTML page found at "+url) + + + + + + + + + + + + + + + def _download_svn(self, url, filename): url = url.split('#',1)[0] # remove any fragment for svn's sake + creds = '' + if url.lower().startswith('svn:') and '@' in url: + scheme, netloc, path, p, q, f = urlparse.urlparse(url) + if not netloc and path.startswith('//') and '/' in path[2:]: + netloc, path = path[2:].split('/',1) + auth, host = urllib.splituser(netloc) + if auth: + if ':' in auth: + user, pw = auth.split(':',1) + creds = " --username=%s --password=%s" % (user, pw) + else: + creds = " --username="+auth + netloc = host + url = urlparse.urlunparse((scheme, netloc, url, p, q, f)) self.info("Doing subversion checkout from %s to %s", url, filename) - os.system("svn checkout -q %s %s" % (url, filename)) + os.system("svn checkout%s -q %s %s" % (creds, url, filename)) return filename def debug(self, msg, *args): @@ -654,6 +683,18 @@ class PackageIndex(Environment): def warn(self, msg, *args): log.warn(msg, *args) + + + + + + + + + + + + # This pattern matches a character entity reference (a decimal numeric # references, a hexadecimal numeric reference, or a named reference). entity_sub = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub -- cgit v1.2.1 From 1509fe5874ad7155482d71b1b922bfdeb522ab58 Mon Sep 17 00:00:00 2001 From: agroszer <none@none> Date: Fri, 15 Apr 2011 10:49:49 +0200 Subject: Fixing #200 --HG-- branch : distribute extra : rebase_source : 4446e76a0bcf2e968abce2020569aecbaab1df01 --- setuptools/package_index.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 459cae2c..6d4047af 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -31,16 +31,25 @@ def parse_bdist_wininst(name): """Return (base,pyversion) or (None,None) for possible .exe name""" lower = name.lower() - base, py_ver = None, None + base, py_ver, plat = None, None, None if lower.endswith('.exe'): if lower.endswith('.win32.exe'): base = name[:-10] + plat = 'win32' elif lower.startswith('.win32-py',-16): py_ver = name[-7:-4] base = name[:-16] + plat = 'win32' + elif lower.endswith('.win-amd64.exe'): + base = name[:-14] + plat = 'win-amd64' + elif lower.startswith('.win-amd64-py',-20): + py_ver = name[-7:-4] + base = name[:-20] + plat = 'win-amd64' + return base,py_ver,plat - return base,py_ver def egg_info_for_url(url): scheme, server, path, parameters, query, fragment = urlparse.urlparse(url) @@ -69,10 +78,10 @@ def distros_for_location(location, basename, metadata=None): return [Distribution.from_location(location, basename, metadata)] if basename.endswith('.exe'): - win_base, py_ver = parse_bdist_wininst(basename) + win_base, py_ver, platform = parse_bdist_wininst(basename) if win_base is not None: return interpret_distro_name( - location, win_base, metadata, py_ver, BINARY_DIST, "win32" + location, win_base, metadata, py_ver, BINARY_DIST, platform ) # Try source distro extensions (.zip, .tgz, etc.) -- cgit v1.2.1 From f679304a95d0b377fd069c56fc975a82da9ee2ca Mon Sep 17 00:00:00 2001 From: Tarek Ziade <tarek@ziade.org> Date: Mon, 16 May 2011 14:22:31 +0200 Subject: Tolerate responses with multiple Content-Length headers - fixes #196 --HG-- branch : distribute extra : rebase_source : b0f0f73fd59017dd9a9bb6e58b2a1fd9e5773dd2 --- setuptools/package_index.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 6d4047af..0230497d 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -579,7 +579,9 @@ class PackageIndex(Environment): bs = self.dl_blocksize size = -1 if "content-length" in headers: - size = int(headers["Content-Length"]) + # Some servers return multiple Content-Length headers :( + content_length = headers.getheaders("Content-Length")[0] + size = int(content_length) self.reporthook(url, filename, blocknum, bs, size) tfp = open(filename,'wb') while True: -- cgit v1.2.1 From b672805ffe0c8ef4be0acc0ce05d5273f25d6533 Mon Sep 17 00:00:00 2001 From: Tarek Ziade <tarek@ziade.org> Date: Thu, 2 Jun 2011 02:10:13 +0200 Subject: make sure we don't use getheaders(). get() works for all py versions - fixes #206 --HG-- branch : distribute extra : rebase_source : c9e414a0642dd5b2222dd3fc4aa1b11a442e48a1 --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 0230497d..f064b110 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -580,7 +580,7 @@ class PackageIndex(Environment): size = -1 if "content-length" in headers: # Some servers return multiple Content-Length headers :( - content_length = headers.getheaders("Content-Length")[0] + content_length = headers.get("Content-Length") size = int(content_length) self.reporthook(url, filename, blocknum, bs, size) tfp = open(filename,'wb') -- cgit v1.2.1 From 58a658b26d1c95b31d02050dcccd648d2e4ce27b Mon Sep 17 00:00:00 2001 From: Vinay Sajip <vinay_sajip@yahoo.co.uk> Date: Mon, 20 Jun 2011 22:55:16 +0100 Subject: Changes to support 2.x and 3.x in the same codebase. --HG-- branch : distribute extra : rebase_source : 7d3608edee54a43789f0574d702fb839628b5071 --- setuptools/package_index.py | 82 +++++++++++++++++++++++++-------------------- 1 file changed, 45 insertions(+), 37 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index f064b110..589dade6 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1,9 +1,12 @@ """PyPI and direct package downloading""" -import sys, os.path, re, urlparse, urllib, urllib2, shutil, random, socket, cStringIO -import httplib +import sys, os.path, re, shutil, random, socket from pkg_resources import * from distutils import log from distutils.errors import DistutilsError +from setuptools.compat import (urllib2, httplib, StringIO, HTTPError, + urlparse, urlunparse, unquote, splituser, + url2pathname, name2codepoint, + unichr, urljoin) try: from hashlib import md5 except ImportError: @@ -52,8 +55,8 @@ def parse_bdist_wininst(name): def egg_info_for_url(url): - scheme, server, path, parameters, query, fragment = urlparse.urlparse(url) - base = urllib2.unquote(path.split('/')[-1]) + scheme, server, path, parameters, query, fragment = urlparse(url) + base = unquote(path.split('/')[-1]) if '#' in base: base, fragment = base.split('#',1) return base,fragment @@ -144,14 +147,14 @@ def find_external_links(url, page): rels = map(str.strip, rel.lower().split(',')) if 'homepage' in rels or 'download' in rels: for match in HREF.finditer(tag): - yield urlparse.urljoin(url, htmldecode(match.group(1))) + yield urljoin(url, htmldecode(match.group(1))) for tag in ("<th>Home Page", "<th>Download URL"): pos = page.find(tag) if pos!=-1: match = HREF.search(page,pos) if match: - yield urlparse.urljoin(url, htmldecode(match.group(1))) + yield urljoin(url, htmldecode(match.group(1))) user_agent = "Python-urllib/%s distribute/%s" % ( sys.version[:3], require('distribute')[0].version @@ -190,7 +193,7 @@ class PackageIndex(Environment): self.debug("Found link: %s", url) if dists or not retrieve or url in self.fetched_urls: - map(self.add, dists) + list(map(self.add, dists)) return # don't need the actual page if not self.url_ok(url): @@ -209,7 +212,7 @@ class PackageIndex(Environment): base = f.url # handle redirects page = f.read() if not isinstance(page, str): # We are in Python 3 and got bytes. We want str. - if isinstance(f, urllib2.HTTPError): + if isinstance(f, HTTPError): # Errors have no charset, assume latin1: charset = 'latin-1' else: @@ -217,7 +220,7 @@ class PackageIndex(Environment): page = page.decode(charset, "ignore") f.close() for match in HREF.finditer(page): - link = urlparse.urljoin(base, htmldecode(match.group(1))) + link = urljoin(base, htmldecode(match.group(1))) self.process_url(link) if url.startswith(self.index_url) and getattr(f,'code',None)!=404: page = self.process_index(url, page) @@ -236,11 +239,11 @@ class PackageIndex(Environment): dists = distros_for_filename(fn) if dists: self.debug("Found: %s", fn) - map(self.add, dists) + list(map(self.add, dists)) def url_ok(self, url, fatal=False): s = URL_SCHEME(url) - if (s and s.group(1).lower()=='file') or self.allows(urlparse.urlparse(url)[1]): + if (s and s.group(1).lower()=='file') or self.allows(urlparse(url)[1]): return True msg = "\nLink to % s ***BLOCKED*** by --allow-hosts\n" if fatal: @@ -256,7 +259,8 @@ class PackageIndex(Environment): self.scan_egg_link(item, entry) def scan_egg_link(self, path, entry): - lines = filter(None, map(str.strip, open(os.path.join(path, entry)))) + lines = [_f for _f in map(str.strip, + open(os.path.join(path, entry))) if _f] if len(lines)==2: for dist in find_distributions(os.path.join(path, lines[0])): dist.location = os.path.join(path, *lines) @@ -268,9 +272,9 @@ class PackageIndex(Environment): def scan(link): # Process a URL to see if it's for a package page if link.startswith(self.index_url): - parts = map( - urllib2.unquote, link[len(self.index_url):].split('/') - ) + parts = list(map( + unquote, link[len(self.index_url):].split('/') + )) if len(parts)==2 and '#' not in parts[1]: # it's a package page, sanitize and index it pkg = safe_name(parts[0]) @@ -282,7 +286,7 @@ class PackageIndex(Environment): # process an index page into the package-page index for match in HREF.finditer(page): try: - scan( urlparse.urljoin(url, htmldecode(match.group(1))) ) + scan( urljoin(url, htmldecode(match.group(1))) ) except ValueError: pass @@ -351,7 +355,7 @@ class PackageIndex(Environment): def check_md5(self, cs, info, filename, tfp): if re.match('md5=[0-9a-f]{32}$', info): self.debug("Validating md5 checksum for %s", filename) - if cs.hexdigest()<>info[4:]: + if cs.hexdigest() != info[4:]: tfp.close() os.unlink(filename) raise DistutilsError( @@ -377,7 +381,7 @@ class PackageIndex(Environment): def prescan(self): """Scan urls scheduled for prescanning (e.g. --find-links)""" if self.to_scan: - map(self.scan_url, self.to_scan) + list(map(self.scan_url, self.to_scan)) self.to_scan = None # from now on, go ahead and process immediately def not_found_in_index(self, requirement): @@ -569,7 +573,7 @@ class PackageIndex(Environment): if '#' in url: url, info = url.split('#', 1) fp = self.open_url(url) - if isinstance(fp, urllib2.HTTPError): + if isinstance(fp, HTTPError): raise DistutilsError( "Can't download %s: %s %s" % (url, fp.code,fp.msg) ) @@ -608,28 +612,33 @@ class PackageIndex(Environment): return local_open(url) try: return open_with_auth(url) - except (ValueError, httplib.InvalidURL), v: + except (ValueError, httplib.InvalidURL): + v = sys.exc_info()[1] msg = ' '.join([str(arg) for arg in v.args]) if warning: self.warn(warning, msg) else: raise DistutilsError('%s %s' % (url, msg)) - except urllib2.HTTPError, v: + except urllib2.HTTPError: + v = sys.exc_info()[1] return v - except urllib2.URLError, v: + except urllib2.URLError: + v = sys.exc_info()[1] if warning: self.warn(warning, v.reason) else: raise DistutilsError("Download error for %s: %s" % (url, v.reason)) - except httplib.BadStatusLine, v: + except httplib.BadStatusLine: + v = sys.exc_info()[1] if warning: self.warn(warning, v.line) else: raise DistutilsError('%s returned a bad status line. ' 'The server might be down, %s' % \ (url, v.line)) - except httplib.HTTPException, v: + except httplib.HTTPException: + v = sys.exc_info()[1] if warning: self.warn(warning, v) else: @@ -639,7 +648,7 @@ class PackageIndex(Environment): def _download_url(self, scheme, url, tmpdir): # Determine download filename # - name = filter(None,urlparse.urlparse(url)[2].split('/')) + name = [_f for _f in urlparse(url)[2].split('/') if _f] if name: name = name[-1] while '..' in name: @@ -657,7 +666,7 @@ class PackageIndex(Environment): if scheme=='svn' or scheme.startswith('svn+'): return self._download_svn(url, filename) elif scheme=='file': - return urllib.url2pathname(urlparse.urlparse(url)[2]) + return url2pathname(urlparse.urlparse(url)[2]) else: self.url_ok(url, True) # raises error if not allowed return self._attempt_download(url, filename) @@ -722,7 +731,6 @@ def decode_entity(match): elif what.startswith('#'): what = int(what[1:]) else: - from htmlentitydefs import name2codepoint what = name2codepoint.get(what, match.group(0)) return uchr(what) @@ -760,16 +768,16 @@ def socket_timeout(timeout=15): def open_with_auth(url): """Open a urllib2 request, handling HTTP authentication""" - scheme, netloc, path, params, query, frag = urlparse.urlparse(url) + scheme, netloc, path, params, query, frag = urlparse(url) if scheme in ('http', 'https'): - auth, host = urllib2.splituser(netloc) + auth, host = splituser(netloc) else: auth = None if auth: - auth = "Basic " + urllib2.unquote(auth).encode('base64').strip() - new_url = urlparse.urlunparse((scheme,host,path,params,query,frag)) + auth = "Basic " + unquote(auth).encode('base64').strip() + new_url = urlunparse((scheme,host,path,params,query,frag)) request = urllib2.Request(new_url) request.add_header("Authorization", auth) else: @@ -781,9 +789,9 @@ def open_with_auth(url): if auth: # Put authentication info back into request URL if same host, # so that links found on the page will work - s2, h2, path2, param2, query2, frag2 = urlparse.urlparse(fp.url) + s2, h2, path2, param2, query2, frag2 = urlparse(fp.url) if s2==scheme and h2==host: - fp.url = urlparse.urlunparse((s2,netloc,path2,param2,query2,frag2)) + fp.url = urlunparse((s2,netloc,path2,param2,query2,frag2)) return fp @@ -805,8 +813,8 @@ def fix_sf_url(url): def local_open(url): """Read a local path, with special support for directories""" - scheme, server, path, param, query, frag = urlparse.urlparse(url) - filename = urllib.url2pathname(path) + scheme, server, path, param, query, frag = urlparse(url) + filename = url2pathname(path) if os.path.isfile(filename): return urllib2.urlopen(url) elif path.endswith('/') and os.path.isdir(filename): @@ -827,8 +835,8 @@ def local_open(url): else: status, message, body = 404, "Path not found", "Not found" - return urllib2.HTTPError(url, status, message, - {'content-type':'text/html'}, cStringIO.StringIO(body)) + return HTTPError(url, status, message, + {'content-type':'text/html'}, StringIO(body)) -- cgit v1.2.1 From edaefa4982e0aac7b07048c4761446ed66fec524 Mon Sep 17 00:00:00 2001 From: Alex Clark <aclark@aclark.net> Date: Sun, 14 Aug 2011 19:18:17 -0400 Subject: Remove extraneous 2nd argument in call to open_url, apparently intended to issue a warning (looks like open_url takes an optional `warning` argument, but I couldn't get that to work either). Fixes #135. This fix is better than the status quo, but probably not as good as issuing a warning instead of failure. --HG-- branch : distribute extra : rebase_source : 39889bf4dd21abbd57207bfafe6f8bad68b1e46f --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index f064b110..c9e3d637 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -198,7 +198,7 @@ class PackageIndex(Environment): return self.info("Reading %s", url) - f = self.open_url(url, "Download error: %s -- Some packages may not be found!") + f = self.open_url(url) if f is None: return self.fetched_urls[url] = self.fetched_urls[f.url] = True -- cgit v1.2.1 From e6a12bbf1a51bb572145fd46715e2a20e54bfc52 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Wed, 17 Aug 2011 03:44:27 -0400 Subject: Include url in warning when processing URL. Fixes #135. --HG-- branch : distribute extra : rebase_source : 1241e1cb548adad562fcf61ce33538712a64aaed --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index f064b110..1dccabcb 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -198,7 +198,7 @@ class PackageIndex(Environment): return self.info("Reading %s", url) - f = self.open_url(url, "Download error: %s -- Some packages may not be found!") + f = self.open_url(url, "Download error on %s: %%s -- Some packages may not be found!" % url) if f is None: return self.fetched_urls[url] = self.fetched_urls[f.url] = True -- cgit v1.2.1 From 67ee093341a7342fcbe29dd99800c373abd872a0 Mon Sep 17 00:00:00 2001 From: Paulo Koch <paulo.koch@gmail.com> Date: Tue, 11 Oct 2011 18:25:09 +0100 Subject: Add support for git URLs. --HG-- branch : distribute extra : rebase_source : f664e8cf2b7b6f78096cd00463f4e2009bdeb3ac --- setuptools/package_index.py | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index d0896feb..da586a27 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -657,6 +657,8 @@ class PackageIndex(Environment): # if scheme=='svn' or scheme.startswith('svn+'): return self._download_svn(url, filename) + elif scheme=='git' or scheme.startswith('git+'): + return self._download_git(url, filename) elif scheme=='file': return urllib.url2pathname(urlparse.urlparse(url)[2]) else: @@ -697,6 +699,15 @@ class PackageIndex(Environment): os.system("svn checkout -q %s %s" % (url, filename)) return filename + def _download_git(self, url, filename): + if url.startswith('git+'): + url = url[4:] + url = url.split('#',1)[0] # remove any fragment for svn's sake + filename = filename.split('#',1)[0] # remove any fragment to get a decent name. + self.info("Doing git clone from %s to %s", url, filename) + os.system("git clone -q %s %s" % (url, filename)) + return filename + def debug(self, msg, *args): log.debug(msg, *args) -- cgit v1.2.1 From 930b2596aaa267e9ba28bb5a1066b20d9e0fcfeb Mon Sep 17 00:00:00 2001 From: Paulo Koch <paulo.koch@gmail.com> Date: Tue, 11 Oct 2011 20:08:25 +0100 Subject: Add support for Mercurial URLs. --HG-- branch : distribute extra : rebase_source : 7c33d5bea70d9300cff635646988b13c007e24c9 --- setuptools/package_index.py | 11 +++++++++++ 1 file changed, 11 insertions(+) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index da586a27..cd89e9a6 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -659,6 +659,8 @@ class PackageIndex(Environment): return self._download_svn(url, filename) elif scheme=='git' or scheme.startswith('git+'): return self._download_git(url, filename) + elif scheme.startswith('hg+'): + return self._download_hg(url, filename) elif scheme=='file': return urllib.url2pathname(urlparse.urlparse(url)[2]) else: @@ -708,6 +710,15 @@ class PackageIndex(Environment): os.system("git clone -q %s %s" % (url, filename)) return filename + def _download_hg(self, url, filename): + if url.startswith('hg+'): + url = url[3:] + url = url.split('#',1)[0] # remove any fragment for svn's sake + filename = filename.split('#',1)[0] # remove any fragment to get a decent name. + self.info("Doing hg clone from %s to %s", url, filename) + os.system("hg clone --quiet %s %s" % (url, filename)) + return filename + def debug(self, msg, *args): log.debug(msg, *args) -- cgit v1.2.1 From bbe00513be4b20a82eb844dab52c15ca5e8e7113 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Thu, 1 Dec 2011 12:55:33 -0500 Subject: Fix issue #262 - package_index.open_with_auth no longer throws LookupError on Python 3. --HG-- branch : distribute extra : rebase_source : ae4a0886ff89d3679f495f51171f94d3770e5d47 --- setuptools/package_index.py | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index bb0ae129..d0896feb 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1,5 +1,6 @@ """PyPI and direct package downloading""" import sys, os.path, re, urlparse, urllib, urllib2, shutil, random, socket, cStringIO +import base64 import httplib from pkg_resources import * from distutils import log @@ -756,6 +757,22 @@ def socket_timeout(timeout=15): return _socket_timeout return _socket_timeout +def _encode_auth(auth): + """ + A function compatible with Python 2.3-3.3 that will encode + auth from a URL suitable for an HTTP header. + >>> _encode_auth('username%3Apassword') + u'dXNlcm5hbWU6cGFzc3dvcmQ=' + """ + auth_s = urllib2.unquote(auth) + # convert to bytes + auth_bytes = auth_s.encode() + # use the legacy interface for Python 2.3 support + encoded_bytes = base64.encodestring(auth_bytes) + # convert back to a string + encoded = encoded_bytes.decode() + # strip the trailing carriage return + return encoded.rstrip() def open_with_auth(url): """Open a urllib2 request, handling HTTP authentication""" @@ -768,7 +785,7 @@ def open_with_auth(url): auth = None if auth: - auth = "Basic " + urllib2.unquote(auth).encode('base64').strip() + auth = "Basic " + _encode_auth(auth) new_url = urlparse.urlunparse((scheme,host,path,params,query,frag)) request = urllib2.Request(new_url) request.add_header("Authorization", auth) -- cgit v1.2.1 From 85097e081f3d987c7acf6ab51e29a307f4f8f167 Mon Sep 17 00:00:00 2001 From: Paulo Koch <paulo.koch@gmail.com> Date: Thu, 13 Sep 2012 00:19:08 +0100 Subject: Make revision specifiable. --HG-- branch : distribute extra : rebase_source : 059208b9bdcd9a0cf2c2f654f25b638a65f2d9d1 --- setuptools/package_index.py | 49 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 40 insertions(+), 9 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index cd89e9a6..e085732d 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -701,22 +701,53 @@ class PackageIndex(Environment): os.system("svn checkout -q %s %s" % (url, filename)) return filename + def _vcs_split_rev_from_url(self, url, pop_prefix=False): + scheme, netloc, path, query, frag = urlparse.urlsplit(url) + + scheme = scheme.split('+', 1)[-1] + + # Some fragment identification fails + path = path.split('#',1)[0] + + rev = None + if '@' in path: + path, rev = path.rsplit('@', 1) + + # Also, discard fragment + url = urlparse.urlunsplit((scheme, netloc, path, query, '')) + + return url, rev + def _download_git(self, url, filename): - if url.startswith('git+'): - url = url[4:] - url = url.split('#',1)[0] # remove any fragment for svn's sake - filename = filename.split('#',1)[0] # remove any fragment to get a decent name. + filename = filename.split('#',1)[0] + url, rev = self._vcs_split_rev_from_url(url, pop_prefix=True) + self.info("Doing git clone from %s to %s", url, filename) - os.system("git clone -q %s %s" % (url, filename)) + os.system("git clone --quiet %s %s" % (url, filename)) + + if rev is not None: + self.info("Checking out %s", rev) + os.system("(cd %s && git checkout --quiet %s)" % ( + filename, + rev, + )) + return filename def _download_hg(self, url, filename): - if url.startswith('hg+'): - url = url[3:] - url = url.split('#',1)[0] # remove any fragment for svn's sake - filename = filename.split('#',1)[0] # remove any fragment to get a decent name. + filename = filename.split('#',1)[0] + url, rev = self._vcs_split_rev_from_url(url, pop_prefix=True) + self.info("Doing hg clone from %s to %s", url, filename) os.system("hg clone --quiet %s %s" % (url, filename)) + + if rev is not None: + self.info("Updating to %s", rev) + os.system("(cd %s && hg up -C -r %s >&-)" % ( + filename, + rev, + )) + return filename def debug(self, msg, *args): -- cgit v1.2.1 From 95907bcc8ddc3f87e46fdc6b9518d3c2f2a015ed Mon Sep 17 00:00:00 2001 From: "Stefan H. Holek" <stefan@epy.co.at> Date: Tue, 9 Oct 2012 18:38:21 +0200 Subject: Fix cause of test failure on Mac OS X. Refs #20. --HG-- branch : distribute extra : rebase_source : 92ba8151d6dfa3755b31139a9b5ada570183731d --- setuptools/package_index.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index d0896feb..ffbffa99 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -779,6 +779,12 @@ def open_with_auth(url): scheme, netloc, path, params, query, frag = urlparse.urlparse(url) + # Double scheme does not raise on Mac OS X as revealed by a + # failing test. We would expect "nonnumeric port". Refs #20. + if sys.platform == 'darwin': + if netloc.endswith(':'): + raise httplib.InvalidURL("nonnumeric port: ''") + if scheme in ('http', 'https'): auth, host = urllib2.splituser(netloc) else: @@ -859,4 +865,4 @@ def local_open(url): -# this line is a kludge to keep the trailing blank lines for pje's editor \ No newline at end of file +# this line is a kludge to keep the trailing blank lines for pje's editor -- cgit v1.2.1 From f8481f4b522f0fbeafbb36a20dd105497f2623ac Mon Sep 17 00:00:00 2001 From: "Stefan H. Holek" <stefan@epy.co.at> Date: Fri, 26 Oct 2012 02:10:48 +0200 Subject: This one also failed on Windows. --HG-- branch : distribute extra : rebase_source : c26a4318302d55dfdd67d379525e84172b440b82 --- setuptools/package_index.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 7a954e21..0ee21e3b 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -834,9 +834,8 @@ def open_with_auth(url): # Double scheme does not raise on Mac OS X as revealed by a # failing test. We would expect "nonnumeric port". Refs #20. - if sys.platform == 'darwin': - if netloc.endswith(':'): - raise httplib.InvalidURL("nonnumeric port: ''") + if netloc.endswith(':'): + raise httplib.InvalidURL("nonnumeric port: ''") if scheme in ('http', 'https'): auth, host = urllib2.splituser(netloc) -- cgit v1.2.1 From 70c61d89a8c7d548a42f843579ca540b39655ca9 Mon Sep 17 00:00:00 2001 From: pje <none@none> Date: Fri, 1 Mar 2013 12:23:20 -0500 Subject: Default index URL to use SSL version of PyPI --HG-- branch : setuptools-0.6 extra : source : b95868385a32c0103133105788c70850656662c6 --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 9a9c5d62..1668e765 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -149,7 +149,7 @@ user_agent = "Python-urllib/%s setuptools/%s" % ( class PackageIndex(Environment): """A distribution index that scans web pages for download URLs""" - def __init__(self, index_url="http://pypi.python.org/simple", hosts=('*',), + def __init__(self, index_url="https://pypi.python.org/simple", hosts=('*',), *args, **kw ): Environment.__init__(self,*args,**kw) -- cgit v1.2.1 From 4febe30d3a406b342ef398c95a8cfec6018aa86e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Sun, 3 Mar 2013 12:46:21 -0500 Subject: Backed out distribute-specific references in package_index module --HG-- branch : Setuptools-Distribute merge extra : source : 619f12674c8d42831a59dc805b7b9cbf375a831d --- setuptools/package_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 0ee21e3b..d4d4631a 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -154,8 +154,8 @@ def find_external_links(url, page): if match: yield urlparse.urljoin(url, htmldecode(match.group(1))) -user_agent = "Python-urllib/%s distribute/%s" % ( - sys.version[:3], require('distribute')[0].version +user_agent = "Python-urllib/%s setuptools/%s" % ( + sys.version[:3], require('setuptools')[0].version ) -- cgit v1.2.1 From 742bd477e51562a8548d3a18bd3695c76a618fc0 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Wed, 20 Mar 2013 13:56:46 -0700 Subject: Copy changes to package_index from 1aae1efe5733 --HG-- branch : Setuptools-Distribute merge extra : source : db6acae7b8cfb92007fe8e7ae4f392481b080a00 --- setuptools/package_index.py | 113 ++++++++++++++++++++++++++++++-------------- 1 file changed, 77 insertions(+), 36 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index da3fed12..9a9c5d62 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1,5 +1,6 @@ """PyPI and direct package downloading""" import sys, os.path, re, urlparse, urllib2, shutil, random, socket, cStringIO +import httplib, urllib from pkg_resources import * from distutils import log from distutils.errors import DistutilsError @@ -8,7 +9,6 @@ try: except ImportError: from md5 import md5 from fnmatch import translate - EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.]+)$') HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I) # this is here to fix emacs' cruddy broken syntax highlighting @@ -42,6 +42,8 @@ def parse_bdist_wininst(name): def egg_info_for_url(url): scheme, server, path, parameters, query, fragment = urlparse.urlparse(url) base = urllib2.unquote(path.split('/')[-1]) + if server=='sourceforge.net' and base=='download': # XXX Yuck + base = urllib2.unquote(path.split('/')[-2]) if '#' in base: base, fragment = base.split('#',1) return base,fragment @@ -64,14 +66,12 @@ def distros_for_location(location, basename, metadata=None): if basename.endswith('.egg') and '-' in basename: # only one, unambiguous interpretation return [Distribution.from_location(location, basename, metadata)] - if basename.endswith('.exe'): win_base, py_ver = parse_bdist_wininst(basename) if win_base is not None: return interpret_distro_name( location, win_base, metadata, py_ver, BINARY_DIST, "win32" ) - # Try source distro extensions (.zip, .tgz, etc.) # for ext in EXTENSIONS: @@ -186,10 +186,10 @@ class PackageIndex(Environment): return self.info("Reading %s", url) + self.fetched_urls[url] = True # prevent multiple fetch attempts f = self.open_url(url, "Download error: %s -- Some packages may not be found!") if f is None: return - self.fetched_urls[url] = self.fetched_urls[f.url] = True - + self.fetched_urls[f.url] = True if 'html' not in f.headers.get('content-type', '').lower(): f.close() # not html, we can't process it return @@ -329,7 +329,7 @@ class PackageIndex(Environment): def check_md5(self, cs, info, filename, tfp): if re.match('md5=[0-9a-f]{32}$', info): self.debug("Validating md5 checksum for %s", filename) - if cs.hexdigest()<>info[4:]: + if cs.hexdigest()!=info[4:]: tfp.close() os.unlink(filename) raise DistutilsError( @@ -409,7 +409,8 @@ class PackageIndex(Environment): def fetch_distribution(self, - requirement, tmpdir, force_scan=False, source=False, develop_ok=False + requirement, tmpdir, force_scan=False, source=False, develop_ok=False, + local_index=None, ): """Obtain a distribution suitable for fulfilling `requirement` @@ -427,15 +428,15 @@ class PackageIndex(Environment): set, development and system eggs (i.e., those using the ``.egg-info`` format) will be ignored. """ - # process a Requirement self.info("Searching for %s", requirement) skipped = {} + dist = None - def find(req): + def find(env, req): # Find a matching distribution; may be called more than once - for dist in self[req.key]: + for dist in env[req.key]: if dist.precedence==DEVELOP_DIST and not develop_ok: if dist not in skipped: @@ -444,23 +445,25 @@ class PackageIndex(Environment): continue if dist in req and (dist.precedence<=SOURCE_DIST or not source): - self.info("Best match: %s", dist) - return dist.clone( - location=self.download(dist.location, tmpdir) - ) + return dist + + if force_scan: self.prescan() self.find_packages(requirement) + dist = find(self, requirement) + + if local_index is not None: + dist = dist or find(local_index, requirement) - dist = find(requirement) if dist is None and self.to_scan is not None: self.prescan() - dist = find(requirement) + dist = find(self, requirement) if dist is None and not force_scan: self.find_packages(requirement) - dist = find(requirement) + dist = find(self, requirement) if dist is None: self.warn( @@ -468,7 +471,10 @@ class PackageIndex(Environment): (source and "a source distribution of " or ""), requirement, ) - return dist + else: + self.info("Best match: %s", dist) + return dist.clone(location=self.download(dist.location, tmpdir)) + def fetch(self, requirement, tmpdir, force_scan=False, source=False): """Obtain a file suitable for fulfilling `requirement` @@ -484,12 +490,6 @@ class PackageIndex(Environment): return None - - - - - - def gen_setup(self, filename, fragment, tmpdir): match = EGG_FRAGMENT.match(fragment) dists = match and [d for d in @@ -550,7 +550,7 @@ class PackageIndex(Environment): bs = self.dl_blocksize size = -1 if "content-length" in headers: - size = int(headers["Content-Length"]) + size = max(map(int,headers.getheaders("Content-Length"))) self.reporthook(url, filename, blocknum, bs, size) tfp = open(filename,'wb') while True: @@ -573,24 +573,25 @@ class PackageIndex(Environment): def open_url(self, url, warning=None): - if url.startswith('file:'): - return local_open(url) + if url.startswith('file:'): return local_open(url) try: return open_with_auth(url) except urllib2.HTTPError, v: return v except urllib2.URLError, v: - if warning: self.warn(warning, v.reason) - else: - raise DistutilsError("Download error for %s: %s" - % (url, v.reason)) + reason = v.reason + except httplib.HTTPException, v: + reason = "%s: %s" % (v.__doc__ or v.__class__.__name__, v) + if warning: + self.warn(warning, reason) + else: + raise DistutilsError("Download error for %s: %s" % (url, reason)) def _download_url(self, scheme, url, tmpdir): # Determine download filename # - name = filter(None,urlparse.urlparse(url)[2].split('/')) + name, fragment = egg_info_for_url(url) if name: - name = name[-1] while '..' in name: name = name.replace('..','.').replace('\\','_') else: @@ -612,7 +613,6 @@ class PackageIndex(Environment): return self._attempt_download(url, filename) - def scan_url(self, url): self.process_url(url, True) @@ -639,10 +639,39 @@ class PackageIndex(Environment): os.unlink(filename) raise DistutilsError("Unexpected HTML page found at "+url) + + + + + + + + + + + + + + + def _download_svn(self, url, filename): url = url.split('#',1)[0] # remove any fragment for svn's sake + creds = '' + if url.lower().startswith('svn:') and '@' in url: + scheme, netloc, path, p, q, f = urlparse.urlparse(url) + if not netloc and path.startswith('//') and '/' in path[2:]: + netloc, path = path[2:].split('/',1) + auth, host = urllib.splituser(netloc) + if auth: + if ':' in auth: + user, pw = auth.split(':',1) + creds = " --username=%s --password=%s" % (user, pw) + else: + creds = " --username="+auth + netloc = host + url = urlparse.urlunparse((scheme, netloc, url, p, q, f)) self.info("Doing subversion checkout from %s to %s", url, filename) - os.system("svn checkout -q %s %s" % (url, filename)) + os.system("svn checkout%s -q %s %s" % (creds, url, filename)) return filename def debug(self, msg, *args): @@ -654,6 +683,18 @@ class PackageIndex(Environment): def warn(self, msg, *args): log.warn(msg, *args) + + + + + + + + + + + + # This pattern matches a character entity reference (a decimal numeric # references, a hexadecimal numeric reference, or a named reference). entity_sub = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub @@ -701,7 +742,7 @@ def open_with_auth(url): scheme, netloc, path, params, query, frag = urlparse.urlparse(url) if scheme in ('http', 'https'): - auth, host = urllib2.splituser(netloc) + auth, host = urllib.splituser(netloc) else: auth = None -- cgit v1.2.1 From 27d44ebd4ca443c9df123f308b46cc1cbd7d2404 Mon Sep 17 00:00:00 2001 From: pje <none@none> Date: Sat, 4 May 2013 16:07:01 -0400 Subject: Low-level SSL verification w/out warnings or options; automatically uses it if it can, won't if it doesn't. (grafted from 8dc579408781836ecf30b637665ad7f2509933a5) --HG-- branch : setuptools-0.6 extra : source : 8dc579408781836ecf30b637665ad7f2509933a5 --- setuptools/package_index.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 1668e765..aab2bb23 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1,6 +1,6 @@ """PyPI and direct package downloading""" import sys, os.path, re, urlparse, urllib2, shutil, random, socket, cStringIO -import httplib, urllib +import httplib, urllib; from setuptools import ssl_support from pkg_resources import * from distutils import log from distutils.errors import DistutilsError @@ -145,12 +145,11 @@ user_agent = "Python-urllib/%s setuptools/%s" % ( urllib2.__version__, require('setuptools')[0].version ) - class PackageIndex(Environment): """A distribution index that scans web pages for download URLs""" def __init__(self, index_url="https://pypi.python.org/simple", hosts=('*',), - *args, **kw + ca_bundle=None, verify_ssl=True, *args, **kw ): Environment.__init__(self,*args,**kw) self.index_url = index_url + "/"[:not index_url.endswith('/')] @@ -159,8 +158,9 @@ class PackageIndex(Environment): self.package_pages = {} self.allows = re.compile('|'.join(map(translate,hosts))).match self.to_scan = [] - - + if verify_ssl and ssl_support.is_available and (ca_bundle or ssl_support.find_ca_bundle()): + self.opener = ssl_support.opener_for(ca_bundle) + else: self.opener = urllib2.urlopen def process_url(self, url, retrieve=False): """Evaluate a URL as a possible download, and maybe retrieve it""" @@ -575,12 +575,13 @@ class PackageIndex(Environment): def open_url(self, url, warning=None): if url.startswith('file:'): return local_open(url) try: - return open_with_auth(url) - except urllib2.HTTPError, v: - return v - except urllib2.URLError, v: - reason = v.reason - except httplib.HTTPException, v: + return open_with_auth(url, self.opener) + except urllib2.HTTPError: + return sys.exc_info()[1] + except urllib2.URLError: + reason = sys.exc_info()[1].reason + except httplib.HTTPException: + v = sys.exc_info()[1] reason = "%s: %s" % (v.__doc__ or v.__class__.__name__, v) if warning: self.warn(warning, reason) @@ -612,7 +613,6 @@ class PackageIndex(Environment): self.url_ok(url, True) # raises error if not allowed return self._attempt_download(url, filename) - def scan_url(self, url): self.process_url(url, True) @@ -736,7 +736,7 @@ def htmldecode(text): -def open_with_auth(url): +def open_with_auth(url, opener=urllib2.urlopen): """Open a urllib2 request, handling HTTP authentication""" scheme, netloc, path, params, query, frag = urlparse.urlparse(url) @@ -755,7 +755,7 @@ def open_with_auth(url): request = urllib2.Request(url) request.add_header('User-Agent', user_agent) - fp = urllib2.urlopen(request) + fp = opener(request) if auth: # Put authentication info back into request URL if same host, -- cgit v1.2.1 From 2b48816298bc0ad2632a393560176ed151149aa5 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Fri, 24 May 2013 20:37:47 -0400 Subject: Restore Python 2.4 compatible syntax --HG-- branch : distribute --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 0a3f9e05..b0388628 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -12,7 +12,7 @@ except ImportError: from md5 import md5 from fnmatch import translate -from .py24compat import wraps +from setuptools.py24compat import wraps EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.]+)$') HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I) -- cgit v1.2.1 From 59d45d94058826e754c7f36fe0605ab9e1160b62 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Fri, 24 May 2013 23:53:44 -0400 Subject: Support get_headers on Python 3 and Python 2 --- setuptools/package_index.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 2701c873..8a3c96de 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -608,7 +608,10 @@ class PackageIndex(Environment): size = -1 if "content-length" in headers: # Some servers return multiple Content-Length headers :( - size = max(map(int,headers.getheaders("Content-Length"))) + if not hasattr(headers, 'get_all'): + # Older versions of Python don't have the get_all method + headers.get_all = headers.getheaders + size = max(map(int,headers.get_all("Content-Length"))) self.reporthook(url, filename, blocknum, bs, size) tfp = open(filename,'wb') while True: -- cgit v1.2.1 From cc51ff77f4914497a98b7223723995f05e8ab1a1 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Sat, 25 May 2013 14:32:23 -0400 Subject: Fix use of getheaders on Python 3 --- setuptools/package_index.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 8a3c96de..133677e8 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -13,6 +13,7 @@ except ImportError: from md5 import md5 from fnmatch import translate from .py24compat import wraps +from setuptools.py27compat import get_all_headers EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.]+)$') HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I) @@ -608,10 +609,8 @@ class PackageIndex(Environment): size = -1 if "content-length" in headers: # Some servers return multiple Content-Length headers :( - if not hasattr(headers, 'get_all'): - # Older versions of Python don't have the get_all method - headers.get_all = headers.getheaders - size = max(map(int,headers.get_all("Content-Length"))) + sizes = get_all_headers(headers, 'Content-Length') + size = max(map(int, sizes)) self.reporthook(url, filename, blocknum, bs, size) tfp = open(filename,'wb') while True: -- cgit v1.2.1 From 8e657eac1ef02faedca99df319fff6b63f4a4305 Mon Sep 17 00:00:00 2001 From: Vinay Sajip <vinay_sajip@yahoo.co.uk> Date: Sat, 15 Jun 2013 15:34:53 +0100 Subject: Initial commit. All tests pass on 2.7, 3.2 and 3.3, though there are some atexit errors in the multiprocessing module in 2.7/3.2 (seemingly unrelated to setuptools). --HG-- branch : single-codebase --- setuptools/package_index.py | 88 ++++++++++++++++++++++++--------------------- 1 file changed, 47 insertions(+), 41 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 4f39c70a..5ee6fd27 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1,12 +1,14 @@ """PyPI and direct package downloading""" -import sys, os.path, re, urlparse, urllib2, shutil, random, socket, cStringIO -import itertools -import base64 -import httplib, urllib -from setuptools import ssl_support +import sys, os.path, re, shutil, random, socket from pkg_resources import * from distutils import log from distutils.errors import DistutilsError + +from setuptools import ssl_support +from setuptools.compat import (urllib2, httplib, StringIO, HTTPError, + urlparse, urlunparse, unquote, splituser, + url2pathname, name2codepoint, ifilterfalse, + unichr, urljoin) try: from hashlib import md5 except ImportError: @@ -57,7 +59,7 @@ def parse_bdist_wininst(name): def egg_info_for_url(url): - scheme, server, path, parameters, query, fragment = urlparse.urlparse(url) + scheme, server, path, parameters, query, fragment = urlparse(url) base = urllib2.unquote(path.split('/')[-1]) if server=='sourceforge.net' and base=='download': # XXX Yuck base = urllib2.unquote(path.split('/')[-2]) @@ -146,7 +148,7 @@ def unique_everseen(iterable, key=None): seen = set() seen_add = seen.add if key is None: - for element in itertools.ifilterfalse(seen.__contains__, iterable): + for element in ifilterfalse(seen.__contains__, iterable): seen_add(element) yield element else: @@ -178,14 +180,14 @@ def find_external_links(url, page): rels = map(str.strip, rel.lower().split(',')) if 'homepage' in rels or 'download' in rels: for match in HREF.finditer(tag): - yield urlparse.urljoin(url, htmldecode(match.group(1))) + yield urljoin(url, htmldecode(match.group(1))) for tag in ("<th>Home Page", "<th>Download URL"): pos = page.find(tag) if pos!=-1: match = HREF.search(page,pos) if match: - yield urlparse.urljoin(url, htmldecode(match.group(1))) + yield urljoin(url, htmldecode(match.group(1))) user_agent = "Python-urllib/%s setuptools/%s" % ( sys.version[:3], require('setuptools')[0].version @@ -224,7 +226,7 @@ class PackageIndex(Environment): self.debug("Found link: %s", url) if dists or not retrieve or url in self.fetched_urls: - map(self.add, dists) + list(map(self.add, dists)) return # don't need the actual page if not self.url_ok(url): @@ -243,7 +245,7 @@ class PackageIndex(Environment): base = f.url # handle redirects page = f.read() if not isinstance(page, str): # We are in Python 3 and got bytes. We want str. - if isinstance(f, urllib2.HTTPError): + if isinstance(f, HTTPError): # Errors have no charset, assume latin1: charset = 'latin-1' else: @@ -251,7 +253,7 @@ class PackageIndex(Environment): page = page.decode(charset, "ignore") f.close() for match in HREF.finditer(page): - link = urlparse.urljoin(base, htmldecode(match.group(1))) + link = urljoin(base, htmldecode(match.group(1))) self.process_url(link) if url.startswith(self.index_url) and getattr(f,'code',None)!=404: page = self.process_index(url, page) @@ -270,11 +272,11 @@ class PackageIndex(Environment): dists = distros_for_filename(fn) if dists: self.debug("Found: %s", fn) - map(self.add, dists) + list(map(self.add, dists)) def url_ok(self, url, fatal=False): s = URL_SCHEME(url) - if (s and s.group(1).lower()=='file') or self.allows(urlparse.urlparse(url)[1]): + if (s and s.group(1).lower()=='file') or self.allows(urlparse(url)[1]): return True msg = "\nLink to % s ***BLOCKED*** by --allow-hosts\n" if fatal: @@ -290,7 +292,7 @@ class PackageIndex(Environment): self.scan_egg_link(item, entry) def scan_egg_link(self, path, entry): - lines = filter(None, map(str.strip, open(os.path.join(path, entry)))) + lines = list(filter(None, map(str.strip, open(os.path.join(path, entry))))) if len(lines)==2: for dist in find_distributions(os.path.join(path, lines[0])): dist.location = os.path.join(path, *lines) @@ -302,9 +304,9 @@ class PackageIndex(Environment): def scan(link): # Process a URL to see if it's for a package page if link.startswith(self.index_url): - parts = map( - urllib2.unquote, link[len(self.index_url):].split('/') - ) + parts = list(map( + unquote, link[len(self.index_url):].split('/') + )) if len(parts)==2 and '#' not in parts[1]: # it's a package page, sanitize and index it pkg = safe_name(parts[0]) @@ -316,7 +318,7 @@ class PackageIndex(Environment): # process an index page into the package-page index for match in HREF.finditer(page): try: - scan( urlparse.urljoin(url, htmldecode(match.group(1))) ) + scan( urljoin(url, htmldecode(match.group(1))) ) except ValueError: pass @@ -411,7 +413,7 @@ class PackageIndex(Environment): def prescan(self): """Scan urls scheduled for prescanning (e.g. --find-links)""" if self.to_scan: - map(self.scan_url, self.to_scan) + list(map(self.scan_url, self.to_scan)) self.to_scan = None # from now on, go ahead and process immediately def not_found_in_index(self, requirement): @@ -598,7 +600,7 @@ class PackageIndex(Environment): if '#' in url: url, info = url.split('#', 1) fp = self.open_url(url) - if isinstance(fp, urllib2.HTTPError): + if isinstance(fp, HTTPError): raise DistutilsError( "Can't download %s: %s %s" % (url, fp.code,fp.msg) ) @@ -637,28 +639,33 @@ class PackageIndex(Environment): return local_open(url) try: return open_with_auth(url, self.opener) - except (ValueError, httplib.InvalidURL), v: + except (ValueError, httplib.InvalidURL): + v = sys.exc_info()[1] msg = ' '.join([str(arg) for arg in v.args]) if warning: self.warn(warning, msg) else: raise DistutilsError('%s %s' % (url, msg)) - except urllib2.HTTPError, v: + except urllib2.HTTPError: + v = sys.exc_info()[1] return v - except urllib2.URLError, v: + except urllib2.URLError: + v = sys.exc_info()[1] if warning: self.warn(warning, v.reason) else: raise DistutilsError("Download error for %s: %s" % (url, v.reason)) - except httplib.BadStatusLine, v: + except httplib.BadStatusLine: + v = sys.exc_info()[1] if warning: self.warn(warning, v.line) else: raise DistutilsError('%s returned a bad status line. ' 'The server might be down, %s' % \ (url, v.line)) - except httplib.HTTPException, v: + except httplib.HTTPException: + v = sys.exc_info()[1] if warning: self.warn(warning, v) else: @@ -689,7 +696,7 @@ class PackageIndex(Environment): elif scheme.startswith('hg+'): return self._download_hg(url, filename) elif scheme=='file': - return urllib.url2pathname(urlparse.urlparse(url)[2]) + return url2pathname(urlparse(url)[2]) else: self.url_ok(url, True) # raises error if not allowed return self._attempt_download(url, filename) @@ -739,7 +746,7 @@ class PackageIndex(Environment): url = url.split('#',1)[0] # remove any fragment for svn's sake creds = '' if url.lower().startswith('svn:') and '@' in url: - scheme, netloc, path, p, q, f = urlparse.urlparse(url) + scheme, netloc, path, p, q, f = urlparse(url) if not netloc and path.startswith('//') and '/' in path[2:]: netloc, path = path[2:].split('/',1) auth, host = urllib.splituser(netloc) @@ -750,13 +757,13 @@ class PackageIndex(Environment): else: creds = " --username="+auth netloc = host - url = urlparse.urlunparse((scheme, netloc, url, p, q, f)) + url = urlunparse((scheme, netloc, url, p, q, f)) self.info("Doing subversion checkout from %s to %s", url, filename) os.system("svn checkout%s -q %s %s" % (creds, url, filename)) return filename def _vcs_split_rev_from_url(self, url, pop_prefix=False): - scheme, netloc, path, query, frag = urlparse.urlsplit(url) + scheme, netloc, path, query, frag = urlsplit(url) scheme = scheme.split('+', 1)[-1] @@ -768,7 +775,7 @@ class PackageIndex(Environment): path, rev = path.rsplit('@', 1) # Also, discard fragment - url = urlparse.urlunsplit((scheme, netloc, path, query, '')) + url = urlunsplit((scheme, netloc, path, query, '')) return url, rev @@ -842,7 +849,6 @@ def decode_entity(match): elif what.startswith('#'): what = int(what[1:]) else: - from htmlentitydefs import name2codepoint what = name2codepoint.get(what, match.group(0)) return uchr(what) @@ -896,7 +902,7 @@ def _encode_auth(auth): def open_with_auth(url, opener=urllib2.urlopen): """Open a urllib2 request, handling HTTP authentication""" - scheme, netloc, path, params, query, frag = urlparse.urlparse(url) + scheme, netloc, path, params, query, frag = urlparse(url) # Double scheme does not raise on Mac OS X as revealed by a # failing test. We would expect "nonnumeric port". Refs #20. @@ -904,13 +910,13 @@ def open_with_auth(url, opener=urllib2.urlopen): raise httplib.InvalidURL("nonnumeric port: ''") if scheme in ('http', 'https'): - auth, host = urllib.splituser(netloc) + auth, host = splituser(netloc) else: auth = None if auth: auth = "Basic " + _encode_auth(auth) - new_url = urlparse.urlunparse((scheme,host,path,params,query,frag)) + new_url = urlunparse((scheme,host,path,params,query,frag)) request = urllib2.Request(new_url) request.add_header("Authorization", auth) else: @@ -922,9 +928,9 @@ def open_with_auth(url, opener=urllib2.urlopen): if auth: # Put authentication info back into request URL if same host, # so that links found on the page will work - s2, h2, path2, param2, query2, frag2 = urlparse.urlparse(fp.url) + s2, h2, path2, param2, query2, frag2 = urlparse(fp.url) if s2==scheme and h2==host: - fp.url = urlparse.urlunparse((s2,netloc,path2,param2,query2,frag2)) + fp.url = urlunparse((s2,netloc,path2,param2,query2,frag2)) return fp @@ -946,8 +952,8 @@ def fix_sf_url(url): def local_open(url): """Read a local path, with special support for directories""" - scheme, server, path, param, query, frag = urlparse.urlparse(url) - filename = urllib.url2pathname(path) + scheme, server, path, param, query, frag = urlparse(url) + filename = url2pathname(path) if os.path.isfile(filename): return urllib2.urlopen(url) elif path.endswith('/') and os.path.isdir(filename): @@ -968,8 +974,8 @@ def local_open(url): else: status, message, body = 404, "Path not found", "Not found" - return urllib2.HTTPError(url, status, message, - {'content-type':'text/html'}, cStringIO.StringIO(body)) + return HTTPError(url, status, message, + {'content-type':'text/html'}, StringIO(body)) -- cgit v1.2.1 From 3c86c861ab9665df0e502f250a7471d09240e413 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Mon, 17 Jun 2013 10:55:09 -0500 Subject: Fix Python3 compatibility issue in filterfalse --HG-- branch : distribute extra : rebase_source : 71e3e89efe6483924a07e7f9819ee86f08dbf1f2 extra : histedit_source : fa75c7bc174b248c74da7b4efd5f6f05c36b6ae4 --- setuptools/package_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 85a1deeb..1e2a086d 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1,6 +1,5 @@ """PyPI and direct package downloading""" import sys, os.path, re, shutil, random, socket -import itertools import base64 from pkg_resources import * from distutils import log @@ -9,6 +8,7 @@ from setuptools.compat import (urllib2, httplib, StringIO, HTTPError, urlparse, urlunparse, unquote, splituser, url2pathname, name2codepoint, unichr, urljoin) +from setuptools.compat import filterfalse try: from hashlib import md5 except ImportError: @@ -148,7 +148,7 @@ def unique_everseen(iterable, key=None): seen = set() seen_add = seen.add if key is None: - for element in itertools.ifilterfalse(seen.__contains__, iterable): + for element in filterfalse(seen.__contains__, iterable): seen_add(element) yield element else: -- cgit v1.2.1 From 744a61f18bbfcbf7dfaa08886185b4595d8b7bcb Mon Sep 17 00:00:00 2001 From: Vinay Sajip <vinay_sajip@yahoo.co.uk> Date: Mon, 17 Jun 2013 19:23:33 +0100 Subject: Misc. updates following 2to3 checks. --HG-- branch : single-codebase --- setuptools/package_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 5ee6fd27..25936b91 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -177,7 +177,7 @@ def find_external_links(url, page): for match in REL.finditer(page): tag, rel = match.groups() - rels = map(str.strip, rel.lower().split(',')) + rels = set(map(str.strip, rel.lower().split(','))) if 'homepage' in rels or 'download' in rels: for match in HREF.finditer(tag): yield urljoin(url, htmldecode(match.group(1))) @@ -749,7 +749,7 @@ class PackageIndex(Environment): scheme, netloc, path, p, q, f = urlparse(url) if not netloc and path.startswith('//') and '/' in path[2:]: netloc, path = path[2:].split('/',1) - auth, host = urllib.splituser(netloc) + auth, host = splituser(netloc) if auth: if ':' in auth: user, pw = auth.split(':',1) -- cgit v1.2.1 From 37e2f6f5395586e12eb57a4b6fd598b29744de0c Mon Sep 17 00:00:00 2001 From: Donald Stufft <donald@stufft.io> Date: Tue, 9 Jul 2013 22:30:25 -0400 Subject: Enable using any guarenteed hash as a hash function --HG-- extra : rebase_source : 1c5040c4a89dfcd4ec8cf2ad64825d5bc73ebe30 --- setuptools/package_index.py | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 61a66c6d..e29a142c 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -11,11 +11,8 @@ from setuptools.compat import (urllib2, httplib, StringIO, HTTPError, url2pathname, name2codepoint, unichr, urljoin) from setuptools.compat import filterfalse -try: - from hashlib import md5 -except ImportError: - from md5 import md5 from fnmatch import translate +from setuptools.py24compat import hashlib from setuptools.py24compat import wraps from setuptools.py27compat import get_all_headers @@ -28,6 +25,7 @@ PYPI_MD5 = re.compile( ) URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):',re.I).match EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split() +_HASH_RE = re.compile(r'(sha1|sha224|sha384|sha256|sha512|md5)=([a-f0-9]+)') __all__ = [ 'PackageIndex', 'distros_for_url', 'parse_bdist_wininst', @@ -387,15 +385,19 @@ class PackageIndex(Environment): - def check_md5(self, cs, info, filename, tfp): - if re.match('md5=[0-9a-f]{32}$', info): - self.debug("Validating md5 checksum for %s", filename) - if cs.hexdigest() != info[4:]: + def check_hash(self, cs, info, filename, tfp): + match = _HASH_RE.search(info) + if match: + hash_name = match.group(1) + hash_data = match.group(2) + self.debug("Validating %s checksum for %s", hash_name, filename) + if cs.hexdigest() != hash_data: tfp.close() os.unlink(filename) raise DistutilsError( - "MD5 validation failed for "+os.path.basename(filename)+ - "; possible download problem?" + "%s validation failed for %s; " + "possible download problem?" % ( + hash_name, os.path.basename(filename)) ) def add_find_links(self, urls): @@ -598,16 +600,19 @@ class PackageIndex(Environment): def _download_to(self, url, filename): self.info("Downloading %s", url) # Download the file - fp, tfp, info = None, None, None + fp, tfp, cs, info = None, None, None, None try: if '#' in url: url, info = url.split('#', 1) + hmatch = _HASH_RE.search(info) + hash_name = hmatch.group(1) + hash_data = hmatch.group(2) + cs = hashlib.new(hash_name) fp = self.open_url(url) if isinstance(fp, HTTPError): raise DistutilsError( "Can't download %s: %s %s" % (url, fp.code,fp.msg) ) - cs = md5() headers = fp.info() blocknum = 0 bs = self.dl_blocksize @@ -621,13 +626,14 @@ class PackageIndex(Environment): while True: block = fp.read(bs) if block: - cs.update(block) + if cs is not None: + cs.update(block) tfp.write(block) blocknum += 1 self.reporthook(url, filename, blocknum, bs, size) else: break - if info: self.check_md5(cs, info, filename, tfp) + if info: self.check_hash(cs, info, filename, tfp) return headers finally: if fp: fp.close() -- cgit v1.2.1 From 95bf90f3dae23f3d672450d94adfbcc55f9e252a Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Mon, 15 Jul 2013 12:06:54 -0400 Subject: Extracted hash-checking functionality into its own classes. Hashes are no longer checked when the proper pattern isn't detected. Fixes #42. --- setuptools/package_index.py | 99 +++++++++++++++++++++++++++++++++------------ 1 file changed, 73 insertions(+), 26 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index e29a142c..4c4a647d 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -25,7 +25,6 @@ PYPI_MD5 = re.compile( ) URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):',re.I).match EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split() -_HASH_RE = re.compile(r'(sha1|sha224|sha384|sha256|sha512|md5)=([a-f0-9]+)') __all__ = [ 'PackageIndex', 'distros_for_url', 'parse_bdist_wininst', @@ -193,6 +192,61 @@ user_agent = "Python-urllib/%s setuptools/%s" % ( sys.version[:3], require('setuptools')[0].version ) +class ContentChecker(object): + """ + A null content checker that defines the interface for checking content + """ + def feed(self, block): + """ + Feed a block of data to the hash. + """ + return + + def check(self): + """ + Check the hash. Return False if validation fails. + """ + return True + + def report(self, reporter, template): + """ + Call reporter with information about the checker (hash name) + substituted into the template. + """ + return + +class HashChecker(ContentChecker): + pattern = re.compile( + r'(?P<hash_name>sha1|sha224|sha384|sha256|sha512|md5)=' + r'(?P<expected>[a-f0-9]+)' + ) + + def __init__(self, hash_name, expected): + self.hash = hashlib.new(hash_name) + self.expected = expected + + @classmethod + def from_url(cls, url): + "Construct a (possibly null) ContentChecker from a URL" + fragment = urlparse(url)[-1] + if not fragment: + return ContentChecker() + match = cls.pattern.search(fragment) + if not match: + return ContentChecker() + return cls(**match.groupdict()) + + def feed(self, block): + self.hash.update(block) + + def check(self): + return self.hash.hexdigest() == self.expected + + def report(self, reporter, template): + msg = template % self.hash.name + return reporter(msg) + + class PackageIndex(Environment): """A distribution index that scans web pages for download URLs""" @@ -385,20 +439,20 @@ class PackageIndex(Environment): - def check_hash(self, cs, info, filename, tfp): - match = _HASH_RE.search(info) - if match: - hash_name = match.group(1) - hash_data = match.group(2) - self.debug("Validating %s checksum for %s", hash_name, filename) - if cs.hexdigest() != hash_data: - tfp.close() - os.unlink(filename) - raise DistutilsError( - "%s validation failed for %s; " - "possible download problem?" % ( - hash_name, os.path.basename(filename)) - ) + def check_hash(self, checker, filename, tfp): + """ + checker is a ContentChecker + """ + checker.report(self.debug, + "Validating %%s checksum for %s" % filename) + if not checker.valid(): + tfp.close() + os.unlink(filename) + raise DistutilsError( + "%s validation failed for %s; " + "possible download problem?" % ( + checker.hash.name, os.path.basename(filename)) + ) def add_find_links(self, urls): """Add `urls` to the list that will be prescanned for searches""" @@ -600,14 +654,9 @@ class PackageIndex(Environment): def _download_to(self, url, filename): self.info("Downloading %s", url) # Download the file - fp, tfp, cs, info = None, None, None, None + fp, tfp, info = None, None, None try: - if '#' in url: - url, info = url.split('#', 1) - hmatch = _HASH_RE.search(info) - hash_name = hmatch.group(1) - hash_data = hmatch.group(2) - cs = hashlib.new(hash_name) + checker = HashChecker.from_url(url) fp = self.open_url(url) if isinstance(fp, HTTPError): raise DistutilsError( @@ -626,14 +675,13 @@ class PackageIndex(Environment): while True: block = fp.read(bs) if block: - if cs is not None: - cs.update(block) + checker.feed(block) tfp.write(block) blocknum += 1 self.reporthook(url, filename, blocknum, bs, size) else: break - if info: self.check_hash(cs, info, filename, tfp) + self.check_hash(checker, filename, tfp) return headers finally: if fp: fp.close() @@ -642,7 +690,6 @@ class PackageIndex(Environment): def reporthook(self, url, filename, blocknum, blksize, size): pass # no-op - def open_url(self, url, warning=None): if url.startswith('file:'): return local_open(url) -- cgit v1.2.1 From 915f05b7445af2c51b63d22e429d9a7397221518 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Mon, 15 Jul 2013 13:31:00 -0400 Subject: Use 'is_valid' instead of simply 'valid' or 'check', which are less clear about the purpose of the method. Fixes AttributeError introduces in 0.9.2. Fixes #42. --- setuptools/package_index.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 4c4a647d..70aabd1b 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -202,7 +202,7 @@ class ContentChecker(object): """ return - def check(self): + def is_valid(self): """ Check the hash. Return False if validation fails. """ @@ -239,7 +239,7 @@ class HashChecker(ContentChecker): def feed(self, block): self.hash.update(block) - def check(self): + def is_valid(self): return self.hash.hexdigest() == self.expected def report(self, reporter, template): @@ -445,7 +445,7 @@ class PackageIndex(Environment): """ checker.report(self.debug, "Validating %%s checksum for %s" % filename) - if not checker.valid(): + if not checker.is_valid(): tfp.close() os.unlink(filename) raise DistutilsError( -- cgit v1.2.1 From 3e72e7f7eacca7db638f7230f93cf696d49c77bf Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Wed, 17 Jul 2013 11:28:44 -0400 Subject: Add compatibility for Python 2.4 when querying the hash name. Fixes #44 --- setuptools/package_index.py | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 70aabd1b..47f00c00 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -242,8 +242,23 @@ class HashChecker(ContentChecker): def is_valid(self): return self.hash.hexdigest() == self.expected + def _get_hash_name(self): + """ + Python 2.4 implementation of MD5 doesn't supply a .name attribute + so provide that name. + + When Python 2.4 is no longer required, replace invocations of this + method with simply 'self.hash.name'. + """ + try: + return self.hash.name + except AttributeError: + if 'md5' in str(type(self.hash)): + return 'md5' + raise + def report(self, reporter, template): - msg = template % self.hash.name + msg = template % self._get_hash_name() return reporter(msg) -- cgit v1.2.1 From 937a7dee0b5de773f164d7e3b143113c270733eb Mon Sep 17 00:00:00 2001 From: Daniel Holth <dholth@fastmail.fm> Date: Mon, 22 Jul 2013 17:32:14 +0000 Subject: package_index.py : fix hash_name --- setuptools/package_index.py | 20 +++----------------- 1 file changed, 3 insertions(+), 17 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 47f00c00..b1b38f03 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -222,7 +222,8 @@ class HashChecker(ContentChecker): ) def __init__(self, hash_name, expected): - self.hash = hashlib.new(hash_name) + self.hash_name = hash_name + self.hash = hashlib.new(hash_name) self.expected = expected @classmethod @@ -242,23 +243,8 @@ class HashChecker(ContentChecker): def is_valid(self): return self.hash.hexdigest() == self.expected - def _get_hash_name(self): - """ - Python 2.4 implementation of MD5 doesn't supply a .name attribute - so provide that name. - - When Python 2.4 is no longer required, replace invocations of this - method with simply 'self.hash.name'. - """ - try: - return self.hash.name - except AttributeError: - if 'md5' in str(type(self.hash)): - return 'md5' - raise - def report(self, reporter, template): - msg = template % self._get_hash_name() + msg = template % self.hash_name return reporter(msg) -- cgit v1.2.1 From a09c0b0efd3b71306b9293837850aec1887f6b98 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Wed, 24 Jul 2013 22:31:56 +0200 Subject: Remove unused imports --- setuptools/package_index.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index b1b38f03..88f04809 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1,6 +1,5 @@ """PyPI and direct package downloading""" -import sys, os.path, re, shutil, random, socket -import itertools +import sys, os.path, re, shutil, socket import base64 from setuptools import ssl_support from pkg_resources import * @@ -223,7 +222,7 @@ class HashChecker(ContentChecker): def __init__(self, hash_name, expected): self.hash_name = hash_name - self.hash = hashlib.new(hash_name) + self.hash = hashlib.new(hash_name) self.expected = expected @classmethod -- cgit v1.2.1 From 9505e8bf8b0ca5cf33befc3f8818cb519c74cc0c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Wed, 24 Jul 2013 22:33:16 +0200 Subject: Clean up imports in package_index --- setuptools/package_index.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 88f04809..76421c3b 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1,6 +1,11 @@ """PyPI and direct package downloading""" -import sys, os.path, re, shutil, socket +import sys +import os +import re +import shutil +import socket import base64 + from setuptools import ssl_support from pkg_resources import * from distutils import log -- cgit v1.2.1 From a58abed83f51c263547294fbfb57e5a44a32e5a3 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Wed, 24 Jul 2013 22:38:26 +0200 Subject: Re-indent several function signatures to reduce linter warnings --- setuptools/package_index.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 76421c3b..4c2e10f9 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -109,9 +109,10 @@ def distros_for_filename(filename, metadata=None): ) -def interpret_distro_name(location, basename, metadata, - py_version=None, precedence=SOURCE_DIST, platform=None -): +def interpret_distro_name( + location, basename, metadata, py_version=None, precedence=SOURCE_DIST, + platform=None + ): """Generate alternative interpretations of a source distro name Note: if `location` is a filesystem filename, you should call @@ -255,9 +256,10 @@ class HashChecker(ContentChecker): class PackageIndex(Environment): """A distribution index that scans web pages for download URLs""" - def __init__(self, index_url="https://pypi.python.org/simple", hosts=('*',), - ca_bundle=None, verify_ssl=True, *args, **kw - ): + def __init__( + self, index_url="https://pypi.python.org/simple", hosts=('*',), + ca_bundle=None, verify_ssl=True, *args, **kw + ): Environment.__init__(self,*args,**kw) self.index_url = index_url + "/"[:not index_url.endswith('/')] self.scanned_urls = {} @@ -378,7 +380,7 @@ class PackageIndex(Environment): # process an index page into the package-page index for match in HREF.finditer(page): try: - scan( urljoin(url, htmldecode(match.group(1))) ) + scan(urljoin(url, htmldecode(match.group(1)))) except ValueError: pass @@ -530,10 +532,10 @@ class PackageIndex(Environment): return getattr(self.fetch_distribution(spec, tmpdir),'location',None) - def fetch_distribution(self, - requirement, tmpdir, force_scan=False, source=False, develop_ok=False, - local_index=None - ): + def fetch_distribution( + self, requirement, tmpdir, force_scan=False, source=False, + develop_ok=False, local_index=None + ): """Obtain a distribution suitable for fulfilling `requirement` `requirement` must be a ``pkg_resources.Requirement`` instance. @@ -616,7 +618,8 @@ class PackageIndex(Environment): def gen_setup(self, filename, fragment, tmpdir): match = EGG_FRAGMENT.match(fragment) - dists = match and [d for d in + dists = match and [ + d for d in interpret_distro_name(filename, match.group(1), None) if d.version ] or [] @@ -722,9 +725,11 @@ class PackageIndex(Environment): if warning: self.warn(warning, v.line) else: - raise DistutilsError('%s returned a bad status line. ' - 'The server might be down, %s' % \ - (url, v.line)) + raise DistutilsError( + '%s returned a bad status line. The server might be ' + 'down, %s' % + (url, v.line) + ) except httplib.HTTPException: v = sys.exc_info()[1] if warning: -- cgit v1.2.1 From 7b42936dc2dacbf91504c495b6cae569a1d88c95 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Wed, 24 Jul 2013 22:40:18 +0200 Subject: Normalize whitespace in package_index.py --- setuptools/package_index.py | 79 ++------------------------------------------- 1 file changed, 2 insertions(+), 77 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 4c2e10f9..9abd9679 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -403,8 +403,6 @@ class PackageIndex(Environment): else: return "" # no sense double-scanning non-package pages - - def need_version_info(self, url): self.scan_all( "Page at %s links to .py file(s) without version info; an index " @@ -435,17 +433,14 @@ class PackageIndex(Environment): self.scan_url(url) def obtain(self, requirement, installer=None): - self.prescan(); self.find_packages(requirement) + self.prescan() + self.find_packages(requirement) for dist in self[requirement.key]: if dist in requirement: return dist self.debug("%s does not match %s", requirement, dist) return super(PackageIndex, self).obtain(requirement,installer) - - - - def check_hash(self, checker, filename, tfp): """ checker is a ContentChecker @@ -531,7 +526,6 @@ class PackageIndex(Environment): ) return getattr(self.fetch_distribution(spec, tmpdir),'location',None) - def fetch_distribution( self, requirement, tmpdir, force_scan=False, source=False, develop_ok=False, local_index=None @@ -573,8 +567,6 @@ class PackageIndex(Environment): if dist in req and (dist.precedence<=SOURCE_DIST or not source): return dist - - if force_scan: self.prescan() self.find_packages(requirement) @@ -601,7 +593,6 @@ class PackageIndex(Environment): self.info("Best match: %s", dist) return dist.clone(location=self.download(dist.location, tmpdir)) - def fetch(self, requirement, tmpdir, force_scan=False, source=False): """Obtain a file suitable for fulfilling `requirement` @@ -615,7 +606,6 @@ class PackageIndex(Environment): return dist.location return None - def gen_setup(self, filename, fragment, tmpdir): match = EGG_FRAGMENT.match(fragment) dists = match and [ @@ -770,7 +760,6 @@ class PackageIndex(Environment): def scan_url(self, url): self.process_url(url, True) - def _attempt_download(self, url, filename): headers = self._download_to(url, filename) if 'html' in headers.get('content-type','').lower(): @@ -793,21 +782,6 @@ class PackageIndex(Environment): os.unlink(filename) raise DistutilsError("Unexpected HTML page found at "+url) - - - - - - - - - - - - - - - def _download_svn(self, url, filename): url = url.split('#',1)[0] # remove any fragment for svn's sake creds = '' @@ -886,18 +860,6 @@ class PackageIndex(Environment): def warn(self, msg, *args): log.warn(msg, *args) - - - - - - - - - - - - # This pattern matches a character entity reference (a decimal numeric # references, a hexadecimal numeric reference, or a named reference). entity_sub = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub @@ -922,20 +884,6 @@ def htmldecode(text): """Decode HTML entities in the given text.""" return entity_sub(decode_entity, text) - - - - - - - - - - - - - - def socket_timeout(timeout=15): def _socket_timeout(func): def _socket_timeout(*args, **kwargs): @@ -1004,15 +952,6 @@ def open_with_auth(url, opener=urllib2.urlopen): open_with_auth = socket_timeout(_SOCKET_TIMEOUT)(open_with_auth) - - - - - - - - - def fix_sf_url(url): return url # backward compatibility @@ -1042,17 +981,3 @@ def local_open(url): return HTTPError(url, status, message, {'content-type':'text/html'}, StringIO(body)) - - - - - - - - - - - - - -# this line is a kludge to keep the trailing blank lines for pje's editor -- cgit v1.2.1 From 9a3cc1cd285136180c9b59c821a01f755ddf8874 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Wed, 24 Jul 2013 22:47:05 +0200 Subject: Replace import * with explicit imports, allowing linter to detect the missing imports reported in #53. --- setuptools/package_index.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 9abd9679..f2ef50d8 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -6,8 +6,12 @@ import shutil import socket import base64 +from pkg_resources import ( + CHECKOUT_DIST, Distribution, BINARY_DIST, normalize_path, SOURCE_DIST, + require, Environment, find_distributions, safe_name, safe_version, + to_filename, Requirement, DEVELOP_DIST, +) from setuptools import ssl_support -from pkg_resources import * from distutils import log from distutils.errors import DistutilsError from setuptools.compat import (urllib2, httplib, StringIO, HTTPError, -- cgit v1.2.1 From d2f68c57b5fbca47a9389c6cf3399f956a6c2aa6 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Wed, 24 Jul 2013 23:01:43 +0200 Subject: Import urlsplit and urlunsplit from compat module. Fixes #53. --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index f2ef50d8..26b6583c 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -17,7 +17,7 @@ from distutils.errors import DistutilsError from setuptools.compat import (urllib2, httplib, StringIO, HTTPError, urlparse, urlunparse, unquote, splituser, url2pathname, name2codepoint, - unichr, urljoin) + unichr, urljoin, urlsplit, urlunsplit) from setuptools.compat import filterfalse from fnmatch import translate from setuptools.py24compat import hashlib -- cgit v1.2.1 From 3d5510a01906bebf369cde304706ce5f52117afe Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Wed, 24 Jul 2013 23:03:19 +0200 Subject: Make PackageIndex._vcs_split_rev_from_url a static method (as it doesn't have any instance or class references). --- setuptools/package_index.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 26b6583c..9c9d76a1 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -806,7 +806,8 @@ class PackageIndex(Environment): os.system("svn checkout%s -q %s %s" % (creds, url, filename)) return filename - def _vcs_split_rev_from_url(self, url, pop_prefix=False): + @staticmethod + def _vcs_split_rev_from_url(url, pop_prefix=False): scheme, netloc, path, query, frag = urlsplit(url) scheme = scheme.split('+', 1)[-1] -- cgit v1.2.1 From f051b3d36871ccd9f0f5fc770047c497f196547a Mon Sep 17 00:00:00 2001 From: Kai Hoppert <kai.hoppert@online.de> Date: Sat, 10 Aug 2013 18:24:23 +0200 Subject: Added .pypirc authentication support for getting eggs --HG-- extra : histedit_source : ebdc22b1f156f8af40265c5772addcfda6ae11d8 --- setuptools/package_index.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 9c9d76a1..ca228997 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -6,6 +6,8 @@ import shutil import socket import base64 +import ConfigParser + from pkg_resources import ( CHECKOUT_DIST, Distribution, BINARY_DIST, normalize_path, SOURCE_DIST, require, Environment, find_distributions, safe_name, safe_version, @@ -918,6 +920,37 @@ def _encode_auth(auth): # strip the trailing carriage return return encoded.rstrip() +class PyPirc: + + def __init__(self): + """ + Extract pypirc authentication information from home directory + """ + self.dict_ = {} + + if os.environ.has_key('HOME'): + rc = os.path.join(os.environ['HOME'], '.pypirc') + if os.path.exists(rc): + config = ConfigParser.ConfigParser({ + 'username' : '', + 'password' : '', + 'repository' : ''}) + config.read(rc) + + for section in config.sections(): + if config.get(section, 'repository').strip(): + value = '%s:%s'%(config.get(section, 'username').strip(), + config.get(section, 'password').strip()) + self.dict_[config.get(section, 'repository').strip()] = value + + def __call__(self, url): + """ """ + for base_url, auth in self.dict_.items(): + if url.startswith(base_url): + return auth + + + def open_with_auth(url, opener=urllib2.urlopen): """Open a urllib2 request, handling HTTP authentication""" @@ -933,6 +966,10 @@ def open_with_auth(url, opener=urllib2.urlopen): else: auth = None + if not auth: + auth = PyPirc()(url) + log.info('Authentication found for URL: %s'%url) + if auth: auth = "Basic " + _encode_auth(auth) new_url = urlunparse((scheme,host,path,params,query,frag)) -- cgit v1.2.1 From 4041d1f4667aa4bd7f7373ff7cc626b0d9f5d10c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Sun, 25 Aug 2013 21:22:34 -0400 Subject: Updated error message reported when `--allow-hosts` blocks a link to provide a less startling user experience. Fixes #71. --- setuptools/package_index.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 9c9d76a1..d949063e 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -343,7 +343,8 @@ class PackageIndex(Environment): s = URL_SCHEME(url) if (s and s.group(1).lower()=='file') or self.allows(urlparse(url)[1]): return True - msg = "\nLink to % s ***BLOCKED*** by --allow-hosts\n" + msg = ("\nNote: Bypassing %s (disallowed host; see " + "http://bit.ly/1dg9ijs for details).\n") if fatal: raise DistutilsError(msg % url) else: -- cgit v1.2.1 From 705e6255fca83f5ff2b194e6529968017cefaaa3 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Fri, 6 Sep 2013 09:33:13 -0400 Subject: Correct 404 errors when URLs contain fragments. Fixes #69. --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 4c4a647d..a09048b4 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -657,7 +657,7 @@ class PackageIndex(Environment): fp, tfp, info = None, None, None try: checker = HashChecker.from_url(url) - fp = self.open_url(url) + fp = self.open_url(strip_fragment(url)) if isinstance(fp, HTTPError): raise DistutilsError( "Can't download %s: %s %s" % (url, fp.code,fp.msg) -- cgit v1.2.1 From 559c85e2d4093e5a3163b7fe01cbb2f0180c9141 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Fri, 6 Sep 2013 09:39:47 -0400 Subject: Add the import, omitted from the previous commit. --- setuptools/package_index.py | 1 + 1 file changed, 1 insertion(+) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index a6672c70..4c9e40a7 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -22,6 +22,7 @@ from setuptools.compat import filterfalse from fnmatch import translate from setuptools.py24compat import hashlib from setuptools.py24compat import wraps +from setuptools.py26compat import strip_fragment from setuptools.py27compat import get_all_headers EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.]+)$') -- cgit v1.2.1 From 45928d50cd992a38a0df25813bd13617c13c1462 Mon Sep 17 00:00:00 2001 From: Jonathan Dye <jonathan@lulz.bz> Date: Thu, 31 Oct 2013 03:35:56 -0600 Subject: fixes a bug where a Basic auth digest header can get encoded with newlines inside --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 4c9e40a7..94873620 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -918,7 +918,7 @@ def _encode_auth(auth): # convert back to a string encoded = encoded_bytes.decode() # strip the trailing carriage return - return encoded.rstrip() + return encoded.replace('\n','') def open_with_auth(url, opener=urllib2.urlopen): """Open a urllib2 request, handling HTTP authentication""" -- cgit v1.2.1 From 3822e3e7ad8cda5012b602f7e8f52cdd44665dd3 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Thu, 14 Nov 2013 09:35:20 -0500 Subject: Delint syntax --HG-- extra : amend_source : 0f58aa917ebc71efd2904638fac3838fd0b0e4dd --- setuptools/package_index.py | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index ca228997..a0bb936d 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -920,29 +920,29 @@ def _encode_auth(auth): # strip the trailing carriage return return encoded.rstrip() -class PyPirc: +class PyPirc(object): def __init__(self): """ - Extract pypirc authentication information from home directory + Extract pypirc authentication information from home directory """ self.dict_ = {} - if os.environ.has_key('HOME'): + if 'HOME' in os.environ: rc = os.path.join(os.environ['HOME'], '.pypirc') if os.path.exists(rc): config = ConfigParser.ConfigParser({ - 'username' : '', - 'password' : '', - 'repository' : ''}) + 'username': '', + 'password': '', + 'repository': ''}) config.read(rc) for section in config.sections(): if config.get(section, 'repository').strip(): - value = '%s:%s'%(config.get(section, 'username').strip(), + value = '%s:%s' % (config.get(section, 'username').strip(), config.get(section, 'password').strip()) self.dict_[config.get(section, 'repository').strip()] = value - + def __call__(self, url): """ """ for base_url, auth in self.dict_.items(): @@ -950,7 +950,6 @@ class PyPirc: return auth - def open_with_auth(url, opener=urllib2.urlopen): """Open a urllib2 request, handling HTTP authentication""" @@ -968,7 +967,7 @@ def open_with_auth(url, opener=urllib2.urlopen): if not auth: auth = PyPirc()(url) - log.info('Authentication found for URL: %s'%url) + log.info('Authentication found for URL: %s' % url) if auth: auth = "Basic " + _encode_auth(auth) -- cgit v1.2.1 From b79e972b8928224516e5361342c4d2f11c83fb24 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Thu, 14 Nov 2013 09:44:07 -0500 Subject: Refactor for flatter code --- setuptools/package_index.py | 38 ++++++++++++++++++++++++-------------- 1 file changed, 24 insertions(+), 14 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index a0bb936d..2e0f2092 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -928,20 +928,30 @@ class PyPirc(object): """ self.dict_ = {} - if 'HOME' in os.environ: - rc = os.path.join(os.environ['HOME'], '.pypirc') - if os.path.exists(rc): - config = ConfigParser.ConfigParser({ - 'username': '', - 'password': '', - 'repository': ''}) - config.read(rc) - - for section in config.sections(): - if config.get(section, 'repository').strip(): - value = '%s:%s' % (config.get(section, 'username').strip(), - config.get(section, 'password').strip()) - self.dict_[config.get(section, 'repository').strip()] = value + if 'HOME' not in os.environ: + return + + rc = os.path.join(os.environ['HOME'], '.pypirc') + if not os.path.exists(rc): + return + + initial = dict.fromkeys(['username', 'password', 'repository'], '') + config = ConfigParser.ConfigParser(initial) + config.read(rc) + + sections_with_repositories = [ + section for section in config.sections() + if config.get(section, 'repository').strip() + ] + + for section in sections_with_repositories: + auth = ( + config.get(section, 'username').strip(), + config.get(section, 'password').strip(), + ) + value = '%s:%s' % auth + repo = config.get(section, 'repository').strip() + self.dict_[repo] = value def __call__(self, url): """ """ -- cgit v1.2.1 From e405a2b4861216e3c6b50c79f7b26289656fdb41 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Thu, 14 Nov 2013 09:52:58 -0500 Subject: Extract 'Credential' class for representing a username/password credential --- setuptools/package_index.py | 22 ++++++++++++++++++---- 1 file changed, 18 insertions(+), 4 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 2e0f2092..3154eccb 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -920,6 +920,21 @@ def _encode_auth(auth): # strip the trailing carriage return return encoded.rstrip() +class Credential(object): + """ + A username/password pair. Use like a namedtuple. + """ + def __init__(self, username, password): + self.username = username + self.password = password + + def __iter__(self): + yield self.username + yield self.password + + def __str__(self): + return '%(username)s:%(password)s' % vars(self) + class PyPirc(object): def __init__(self): @@ -945,13 +960,12 @@ class PyPirc(object): ] for section in sections_with_repositories: - auth = ( + cred = Credential( config.get(section, 'username').strip(), config.get(section, 'password').strip(), ) - value = '%s:%s' % auth repo = config.get(section, 'repository').strip() - self.dict_[repo] = value + self.dict_[repo] = cred def __call__(self, url): """ """ @@ -976,7 +990,7 @@ def open_with_auth(url, opener=urllib2.urlopen): auth = None if not auth: - auth = PyPirc()(url) + auth = str(PyPirc()(url)) log.info('Authentication found for URL: %s' % url) if auth: -- cgit v1.2.1 From fccf9357c484e4b475348dfbef321e544e2a3cab Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Thu, 14 Nov 2013 09:56:22 -0500 Subject: Use 'expanduser' for better compatibilty. --- setuptools/package_index.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 3154eccb..5f328a03 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -943,10 +943,7 @@ class PyPirc(object): """ self.dict_ = {} - if 'HOME' not in os.environ: - return - - rc = os.path.join(os.environ['HOME'], '.pypirc') + rc = os.path.join(os.path.expanduser('~'), '.pypirc') if not os.path.exists(rc): return -- cgit v1.2.1 From bf742c3f6069802d26c09118dd6c36d55634e22d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Thu, 14 Nov 2013 10:08:50 -0500 Subject: Derive PyPirc from ConfigParser for more general purpose use. --- setuptools/package_index.py | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 5f328a03..5b8dc357 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -935,34 +935,35 @@ class Credential(object): def __str__(self): return '%(username)s:%(password)s' % vars(self) -class PyPirc(object): +class PyPirc(ConfigParser.ConfigParser): def __init__(self): """ - Extract pypirc authentication information from home directory + Load from ~/.pypirc """ - self.dict_ = {} + defaults = dict.fromkeys(['username', 'password', 'repository'], '') + super(PyPirc, self).__init__(defaults) rc = os.path.join(os.path.expanduser('~'), '.pypirc') - if not os.path.exists(rc): - return - - initial = dict.fromkeys(['username', 'password', 'repository'], '') - config = ConfigParser.ConfigParser(initial) - config.read(rc) + if os.path.exists(rc): + self.read(rc) + @property + def dict_(self): + dict_ = {} sections_with_repositories = [ - section for section in config.sections() - if config.get(section, 'repository').strip() + section for section in self.sections() + if self.get(section, 'repository').strip() ] for section in sections_with_repositories: cred = Credential( - config.get(section, 'username').strip(), - config.get(section, 'password').strip(), + self.get(section, 'username').strip(), + self.get(section, 'password').strip(), ) - repo = config.get(section, 'repository').strip() - self.dict_[repo] = cred + repo = self.get(section, 'repository').strip() + dict_[repo] = cred + return dict_ def __call__(self, url): """ """ -- cgit v1.2.1 From b7ee88761b75f2c63eb5553f7b28e5e8a88be186 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Thu, 14 Nov 2013 10:20:22 -0500 Subject: Replaced __call__ with find_credential (for clarity of purpose). Removed dict_. --- setuptools/package_index.py | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 5b8dc357..c63ae29a 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -949,8 +949,8 @@ class PyPirc(ConfigParser.ConfigParser): self.read(rc) @property - def dict_(self): - dict_ = {} + def creds_by_repository(self): + creds = {} sections_with_repositories = [ section for section in self.sections() if self.get(section, 'repository').strip() @@ -962,14 +962,17 @@ class PyPirc(ConfigParser.ConfigParser): self.get(section, 'password').strip(), ) repo = self.get(section, 'repository').strip() - dict_[repo] = cred - return dict_ + creds[repo] = cred + return creds - def __call__(self, url): - """ """ - for base_url, auth in self.dict_.items(): - if url.startswith(base_url): - return auth + def find_credential(self, url): + """ + If the URL indicated appears to be a repository defined in this + config, return the credential for that repository. + """ + for repository, cred in self.creds_by_repository.items(): + if url.startswith(repository): + return cred def open_with_auth(url, opener=urllib2.urlopen): @@ -988,8 +991,10 @@ def open_with_auth(url, opener=urllib2.urlopen): auth = None if not auth: - auth = str(PyPirc()(url)) - log.info('Authentication found for URL: %s' % url) + cred = PyPirc().find_credential(url) + if cred: + auth = str(cred) + log.info('Authentication found for URL: %s' % url) if auth: auth = "Basic " + _encode_auth(auth) -- cgit v1.2.1 From 113f55577768c9434b33b799521e4f70fdecd8de Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Thu, 14 Nov 2013 10:22:59 -0500 Subject: Get ConfigParser module from compat module for Python 3 compatibility. --- setuptools/package_index.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index c63ae29a..482ba38b 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -6,8 +6,6 @@ import shutil import socket import base64 -import ConfigParser - from pkg_resources import ( CHECKOUT_DIST, Distribution, BINARY_DIST, normalize_path, SOURCE_DIST, require, Environment, find_distributions, safe_name, safe_version, @@ -19,7 +17,8 @@ from distutils.errors import DistutilsError from setuptools.compat import (urllib2, httplib, StringIO, HTTPError, urlparse, urlunparse, unquote, splituser, url2pathname, name2codepoint, - unichr, urljoin, urlsplit, urlunsplit) + unichr, urljoin, urlsplit, urlunsplit, + ConfigParser) from setuptools.compat import filterfalse from fnmatch import translate from setuptools.py24compat import hashlib -- cgit v1.2.1 From 117a8a57a9a06521f028eb4e1afc1f6fd4dd8dc8 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Thu, 14 Nov 2013 10:28:27 -0500 Subject: Construct the 'creds' dictionary directly rather than building imperatively. --- setuptools/package_index.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 482ba38b..0a409604 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -949,20 +949,19 @@ class PyPirc(ConfigParser.ConfigParser): @property def creds_by_repository(self): - creds = {} sections_with_repositories = [ section for section in self.sections() if self.get(section, 'repository').strip() ] - for section in sections_with_repositories: - cred = Credential( - self.get(section, 'username').strip(), - self.get(section, 'password').strip(), - ) - repo = self.get(section, 'repository').strip() - creds[repo] = cred - return creds + return dict(map(self._get_repo_cred, sections_with_repositories)) + + def _get_repo_cred(self, section): + repo = self.get(section, 'repository').strip() + return repo, Credential( + self.get(section, 'username').strip(), + self.get(section, 'password').strip(), + ) def find_credential(self, url): """ -- cgit v1.2.1 From 75345928c3e2cb0f88cdf44d33752e1ed6b19290 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Thu, 14 Nov 2013 10:29:59 -0500 Subject: Renamed class for proper capitalization and for clarity. --- setuptools/package_index.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 0a409604..a61d3c74 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -934,14 +934,14 @@ class Credential(object): def __str__(self): return '%(username)s:%(password)s' % vars(self) -class PyPirc(ConfigParser.ConfigParser): +class PyPIConfig(ConfigParser.ConfigParser): def __init__(self): """ Load from ~/.pypirc """ defaults = dict.fromkeys(['username', 'password', 'repository'], '') - super(PyPirc, self).__init__(defaults) + super(PyPIConfig, self).__init__(defaults) rc = os.path.join(os.path.expanduser('~'), '.pypirc') if os.path.exists(rc): @@ -989,7 +989,7 @@ def open_with_auth(url, opener=urllib2.urlopen): auth = None if not auth: - cred = PyPirc().find_credential(url) + cred = PyPIConfig().find_credential(url) if cred: auth = str(cred) log.info('Authentication found for URL: %s' % url) -- cgit v1.2.1 From cee7279fb782fd97b8e7ce06642054ee45ca0564 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Thu, 14 Nov 2013 10:34:14 -0500 Subject: Updated message when credentials used from .pypirc --- setuptools/package_index.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index a61d3c74..231311ec 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -992,7 +992,8 @@ def open_with_auth(url, opener=urllib2.urlopen): cred = PyPIConfig().find_credential(url) if cred: auth = str(cred) - log.info('Authentication found for URL: %s' % url) + info = cred.username, url + log.info('Authenticating as %s for %s (from .pypirc)' % info) if auth: auth = "Basic " + _encode_auth(auth) -- cgit v1.2.1 From 9b316f1fa946fa24d770ba8c03dcfeec700d37cc Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Thu, 14 Nov 2013 14:48:29 -0500 Subject: ConfigParser is not a new style class. --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 26d1d4ad..bc276a26 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -943,7 +943,7 @@ class PyPIConfig(ConfigParser.ConfigParser): Load from ~/.pypirc """ defaults = dict.fromkeys(['username', 'password', 'repository'], '') - super(PyPIConfig, self).__init__(defaults) + ConfigParser.ConfigParser.__init__(self, defaults) rc = os.path.join(os.path.expanduser('~'), '.pypirc') if os.path.exists(rc): -- cgit v1.2.1 From 7ade322c4e685aef1adab02ca3bad6444acc2466 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Thu, 14 Nov 2013 15:12:31 -0500 Subject: Update docstring to reflect failure reported in pull request #21 --- setuptools/package_index.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index bc276a26..2e8f62e2 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -908,8 +908,13 @@ def _encode_auth(auth): """ A function compatible with Python 2.3-3.3 that will encode auth from a URL suitable for an HTTP header. - >>> _encode_auth('username%3Apassword') - u'dXNlcm5hbWU6cGFzc3dvcmQ=' + >>> str(_encode_auth('username%3Apassword')) + 'dXNlcm5hbWU6cGFzc3dvcmQ=' + + Long auth strings should not cause a newline to be inserted. + >>> long_auth = 'username:' + 'password'*10 + >>> chr(10) in str(_encode_auth(long_auth)) + False """ auth_s = unquote(auth) # convert to bytes -- cgit v1.2.1 From 47e5462bc732ffd5f363b4a60d5a6dd9efa0af3b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Sun, 24 Nov 2013 15:07:33 -0500 Subject: Remove py24compat module --- setuptools/package_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 4a3f49c7..12a062b5 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -5,6 +5,8 @@ import re import shutil import socket import base64 +import hashlib +from functools import wraps from pkg_resources import ( CHECKOUT_DIST, Distribution, BINARY_DIST, normalize_path, SOURCE_DIST, @@ -21,8 +23,6 @@ from setuptools.compat import (urllib2, httplib, StringIO, HTTPError, ConfigParser) from setuptools.compat import filterfalse from fnmatch import translate -from setuptools.py24compat import hashlib -from setuptools.py24compat import wraps from setuptools.py26compat import strip_fragment from setuptools.py27compat import get_all_headers -- cgit v1.2.1 From 35d9509cd23b7830c91398fa5b3f2ffe6ffe7403 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Sun, 1 Dec 2013 05:49:33 -0500 Subject: Reindent for clarity --- setuptools/package_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 4a3f49c7..4ffafa28 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1053,5 +1053,5 @@ def local_open(url): else: status, message, body = 404, "Path not found", "Not found" - return HTTPError(url, status, message, - {'content-type':'text/html'}, StringIO(body)) + headers = {'content-type': 'text/html'} + return HTTPError(url, status, message, headers, StringIO(body)) -- cgit v1.2.1 From ec3014377e1369116da3bf71b6c5acaa3168b9dc Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Sun, 1 Dec 2013 06:12:06 -0500 Subject: Corrected a TypeError when reading a local package index on Python 3. Fixes #116. --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 4ffafa28..ef247cf3 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1039,7 +1039,7 @@ def local_open(url): files = [] for f in os.listdir(filename): if f=='index.html': - fp = open(os.path.join(filename,f),'rb') + fp = open(os.path.join(filename,f),'r') body = fp.read() fp.close() break -- cgit v1.2.1 From 379096d8a1e4be86f8b2e36a0ec66fa75e2edbb8 Mon Sep 17 00:00:00 2001 From: Menghan Zheng <menghan412@gmail.com> Date: Tue, 24 Dec 2013 14:16:43 +0800 Subject: fix setuptools don't search download location in dependency_links bug --- setuptools/package_index.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 0e51b72c..167c34e5 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -582,8 +582,9 @@ class PackageIndex(Environment): if local_index is not None: dist = dist or find(requirement, local_index) - if dist is None and self.to_scan is not None: - self.prescan() + if dist is None: + if self.to_scan is not None: + self.prescan() dist = find(requirement) if dist is None and not force_scan: -- cgit v1.2.1 From f9f94d714cf97df731a49b614b35d146536449f7 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Sat, 17 May 2014 23:47:08 -0400 Subject: Use context manager --HG-- extra : amend_source : 5e98bee2918d9eeb073c8c896a849c5f68da6634 --- setuptools/package_index.py | 47 +++++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 25 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 167c34e5..58572ce6 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -632,16 +632,15 @@ class PackageIndex(Environment): shutil.copy2(filename, dst) filename=dst - file = open(os.path.join(tmpdir, 'setup.py'), 'w') - file.write( - "from setuptools import setup\n" - "setup(name=%r, version=%r, py_modules=[%r])\n" - % ( - dists[0].project_name, dists[0].version, - os.path.splitext(basename)[0] + with open(os.path.join(tmpdir, 'setup.py'), 'w') as file: + file.write( + "from setuptools import setup\n" + "setup(name=%r, version=%r, py_modules=[%r])\n" + % ( + dists[0].project_name, dists[0].version, + os.path.splitext(basename)[0] + ) ) - ) - file.close() return filename elif match: @@ -660,7 +659,7 @@ class PackageIndex(Environment): def _download_to(self, url, filename): self.info("Downloading %s", url) # Download the file - fp, tfp, info = None, None, None + fp, info = None, None try: checker = HashChecker.from_url(url) fp = self.open_url(strip_fragment(url)) @@ -677,21 +676,20 @@ class PackageIndex(Environment): sizes = get_all_headers(headers, 'Content-Length') size = max(map(int, sizes)) self.reporthook(url, filename, blocknum, bs, size) - tfp = open(filename,'wb') - while True: - block = fp.read(bs) - if block: - checker.feed(block) - tfp.write(block) - blocknum += 1 - self.reporthook(url, filename, blocknum, bs, size) - else: - break - self.check_hash(checker, filename, tfp) + with open(filename,'wb') as tfp: + while True: + block = fp.read(bs) + if block: + checker.feed(block) + tfp.write(block) + blocknum += 1 + self.reporthook(url, filename, blocknum, bs, size) + else: + break + self.check_hash(checker, filename, tfp) return headers finally: if fp: fp.close() - if tfp: tfp.close() def reporthook(self, url, filename, blocknum, blksize, size): pass # no-op @@ -1040,9 +1038,8 @@ def local_open(url): files = [] for f in os.listdir(filename): if f=='index.html': - fp = open(os.path.join(filename,f),'r') - body = fp.read() - fp.close() + with open(os.path.join(filename,f),'r') as fp: + body = fp.read() break elif os.path.isdir(os.path.join(filename,f)): f+='/' -- cgit v1.2.1 From b49435397a5094f94678adf3549cc8941aa469b7 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Sat, 5 Jul 2014 15:06:51 -0400 Subject: Use six for Python 2 compatibility --HG-- branch : feature/issue-229 extra : source : 7b1997ececc5772798ce33a0f8e77387cb55a977 --- setuptools/package_index.py | 101 +++++++++++++++++++++++--------------------- 1 file changed, 54 insertions(+), 47 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 58572ce6..9c899218 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -8,6 +8,14 @@ import base64 import hashlib from functools import wraps +try: + from urllib.parse import splituser +except ImportError: + from urllib2 import splituser + +import six +from six.moves import urllib, http_client + from pkg_resources import ( CHECKOUT_DIST, Distribution, BINARY_DIST, normalize_path, SOURCE_DIST, require, Environment, find_distributions, safe_name, safe_version, @@ -16,12 +24,6 @@ from pkg_resources import ( from setuptools import ssl_support from distutils import log from distutils.errors import DistutilsError -from setuptools.compat import (urllib2, httplib, StringIO, HTTPError, - urlparse, urlunparse, unquote, splituser, - url2pathname, name2codepoint, - unichr, urljoin, urlsplit, urlunsplit, - ConfigParser) -from setuptools.compat import filterfalse from fnmatch import translate from setuptools.py26compat import strip_fragment from setuptools.py27compat import get_all_headers @@ -68,10 +70,11 @@ def parse_bdist_wininst(name): def egg_info_for_url(url): - scheme, server, path, parameters, query, fragment = urlparse(url) - base = unquote(path.split('/')[-1]) + parts = urllib.parse.urlparse(url) + scheme, server, path, parameters, query, fragment = parts + base = urllib.parse.unquote(path.split('/')[-1]) if server=='sourceforge.net' and base=='download': # XXX Yuck - base = unquote(path.split('/')[-2]) + base = urllib.parse.unquote(path.split('/')[-2]) if '#' in base: base, fragment = base.split('#',1) return base,fragment @@ -158,7 +161,7 @@ def unique_everseen(iterable, key=None): seen = set() seen_add = seen.add if key is None: - for element in filterfalse(seen.__contains__, iterable): + for element in six.moves.filterfalse(seen.__contains__, iterable): seen_add(element) yield element else: @@ -190,14 +193,14 @@ def find_external_links(url, page): rels = set(map(str.strip, rel.lower().split(','))) if 'homepage' in rels or 'download' in rels: for match in HREF.finditer(tag): - yield urljoin(url, htmldecode(match.group(1))) + yield urllib.parse.urljoin(url, htmldecode(match.group(1))) for tag in ("<th>Home Page", "<th>Download URL"): pos = page.find(tag) if pos!=-1: match = HREF.search(page,pos) if match: - yield urljoin(url, htmldecode(match.group(1))) + yield urllib.parse.urljoin(url, htmldecode(match.group(1))) user_agent = "Python-urllib/%s setuptools/%s" % ( sys.version[:3], require('setuptools')[0].version @@ -240,7 +243,7 @@ class HashChecker(ContentChecker): @classmethod def from_url(cls, url): "Construct a (possibly null) ContentChecker from a URL" - fragment = urlparse(url)[-1] + fragment = urllib.parse.urlparse(url)[-1] if not fragment: return ContentChecker() match = cls.pattern.search(fragment) @@ -275,7 +278,7 @@ class PackageIndex(Environment): self.to_scan = [] if verify_ssl and ssl_support.is_available and (ca_bundle or ssl_support.find_ca_bundle()): self.opener = ssl_support.opener_for(ca_bundle) - else: self.opener = urllib2.urlopen + else: self.opener = urllib.request.urlopen def process_url(self, url, retrieve=False): """Evaluate a URL as a possible download, and maybe retrieve it""" @@ -312,7 +315,7 @@ class PackageIndex(Environment): base = f.url # handle redirects page = f.read() if not isinstance(page, str): # We are in Python 3 and got bytes. We want str. - if isinstance(f, HTTPError): + if isinstance(f, urllib.request.HTTPError): # Errors have no charset, assume latin1: charset = 'latin-1' else: @@ -320,7 +323,7 @@ class PackageIndex(Environment): page = page.decode(charset, "ignore") f.close() for match in HREF.finditer(page): - link = urljoin(base, htmldecode(match.group(1))) + link = urllib.parse.urljoin(base, htmldecode(match.group(1))) self.process_url(link) if url.startswith(self.index_url) and getattr(f,'code',None)!=404: page = self.process_index(url, page) @@ -343,7 +346,7 @@ class PackageIndex(Environment): def url_ok(self, url, fatal=False): s = URL_SCHEME(url) - if (s and s.group(1).lower()=='file') or self.allows(urlparse(url)[1]): + if (s and s.group(1).lower()=='file') or self.allows(urllib.parse.urlparse(url)[1]): return True msg = ("\nNote: Bypassing %s (disallowed host; see " "http://bit.ly/1dg9ijs for details).\n") @@ -374,7 +377,7 @@ class PackageIndex(Environment): # Process a URL to see if it's for a package page if link.startswith(self.index_url): parts = list(map( - unquote, link[len(self.index_url):].split('/') + urllib.parse.unquote, link[len(self.index_url):].split('/') )) if len(parts)==2 and '#' not in parts[1]: # it's a package page, sanitize and index it @@ -387,7 +390,7 @@ class PackageIndex(Environment): # process an index page into the package-page index for match in HREF.finditer(page): try: - scan(urljoin(url, htmldecode(match.group(1)))) + scan(urllib.parse.urljoin(url, htmldecode(match.group(1)))) except ValueError: pass @@ -663,7 +666,7 @@ class PackageIndex(Environment): try: checker = HashChecker.from_url(url) fp = self.open_url(strip_fragment(url)) - if isinstance(fp, HTTPError): + if isinstance(fp, urllib.request.HTTPError): raise DistutilsError( "Can't download %s: %s %s" % (url, fp.code,fp.msg) ) @@ -699,24 +702,24 @@ class PackageIndex(Environment): return local_open(url) try: return open_with_auth(url, self.opener) - except (ValueError, httplib.InvalidURL): + except (ValueError, http_client.InvalidURL): v = sys.exc_info()[1] msg = ' '.join([str(arg) for arg in v.args]) if warning: self.warn(warning, msg) else: raise DistutilsError('%s %s' % (url, msg)) - except urllib2.HTTPError: + except urllib.request.HTTPError: v = sys.exc_info()[1] return v - except urllib2.URLError: + except urllib.request.URLError: v = sys.exc_info()[1] if warning: self.warn(warning, v.reason) else: raise DistutilsError("Download error for %s: %s" % (url, v.reason)) - except httplib.BadStatusLine: + except http_client.BadStatusLine: v = sys.exc_info()[1] if warning: self.warn(warning, v.line) @@ -726,7 +729,7 @@ class PackageIndex(Environment): 'down, %s' % (url, v.line) ) - except httplib.HTTPException: + except http_client.HTTPException: v = sys.exc_info()[1] if warning: self.warn(warning, v) @@ -758,7 +761,7 @@ class PackageIndex(Environment): elif scheme.startswith('hg+'): return self._download_hg(url, filename) elif scheme=='file': - return url2pathname(urlparse(url)[2]) + return urllib.request.url2pathname(urllib.parse.urlparse(url)[2]) else: self.url_ok(url, True) # raises error if not allowed return self._attempt_download(url, filename) @@ -792,7 +795,7 @@ class PackageIndex(Environment): url = url.split('#',1)[0] # remove any fragment for svn's sake creds = '' if url.lower().startswith('svn:') and '@' in url: - scheme, netloc, path, p, q, f = urlparse(url) + scheme, netloc, path, p, q, f = urllib.parse.urlparse(url) if not netloc and path.startswith('//') and '/' in path[2:]: netloc, path = path[2:].split('/',1) auth, host = splituser(netloc) @@ -803,14 +806,15 @@ class PackageIndex(Environment): else: creds = " --username="+auth netloc = host - url = urlunparse((scheme, netloc, url, p, q, f)) + parts = scheme, netloc, url, p, q, f + url = urllib.parse.urlunparse(parts) self.info("Doing subversion checkout from %s to %s", url, filename) os.system("svn checkout%s -q %s %s" % (creds, url, filename)) return filename @staticmethod def _vcs_split_rev_from_url(url, pop_prefix=False): - scheme, netloc, path, query, frag = urlsplit(url) + scheme, netloc, path, query, frag = urllib.parse.urlsplit(url) scheme = scheme.split('+', 1)[-1] @@ -822,7 +826,7 @@ class PackageIndex(Environment): path, rev = path.rsplit('@', 1) # Also, discard fragment - url = urlunsplit((scheme, netloc, path, query, '')) + url = urllib.parse.urlunsplit((scheme, netloc, path, query, '')) return url, rev @@ -874,7 +878,7 @@ entity_sub = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub def uchr(c): if not isinstance(c, int): return c - if c>255: return unichr(c) + if c>255: return six.unichr(c) return chr(c) def decode_entity(match): @@ -884,7 +888,7 @@ def decode_entity(match): elif what.startswith('#'): what = int(what[1:]) else: - what = name2codepoint.get(what, match.group(0)) + what = six.moves.html_entities.name2codepoint.get(what, match.group(0)) return uchr(what) def htmldecode(text): @@ -915,7 +919,7 @@ def _encode_auth(auth): >>> chr(10) in str(_encode_auth(long_auth)) False """ - auth_s = unquote(auth) + auth_s = urllib.parse.unquote(auth) # convert to bytes auth_bytes = auth_s.encode() # use the legacy interface for Python 2.3 support @@ -940,14 +944,14 @@ class Credential(object): def __str__(self): return '%(username)s:%(password)s' % vars(self) -class PyPIConfig(ConfigParser.ConfigParser): +class PyPIConfig(six.moves.configparser.ConfigParser): def __init__(self): """ Load from ~/.pypirc """ defaults = dict.fromkeys(['username', 'password', 'repository'], '') - ConfigParser.ConfigParser.__init__(self, defaults) + six.moves.configparser.ConfigParser.__init__(self, defaults) rc = os.path.join(os.path.expanduser('~'), '.pypirc') if os.path.exists(rc): @@ -979,15 +983,15 @@ class PyPIConfig(ConfigParser.ConfigParser): return cred -def open_with_auth(url, opener=urllib2.urlopen): +def open_with_auth(url, opener=urllib.request.urlopen): """Open a urllib2 request, handling HTTP authentication""" - scheme, netloc, path, params, query, frag = urlparse(url) + scheme, netloc, path, params, query, frag = urllib.parse.urlparse(url) # Double scheme does not raise on Mac OS X as revealed by a # failing test. We would expect "nonnumeric port". Refs #20. if netloc.endswith(':'): - raise httplib.InvalidURL("nonnumeric port: ''") + raise http_client.InvalidURL("nonnumeric port: ''") if scheme in ('http', 'https'): auth, host = splituser(netloc) @@ -1003,11 +1007,12 @@ def open_with_auth(url, opener=urllib2.urlopen): if auth: auth = "Basic " + _encode_auth(auth) - new_url = urlunparse((scheme,host,path,params,query,frag)) - request = urllib2.Request(new_url) + parts = scheme, host, path, params, query, frag + new_url = urllib.parse.urlunparse(parts) + request = urllib.request.Request(new_url) request.add_header("Authorization", auth) else: - request = urllib2.Request(url) + request = urllib.request.Request(url) request.add_header('User-Agent', user_agent) fp = opener(request) @@ -1015,9 +1020,10 @@ def open_with_auth(url, opener=urllib2.urlopen): if auth: # Put authentication info back into request URL if same host, # so that links found on the page will work - s2, h2, path2, param2, query2, frag2 = urlparse(fp.url) + s2, h2, path2, param2, query2, frag2 = urllib.parse.urlparse(fp.url) if s2==scheme and h2==host: - fp.url = urlunparse((s2,netloc,path2,param2,query2,frag2)) + parts = s2, netloc, path2, param2, query2, frag2 + fp.url = urllib.parse.urlunparse(parts) return fp @@ -1030,10 +1036,10 @@ def fix_sf_url(url): def local_open(url): """Read a local path, with special support for directories""" - scheme, server, path, param, query, frag = urlparse(url) - filename = url2pathname(path) + scheme, server, path, param, query, frag = urllib.parse.urlparse(url) + filename = urllib.request.url2pathname(path) if os.path.isfile(filename): - return urllib2.urlopen(url) + return urllib.request.urlopen(url) elif path.endswith('/') and os.path.isdir(filename): files = [] for f in os.listdir(filename): @@ -1052,4 +1058,5 @@ def local_open(url): status, message, body = 404, "Path not found", "Not found" headers = {'content-type': 'text/html'} - return HTTPError(url, status, message, headers, StringIO(body)) + body_stream = six.StringIO(body) + return urllib.error.HTTPError(url, status, message, headers, body_stream) -- cgit v1.2.1 From 96aed21b42121acde68dd6b3732c3fbae0903569 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Sat, 5 Jul 2014 21:17:52 -0400 Subject: Correct incorrect module for urllib.errors --HG-- branch : feature/issue-229 extra : rebase_source : 93a1adbc64455d9cf7c3c8cceb7bbd283c06ac5b --- setuptools/package_index.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 9c899218..a14c8ac6 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -315,7 +315,7 @@ class PackageIndex(Environment): base = f.url # handle redirects page = f.read() if not isinstance(page, str): # We are in Python 3 and got bytes. We want str. - if isinstance(f, urllib.request.HTTPError): + if isinstance(f, urllib.error.HTTPError): # Errors have no charset, assume latin1: charset = 'latin-1' else: @@ -666,7 +666,7 @@ class PackageIndex(Environment): try: checker = HashChecker.from_url(url) fp = self.open_url(strip_fragment(url)) - if isinstance(fp, urllib.request.HTTPError): + if isinstance(fp, urllib.error.HTTPError): raise DistutilsError( "Can't download %s: %s %s" % (url, fp.code,fp.msg) ) @@ -709,10 +709,10 @@ class PackageIndex(Environment): self.warn(warning, msg) else: raise DistutilsError('%s %s' % (url, msg)) - except urllib.request.HTTPError: + except urllib.error.HTTPError: v = sys.exc_info()[1] return v - except urllib.request.URLError: + except urllib.error.URLError: v = sys.exc_info()[1] if warning: self.warn(warning, v.reason) -- cgit v1.2.1 From 9d6a6e5927ae0e67164383e419f3145fc154467e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Sun, 4 Jan 2015 11:35:16 -0500 Subject: Use except/as, now supported by Python 2.6 --- setuptools/package_index.py | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 58572ce6..5ed19130 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -699,25 +699,21 @@ class PackageIndex(Environment): return local_open(url) try: return open_with_auth(url, self.opener) - except (ValueError, httplib.InvalidURL): - v = sys.exc_info()[1] + except (ValueError, httplib.InvalidURL) as v: msg = ' '.join([str(arg) for arg in v.args]) if warning: self.warn(warning, msg) else: raise DistutilsError('%s %s' % (url, msg)) - except urllib2.HTTPError: - v = sys.exc_info()[1] + except urllib2.HTTPError as v: return v - except urllib2.URLError: - v = sys.exc_info()[1] + except urllib2.URLError as v: if warning: self.warn(warning, v.reason) else: raise DistutilsError("Download error for %s: %s" % (url, v.reason)) - except httplib.BadStatusLine: - v = sys.exc_info()[1] + except httplib.BadStatusLine as v: if warning: self.warn(warning, v.line) else: @@ -726,8 +722,7 @@ class PackageIndex(Environment): 'down, %s' % (url, v.line) ) - except httplib.HTTPException: - v = sys.exc_info()[1] + except httplib.HTTPException as v: if warning: self.warn(warning, v) else: -- cgit v1.2.1 From a631a0700d2ebcc9ee98c91d474f351974cc0b92 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Fri, 20 Mar 2015 13:30:51 -0400 Subject: Match Python 3 for bdist_dumb. --- setuptools/package_index.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 5ed19130..963b4b41 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -140,8 +140,9 @@ def interpret_distro_name( parts = basename.split('-') if not py_version: for i,p in enumerate(parts[2:]): - if len(p)==5 and p.startswith('py2.'): - return # It's a bdist_dumb, not an sdist -- bail out + if p.match('py\d\.\d'): + # It's a bdist_dumb, not an sdist -- bail out + return for p in range(1,len(parts)+1): yield Distribution( -- cgit v1.2.1 From 00477b5b3e1f42b5ad9cdc553652e04beb675dbb Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Fri, 20 Mar 2015 13:31:38 -0400 Subject: Remove unused variable --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 963b4b41..c117cb62 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -139,7 +139,7 @@ def interpret_distro_name( parts = basename.split('-') if not py_version: - for i,p in enumerate(parts[2:]): + for p in parts[2:]: if p.match('py\d\.\d'): # It's a bdist_dumb, not an sdist -- bail out return -- cgit v1.2.1 From edf097e299d4382700f819288cb08390c53e109d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Fri, 20 Mar 2015 13:34:15 -0400 Subject: Flat is better than nested. --- setuptools/package_index.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index c117cb62..51769cdf 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -138,11 +138,9 @@ def interpret_distro_name( # versions in distribution archive names (sdist and bdist). parts = basename.split('-') - if not py_version: - for p in parts[2:]: - if p.match('py\d\.\d'): - # It's a bdist_dumb, not an sdist -- bail out - return + if not py_version and any(p.match('py\d\.\d') for p in parts[2:]): + # it is a bdist_dumb, not an sdist -- bail out + return for p in range(1,len(parts)+1): yield Distribution( -- cgit v1.2.1 From 01bb6e0e9217255cd835cefb8162463dac5d8e6d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Fri, 20 Mar 2015 15:58:37 -0400 Subject: Correct regex usage --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 963b4b41..95eb1cf3 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -140,7 +140,7 @@ def interpret_distro_name( parts = basename.split('-') if not py_version: for i,p in enumerate(parts[2:]): - if p.match('py\d\.\d'): + if re.match('py\d\.\d$', p): # It's a bdist_dumb, not an sdist -- bail out return -- cgit v1.2.1 From bb967deca4ff251c5aa8f0404b1223e94d9a301b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" <jaraco@jaraco.com> Date: Tue, 8 Dec 2015 19:04:13 -0500 Subject: Don't rely on repr for an HTML attribute value (could end up with 'u' prefix). Fixes #471. --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index cabf1039..dd2df229 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1037,7 +1037,7 @@ def local_open(url): break elif os.path.isdir(os.path.join(filename,f)): f+='/' - files.append("<a href=%r>%s</a>" % (f,f)) + files.append('<a href="{name}">{name}</a>'.format(name=f)) else: body = ("<html><head><title>%s" % url) + \ "%s" % '\n'.join(files) -- cgit v1.2.1 From 59e47d76244bf25a9c385800328394e97be74b48 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 8 Dec 2015 19:06:03 -0500 Subject: Update syntax for modern style --- setuptools/package_index.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index dd2df229..330b8391 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1031,12 +1031,12 @@ def local_open(url): elif path.endswith('/') and os.path.isdir(filename): files = [] for f in os.listdir(filename): - if f=='index.html': - with open(os.path.join(filename,f),'r') as fp: + if f == 'index.html': + with open(os.path.join(filename, f), 'r') as fp: body = fp.read() break - elif os.path.isdir(os.path.join(filename,f)): - f+='/' + elif os.path.isdir(os.path.join(filename, f)): + f += '/' files.append('{name}'.format(name=f)) else: body = ("%s" % url) + \ -- cgit v1.2.1 From 5a662289e866fb9bac77736ffede2fca05c79367 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 8 Dec 2015 19:06:41 -0500 Subject: Extract variable --- setuptools/package_index.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 330b8391..1c156f07 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1031,11 +1031,12 @@ def local_open(url): elif path.endswith('/') and os.path.isdir(filename): files = [] for f in os.listdir(filename): + filepath = os.path.join(filename, f) if f == 'index.html': - with open(os.path.join(filename, f), 'r') as fp: + with open(filepath, 'r') as fp: body = fp.read() break - elif os.path.isdir(os.path.join(filename, f)): + elif os.path.isdir(filepath): f += '/' files.append('{name}'.format(name=f)) else: -- cgit v1.2.1 From 95ac77a928a08c836c6f4ccc066a797357207abc Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 8 Dec 2015 19:14:04 -0500 Subject: Use new string formatting here as well --- setuptools/package_index.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 1c156f07..525cb645 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1040,8 +1040,9 @@ def local_open(url): f += '/' files.append('{name}'.format(name=f)) else: - body = ("%s" % url) + \ - "%s" % '\n'.join(files) + tmpl = ("{url}" + "{files}") + body = tmpl.format(url=url, files='\n'.join(files)) status, message = 200, "OK" else: status, message, body = 404, "Path not found", "Not found" -- cgit v1.2.1 From 5ba1d009e2c44b63fc2d2e1de09fd1d768c10fca Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 11 Dec 2015 10:13:30 -0500 Subject: Use context for opening file. --- setuptools/package_index.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 525cb645..f81b8d78 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -359,8 +359,9 @@ class PackageIndex(Environment): self.scan_egg_link(item, entry) def scan_egg_link(self, path, entry): - lines = [_f for _f in map(str.strip, - open(os.path.join(path, entry))) if _f] + with open(os.path.join(path, entry)) as raw_lines: + # filter non-empty lines + lines = list(filter(None, map(str.strip, raw_lines))) if len(lines)==2: for dist in find_distributions(os.path.join(path, lines[0])): dist.location = os.path.join(path, *lines) -- cgit v1.2.1 From cd28f4e0eddf3f571cb0efe05a23fa4a7a254de7 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 11 Dec 2015 10:16:25 -0500 Subject: Replace nested code with short-circuit return. --- setuptools/package_index.py | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index f81b8d78..5adb8c2b 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -362,11 +362,15 @@ class PackageIndex(Environment): with open(os.path.join(path, entry)) as raw_lines: # filter non-empty lines lines = list(filter(None, map(str.strip, raw_lines))) - if len(lines)==2: - for dist in find_distributions(os.path.join(path, lines[0])): - dist.location = os.path.join(path, *lines) - dist.precedence = SOURCE_DIST - self.add(dist) + + if len(lines) != 2: + # format is not recognized; punt + return + + for dist in find_distributions(os.path.join(path, lines[0])): + dist.location = os.path.join(path, *lines) + dist.precedence = SOURCE_DIST + self.add(dist) def process_index(self,url,page): """Process the contents of a PyPI page""" -- cgit v1.2.1 From 2f7d95690e53fe92a82b055b82c4dc02b6a3339c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 11 Dec 2015 10:18:38 -0500 Subject: Extract variables for improved documentation. --- setuptools/package_index.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 5adb8c2b..36231f11 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -367,7 +367,9 @@ class PackageIndex(Environment): # format is not recognized; punt return - for dist in find_distributions(os.path.join(path, lines[0])): + egg_path, setup_path = lines + + for dist in find_distributions(os.path.join(path, egg_path)): dist.location = os.path.join(path, *lines) dist.precedence = SOURCE_DIST self.add(dist) -- cgit v1.2.1 From dacc24690e7c8503744ce433aae6dd74a50e2337 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 11 Dec 2015 10:26:16 -0500 Subject: Extract if in for loop --- setuptools/package_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 36231f11..00ce3ea5 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -352,8 +352,8 @@ class PackageIndex(Environment): self.warn(msg, url) def scan_egg_links(self, search_path): - for item in search_path: - if os.path.isdir(item): + dirs = filter(os.path.isdir, search_path) + for item in dirs: for entry in os.listdir(item): if entry.endswith('.egg-link'): self.scan_egg_link(item, entry) -- cgit v1.2.1 From 6b8c9181fbb0ef5b223ba45ed361991c601d1d13 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 11 Dec 2015 10:28:36 -0500 Subject: Get filter from the future --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 00ce3ea5..095688f9 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -20,7 +20,7 @@ from setuptools.compat import (urllib2, httplib, StringIO, HTTPError, urlparse, urlunparse, unquote, splituser, url2pathname, name2codepoint, unichr, urljoin, urlsplit, urlunsplit, - ConfigParser) + ConfigParser, filter) from setuptools.compat import filterfalse from fnmatch import translate from setuptools.py26compat import strip_fragment -- cgit v1.2.1 From 6667ea387d5694cf936644898acd20000ca7b1ec Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 11 Dec 2015 10:34:36 -0500 Subject: Replace nested for loop with dual-for generator expression. --- setuptools/package_index.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 095688f9..3456f715 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -6,6 +6,7 @@ import shutil import socket import base64 import hashlib +import itertools from functools import wraps from pkg_resources import ( @@ -353,10 +354,13 @@ class PackageIndex(Environment): def scan_egg_links(self, search_path): dirs = filter(os.path.isdir, search_path) - for item in dirs: - for entry in os.listdir(item): - if entry.endswith('.egg-link'): - self.scan_egg_link(item, entry) + egg_links = ( + (path, entry) + for path in dirs + for entry in os.listdir(path) + if entry.endswith('.egg-link') + ) + list(itertools.starmap(self.scan_egg_link, egg_links)) def scan_egg_link(self, path, entry): with open(os.path.join(path, entry)) as raw_lines: -- cgit v1.2.1 From e99626c4eadf2e45ca5d729aaa3a5b4bb667536b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 11 Dec 2015 10:36:25 -0500 Subject: Reclaim space from hanging indents --- setuptools/package_index.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 3456f715..b0837628 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -17,11 +17,11 @@ from pkg_resources import ( from setuptools import ssl_support from distutils import log from distutils.errors import DistutilsError -from setuptools.compat import (urllib2, httplib, StringIO, HTTPError, - urlparse, urlunparse, unquote, splituser, - url2pathname, name2codepoint, - unichr, urljoin, urlsplit, urlunsplit, - ConfigParser, filter) +from setuptools.compat import ( + urllib2, httplib, StringIO, HTTPError, urlparse, urlunparse, unquote, + splituser, url2pathname, name2codepoint, unichr, urljoin, urlsplit, + urlunsplit, ConfigParser, filter, +) from setuptools.compat import filterfalse from fnmatch import translate from setuptools.py26compat import strip_fragment -- cgit v1.2.1 From 6a10b4afbad5dca346b7fc7a5d8bb6d08d3286b2 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 11 Dec 2015 10:37:09 -0500 Subject: Also use map from the future --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index b0837628..7c071457 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -20,7 +20,7 @@ from distutils.errors import DistutilsError from setuptools.compat import ( urllib2, httplib, StringIO, HTTPError, urlparse, urlunparse, unquote, splituser, url2pathname, name2codepoint, unichr, urljoin, urlsplit, - urlunsplit, ConfigParser, filter, + urlunsplit, ConfigParser, filter, map, ) from setuptools.compat import filterfalse from fnmatch import translate -- cgit v1.2.1 From ca4321bb5833e2daa7bee1a32a3dee37cb49f012 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 14 Dec 2015 04:55:43 -0500 Subject: Use the modern name for the configparser module --- setuptools/package_index.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 7c071457..a26b58bc 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -20,7 +20,7 @@ from distutils.errors import DistutilsError from setuptools.compat import ( urllib2, httplib, StringIO, HTTPError, urlparse, urlunparse, unquote, splituser, url2pathname, name2codepoint, unichr, urljoin, urlsplit, - urlunsplit, ConfigParser, filter, map, + urlunsplit, configparser, filter, map, ) from setuptools.compat import filterfalse from fnmatch import translate @@ -945,14 +945,14 @@ class Credential(object): def __str__(self): return '%(username)s:%(password)s' % vars(self) -class PyPIConfig(ConfigParser.ConfigParser): +class PyPIConfig(configparser.ConfigParser): def __init__(self): """ Load from ~/.pypirc """ defaults = dict.fromkeys(['username', 'password', 'repository'], '') - ConfigParser.ConfigParser.__init__(self, defaults) + configparser.ConfigParser.__init__(self, defaults) rc = os.path.join(os.path.expanduser('~'), '.pypirc') if os.path.exists(rc): -- cgit v1.2.1 From 80adba1cb4b07f6d182c907b5c5f796eb7b6b93d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 14 Dec 2015 05:31:46 -0500 Subject: Use SafeConfigParser in PyPIConfig file. Allows percent signs to be specified using two percent signs. Fixes #442. --- setuptools/package_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index a26b58bc..322f9a61 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -945,14 +945,14 @@ class Credential(object): def __str__(self): return '%(username)s:%(password)s' % vars(self) -class PyPIConfig(configparser.ConfigParser): +class PyPIConfig(configparser.SafeConfigParser): def __init__(self): """ Load from ~/.pypirc """ defaults = dict.fromkeys(['username', 'password', 'repository'], '') - configparser.ConfigParser.__init__(self, defaults) + configparser.SafeConfigParser.__init__(self, defaults) rc = os.path.join(os.path.expanduser('~'), '.pypirc') if os.path.exists(rc): -- cgit v1.2.1 From fbcbb51009bc76df9f2c66547439474799b9ab15 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 14 Dec 2015 05:38:30 -0500 Subject: Use RawConfigParser instead of SafeConfigParser in PyPIConfig class. Interpolated values are no longer supported. Since backward compatibility could not be retained in either case, prefer the simpler, more direct format. Ref #442. --- setuptools/package_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 322f9a61..2c565e88 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -945,14 +945,14 @@ class Credential(object): def __str__(self): return '%(username)s:%(password)s' % vars(self) -class PyPIConfig(configparser.SafeConfigParser): +class PyPIConfig(configparser.RawConfigParser): def __init__(self): """ Load from ~/.pypirc """ defaults = dict.fromkeys(['username', 'password', 'repository'], '') - configparser.SafeConfigParser.__init__(self, defaults) + configparser.RawConfigParser.__init__(self, defaults) rc = os.path.join(os.path.expanduser('~'), '.pypirc') if os.path.exists(rc): -- cgit v1.2.1 From 06872bb0bbbeb953e90bd0941444b0d499056557 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 31 Dec 2015 11:51:01 -0500 Subject: Update vendoring technique to match that used for packaging. Ref #229. --HG-- branch : feature/issue-229 --- setuptools/package_index.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 657b467f..08c36890 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -14,8 +14,14 @@ try: except ImportError: from urllib2 import splituser -import six -from six.moves import urllib, http_client, configparser +try: + from setuptools._vendor import six + from setuptools._vendor.six.moves import urllib, http_client, configparser +except ImportError: + # fallback to naturally-installed version; allows system packagers to + # omit vendored packages. + import six + from six.moves import urllib, http_client, configparser from pkg_resources import ( CHECKOUT_DIST, Distribution, BINARY_DIST, normalize_path, SOURCE_DIST, -- cgit v1.2.1 From 952c1bafda1929c74c737646aa025e6ffad6632e Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 31 Dec 2015 16:30:47 -0500 Subject: Modeling after Astropy's technique for bundling libraries, the imports are now much cleaner. Thanks @embray. Ref #229. --HG-- branch : feature/issue-229 --- setuptools/package_index.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 08c36890..ea136c09 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -14,14 +14,8 @@ try: except ImportError: from urllib2 import splituser -try: - from setuptools._vendor import six - from setuptools._vendor.six.moves import urllib, http_client, configparser -except ImportError: - # fallback to naturally-installed version; allows system packagers to - # omit vendored packages. - import six - from six.moves import urllib, http_client, configparser +from setuptools.extern import six +from setuptools.extern.six.moves import urllib, http_client, configparser from pkg_resources import ( CHECKOUT_DIST, Distribution, BINARY_DIST, normalize_path, SOURCE_DIST, -- cgit v1.2.1 From 8af3b6ef5b4173a0d0d6735147c98c882ae98344 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 16 Jan 2016 06:54:00 -0500 Subject: Always use Python 3 version of map --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index ea136c09..c53343e4 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -15,7 +15,7 @@ except ImportError: from urllib2 import splituser from setuptools.extern import six -from setuptools.extern.six.moves import urllib, http_client, configparser +from setuptools.extern.six.moves import urllib, http_client, configparser, map from pkg_resources import ( CHECKOUT_DIST, Distribution, BINARY_DIST, normalize_path, SOURCE_DIST, -- cgit v1.2.1 From 094a51dd604d96656ab68f0f64e3169ceebab59c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 1 Jun 2016 08:59:16 -0400 Subject: Move setuptools to beginning of user-agent header. Fixes #598. --- setuptools/package_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index c53343e4..567ab53a 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -202,8 +202,8 @@ def find_external_links(url, page): if match: yield urllib.parse.urljoin(url, htmldecode(match.group(1))) -user_agent = "Python-urllib/%s setuptools/%s" % ( - sys.version[:3], require('setuptools')[0].version +user_agent = "setuptools/%s Python-urllib/%s" % ( + require('setuptools')[0].version, sys.version[:3], ) class ContentChecker(object): -- cgit v1.2.1 From e53e6ea4b46eb0a746fc1549be5ba90c0d3bc7dc Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 1 Jun 2016 09:06:09 -0400 Subject: Use newer string formatting for rendering user agent. Re-use __version__ from main package. --- setuptools/package_index.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 567ab53a..e87504db 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -17,6 +17,7 @@ except ImportError: from setuptools.extern import six from setuptools.extern.six.moves import urllib, http_client, configparser, map +import setuptools from pkg_resources import ( CHECKOUT_DIST, Distribution, BINARY_DIST, normalize_path, SOURCE_DIST, require, Environment, find_distributions, safe_name, safe_version, @@ -46,6 +47,11 @@ __all__ = [ _SOCKET_TIMEOUT = 15 + +_tmpl = "setuptools/{setuptools.__version__} Python-urllib/{py_major}" +user_agent = _tmpl.format(py_major=sys.version[:3], **globals()) + + def parse_bdist_wininst(name): """Return (base,pyversion) or (None,None) for possible .exe name""" @@ -202,9 +208,6 @@ def find_external_links(url, page): if match: yield urllib.parse.urljoin(url, htmldecode(match.group(1))) -user_agent = "setuptools/%s Python-urllib/%s" % ( - require('setuptools')[0].version, sys.version[:3], -) class ContentChecker(object): """ -- cgit v1.2.1 From 6d11e88f938f09ef16db4c6064b6e74acba4db1d Mon Sep 17 00:00:00 2001 From: stepshal Date: Tue, 12 Jul 2016 22:00:43 +0700 Subject: Fix quantity of blank lines after code object. --- setuptools/package_index.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index e87504db..e9b304b1 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -85,6 +85,7 @@ def egg_info_for_url(url): if '#' in base: base, fragment = base.split('#',1) return base,fragment + def distros_for_url(url, metadata=None): """Yield egg or source distribution objects that might be found at a URL""" base, fragment = egg_info_for_url(url) @@ -97,6 +98,7 @@ def distros_for_url(url, metadata=None): ): yield dist + def distros_for_location(location, basename, metadata=None): """Yield egg or source distribution objects based on basename""" if basename.endswith('.egg.zip'): @@ -118,6 +120,7 @@ def distros_for_location(location, basename, metadata=None): return interpret_distro_name(location, basename, metadata) return [] # no extension matched + def distros_for_filename(filename, metadata=None): """Yield possible egg or source distribution objects based on a filename""" return distros_for_location( @@ -177,6 +180,7 @@ def unique_everseen(iterable, key=None): seen_add(k) yield element + def unique_values(func): """ Wrap a function returning an iterable such that the resulting iterable @@ -190,6 +194,7 @@ def unique_values(func): REL = re.compile("""<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>""", re.I) # this line is here to fix emacs' cruddy broken syntax highlighting + @unique_values def find_external_links(url, page): """Find rel="homepage" and rel="download" links in `page`, yielding URLs""" @@ -213,6 +218,7 @@ class ContentChecker(object): """ A null content checker that defines the interface for checking content """ + def feed(self, block): """ Feed a block of data to the hash. @@ -232,6 +238,7 @@ class ContentChecker(object): """ return + class HashChecker(ContentChecker): pattern = re.compile( r'(?Psha1|sha224|sha384|sha256|sha512|md5)=' @@ -672,6 +679,7 @@ class PackageIndex(Environment): ) dl_blocksize = 8192 + def _download_to(self, url, filename): self.info("Downloading %s", url) # Download the file @@ -883,12 +891,14 @@ class PackageIndex(Environment): # references, a hexadecimal numeric reference, or a named reference). entity_sub = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub + def uchr(c): if not isinstance(c, int): return c if c>255: return six.unichr(c) return chr(c) + def decode_entity(match): what = match.group(1) if what.startswith('#x'): @@ -899,10 +909,12 @@ def decode_entity(match): what = six.moves.html_entities.name2codepoint.get(what, match.group(0)) return uchr(what) + def htmldecode(text): """Decode HTML entities in the given text.""" return entity_sub(decode_entity, text) + def socket_timeout(timeout=15): def _socket_timeout(func): def _socket_timeout(*args, **kwargs): @@ -915,6 +927,7 @@ def socket_timeout(timeout=15): return _socket_timeout return _socket_timeout + def _encode_auth(auth): """ A function compatible with Python 2.3-3.3 that will encode @@ -937,10 +950,12 @@ def _encode_auth(auth): # strip the trailing carriage return return encoded.replace('\n','') + class Credential(object): """ A username/password pair. Use like a namedtuple. """ + def __init__(self, username, password): self.username = username self.password = password @@ -952,6 +967,7 @@ class Credential(object): def __str__(self): return '%(username)s:%(password)s' % vars(self) + class PyPIConfig(configparser.RawConfigParser): def __init__(self): @@ -1042,6 +1058,7 @@ open_with_auth = socket_timeout(_SOCKET_TIMEOUT)(open_with_auth) def fix_sf_url(url): return url # backward compatibility + def local_open(url): """Read a local path, with special support for directories""" scheme, server, path, param, query, frag = urllib.parse.urlparse(url) -- cgit v1.2.1 From 053a3a12cf0cc902e0f869b8cc4cff997f73fc84 Mon Sep 17 00:00:00 2001 From: stepshal Date: Thu, 14 Jul 2016 06:59:30 +0700 Subject: Add missing blank line. --- setuptools/package_index.py | 1 + 1 file changed, 1 insertion(+) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index e9b304b1..7fb8b954 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -162,6 +162,7 @@ def interpret_distro_name( platform = platform ) + # From Python 2.7 docs def unique_everseen(iterable, key=None): "List unique elements, preserving order. Remember all elements ever seen." -- cgit v1.2.1 From f749ccab1e55723848946c9aba5c3eddebe0b24e Mon Sep 17 00:00:00 2001 From: stepshal Date: Thu, 14 Jul 2016 09:26:06 +0700 Subject: Add missing whitespace. --- setuptools/package_index.py | 74 ++++++++++++++++++++++----------------------- 1 file changed, 37 insertions(+), 37 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index e9b304b1..f3ee5ec0 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -37,7 +37,7 @@ PYPI_MD5 = re.compile( '([^<]+)\n\s+\\(md5\\)' ) -URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):',re.I).match +URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):', re.I).match EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split() __all__ = [ @@ -62,18 +62,18 @@ def parse_bdist_wininst(name): if lower.endswith('.win32.exe'): base = name[:-10] plat = 'win32' - elif lower.startswith('.win32-py',-16): + elif lower.startswith('.win32-py', -16): py_ver = name[-7:-4] base = name[:-16] plat = 'win32' elif lower.endswith('.win-amd64.exe'): base = name[:-14] plat = 'win-amd64' - elif lower.startswith('.win-amd64-py',-20): + elif lower.startswith('.win-amd64-py', -20): py_ver = name[-7:-4] base = name[:-20] plat = 'win-amd64' - return base,py_ver,plat + return base, py_ver, plat def egg_info_for_url(url): @@ -82,8 +82,8 @@ def egg_info_for_url(url): base = urllib.parse.unquote(path.split('/')[-1]) if server=='sourceforge.net' and base=='download': # XXX Yuck base = urllib.parse.unquote(path.split('/')[-2]) - if '#' in base: base, fragment = base.split('#',1) - return base,fragment + if '#' in base: base, fragment = base.split('#', 1) + return base, fragment def distros_for_url(url, metadata=None): @@ -155,7 +155,7 @@ def interpret_distro_name( # it is a bdist_dumb, not an sdist -- bail out return - for p in range(1,len(parts)+1): + for p in range(1, len(parts)+1): yield Distribution( location, metadata, '-'.join(parts[:p]), '-'.join(parts[p:]), py_version=py_version, precedence = precedence, @@ -209,7 +209,7 @@ def find_external_links(url, page): for tag in ("Home Page", "Download URL"): pos = page.find(tag) if pos!=-1: - match = HREF.search(page,pos) + match = HREF.search(page, pos) if match: yield urllib.parse.urljoin(url, htmldecode(match.group(1))) @@ -279,12 +279,12 @@ class PackageIndex(Environment): self, index_url="https://pypi.python.org/simple", hosts=('*',), ca_bundle=None, verify_ssl=True, *args, **kw ): - Environment.__init__(self,*args,**kw) + Environment.__init__(self, *args, **kw) self.index_url = index_url + "/"[:not index_url.endswith('/')] self.scanned_urls = {} self.fetched_urls = {} self.package_pages = {} - self.allows = re.compile('|'.join(map(translate,hosts))).match + self.allows = re.compile('|'.join(map(translate, hosts))).match self.to_scan = [] if verify_ssl and ssl_support.is_available and (ca_bundle or ssl_support.find_ca_bundle()): self.opener = ssl_support.opener_for(ca_bundle) @@ -335,7 +335,7 @@ class PackageIndex(Environment): for match in HREF.finditer(page): link = urllib.parse.urljoin(base, htmldecode(match.group(1))) self.process_url(link) - if url.startswith(self.index_url) and getattr(f,'code',None)!=404: + if url.startswith(self.index_url) and getattr(f, 'code', None)!=404: page = self.process_index(url, page) def process_filename(self, fn, nested=False): @@ -347,7 +347,7 @@ class PackageIndex(Environment): if os.path.isdir(fn) and not nested: path = os.path.realpath(fn) for item in os.listdir(path): - self.process_filename(os.path.join(path,item), True) + self.process_filename(os.path.join(path, item), True) dists = distros_for_filename(fn) if dists: @@ -391,7 +391,7 @@ class PackageIndex(Environment): dist.precedence = SOURCE_DIST self.add(dist) - def process_index(self,url,page): + def process_index(self, url, page): """Process the contents of a PyPI page""" def scan(link): # Process a URL to see if it's for a package page @@ -403,7 +403,7 @@ class PackageIndex(Environment): # it's a package page, sanitize and index it pkg = safe_name(parts[0]) ver = safe_version(parts[1]) - self.package_pages.setdefault(pkg.lower(),{})[link] = True + self.package_pages.setdefault(pkg.lower(), {})[link] = True return to_filename(pkg), to_filename(ver) return None, None @@ -422,13 +422,13 @@ class PackageIndex(Environment): base, frag = egg_info_for_url(new_url) if base.endswith('.py') and not frag: if ver: - new_url+='#egg=%s-%s' % (pkg,ver) + new_url+='#egg=%s-%s' % (pkg, ver) else: self.need_version_info(url) self.scan_url(new_url) return PYPI_MD5.sub( - lambda m: '%s' % m.group(1,3,2), page + lambda m: '%s' % m.group(1, 3, 2), page ) else: return "" # no sense double-scanning non-package pages @@ -441,7 +441,7 @@ class PackageIndex(Environment): def scan_all(self, msg=None, *args): if self.index_url not in self.fetched_urls: - if msg: self.warn(msg,*args) + if msg: self.warn(msg, *args) self.info( "Scanning index of all packages (this may take a while)" ) @@ -458,7 +458,7 @@ class PackageIndex(Environment): # We couldn't find the target package, so search the index page too self.not_found_in_index(requirement) - for url in list(self.package_pages.get(requirement.key,())): + for url in list(self.package_pages.get(requirement.key, ())): # scan each page that might be related to the desired package self.scan_url(url) @@ -469,7 +469,7 @@ class PackageIndex(Environment): if dist in requirement: return dist self.debug("%s does not match %s", requirement, dist) - return super(PackageIndex, self).obtain(requirement,installer) + return super(PackageIndex, self).obtain(requirement, installer) def check_hash(self, checker, filename, tfp): """ @@ -534,14 +534,14 @@ class PackageIndex(Environment): of `tmpdir`, and the local filename is returned. Various errors may be raised if a problem occurs during downloading. """ - if not isinstance(spec,Requirement): + if not isinstance(spec, Requirement): scheme = URL_SCHEME(spec) if scheme: # It's a url, download it to tmpdir found = self._download_url(scheme.group(1), spec, tmpdir) base, fragment = egg_info_for_url(spec) if base.endswith('.py'): - found = self.gen_setup(found,fragment,tmpdir) + found = self.gen_setup(found, fragment, tmpdir) return found elif os.path.exists(spec): # Existing file or directory, just return it @@ -554,7 +554,7 @@ class PackageIndex(Environment): "Not a URL, existing file, or requirement spec: %r" % (spec,) ) - return getattr(self.fetch_distribution(spec, tmpdir),'location',None) + return getattr(self.fetch_distribution(spec, tmpdir), 'location', None) def fetch_distribution( self, requirement, tmpdir, force_scan=False, source=False, @@ -590,7 +590,7 @@ class PackageIndex(Environment): if dist.precedence==DEVELOP_DIST and not develop_ok: if dist not in skipped: - self.warn("Skipping development or system egg: %s",dist) + self.warn("Skipping development or system egg: %s", dist) skipped[dist] = 1 continue @@ -632,7 +632,7 @@ class PackageIndex(Environment): ``location`` of the downloaded distribution instead of a distribution object. """ - dist = self.fetch_distribution(requirement,tmpdir,force_scan,source) + dist = self.fetch_distribution(requirement, tmpdir, force_scan, source) if dist is not None: return dist.location return None @@ -670,7 +670,7 @@ class PackageIndex(Environment): raise DistutilsError( "Can't unambiguously interpret project/version identifier %r; " "any dashes in the name or version should be escaped using " - "underscores. %r" % (fragment,dists) + "underscores. %r" % (fragment, dists) ) else: raise DistutilsError( @@ -689,7 +689,7 @@ class PackageIndex(Environment): fp = self.open_url(strip_fragment(url)) if isinstance(fp, urllib.error.HTTPError): raise DistutilsError( - "Can't download %s: %s %s" % (url, fp.code,fp.msg) + "Can't download %s: %s %s" % (url, fp.code, fp.msg) ) headers = fp.info() blocknum = 0 @@ -700,7 +700,7 @@ class PackageIndex(Environment): sizes = get_all_headers(headers, 'Content-Length') size = max(map(int, sizes)) self.reporthook(url, filename, blocknum, bs, size) - with open(filename,'wb') as tfp: + with open(filename, 'wb') as tfp: while True: block = fp.read(bs) if block: @@ -759,14 +759,14 @@ class PackageIndex(Environment): name, fragment = egg_info_for_url(url) if name: while '..' in name: - name = name.replace('..','.').replace('\\','_') + name = name.replace('..', '.').replace('\\', '_') else: name = "__downloaded__" # default if URL has no path contents if name.endswith('.egg.zip'): name = name[:-4] # strip the extra .zip before download - filename = os.path.join(tmpdir,name) + filename = os.path.join(tmpdir, name) # Download the file # @@ -787,7 +787,7 @@ class PackageIndex(Environment): def _attempt_download(self, url, filename): headers = self._download_to(url, filename) - if 'html' in headers.get('content-type','').lower(): + if 'html' in headers.get('content-type', '').lower(): return self._download_html(url, headers, filename) else: return filename @@ -808,16 +808,16 @@ class PackageIndex(Environment): raise DistutilsError("Unexpected HTML page found at "+url) def _download_svn(self, url, filename): - url = url.split('#',1)[0] # remove any fragment for svn's sake + url = url.split('#', 1)[0] # remove any fragment for svn's sake creds = '' if url.lower().startswith('svn:') and '@' in url: scheme, netloc, path, p, q, f = urllib.parse.urlparse(url) if not netloc and path.startswith('//') and '/' in path[2:]: - netloc, path = path[2:].split('/',1) + netloc, path = path[2:].split('/', 1) auth, host = splituser(netloc) if auth: if ':' in auth: - user, pw = auth.split(':',1) + user, pw = auth.split(':', 1) creds = " --username=%s --password=%s" % (user, pw) else: creds = " --username="+auth @@ -835,7 +835,7 @@ class PackageIndex(Environment): scheme = scheme.split('+', 1)[-1] # Some fragment identification fails - path = path.split('#',1)[0] + path = path.split('#', 1)[0] rev = None if '@' in path: @@ -847,7 +847,7 @@ class PackageIndex(Environment): return url, rev def _download_git(self, url, filename): - filename = filename.split('#',1)[0] + filename = filename.split('#', 1)[0] url, rev = self._vcs_split_rev_from_url(url, pop_prefix=True) self.info("Doing git clone from %s to %s", url, filename) @@ -863,7 +863,7 @@ class PackageIndex(Environment): return filename def _download_hg(self, url, filename): - filename = filename.split('#',1)[0] + filename = filename.split('#', 1)[0] url, rev = self._vcs_split_rev_from_url(url, pop_prefix=True) self.info("Doing hg clone from %s to %s", url, filename) @@ -948,7 +948,7 @@ def _encode_auth(auth): # convert back to a string encoded = encoded_bytes.decode() # strip the trailing carriage return - return encoded.replace('\n','') + return encoded.replace('\n', '') class Credential(object): -- cgit v1.2.1 From dc2d1dc249bec8e3a864e2aa6002a8e27adc4b7c Mon Sep 17 00:00:00 2001 From: stepshal Date: Thu, 14 Jul 2016 12:11:49 +0700 Subject: Fix missing whitespace around operator. --- setuptools/package_index.py | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index cdedef83..4590e93f 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -80,7 +80,7 @@ def egg_info_for_url(url): parts = urllib.parse.urlparse(url) scheme, server, path, parameters, query, fragment = parts base = urllib.parse.unquote(path.split('/')[-1]) - if server=='sourceforge.net' and base=='download': # XXX Yuck + if server == 'sourceforge.net' and base == 'download': # XXX Yuck base = urllib.parse.unquote(path.split('/')[-2]) if '#' in base: base, fragment = base.split('#', 1) return base, fragment @@ -155,7 +155,7 @@ def interpret_distro_name( # it is a bdist_dumb, not an sdist -- bail out return - for p in range(1, len(parts)+1): + for p in range(1, len(parts) + 1): yield Distribution( location, metadata, '-'.join(parts[:p]), '-'.join(parts[p:]), py_version=py_version, precedence = precedence, @@ -209,7 +209,7 @@ def find_external_links(url, page): for tag in ("Home Page", "Download URL"): pos = page.find(tag) - if pos!=-1: + if pos != -1: match = HREF.search(page, pos) if match: yield urllib.parse.urljoin(url, htmldecode(match.group(1))) @@ -336,7 +336,7 @@ class PackageIndex(Environment): for match in HREF.finditer(page): link = urllib.parse.urljoin(base, htmldecode(match.group(1))) self.process_url(link) - if url.startswith(self.index_url) and getattr(f, 'code', None)!=404: + if url.startswith(self.index_url) and getattr(f, 'code', None) != 404: page = self.process_index(url, page) def process_filename(self, fn, nested=False): @@ -357,7 +357,7 @@ class PackageIndex(Environment): def url_ok(self, url, fatal=False): s = URL_SCHEME(url) - if (s and s.group(1).lower()=='file') or self.allows(urllib.parse.urlparse(url)[1]): + if (s and s.group(1).lower() == 'file') or self.allows(urllib.parse.urlparse(url)[1]): return True msg = ("\nNote: Bypassing %s (disallowed host; see " "http://bit.ly/1dg9ijs for details).\n") @@ -400,7 +400,7 @@ class PackageIndex(Environment): parts = list(map( urllib.parse.unquote, link[len(self.index_url):].split('/') )) - if len(parts)==2 and '#' not in parts[1]: + if len(parts) == 2 and '#' not in parts[1]: # it's a package page, sanitize and index it pkg = safe_name(parts[0]) ver = safe_version(parts[1]) @@ -423,7 +423,7 @@ class PackageIndex(Environment): base, frag = egg_info_for_url(new_url) if base.endswith('.py') and not frag: if ver: - new_url+='#egg=%s-%s' % (pkg, ver) + new_url += '#egg=%s-%s' % (pkg, ver) else: self.need_version_info(url) self.scan_url(new_url) @@ -449,11 +449,11 @@ class PackageIndex(Environment): self.scan_url(self.index_url) def find_packages(self, requirement): - self.scan_url(self.index_url + requirement.unsafe_name+'/') + self.scan_url(self.index_url + requirement.unsafe_name + '/') if not self.package_pages.get(requirement.key): # Fall back to safe version of the name - self.scan_url(self.index_url + requirement.project_name+'/') + self.scan_url(self.index_url + requirement.project_name + '/') if not self.package_pages.get(requirement.key): # We couldn't find the target package, so search the index page too @@ -589,13 +589,13 @@ class PackageIndex(Environment): for dist in env[req.key]: - if dist.precedence==DEVELOP_DIST and not develop_ok: + if dist.precedence == DEVELOP_DIST and not develop_ok: if dist not in skipped: self.warn("Skipping development or system egg: %s", dist) skipped[dist] = 1 continue - if dist in req and (dist.precedence<=SOURCE_DIST or not source): + if dist in req and (dist.precedence <= SOURCE_DIST or not source): return dist if force_scan: @@ -645,7 +645,7 @@ class PackageIndex(Environment): interpret_distro_name(filename, match.group(1), None) if d.version ] or [] - if len(dists)==1: # unambiguous ``#egg`` fragment + if len(dists) == 1: # unambiguous ``#egg`` fragment basename = os.path.basename(filename) # Make sure the file has been downloaded to the temp dir. @@ -654,7 +654,7 @@ class PackageIndex(Environment): from setuptools.command.easy_install import samefile if not samefile(filename, dst): shutil.copy2(filename, dst) - filename=dst + filename = dst with open(os.path.join(tmpdir, 'setup.py'), 'w') as file: file.write( @@ -771,13 +771,13 @@ class PackageIndex(Environment): # Download the file # - if scheme=='svn' or scheme.startswith('svn+'): + if scheme == 'svn' or scheme.startswith('svn+'): return self._download_svn(url, filename) - elif scheme=='git' or scheme.startswith('git+'): + elif scheme == 'git' or scheme.startswith('git+'): return self._download_git(url, filename) elif scheme.startswith('hg+'): return self._download_hg(url, filename) - elif scheme=='file': + elif scheme == 'file': return urllib.request.url2pathname(urllib.parse.urlparse(url)[2]) else: self.url_ok(url, True) # raises error if not allowed @@ -806,7 +806,7 @@ class PackageIndex(Environment): break # not an index page file.close() os.unlink(filename) - raise DistutilsError("Unexpected HTML page found at "+url) + raise DistutilsError("Unexpected HTML page found at " + url) def _download_svn(self, url, filename): url = url.split('#', 1)[0] # remove any fragment for svn's sake @@ -821,7 +821,7 @@ class PackageIndex(Environment): user, pw = auth.split(':', 1) creds = " --username=%s --password=%s" % (user, pw) else: - creds = " --username="+auth + creds = " --username=" + auth netloc = host parts = scheme, netloc, url, p, q, f url = urllib.parse.urlunparse(parts) @@ -896,7 +896,7 @@ entity_sub = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub def uchr(c): if not isinstance(c, int): return c - if c>255: return six.unichr(c) + if c > 255: return six.unichr(c) return chr(c) @@ -1046,7 +1046,7 @@ def open_with_auth(url, opener=urllib.request.urlopen): # Put authentication info back into request URL if same host, # so that links found on the page will work s2, h2, path2, param2, query2, frag2 = urllib.parse.urlparse(fp.url) - if s2==scheme and h2==host: + if s2 == scheme and h2 == host: parts = s2, netloc, path2, param2, query2, frag2 fp.url = urllib.parse.urlunparse(parts) -- cgit v1.2.1 From 93612718350a0dd0bb09d21986ef9333a6f5cb1f Mon Sep 17 00:00:00 2001 From: stepshal Date: Thu, 14 Jul 2016 12:32:22 +0700 Subject: Fix spacing after comment hash. --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index cdedef83..4606c5ae 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -325,7 +325,7 @@ class PackageIndex(Environment): base = f.url # handle redirects page = f.read() - if not isinstance(page, str): # We are in Python 3 and got bytes. We want str. + if not isinstance(page, str): # We are in Python 3 and got bytes. We want str. if isinstance(f, urllib.error.HTTPError): # Errors have no charset, assume latin1: charset = 'latin-1' -- cgit v1.2.1 From 2e1ca9656b566b57c7736ef229e9d46c0aa7216c Mon Sep 17 00:00:00 2001 From: stepshal Date: Thu, 14 Jul 2016 12:47:40 +0700 Subject: Remove whitespace around parameter '=' sign. --- setuptools/package_index.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index cdedef83..45ea49ad 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -94,7 +94,7 @@ def distros_for_url(url, metadata=None): match = EGG_FRAGMENT.match(fragment) if match: for dist in interpret_distro_name( - url, match.group(1), metadata, precedence = CHECKOUT_DIST + url, match.group(1), metadata, precedence=CHECKOUT_DIST ): yield dist @@ -158,8 +158,8 @@ def interpret_distro_name( for p in range(1, len(parts)+1): yield Distribution( location, metadata, '-'.join(parts[:p]), '-'.join(parts[p:]), - py_version=py_version, precedence = precedence, - platform = platform + py_version=py_version, precedence=precedence, + platform=platform ) -- cgit v1.2.1 From 64335b63f9e03e71d0acd885b8bfd0b4b7a60aa8 Mon Sep 17 00:00:00 2001 From: stepshal Date: Thu, 21 Jul 2016 04:13:28 +0700 Subject: Put colon-separated compound statement on separate lines. --- setuptools/package_index.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 8764faa6..77f5f96e 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -82,14 +82,16 @@ def egg_info_for_url(url): base = urllib.parse.unquote(path.split('/')[-1]) if server == 'sourceforge.net' and base == 'download': # XXX Yuck base = urllib.parse.unquote(path.split('/')[-2]) - if '#' in base: base, fragment = base.split('#', 1) + if '#' in base: + base, fragment = base.split('#', 1) return base, fragment def distros_for_url(url, metadata=None): """Yield egg or source distribution objects that might be found at a URL""" base, fragment = egg_info_for_url(url) - for dist in distros_for_location(url, base, metadata): yield dist + for dist in distros_for_location(url, base, metadata): + yield dist if fragment: match = EGG_FRAGMENT.match(fragment) if match: @@ -289,7 +291,8 @@ class PackageIndex(Environment): self.to_scan = [] if verify_ssl and ssl_support.is_available and (ca_bundle or ssl_support.find_ca_bundle()): self.opener = ssl_support.opener_for(ca_bundle) - else: self.opener = urllib.request.urlopen + else: + self.opener = urllib.request.urlopen def process_url(self, url, retrieve=False): """Evaluate a URL as a possible download, and maybe retrieve it""" @@ -317,7 +320,8 @@ class PackageIndex(Environment): self.info("Reading %s", url) self.fetched_urls[url] = True # prevent multiple fetch attempts f = self.open_url(url, "Download error on %s: %%s -- Some packages may not be found!" % url) - if f is None: return + if f is None: + return self.fetched_urls[f.url] = True if 'html' not in f.headers.get('content-type', '').lower(): f.close() # not html, we can't process it @@ -442,7 +446,8 @@ class PackageIndex(Environment): def scan_all(self, msg=None, *args): if self.index_url not in self.fetched_urls: - if msg: self.warn(msg, *args) + if msg: + self.warn(msg, *args) self.info( "Scanning index of all packages (this may take a while)" ) @@ -714,7 +719,8 @@ class PackageIndex(Environment): self.check_hash(checker, filename, tfp) return headers finally: - if fp: fp.close() + if fp: + fp.close() def reporthook(self, url, filename, blocknum, blksize, size): pass # no-op @@ -896,7 +902,8 @@ entity_sub = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub def uchr(c): if not isinstance(c, int): return c - if c > 255: return six.unichr(c) + if c > 255: + return six.unichr(c) return chr(c) -- cgit v1.2.1 From 39bf3155d47c0024240be414a611dcb6d549f53c Mon Sep 17 00:00:00 2001 From: stepshal Date: Thu, 21 Jul 2016 09:37:34 +0700 Subject: Add missing blank lines after class or function definition. --- setuptools/package_index.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 77f5f96e..0ea09bd6 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -194,6 +194,7 @@ def unique_values(func): return unique_everseen(func(*args, **kwargs)) return wrapper + REL = re.compile("""<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>""", re.I) # this line is here to fix emacs' cruddy broken syntax highlighting @@ -894,6 +895,7 @@ class PackageIndex(Environment): def warn(self, msg, *args): log.warn(msg, *args) + # This pattern matches a character entity reference (a decimal numeric # references, a hexadecimal numeric reference, or a named reference). entity_sub = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub @@ -1059,6 +1061,7 @@ def open_with_auth(url, opener=urllib.request.urlopen): return fp + # adding a timeout to avoid freezing package_index open_with_auth = socket_timeout(_SOCKET_TIMEOUT)(open_with_auth) -- cgit v1.2.1 From 3fcd44be1e8d3d8b9f229333e8f43d4893658183 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Wed, 8 Jun 2016 16:56:06 +0200 Subject: check if a download is successfull before deciding not to try the next possible dist --- setuptools/package_index.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 0ea09bd6..8f8bf6ed 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -602,24 +602,26 @@ class PackageIndex(Environment): continue if dist in req and (dist.precedence <= SOURCE_DIST or not source): - return dist + mylocation = self.download(dist.location, tmpdir) + if os.path.exists(mylocation): + return dist, mylocation if force_scan: self.prescan() self.find_packages(requirement) - dist = find(requirement) + dist, mylocation = find(requirement) if local_index is not None: - dist = dist or find(requirement, local_index) + dist, mylocation = dist, mylocation if dist else find(requirement, local_index) if dist is None: if self.to_scan is not None: self.prescan() - dist = find(requirement) + dist, mylocation = find(requirement) if dist is None and not force_scan: self.find_packages(requirement) - dist = find(requirement) + dist, mylocation = find(requirement) if dist is None: self.warn( @@ -629,7 +631,7 @@ class PackageIndex(Environment): ) else: self.info("Best match: %s", dist) - return dist.clone(location=self.download(dist.location, tmpdir)) + return dist.clone(location=mylocation) def fetch(self, requirement, tmpdir, force_scan=False, source=False): """Obtain a file suitable for fulfilling `requirement` -- cgit v1.2.1 From 1e5bb5e680d00fcfc74078d0ea2d12760879b3c7 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Wed, 8 Jun 2016 18:04:00 +0200 Subject: addressed remarks, None, None since a tuple is expected --- setuptools/package_index.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 8f8bf6ed..a6918123 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -605,14 +605,15 @@ class PackageIndex(Environment): mylocation = self.download(dist.location, tmpdir) if os.path.exists(mylocation): return dist, mylocation + return None, None if force_scan: self.prescan() self.find_packages(requirement) dist, mylocation = find(requirement) - if local_index is not None: - dist, mylocation = dist, mylocation if dist else find(requirement, local_index) + if not dist and local_index is not None: + dist, mylocation = find(requirement, local_index) if dist is None: if self.to_scan is not None: -- cgit v1.2.1 From c4ef24f6bac7b4c12b3cfc71258524185d220fc7 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Fri, 24 Jun 2016 17:25:02 +0200 Subject: don't return a tuple, add a new attribute to current returnvalue --- setuptools/package_index.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index a6918123..b7247120 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -604,25 +604,25 @@ class PackageIndex(Environment): if dist in req and (dist.precedence <= SOURCE_DIST or not source): mylocation = self.download(dist.location, tmpdir) if os.path.exists(mylocation): - return dist, mylocation - return None, None + dist._location = mylocation + return dist if force_scan: self.prescan() self.find_packages(requirement) - dist, mylocation = find(requirement) + dist = find(requirement) if not dist and local_index is not None: - dist, mylocation = find(requirement, local_index) + dist = find(requirement, local_index) if dist is None: if self.to_scan is not None: self.prescan() - dist, mylocation = find(requirement) + dist = find(requirement) if dist is None and not force_scan: self.find_packages(requirement) - dist, mylocation = find(requirement) + dist = find(requirement) if dist is None: self.warn( @@ -632,7 +632,7 @@ class PackageIndex(Environment): ) else: self.info("Best match: %s", dist) - return dist.clone(location=mylocation) + return dist.clone(location=dist._location) def fetch(self, requirement, tmpdir, force_scan=False, source=False): """Obtain a file suitable for fulfilling `requirement` -- cgit v1.2.1 From e2124c84b83e8ad6adb3dd8e8ed846eea4f7601f Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Fri, 24 Jun 2016 17:30:02 +0200 Subject: pick better names for variables --- setuptools/package_index.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index b7247120..89eb5786 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -602,9 +602,8 @@ class PackageIndex(Environment): continue if dist in req and (dist.precedence <= SOURCE_DIST or not source): - mylocation = self.download(dist.location, tmpdir) - if os.path.exists(mylocation): - dist._location = mylocation + dist.download_location = self.download(dist.location, tmpdir) + if os.path.exists(dist.download_location): return dist if force_scan: @@ -632,7 +631,7 @@ class PackageIndex(Environment): ) else: self.info("Best match: %s", dist) - return dist.clone(location=dist._location) + return dist.clone(location=dist.download_location) def fetch(self, requirement, tmpdir, force_scan=False, source=False): """Obtain a file suitable for fulfilling `requirement` -- cgit v1.2.1 From e2af2337adc2996e8aabaef2780dea8ac3e88487 Mon Sep 17 00:00:00 2001 From: Jens Timmerman Date: Fri, 24 Jun 2016 17:45:31 +0200 Subject: specify that no 'working' links were found, previous behaviour didn't check if downloads could succeed --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 89eb5786..8d965f49 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -625,7 +625,7 @@ class PackageIndex(Environment): if dist is None: self.warn( - "No local packages or download links found for %s%s", + "No local packages or working download links found for %s%s", (source and "a source distribution of " or ""), requirement, ) -- cgit v1.2.1 From 18c0fdab26ba13a9cf142408d7f1f2566be21fa7 Mon Sep 17 00:00:00 2001 From: Valentin Valls Date: Wed, 3 Aug 2016 13:55:56 +0200 Subject: Fix logging using arguments instead of formatter --- setuptools/package_index.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 8d965f49..9ef782cb 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1038,8 +1038,7 @@ def open_with_auth(url, opener=urllib.request.urlopen): cred = PyPIConfig().find_credential(url) if cred: auth = str(cred) - info = cred.username, url - log.info('Authenticating as %s for %s (from .pypirc)' % info) + log.info('Authenticating as %s for %s (from .pypirc)', cred.username, url) if auth: auth = "Basic " + _encode_auth(auth) -- cgit v1.2.1 From 5f3dac49084b38a248682475635f866e795f484c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 3 Aug 2016 22:08:08 -0400 Subject: In package_index, reindent long lines. --- setuptools/package_index.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 9ef782cb..82cd608f 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -290,7 +290,12 @@ class PackageIndex(Environment): self.package_pages = {} self.allows = re.compile('|'.join(map(translate, hosts))).match self.to_scan = [] - if verify_ssl and ssl_support.is_available and (ca_bundle or ssl_support.find_ca_bundle()): + use_ssl = ( + verify_ssl + and ssl_support.is_available + and (ca_bundle or ssl_support.find_ca_bundle()) + ) + if use_ssl: self.opener = ssl_support.opener_for(ca_bundle) else: self.opener = urllib.request.urlopen @@ -320,7 +325,8 @@ class PackageIndex(Environment): self.info("Reading %s", url) self.fetched_urls[url] = True # prevent multiple fetch attempts - f = self.open_url(url, "Download error on %s: %%s -- Some packages may not be found!" % url) + tmpl = "Download error on %s: %%s -- Some packages may not be found!" + f = self.open_url(url, tmpl % url) if f is None: return self.fetched_urls[f.url] = True @@ -362,7 +368,8 @@ class PackageIndex(Environment): def url_ok(self, url, fatal=False): s = URL_SCHEME(url) - if (s and s.group(1).lower() == 'file') or self.allows(urllib.parse.urlparse(url)[1]): + is_file = s and s.group(1).lower() == 'file' + if is_file or self.allows(urllib.parse.urlparse(url)[1]): return True msg = ("\nNote: Bypassing %s (disallowed host; see " "http://bit.ly/1dg9ijs for details).\n") @@ -1038,7 +1045,8 @@ def open_with_auth(url, opener=urllib.request.urlopen): cred = PyPIConfig().find_credential(url) if cred: auth = str(cred) - log.info('Authenticating as %s for %s (from .pypirc)', cred.username, url) + info = cred.username, url + log.info('Authenticating as %s for %s (from .pypirc)', *info) if auth: auth = "Basic " + _encode_auth(auth) -- cgit v1.2.1 From 1f23f9a25e6c91554954185e84497056062093be Mon Sep 17 00:00:00 2001 From: Steve Kowalik Date: Tue, 13 Sep 2016 10:54:45 +1200 Subject: Don't duplicate error case in package_index easy_install has code to handle parsing a requirement, catching the ValueError and then raising a DistUtilsError. This code was entirely duplicated in package_index, so I've slightly refactored to remove the duplication. --- setuptools/package_index.py | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 82cd608f..3fb39269 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -52,6 +52,15 @@ _tmpl = "setuptools/{setuptools.__version__} Python-urllib/{py_major}" user_agent = _tmpl.format(py_major=sys.version[:3], **globals()) +def parse_requirement_arg(spec): + try: + return Requirement.parse(spec) + except ValueError: + raise DistutilsError( + "Not a URL, existing file, or requirement spec: %r" % (spec,) + ) + + def parse_bdist_wininst(name): """Return (base,pyversion) or (None,None) for possible .exe name""" @@ -561,13 +570,7 @@ class PackageIndex(Environment): # Existing file or directory, just return it return spec else: - try: - spec = Requirement.parse(spec) - except ValueError: - raise DistutilsError( - "Not a URL, existing file, or requirement spec: %r" % - (spec,) - ) + spec = parse_requirement_arg(spec) return getattr(self.fetch_distribution(spec, tmpdir), 'location', None) def fetch_distribution( -- cgit v1.2.1 From 55c36fdfa8fdc3b4581e0331cc91a069a001da51 Mon Sep 17 00:00:00 2001 From: stepshal Date: Sun, 18 Sep 2016 04:25:06 +0700 Subject: Add missing whitespace. --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 3fb39269..3e8d6818 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -293,7 +293,7 @@ class PackageIndex(Environment): ca_bundle=None, verify_ssl=True, *args, **kw ): Environment.__init__(self, *args, **kw) - self.index_url = index_url + "/"[:not index_url.endswith('/')] + self.index_url = index_url + "/" [:not index_url.endswith('/')] self.scanned_urls = {} self.fetched_urls = {} self.package_pages = {} -- cgit v1.2.1 From 31bd37c6ac8de9e8c1bacebc2d8e1215df91eb96 Mon Sep 17 00:00:00 2001 From: stepshal Date: Tue, 18 Oct 2016 20:24:35 +0700 Subject: Fix quantity of blank lines. --- setuptools/package_index.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 3e8d6818..3bb97154 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -47,7 +47,6 @@ __all__ = [ _SOCKET_TIMEOUT = 15 - _tmpl = "setuptools/{setuptools.__version__} Python-urllib/{py_major}" user_agent = _tmpl.format(py_major=sys.version[:3], **globals()) @@ -198,9 +197,11 @@ def unique_values(func): Wrap a function returning an iterable such that the resulting iterable only ever yields unique items. """ + @wraps(func) def wrapper(*args, **kwargs): return unique_everseen(func(*args, **kwargs)) + return wrapper @@ -415,6 +416,7 @@ class PackageIndex(Environment): def process_index(self, url, page): """Process the contents of a PyPI page""" + def scan(link): # Process a URL to see if it's for a package page if link.startswith(self.index_url): @@ -946,7 +948,9 @@ def socket_timeout(timeout=15): return func(*args, **kwargs) finally: socket.setdefaulttimeout(old_timeout) + return _socket_timeout + return _socket_timeout @@ -991,7 +995,6 @@ class Credential(object): class PyPIConfig(configparser.RawConfigParser): - def __init__(self): """ Load from ~/.pypirc -- cgit v1.2.1 From df1bd4e17a082b9b634f62d799807a18e526a7c0 Mon Sep 17 00:00:00 2001 From: stepshal Date: Wed, 19 Oct 2016 00:10:40 +0700 Subject: Fix spacing after comment hash. --- setuptools/package_index.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 3bb97154..e5249b27 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -88,7 +88,7 @@ def egg_info_for_url(url): parts = urllib.parse.urlparse(url) scheme, server, path, parameters, query, fragment = parts base = urllib.parse.unquote(path.split('/')[-1]) - if server == 'sourceforge.net' and base == 'download': # XXX Yuck + if server == 'sourceforge.net' and base == 'download': # XXX Yuck base = urllib.parse.unquote(path.split('/')[-2]) if '#' in base: base, fragment = base.split('#', 1) @@ -112,7 +112,7 @@ def distros_for_url(url, metadata=None): def distros_for_location(location, basename, metadata=None): """Yield egg or source distribution objects based on basename""" if basename.endswith('.egg.zip'): - basename = basename[:-4] # strip the .zip + basename = basename[:-4] # strip the .zip if basename.endswith('.egg') and '-' in basename: # only one, unambiguous interpretation return [Distribution.from_location(location, basename, metadata)] @@ -334,17 +334,17 @@ class PackageIndex(Environment): return self.info("Reading %s", url) - self.fetched_urls[url] = True # prevent multiple fetch attempts + self.fetched_urls[url] = True # prevent multiple fetch attempts tmpl = "Download error on %s: %%s -- Some packages may not be found!" f = self.open_url(url, tmpl % url) if f is None: return self.fetched_urls[f.url] = True if 'html' not in f.headers.get('content-type', '').lower(): - f.close() # not html, we can't process it + f.close() # not html, we can't process it return - base = f.url # handle redirects + base = f.url # handle redirects page = f.read() if not isinstance(page, str): # We are in Python 3 and got bytes. We want str. if isinstance(f, urllib.error.HTTPError): @@ -438,7 +438,7 @@ class PackageIndex(Environment): except ValueError: pass - pkg, ver = scan(url) # ensure this page is in the page index + pkg, ver = scan(url) # ensure this page is in the page index if pkg: # process individual package page for new_url in find_external_links(url, page): @@ -455,7 +455,7 @@ class PackageIndex(Environment): lambda m: '%s' % m.group(1, 3, 2), page ) else: - return "" # no sense double-scanning non-package pages + return "" # no sense double-scanning non-package pages def need_version_info(self, url): self.scan_all( @@ -530,12 +530,12 @@ class PackageIndex(Environment): """Scan urls scheduled for prescanning (e.g. --find-links)""" if self.to_scan: list(map(self.scan_url, self.to_scan)) - self.to_scan = None # from now on, go ahead and process immediately + self.to_scan = None # from now on, go ahead and process immediately def not_found_in_index(self, requirement): - if self[requirement.key]: # we've seen at least one distro + if self[requirement.key]: # we've seen at least one distro meth, msg = self.info, "Couldn't retrieve index page for %r" - else: # no distros seen for this name, might be misspelled + else: # no distros seen for this name, might be misspelled meth, msg = (self.warn, "Couldn't find index page for %r (maybe misspelled?)") meth(msg, requirement.unsafe_name) @@ -665,7 +665,7 @@ class PackageIndex(Environment): interpret_distro_name(filename, match.group(1), None) if d.version ] or [] - if len(dists) == 1: # unambiguous ``#egg`` fragment + if len(dists) == 1: # unambiguous ``#egg`` fragment basename = os.path.basename(filename) # Make sure the file has been downloaded to the temp dir. @@ -738,7 +738,7 @@ class PackageIndex(Environment): fp.close() def reporthook(self, url, filename, blocknum, blksize, size): - pass # no-op + pass # no-op def open_url(self, url, warning=None): if url.startswith('file:'): @@ -783,10 +783,10 @@ class PackageIndex(Environment): while '..' in name: name = name.replace('..', '.').replace('\\', '_') else: - name = "__downloaded__" # default if URL has no path contents + name = "__downloaded__" # default if URL has no path contents if name.endswith('.egg.zip'): - name = name[:-4] # strip the extra .zip before download + name = name[:-4] # strip the extra .zip before download filename = os.path.join(tmpdir, name) @@ -801,7 +801,7 @@ class PackageIndex(Environment): elif scheme == 'file': return urllib.request.url2pathname(urllib.parse.urlparse(url)[2]) else: - self.url_ok(url, True) # raises error if not allowed + self.url_ok(url, True) # raises error if not allowed return self._attempt_download(url, filename) def scan_url(self, url): @@ -824,13 +824,13 @@ class PackageIndex(Environment): file.close() os.unlink(filename) return self._download_svn(url, filename) - break # not an index page + break # not an index page file.close() os.unlink(filename) raise DistutilsError("Unexpected HTML page found at " + url) def _download_svn(self, url, filename): - url = url.split('#', 1)[0] # remove any fragment for svn's sake + url = url.split('#', 1)[0] # remove any fragment for svn's sake creds = '' if url.lower().startswith('svn:') and '@' in url: scheme, netloc, path, p, q, f = urllib.parse.urlparse(url) @@ -1082,7 +1082,7 @@ open_with_auth = socket_timeout(_SOCKET_TIMEOUT)(open_with_auth) def fix_sf_url(url): - return url # backward compatibility + return url # backward compatibility def local_open(url): -- cgit v1.2.1 From dd25d1f858cd02211f4d105ab19d6c3ce90d4048 Mon Sep 17 00:00:00 2001 From: stepshal Date: Tue, 25 Oct 2016 08:37:18 +0700 Subject: Fix spacing after comment hash. --- setuptools/package_index.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index e5249b27..024fab98 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -515,10 +515,10 @@ class PackageIndex(Environment): """Add `urls` to the list that will be prescanned for searches""" for url in urls: if ( - self.to_scan is None # if we have already "gone online" - or not URL_SCHEME(url) # or it's a local file/directory + self.to_scan is None # if we have already "gone online" + or not URL_SCHEME(url) # or it's a local file/directory or url.startswith('file:') - or list(distros_for_url(url)) # or a direct package link + or list(distros_for_url(url)) # or a direct package link ): # then go ahead and process it now self.scan_url(url) -- cgit v1.2.1 From 161c0a5072f6049f6e4790969a387e9aae41ad52 Mon Sep 17 00:00:00 2001 From: Julien Muchembled Date: Mon, 14 Nov 2016 00:25:57 +0100 Subject: package_index: fix bug not catching some network timeouts There are already so many exceptions catched, like socket errors (e.g. failure in name resolution) or HTTP errors. Depending on when a timeout occurs, it is either catched (URLError during the SSL handshake) or not (socket.error while getting a HTTP response). When used by buildout, this fixes random failures when running in newest mode (which is the default case), or when the requested version is available in the download-cache. --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 024fab98..d80d43bc 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -768,7 +768,7 @@ class PackageIndex(Environment): 'down, %s' % (url, v.line) ) - except http_client.HTTPException as v: + except (http_client.HTTPException, socket.error) as v: if warning: self.warn(warning, v) else: -- cgit v1.2.1 From f14930e66601b462699c44384c482cd966f53b8f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 9 Dec 2016 08:16:33 -0500 Subject: Drop support for Python 2.6, removing lots of compatibility code for a leaner, cleaner codebase. Fixes #878. --- setuptools/package_index.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index d80d43bc..d2f27ca6 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -27,7 +27,6 @@ from setuptools import ssl_support from distutils import log from distutils.errors import DistutilsError from fnmatch import translate -from setuptools.py26compat import strip_fragment from setuptools.py27compat import get_all_headers EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.]+)$') @@ -707,7 +706,7 @@ class PackageIndex(Environment): fp, info = None, None try: checker = HashChecker.from_url(url) - fp = self.open_url(strip_fragment(url)) + fp = self.open_url(url) if isinstance(fp, urllib.error.HTTPError): raise DistutilsError( "Can't download %s: %s %s" % (url, fp.code, fp.msg) -- cgit v1.2.1 From ff371f18f0076bc63da05334f7e551c1cc29e10d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 1 Jan 2017 22:34:28 -0500 Subject: Strip out vendored packages and require them instead. Ref #581. --- setuptools/package_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index d80d43bc..65c3c433 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -14,8 +14,8 @@ try: except ImportError: from urllib2 import splituser -from setuptools.extern import six -from setuptools.extern.six.moves import urllib, http_client, configparser, map +import six +from six.moves import urllib, http_client, configparser, map import setuptools from pkg_resources import ( -- cgit v1.2.1 From b9ca305f48b5da7cfe0f831fdc32dc0aa3890035 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 16 Jan 2017 14:37:02 -0500 Subject: Remove unused imports --- setuptools/package_index.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index d80d43bc..5c25c9d6 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -17,10 +17,9 @@ except ImportError: from setuptools.extern import six from setuptools.extern.six.moves import urllib, http_client, configparser, map -import setuptools from pkg_resources import ( CHECKOUT_DIST, Distribution, BINARY_DIST, normalize_path, SOURCE_DIST, - require, Environment, find_distributions, safe_name, safe_version, + Environment, find_distributions, safe_name, safe_version, to_filename, Requirement, DEVELOP_DIST, ) from setuptools import ssl_support -- cgit v1.2.1 From 552ea0d4a08ad24e06c2aa5b1746e669be1c48f5 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 16 Jan 2017 14:50:30 -0500 Subject: Restore setuptools import, falsely identified as an unused import by linter. --- setuptools/package_index.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 5c25c9d6..45fac01a 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -17,6 +17,7 @@ except ImportError: from setuptools.extern import six from setuptools.extern.six.moves import urllib, http_client, configparser, map +import setuptools from pkg_resources import ( CHECKOUT_DIST, Distribution, BINARY_DIST, normalize_path, SOURCE_DIST, Environment, find_distributions, safe_name, safe_version, @@ -47,7 +48,7 @@ __all__ = [ _SOCKET_TIMEOUT = 15 _tmpl = "setuptools/{setuptools.__version__} Python-urllib/{py_major}" -user_agent = _tmpl.format(py_major=sys.version[:3], **globals()) +user_agent = _tmpl.format(py_major=sys.version[:3], setuptools=setuptools) def parse_requirement_arg(spec): -- cgit v1.2.1 From 5f235ac251a88a1704166f2e2c12903bafad8adb Mon Sep 17 00:00:00 2001 From: Moriyoshi Koizumi Date: Sat, 11 Feb 2017 12:14:09 +0900 Subject: A local version label starts with '+' sign, as per https://www.python.org/dev/peps/pep-0440/#id23 --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 42361058..b19893e8 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -30,7 +30,7 @@ from fnmatch import translate from setuptools.py26compat import strip_fragment from setuptools.py27compat import get_all_headers -EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.]+)$') +EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.+]+)$') HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I) # this is here to fix emacs' cruddy broken syntax highlighting PYPI_MD5 = re.compile( -- cgit v1.2.1 From f1ca29cc9332a1bb59e250aa280ddc5239b5457e Mon Sep 17 00:00:00 2001 From: Moriyoshi Koizumi Date: Mon, 13 Feb 2017 01:44:17 +0900 Subject: An epoch starts with a number followed by '!'. --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index b19893e8..3544dd54 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -30,7 +30,7 @@ from fnmatch import translate from setuptools.py26compat import strip_fragment from setuptools.py27compat import get_all_headers -EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.+]+)$') +EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.+!]+)$') HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I) # this is here to fix emacs' cruddy broken syntax highlighting PYPI_MD5 = re.compile( -- cgit v1.2.1 From 3d0cc355fb5e8012cb8c72f0e25042a5a44f31d6 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 24 Feb 2017 11:49:51 -0500 Subject: Revert "Merge pull request #933 from pypa/feature/581-depend-not-bundle" This reverts commit 089cdeb489a0fa94d11b7307b54210ef9aa40511, reversing changes made to aaec654d804cb78dbb6391afff721a63f26a71cd. --- setuptools/package_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 3544dd54..faef5377 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -14,8 +14,8 @@ try: except ImportError: from urllib2 import splituser -import six -from six.moves import urllib, http_client, configparser, map +from setuptools.extern import six +from setuptools.extern.six.moves import urllib, http_client, configparser, map import setuptools from pkg_resources import ( -- cgit v1.2.1 From e753cb42481783ac858ceb518aaac1472075063c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Skytt=C3=A4?= Date: Fri, 24 Feb 2017 10:55:44 +0200 Subject: Python 3.6 invalid escape sequence deprecation fixes --- setuptools/package_index.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 3544dd54..5d397b67 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -34,8 +34,8 @@ EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.+!]+)$') HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I) # this is here to fix emacs' cruddy broken syntax highlighting PYPI_MD5 = re.compile( - '([^<]+)\n\s+\\(md5\\)' + '([^<]+)\n\\s+\\(md5\\)' ) URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):', re.I).match EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split() @@ -161,7 +161,7 @@ def interpret_distro_name( # versions in distribution archive names (sdist and bdist). parts = basename.split('-') - if not py_version and any(re.match('py\d\.\d$', p) for p in parts[2:]): + if not py_version and any(re.match(r'py\d\.\d$', p) for p in parts[2:]): # it is a bdist_dumb, not an sdist -- bail out return @@ -205,7 +205,7 @@ def unique_values(func): return wrapper -REL = re.compile("""<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>""", re.I) +REL = re.compile(r"""<([^>]*\srel\s*=\s*['"]?([^'">]+)[^>]*)>""", re.I) # this line is here to fix emacs' cruddy broken syntax highlighting -- cgit v1.2.1 From 5d163b5bf846b86dfebf335548deb1398262c8b8 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 13 Sep 2017 15:41:30 -0400 Subject: Use platform-friendly syntax for Mercurial checkouts. Fixes #170. --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 2acc817a..a6363b18 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -893,7 +893,7 @@ class PackageIndex(Environment): if rev is not None: self.info("Updating to %s", rev) - os.system("(cd %s && hg up -C -r %s >&-)" % ( + os.system("(cd %s && hg up -C -r %s -q)" % ( filename, rev, )) -- cgit v1.2.1 From 2ef955096db1999fb9caffe5d642dc2cefd2c1ca Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 14 Nov 2017 17:40:10 -0500 Subject: Update broken link. Fixes #1197. --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index a6363b18..5dbb4805 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -382,7 +382,7 @@ class PackageIndex(Environment): if is_file or self.allows(urllib.parse.urlparse(url)[1]): return True msg = ("\nNote: Bypassing %s (disallowed host; see " - "http://bit.ly/1dg9ijs for details).\n") + "http://bit.ly/2hrImnY for details).\n") if fatal: raise DistutilsError(msg % url) else: -- cgit v1.2.1 From b27fc068fe8eb409851f3f92c1834e36073759c1 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 14 Nov 2017 17:45:18 -0500 Subject: Feed the hobgoblins (delint). --- setuptools/package_index.py | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 5dbb4805..e0aeb309 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -141,7 +141,7 @@ def distros_for_filename(filename, metadata=None): def interpret_distro_name( location, basename, metadata, py_version=None, precedence=SOURCE_DIST, platform=None - ): +): """Generate alternative interpretations of a source distro name Note: if `location` is a filesystem filename, you should call @@ -292,7 +292,7 @@ class PackageIndex(Environment): def __init__( self, index_url="https://pypi.python.org/simple", hosts=('*',), ca_bundle=None, verify_ssl=True, *args, **kw - ): + ): Environment.__init__(self, *args, **kw) self.index_url = index_url + "/" [:not index_url.endswith('/')] self.scanned_urls = {} @@ -346,7 +346,8 @@ class PackageIndex(Environment): base = f.url # handle redirects page = f.read() - if not isinstance(page, str): # We are in Python 3 and got bytes. We want str. + if not isinstance(page, str): + # In Python 3 and got bytes but want str. if isinstance(f, urllib.error.HTTPError): # Errors have no charset, assume latin1: charset = 'latin-1' @@ -381,7 +382,8 @@ class PackageIndex(Environment): is_file = s and s.group(1).lower() == 'file' if is_file or self.allows(urllib.parse.urlparse(url)[1]): return True - msg = ("\nNote: Bypassing %s (disallowed host; see " + msg = ( + "\nNote: Bypassing %s (disallowed host; see " "http://bit.ly/2hrImnY for details).\n") if fatal: raise DistutilsError(msg % url) @@ -500,15 +502,16 @@ class PackageIndex(Environment): """ checker is a ContentChecker """ - checker.report(self.debug, + checker.report( + self.debug, "Validating %%s checksum for %s" % filename) if not checker.is_valid(): tfp.close() os.unlink(filename) raise DistutilsError( "%s validation failed for %s; " - "possible download problem?" % ( - checker.hash.name, os.path.basename(filename)) + "possible download problem?" + % (checker.hash.name, os.path.basename(filename)) ) def add_find_links(self, urls): @@ -536,7 +539,8 @@ class PackageIndex(Environment): if self[requirement.key]: # we've seen at least one distro meth, msg = self.info, "Couldn't retrieve index page for %r" else: # no distros seen for this name, might be misspelled - meth, msg = (self.warn, + meth, msg = ( + self.warn, "Couldn't find index page for %r (maybe misspelled?)") meth(msg, requirement.unsafe_name) self.scan_all() @@ -577,8 +581,7 @@ class PackageIndex(Environment): def fetch_distribution( self, requirement, tmpdir, force_scan=False, source=False, - develop_ok=False, local_index=None - ): + develop_ok=False, local_index=None): """Obtain a distribution suitable for fulfilling `requirement` `requirement` must be a ``pkg_resources.Requirement`` instance. @@ -609,12 +612,19 @@ class PackageIndex(Environment): if dist.precedence == DEVELOP_DIST and not develop_ok: if dist not in skipped: - self.warn("Skipping development or system egg: %s", dist) + self.warn( + "Skipping development or system egg: %s", dist, + ) skipped[dist] = 1 continue - if dist in req and (dist.precedence <= SOURCE_DIST or not source): - dist.download_location = self.download(dist.location, tmpdir) + test = ( + dist in req + and (dist.precedence <= SOURCE_DIST or not source) + ) + if test: + loc = self.download(dist.location, tmpdir) + dist.download_location = loc if os.path.exists(dist.download_location): return dist @@ -704,7 +714,7 @@ class PackageIndex(Environment): def _download_to(self, url, filename): self.info("Downloading %s", url) # Download the file - fp, info = None, None + fp = None try: checker = HashChecker.from_url(url) fp = self.open_url(strip_fragment(url)) @@ -1103,7 +1113,8 @@ def local_open(url): f += '/' files.append('{name}'.format(name=f)) else: - tmpl = ("{url}" + tmpl = ( + "{url}" "{files}") body = tmpl.format(url=url, files='\n'.join(files)) status, message = 200, "OK" -- cgit v1.2.1 From 118edbb2b715c96620b51018c1d28e81f2318053 Mon Sep 17 00:00:00 2001 From: Benoit Pierre Date: Thu, 2 Nov 2017 21:19:39 +0100 Subject: easy_install: add support for installing from wheels Note: wheels are installed as eggs, so each install is self-contained and multiple versions of the same package can be installed at the same time. Limitations: - headers are not supported - resulting egg metadata requirements have their markers stripped --- setuptools/package_index.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index fe2ef50f..ad743307 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -21,13 +21,14 @@ import setuptools from pkg_resources import ( CHECKOUT_DIST, Distribution, BINARY_DIST, normalize_path, SOURCE_DIST, Environment, find_distributions, safe_name, safe_version, - to_filename, Requirement, DEVELOP_DIST, + to_filename, Requirement, DEVELOP_DIST, EGG_DIST, ) from setuptools import ssl_support from distutils import log from distutils.errors import DistutilsError from fnmatch import translate from setuptools.py27compat import get_all_headers +from setuptools.wheel import Wheel EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.+!]+)$') HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I) @@ -115,6 +116,17 @@ def distros_for_location(location, basename, metadata=None): if basename.endswith('.egg') and '-' in basename: # only one, unambiguous interpretation return [Distribution.from_location(location, basename, metadata)] + if basename.endswith('.whl') and '-' in basename: + wheel = Wheel(basename) + if not wheel.is_compatible(): + return [] + return [Distribution( + location=location, + project_name=wheel.project_name, + version=wheel.version, + # Increase priority over eggs. + precedence=EGG_DIST + 1, + )] if basename.endswith('.exe'): win_base, py_ver, platform = parse_bdist_wininst(basename) if win_base is not None: -- cgit v1.2.1 From e141c08edc47834b04eb0fd7c1510ee04b682921 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 17 Mar 2018 11:38:32 -0400 Subject: Use six for splituser --- setuptools/package_index.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index ad743307..64a11c9b 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -9,11 +9,6 @@ import hashlib import itertools from functools import wraps -try: - from urllib.parse import splituser -except ImportError: - from urllib2 import splituser - from setuptools.extern import six from setuptools.extern.six.moves import urllib, http_client, configparser, map @@ -857,7 +852,7 @@ class PackageIndex(Environment): scheme, netloc, path, p, q, f = urllib.parse.urlparse(url) if not netloc and path.startswith('//') and '/' in path[2:]: netloc, path = path[2:].split('/', 1) - auth, host = splituser(netloc) + auth, host = urllib.parse.splituser(netloc) if auth: if ':' in auth: user, pw = auth.split(':', 1) @@ -1064,7 +1059,7 @@ def open_with_auth(url, opener=urllib.request.urlopen): raise http_client.InvalidURL("nonnumeric port: ''") if scheme in ('http', 'https'): - auth, host = splituser(netloc) + auth, host = urllib.parse.splituser(netloc) else: auth = None -- cgit v1.2.1 From 6fba91c17b264beec674bb25019e3888fe5ba305 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 17 Mar 2018 11:53:22 -0400 Subject: Rely on stdlib to decode entities. --- setuptools/package_index.py | 17 ++--------------- 1 file changed, 2 insertions(+), 15 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 64a11c9b..914b5e61 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -23,6 +23,7 @@ from distutils import log from distutils.errors import DistutilsError from fnmatch import translate from setuptools.py27compat import get_all_headers +from setuptools.py33compat import unescape from setuptools.wheel import Wheel EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.+!]+)$') @@ -931,23 +932,9 @@ class PackageIndex(Environment): entity_sub = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub -def uchr(c): - if not isinstance(c, int): - return c - if c > 255: - return six.unichr(c) - return chr(c) - - def decode_entity(match): what = match.group(1) - if what.startswith('#x'): - what = int(what[2:], 16) - elif what.startswith('#'): - what = int(what[1:]) - else: - what = six.moves.html_entities.name2codepoint.get(what, match.group(0)) - return uchr(what) + return unescape(what) def htmldecode(text): -- cgit v1.2.1 From 06736eec5894df69dc267876cf78381804df3ada Mon Sep 17 00:00:00 2001 From: Jon Dufresne Date: Thu, 26 Apr 2018 06:27:19 -0700 Subject: Update all pypi.python.org URLs to pypi.org For details on the new PyPI, see the blog post: https://pythoninsider.blogspot.ca/2018/04/new-pypi-launched-legacy-pypi-shutting.html --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 914b5e61..b6407be3 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -297,7 +297,7 @@ class PackageIndex(Environment): """A distribution index that scans web pages for download URLs""" def __init__( - self, index_url="https://pypi.python.org/simple", hosts=('*',), + self, index_url="https://pypi.org/simple/", hosts=('*',), ca_bundle=None, verify_ssl=True, *args, **kw ): Environment.__init__(self, *args, **kw) -- cgit v1.2.1 From cca86c7f1d4040834c3265ccecdd9e21b4036df5 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 3 Jun 2018 09:50:25 -0400 Subject: Use Python 3 syntax for new-style clasess --- setuptools/package_index.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index b6407be3..ed4162cd 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -26,6 +26,8 @@ from setuptools.py27compat import get_all_headers from setuptools.py33compat import unescape from setuptools.wheel import Wheel +__metaclass__ = type + EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.+!]+)$') HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I) # this is here to fix emacs' cruddy broken syntax highlighting @@ -235,7 +237,7 @@ def find_external_links(url, page): yield urllib.parse.urljoin(url, htmldecode(match.group(1))) -class ContentChecker(object): +class ContentChecker: """ A null content checker that defines the interface for checking content """ @@ -980,7 +982,7 @@ def _encode_auth(auth): return encoded.replace('\n', '') -class Credential(object): +class Credential: """ A username/password pair. Use like a namedtuple. """ -- cgit v1.2.1 From 3174f82b943f53e04ad7a5a6b2aa67ca2b2d44c5 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 3 Jun 2018 10:05:20 -0400 Subject: Use raw strings for regexes --- setuptools/package_index.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index ed4162cd..9bf0421f 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -29,11 +29,11 @@ from setuptools.wheel import Wheel __metaclass__ = type EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.+!]+)$') -HREF = re.compile("""href\\s*=\\s*['"]?([^'"> ]+)""", re.I) +HREF = re.compile(r"""href\s*=\s*['"]?([^'"> ]+)""", re.I) # this is here to fix emacs' cruddy broken syntax highlighting PYPI_MD5 = re.compile( - '([^<]+)\n\\s+\\(md5\\)' + r'([^<]+)\n\s+\(md5\)' ) URL_SCHEME = re.compile('([-+.a-z0-9]{2,}):', re.I).match EXTENSIONS = ".tar.gz .tar.bz2 .tar .zip .tgz".split() -- cgit v1.2.1 From 9f379bbf78fcc1af1714c4a6c6f2afe172e0ed05 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 3 Jun 2018 10:06:10 -0400 Subject: Remove stale comment, added in 8cc0d5c2 and made meaningless in 26eee297. --- setuptools/package_index.py | 1 - 1 file changed, 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 9bf0421f..619649b9 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -30,7 +30,6 @@ __metaclass__ = type EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.+!]+)$') HREF = re.compile(r"""href\s*=\s*['"]?([^'"> ]+)""", re.I) -# this is here to fix emacs' cruddy broken syntax highlighting PYPI_MD5 = re.compile( r'([^<]+)\n\s+\(md5\)' -- cgit v1.2.1 From 95880b3ebbd61367d2c4b3409e7789e1893c2d79 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 4 Jun 2018 21:16:43 -0400 Subject: Add test and adjust match. Fixes #1366. --- setuptools/package_index.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 619649b9..cda54b71 100755 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -934,12 +934,19 @@ entity_sub = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub def decode_entity(match): - what = match.group(1) + what = match.group(0) return unescape(what) def htmldecode(text): - """Decode HTML entities in the given text.""" + """ + Decode HTML entities in the given text. + + >>> htmldecode( + ... 'https://../package_name-0.1.2.tar.gz' + ... '?tokena=A&tokenb=B">package_name-0.1.2.tar.gz') + 'https://../package_name-0.1.2.tar.gz?tokena=A&tokenb=B">package_name-0.1.2.tar.gz' + """ return entity_sub(decode_entity, text) -- cgit v1.2.1 From 760e2e1df9c9c9d1fc072e7b6ad9df4c32bfc835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 27 Jul 2018 14:36:34 +0200 Subject: Remove spurious executable permissions --- setuptools/package_index.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 setuptools/package_index.py (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py old mode 100755 new mode 100644 -- cgit v1.2.1 From f325331100dc82916cba35b7310030fe6f337767 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 23 Sep 2018 11:35:24 -0400 Subject: Use preferred interface, fixing DeprecationWarning on later Pythons. --- setuptools/package_index.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index cda54b71..e650ac6f 100644 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -980,8 +980,7 @@ def _encode_auth(auth): auth_s = urllib.parse.unquote(auth) # convert to bytes auth_bytes = auth_s.encode() - # use the legacy interface for Python 2.3 support - encoded_bytes = base64.encodestring(auth_bytes) + encoded_bytes = base64.b64encode(auth_bytes) # convert back to a string encoded = encoded_bytes.decode() # strip the trailing carriage return -- cgit v1.2.1 From c2f72efd261bf89372dfa27b1c115012e74bd525 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 23 Sep 2018 12:44:23 -0400 Subject: Deprecate Subversion download functionality. Ref #1502. Used UserWarning instead of DeprecationWarning so it's visible to users who might be relying on this functionality. --- setuptools/package_index.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index e650ac6f..1608b91a 100644 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -7,6 +7,7 @@ import socket import base64 import hashlib import itertools +import warnings from functools import wraps from setuptools.extern import six @@ -848,6 +849,7 @@ class PackageIndex(Environment): raise DistutilsError("Unexpected HTML page found at " + url) def _download_svn(self, url, filename): + warnings.warn("SVN download support is deprecated", UserWarning) url = url.split('#', 1)[0] # remove any fragment for svn's sake creds = '' if url.lower().startswith('svn:') and '@' in url: -- cgit v1.2.1 From 3a351aa00e2110c3b3df0302b1ed6bcf66a1f8d8 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 23 Sep 2018 12:32:40 -0400 Subject: Remove use of splituser in packag_index.py. Fixes #1499. --- setuptools/package_index.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 1608b91a..7e9517ce 100644 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -850,13 +850,16 @@ class PackageIndex(Environment): def _download_svn(self, url, filename): warnings.warn("SVN download support is deprecated", UserWarning) + def splituser(host): + user, delim, host = host.rpartition('@') + return user, host url = url.split('#', 1)[0] # remove any fragment for svn's sake creds = '' if url.lower().startswith('svn:') and '@' in url: scheme, netloc, path, p, q, f = urllib.parse.urlparse(url) if not netloc and path.startswith('//') and '/' in path[2:]: netloc, path = path[2:].split('/', 1) - auth, host = urllib.parse.splituser(netloc) + auth, host = splituser(netloc) if auth: if ':' in auth: user, pw = auth.split(':', 1) @@ -1047,15 +1050,16 @@ class PyPIConfig(configparser.RawConfigParser): def open_with_auth(url, opener=urllib.request.urlopen): """Open a urllib2 request, handling HTTP authentication""" - scheme, netloc, path, params, query, frag = urllib.parse.urlparse(url) + parsed = urllib.parse.urlparse(url) + scheme, netloc, path, params, query, frag = parsed # Double scheme does not raise on Mac OS X as revealed by a # failing test. We would expect "nonnumeric port". Refs #20. if netloc.endswith(':'): raise http_client.InvalidURL("nonnumeric port: ''") - if scheme in ('http', 'https'): - auth, host = urllib.parse.splituser(netloc) + if scheme in ('http', 'https') and parsed.username: + auth = ':'.join((parsed.username, parsed.password)) else: auth = None @@ -1068,7 +1072,7 @@ def open_with_auth(url, opener=urllib.request.urlopen): if auth: auth = "Basic " + _encode_auth(auth) - parts = scheme, host, path, params, query, frag + parts = scheme, parsed.hostname, path, params, query, frag new_url = urllib.parse.urlunparse(parts) request = urllib.request.Request(new_url) request.add_header("Authorization", auth) @@ -1082,7 +1086,7 @@ def open_with_auth(url, opener=urllib.request.urlopen): # Put authentication info back into request URL if same host, # so that links found on the page will work s2, h2, path2, param2, query2, frag2 = urllib.parse.urlparse(fp.url) - if s2 == scheme and h2 == host: + if s2 == scheme and h2 == parsed.hostname: parts = s2, netloc, path2, param2, query2, frag2 fp.url = urllib.parse.urlunparse(parts) -- cgit v1.2.1 From 260bbe545e15ea75782c540c421da6c0a67abfd5 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Thu, 31 Jan 2019 20:46:48 -0500 Subject: Ensure a specified port in package_index isn't lost in the parse/unparse of the URL when auth is present. Fixes #1663. --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 7e9517ce..ea76c005 100644 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1072,7 +1072,7 @@ def open_with_auth(url, opener=urllib.request.urlopen): if auth: auth = "Basic " + _encode_auth(auth) - parts = scheme, parsed.hostname, path, params, query, frag + parts = scheme, netloc, path, params, query, frag new_url = urllib.parse.urlunparse(parts) request = urllib.request.Request(new_url) request.add_header("Authorization", auth) -- cgit v1.2.1 From 0830a69efde3d561c2025bdfa1fae10dcbbcc8ed Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 3 Feb 2019 09:22:19 -0500 Subject: Revert to using a copy of splituser from Python 3.8. Using urllib.parse.urlparse is clumsy and causes problems as reported in #1663 and #1668. Alternative to #1499 and fixes #1668. --- setuptools/package_index.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index ea76c005..05c9e341 100644 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -850,16 +850,13 @@ class PackageIndex(Environment): def _download_svn(self, url, filename): warnings.warn("SVN download support is deprecated", UserWarning) - def splituser(host): - user, delim, host = host.rpartition('@') - return user, host url = url.split('#', 1)[0] # remove any fragment for svn's sake creds = '' if url.lower().startswith('svn:') and '@' in url: scheme, netloc, path, p, q, f = urllib.parse.urlparse(url) if not netloc and path.startswith('//') and '/' in path[2:]: netloc, path = path[2:].split('/', 1) - auth, host = splituser(netloc) + auth, host = _splituser(netloc) if auth: if ':' in auth: user, pw = auth.split(':', 1) @@ -1058,8 +1055,8 @@ def open_with_auth(url, opener=urllib.request.urlopen): if netloc.endswith(':'): raise http_client.InvalidURL("nonnumeric port: ''") - if scheme in ('http', 'https') and parsed.username: - auth = ':'.join((parsed.username, parsed.password)) + if scheme in ('http', 'https'): + auth, address = _splituser(netloc) else: auth = None @@ -1072,7 +1069,7 @@ def open_with_auth(url, opener=urllib.request.urlopen): if auth: auth = "Basic " + _encode_auth(auth) - parts = scheme, netloc, path, params, query, frag + parts = scheme, address, path, params, query, frag new_url = urllib.parse.urlunparse(parts) request = urllib.request.Request(new_url) request.add_header("Authorization", auth) @@ -1093,6 +1090,13 @@ def open_with_auth(url, opener=urllib.request.urlopen): return fp +# copy of urllib.parse._splituser from Python 3.8 +def _splituser(host): + """splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'.""" + user, delim, host = host.rpartition('@') + return (user if delim else None), host + + # adding a timeout to avoid freezing package_index open_with_auth = socket_timeout(_SOCKET_TIMEOUT)(open_with_auth) -- cgit v1.2.1 From be840c2fe49766e3311475441b123a4eb3ba473a Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 3 Feb 2019 10:17:02 -0500 Subject: Also restore port consideration when re-injecting credentials for found links. --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 05c9e341..705a47cf 100644 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1083,7 +1083,7 @@ def open_with_auth(url, opener=urllib.request.urlopen): # Put authentication info back into request URL if same host, # so that links found on the page will work s2, h2, path2, param2, query2, frag2 = urllib.parse.urlparse(fp.url) - if s2 == scheme and h2 == parsed.hostname: + if s2 == scheme and h2 == address: parts = s2, netloc, path2, param2, query2, frag2 fp.url = urllib.parse.urlunparse(parts) -- cgit v1.2.1 From 59aeb62614ab07acb4b9520d81179d6e647dfbb7 Mon Sep 17 00:00:00 2001 From: 2xB <31772910+2xB@users.noreply.github.com> Date: Fri, 12 Apr 2019 00:32:12 +0200 Subject: FIX: git and hg revision checkout under Windows Windows does not change the working directory for a process via `cd .. && ` (see e.g. this question: https://stackoverflow.com/q/55641332/8575607 ). This commit replaces the use of `cd .. &&` with arguments provided by `git` and `hg` respectively. --- setuptools/package_index.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 705a47cf..6b06f2ca 100644 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -897,7 +897,7 @@ class PackageIndex(Environment): if rev is not None: self.info("Checking out %s", rev) - os.system("(cd %s && git checkout --quiet %s)" % ( + os.system("git -C %s checkout --quiet %s" % ( filename, rev, )) @@ -913,7 +913,7 @@ class PackageIndex(Environment): if rev is not None: self.info("Updating to %s", rev) - os.system("(cd %s && hg up -C -r %s -q)" % ( + os.system("hg --cwd %s up -C -r %s -q" % ( filename, rev, )) -- cgit v1.2.1 From 43add1d3f5138e38adc4940647cc6eae94fb6123 Mon Sep 17 00:00:00 2001 From: Anthony Sottile Date: Sat, 17 Aug 2019 19:14:48 -0700 Subject: Fixes for python3.10 --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 6b06f2ca..f419d471 100644 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -46,7 +46,7 @@ __all__ = [ _SOCKET_TIMEOUT = 15 _tmpl = "setuptools/{setuptools.__version__} Python-urllib/{py_major}" -user_agent = _tmpl.format(py_major=sys.version[:3], setuptools=setuptools) +user_agent = _tmpl.format(py_major='{}.{}'.format(*sys.version_info), setuptools=setuptools) def parse_requirement_arg(spec): -- cgit v1.2.1 From b2ab6f7c90b4daf5c05f08be230f329e8cb2f8e2 Mon Sep 17 00:00:00 2001 From: Andrew Taylor Date: Tue, 29 Oct 2019 17:03:11 -0600 Subject: Add info message when authentication error encountered processing package index. --- setuptools/package_index.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index f419d471..d9668e4e 100644 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -348,6 +348,8 @@ class PackageIndex(Environment): f = self.open_url(url, tmpl % url) if f is None: return + if isinstance(f, urllib.error.HTTPError) and f.code == 401: + self.info("Authentication error: %s" % f.msg) self.fetched_urls[f.url] = True if 'html' not in f.headers.get('content-type', '').lower(): f.close() # not html, we can't process it -- cgit v1.2.1 From 3d4d8b9dde61b87271861b8c7ebeb168ac4fa72b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 19 Jan 2020 12:46:30 -0500 Subject: =?UTF-8?q?=F0=9F=91=B9=20Feed=20the=20hobgoblins=20(delint).?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- setuptools/package_index.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index f419d471..82eb4516 100644 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -46,7 +46,8 @@ __all__ = [ _SOCKET_TIMEOUT = 15 _tmpl = "setuptools/{setuptools.__version__} Python-urllib/{py_major}" -user_agent = _tmpl.format(py_major='{}.{}'.format(*sys.version_info), setuptools=setuptools) +user_agent = _tmpl.format( + py_major='{}.{}'.format(*sys.version_info), setuptools=setuptools) def parse_requirement_arg(spec): @@ -1092,7 +1093,8 @@ def open_with_auth(url, opener=urllib.request.urlopen): # copy of urllib.parse._splituser from Python 3.8 def _splituser(host): - """splituser('user[:passwd]@host[:port]') --> 'user[:passwd]', 'host[:port]'.""" + """splituser('user[:passwd]@host[:port]') + --> 'user[:passwd]', 'host[:port]'.""" user, delim, host = host.rpartition('@') return (user if delim else None), host -- cgit v1.2.1 From bafe5abd8c715fa18a06bfd62e685802a2a74ce8 Mon Sep 17 00:00:00 2001 From: Reece Dunham Date: Sat, 11 Apr 2020 18:11:31 +0000 Subject: change: Mac OS X -> macOS Signed-off-by: Reece Dunham --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 7a802413..0744ea2a 100644 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -1053,7 +1053,7 @@ def open_with_auth(url, opener=urllib.request.urlopen): parsed = urllib.parse.urlparse(url) scheme, netloc, path, params, query, frag = parsed - # Double scheme does not raise on Mac OS X as revealed by a + # Double scheme does not raise on macOS as revealed by a # failing test. We would expect "nonnumeric port". Refs #20. if netloc.endswith(':'): raise http_client.InvalidURL("nonnumeric port: ''") -- cgit v1.2.1 From 6588760710433951a1df48c298bdc49fa5bb7c6c Mon Sep 17 00:00:00 2001 From: Ram Rachum Date: Thu, 11 Jun 2020 22:09:29 +0300 Subject: Fix exception causes in package_index.py --- setuptools/package_index.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 0744ea2a..1702c7c6 100644 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -53,10 +53,10 @@ user_agent = _tmpl.format( def parse_requirement_arg(spec): try: return Requirement.parse(spec) - except ValueError: + except ValueError as e: raise DistutilsError( "Not a URL, existing file, or requirement spec: %r" % (spec,) - ) + ) from e def parse_bdist_wininst(name): @@ -772,7 +772,7 @@ class PackageIndex(Environment): if warning: self.warn(warning, msg) else: - raise DistutilsError('%s %s' % (url, msg)) + raise DistutilsError('%s %s' % (url, msg)) from v except urllib.error.HTTPError as v: return v except urllib.error.URLError as v: @@ -780,7 +780,7 @@ class PackageIndex(Environment): self.warn(warning, v.reason) else: raise DistutilsError("Download error for %s: %s" - % (url, v.reason)) + % (url, v.reason)) from v except http_client.BadStatusLine as v: if warning: self.warn(warning, v.line) @@ -789,13 +789,13 @@ class PackageIndex(Environment): '%s returned a bad status line. The server might be ' 'down, %s' % (url, v.line) - ) + ) from v except (http_client.HTTPException, socket.error) as v: if warning: self.warn(warning, v) else: raise DistutilsError("Download error for %s: %s" - % (url, v)) + % (url, v)) from v def _download_url(self, scheme, url, tmpdir): # Determine download filename -- cgit v1.2.1 From fb7ab81a3d080422687bad71f9ae9d36eeefbee2 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 16 Aug 2020 00:29:24 -0400 Subject: Remove Python 2 compatibility --- setuptools/package_index.py | 33 ++++++++++++++++----------------- 1 file changed, 16 insertions(+), 17 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 1702c7c6..3979b131 100644 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -2,17 +2,21 @@ import sys import os import re +import io import shutil import socket import base64 import hashlib import itertools import warnings +import configparser +import html +import http.client +import urllib.parse +import urllib.request +import urllib.error from functools import wraps -from setuptools.extern import six -from setuptools.extern.six.moves import urllib, http_client, configparser, map - import setuptools from pkg_resources import ( CHECKOUT_DIST, Distribution, BINARY_DIST, normalize_path, SOURCE_DIST, @@ -23,12 +27,8 @@ from setuptools import ssl_support from distutils import log from distutils.errors import DistutilsError from fnmatch import translate -from setuptools.py27compat import get_all_headers -from setuptools.py33compat import unescape from setuptools.wheel import Wheel -__metaclass__ = type - EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.+!]+)$') HREF = re.compile(r"""href\s*=\s*['"]?([^'"> ]+)""", re.I) PYPI_MD5 = re.compile( @@ -191,7 +191,7 @@ def unique_everseen(iterable, key=None): seen = set() seen_add = seen.add if key is None: - for element in six.moves.filterfalse(seen.__contains__, iterable): + for element in itertools.filterfalse(seen.__contains__, iterable): seen_add(element) yield element else: @@ -740,7 +740,7 @@ class PackageIndex(Environment): size = -1 if "content-length" in headers: # Some servers return multiple Content-Length headers :( - sizes = get_all_headers(headers, 'Content-Length') + sizes = headers.get_all('Content-Length') size = max(map(int, sizes)) self.reporthook(url, filename, blocknum, bs, size) with open(filename, 'wb') as tfp: @@ -767,7 +767,7 @@ class PackageIndex(Environment): return local_open(url) try: return open_with_auth(url, self.opener) - except (ValueError, http_client.InvalidURL) as v: + except (ValueError, http.client.InvalidURL) as v: msg = ' '.join([str(arg) for arg in v.args]) if warning: self.warn(warning, msg) @@ -781,7 +781,7 @@ class PackageIndex(Environment): else: raise DistutilsError("Download error for %s: %s" % (url, v.reason)) from v - except http_client.BadStatusLine as v: + except http.client.BadStatusLine as v: if warning: self.warn(warning, v.line) else: @@ -790,7 +790,7 @@ class PackageIndex(Environment): 'down, %s' % (url, v.line) ) from v - except (http_client.HTTPException, socket.error) as v: + except (http.client.HTTPException, socket.error) as v: if warning: self.warn(warning, v) else: @@ -940,7 +940,7 @@ entity_sub = re.compile(r'&(#(\d+|x[\da-fA-F]+)|[\w.:-]+);?').sub def decode_entity(match): what = match.group(0) - return unescape(what) + return html.unescape(what) def htmldecode(text): @@ -972,8 +972,7 @@ def socket_timeout(timeout=15): def _encode_auth(auth): """ - A function compatible with Python 2.3-3.3 that will encode - auth from a URL suitable for an HTTP header. + Encode auth from a URL suitable for an HTTP header. >>> str(_encode_auth('username%3Apassword')) 'dXNlcm5hbWU6cGFzc3dvcmQ=' @@ -1056,7 +1055,7 @@ def open_with_auth(url, opener=urllib.request.urlopen): # Double scheme does not raise on macOS as revealed by a # failing test. We would expect "nonnumeric port". Refs #20. if netloc.endswith(':'): - raise http_client.InvalidURL("nonnumeric port: ''") + raise http.client.InvalidURL("nonnumeric port: ''") if scheme in ('http', 'https'): auth, address = _splituser(netloc) @@ -1136,5 +1135,5 @@ def local_open(url): status, message, body = 404, "Path not found", "Not found" headers = {'content-type': 'text/html'} - body_stream = six.StringIO(body) + body_stream = io.StringIO(body) return urllib.error.HTTPError(url, status, message, headers, body_stream) -- cgit v1.2.1 From c225c4c0f64bf044f2f82693df097ad07f9c12bd Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Thu, 31 Dec 2020 18:04:46 +0100 Subject: Simplify `PackageIndex.process_index` --- setuptools/package_index.py | 68 ++++++++++++++++++++++++--------------------- 1 file changed, 36 insertions(+), 32 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 3979b131..713391af 100644 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -428,49 +428,53 @@ class PackageIndex(Environment): dist.precedence = SOURCE_DIST self.add(dist) + def _scan(self, link): + # Process a URL to see if it's for a package page + NO_MATCH_SENTINEL = None, None + if not link.startswith(self.index_url): + return NO_MATCH_SENTINEL + + parts = list(map( + urllib.parse.unquote, link[len(self.index_url):].split('/') + )) + if len(parts) != 2 or '#' in parts[1]: + return NO_MATCH_SENTINEL + + # it's a package page, sanitize and index it + pkg = safe_name(parts[0]) + ver = safe_version(parts[1]) + self.package_pages.setdefault(pkg.lower(), {})[link] = True + return to_filename(pkg), to_filename(ver) + def process_index(self, url, page): """Process the contents of a PyPI page""" - def scan(link): - # Process a URL to see if it's for a package page - if link.startswith(self.index_url): - parts = list(map( - urllib.parse.unquote, link[len(self.index_url):].split('/') - )) - if len(parts) == 2 and '#' not in parts[1]: - # it's a package page, sanitize and index it - pkg = safe_name(parts[0]) - ver = safe_version(parts[1]) - self.package_pages.setdefault(pkg.lower(), {})[link] = True - return to_filename(pkg), to_filename(ver) - return None, None - # process an index page into the package-page index for match in HREF.finditer(page): try: - scan(urllib.parse.urljoin(url, htmldecode(match.group(1)))) + self._scan(urllib.parse.urljoin(url, htmldecode(match.group(1)))) except ValueError: pass - pkg, ver = scan(url) # ensure this page is in the page index - if pkg: - # process individual package page - for new_url in find_external_links(url, page): - # Process the found URL - base, frag = egg_info_for_url(new_url) - if base.endswith('.py') and not frag: - if ver: - new_url += '#egg=%s-%s' % (pkg, ver) - else: - self.need_version_info(url) - self.scan_url(new_url) - - return PYPI_MD5.sub( - lambda m: '%s' % m.group(1, 3, 2), page - ) - else: + pkg, ver = self._scan(url) # ensure this page is in the page index + if not pkg: return "" # no sense double-scanning non-package pages + # process individual package page + for new_url in find_external_links(url, page): + # Process the found URL + base, frag = egg_info_for_url(new_url) + if base.endswith('.py') and not frag: + if ver: + new_url += '#egg=%s-%s' % (pkg, ver) + else: + self.need_version_info(url) + self.scan_url(new_url) + + return PYPI_MD5.sub( + lambda m: '%s' % m.group(1, 3, 2), page + ) + def need_version_info(self, url): self.scan_all( "Page at %s links to .py file(s) without version info; an index " -- cgit v1.2.1 From fc891f5cf6d93ad533e2afb5e15a2952408ab358 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Thu, 31 Dec 2020 13:08:18 +0100 Subject: Apply noqa C901 comments to overly complex code --- setuptools/package_index.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 713391af..123e9582 100644 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -320,7 +320,8 @@ class PackageIndex(Environment): else: self.opener = urllib.request.urlopen - def process_url(self, url, retrieve=False): + # FIXME: 'PackageIndex.process_url' is too complex (14) + def process_url(self, url, retrieve=False): # noqa: C901 """Evaluate a URL as a possible download, and maybe retrieve it""" if url in self.scanned_urls and not retrieve: return @@ -595,7 +596,7 @@ class PackageIndex(Environment): spec = parse_requirement_arg(spec) return getattr(self.fetch_distribution(spec, tmpdir), 'location', None) - def fetch_distribution( + def fetch_distribution( # noqa: C901 # is too complex (14) # FIXME self, requirement, tmpdir, force_scan=False, source=False, develop_ok=False, local_index=None): """Obtain a distribution suitable for fulfilling `requirement` @@ -766,7 +767,8 @@ class PackageIndex(Environment): def reporthook(self, url, filename, blocknum, blksize, size): pass # no-op - def open_url(self, url, warning=None): + # FIXME: + def open_url(self, url, warning=None): # noqa: C901 # is too complex (12) if url.startswith('file:'): return local_open(url) try: -- cgit v1.2.1 From c063b3af1ecbcea0f0cc63b941f5e2038be9efcb Mon Sep 17 00:00:00 2001 From: luz paz Date: Fri, 14 May 2021 08:34:17 -0400 Subject: Fix misc. doc typos Found via `codespell` --- setuptools/package_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 123e9582..8bec05e7 100644 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -161,7 +161,7 @@ def interpret_distro_name( # Generate alternative interpretations of a source distro name # Because some packages are ambiguous as to name/versions split # e.g. "adns-python-1.1.0", "egenix-mx-commercial", etc. - # So, we generate each possible interepretation (e.g. "adns, python-1.1.0" + # So, we generate each possible interpretation (e.g. "adns, python-1.1.0" # "adns-python, 1.1.0", and "adns-python-1.1.0, no version"). In practice, # the spurious interpretations should be ignored, because in the event # there's also an "adns" package, the spurious "python-1.1.0" version will -- cgit v1.2.1 From 9c2cf25a13bf33a3fd706c97064c0d2fa22be179 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 22 May 2021 19:19:05 -0400 Subject: Use unique_everseen from more_itertools. --- setuptools/package_index.py | 21 ++------------------- 1 file changed, 2 insertions(+), 19 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index 8bec05e7..d1f13378 100644 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -28,6 +28,8 @@ from distutils import log from distutils.errors import DistutilsError from fnmatch import translate from setuptools.wheel import Wheel +from setuptools.extern.more_itertools import unique_everseen + EGG_FRAGMENT = re.compile(r'^egg=([-A-Za-z0-9_.+!]+)$') HREF = re.compile(r"""href\s*=\s*['"]?([^'"> ]+)""", re.I) @@ -183,25 +185,6 @@ def interpret_distro_name( ) -# From Python 2.7 docs -def unique_everseen(iterable, key=None): - "List unique elements, preserving order. Remember all elements ever seen." - # unique_everseen('AAAABBBCCDAABBB') --> A B C D - # unique_everseen('ABBCcAD', str.lower) --> A B C D - seen = set() - seen_add = seen.add - if key is None: - for element in itertools.filterfalse(seen.__contains__, iterable): - seen_add(element) - yield element - else: - for element in iterable: - k = key(element) - if k not in seen: - seen_add(k) - yield element - - def unique_values(func): """ Wrap a function returning an iterable such that the resulting iterable -- cgit v1.2.1 From ff0a836838adb64de7c9c50679f9213d397c867a Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 4 Jul 2021 16:54:24 -0400 Subject: Remove ssl_support. Fixes #2715. --- setuptools/package_index.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) (limited to 'setuptools/package_index.py') diff --git a/setuptools/package_index.py b/setuptools/package_index.py index d1f13378..d818f44a 100644 --- a/setuptools/package_index.py +++ b/setuptools/package_index.py @@ -23,7 +23,6 @@ from pkg_resources import ( Environment, find_distributions, safe_name, safe_version, to_filename, Requirement, DEVELOP_DIST, EGG_DIST, ) -from setuptools import ssl_support from distutils import log from distutils.errors import DistutilsError from fnmatch import translate @@ -293,15 +292,7 @@ class PackageIndex(Environment): self.package_pages = {} self.allows = re.compile('|'.join(map(translate, hosts))).match self.to_scan = [] - use_ssl = ( - verify_ssl - and ssl_support.is_available - and (ca_bundle or ssl_support.find_ca_bundle()) - ) - if use_ssl: - self.opener = ssl_support.opener_for(ca_bundle) - else: - self.opener = urllib.request.urlopen + self.opener = urllib.request.urlopen # FIXME: 'PackageIndex.process_url' is too complex (14) def process_url(self, url, retrieve=False): # noqa: C901 -- cgit v1.2.1