From 43e707381ef328a5e7cb251463edb391aec5fd32 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Sun, 12 Jun 2005 01:12:34 +0000 Subject: Split setup-running and archive-extraction utilities into separate modules, for easy use by tools other than EasyInstall. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041049 --- setuptools/archive_util.py | 164 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 164 insertions(+) create mode 100755 setuptools/archive_util.py (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py new file mode 100755 index 00000000..f39d073e --- /dev/null +++ b/setuptools/archive_util.py @@ -0,0 +1,164 @@ +"""Utilities for extracting common archive formats""" + + +__all__ = [ + "unpack_archive", "unpack_zipfile", "unpack_tarfile", "default_filter", + "UnrecognizedFormat", "extraction_drivers" +] + +import zipfile, tarfile, os +from pkg_resources import ensure_directory + +class UnrecognizedFormat(RuntimeError): + """Couldn't recognize the archive type""" + +def default_filter(src,dst): + """The default progress/filter callback; returns True for all files""" + + return True + + + + + + + + + + + + + + + + + + + + + + + +def unpack_archive(filename, extract_dir, progress_filter=default_filter, + drivers=None +): + """Unpack `filename` to `extract_dir`, or raise ``UnrecognizedFormat`` + + `progress_filter` is a function taking two arguments: a source path + internal to the archive ('/'-separated), and a filesystem path where it + will be extracted. The callback must return a true value, or else that + file or directory will be skipped. The callback can thus be used to + report on the progress of the extraction, as well as to filter the items + extracted. + + `drivers`, if supplied, must be a non-empty sequence of functions with the + same signature as this function (minus the `drivers` argument), that raise + ``UnrecognizedFormat`` if they do not support extracting the designated + archive type. The `drivers` are tried in sequence until one is found that + does not raise an error, or until all are exhausted (in which case + ``UnrecognizedFormat`` is raised). If you do not supply a sequence of + drivers, the module's ``extraction_drivers`` constant will be used, which + means that ``unpack_zipfile`` and ``unpack_tarfile`` will be tried, in that + order. + """ + for driver in drivers or extraction_drivers: + try: + driver(filename, extract_dir, progress_filter) + except UnrecognizedFormat: + continue + else: + return + else: + raise UnrecognizedFormat( + "Not a recognized archive type: %s" % filename + ) + + + + + + + + +def unpack_zipfile(filename, extract_dir, progress_filter=default_filter): + """Unpack zip `filename` to `extract_dir` + + Raises ``UnrecognizedFormat`` if `filename` is not a zipfile (as determined + by ``zipfile.is_zipfile()``). See ``unpack_archive()`` for an explanation + of the `progress_filter` argument. + """ + + if not zipfile.is_zipfile(filename): + raise UnrecognizedFormat("%s is not a zip file" % (filename,)) + + z = zipfile.ZipFile(filename) + try: + for info in z.infolist(): + name = info.filename + + # don't extract absolute paths or ones with .. in them + if name.startswith('/') or '..' in name: + continue + + target = os.path.join(extract_dir,name) + if not progress_filter(name,target): + continue + if name.endswith('/'): + # directory + ensure_directory(target) + else: + # file + ensure_directory(target) + data = z.read(info.filename) + f = open(target,'wb') + try: + f.write(data) + finally: + f.close() + del data + finally: + z.close() + + + +def unpack_tarfile(filename, extract_dir, progress_filter=default_filter): + """Unpack tar/tar.gz/tar.bz2 `filename` to `extract_dir` + + Raises ``UnrecognizedFormat`` if `filename` is not a tarfile (as determined + by ``tarfile.open()``). See ``unpack_archive()`` for an explanation + of the `progress_filter` argument. + """ + + try: + tarobj = tarfile.open(filename) + except tarfile.TarError: + raise UnrecognizedFormat( + "%s is not a compressed or uncompressed tar file" % (filename,) + ) + + try: + tarobj.chown = lambda *args: None # don't do any chowning! + for member in tarobj: + if member.isfile() or member.isdir(): + name = member.name + # don't extract absolute paths or ones with .. in them + if not name.startswith('/') and '..' not in name: + dst = os.path.join(extract_dir, *name.split('/')) + if progress_filter(name, dst): + tarobj.extract(member,extract_dir) + return True + finally: + tarobj.close() + + + + +extraction_drivers = unpack_zipfile, unpack_tarfile + + + + + + + + -- cgit v1.2.1 From 0398b84fb305563ce2506c626f87c8a2ff56a82c Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Wed, 15 Jun 2005 02:11:09 +0000 Subject: Enhance unpack_* utilities to allow on-the-fly redirection of where files are extracted to. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041066 --- setuptools/archive_util.py | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index f39d073e..4def0a70 100755 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -13,9 +13,9 @@ class UnrecognizedFormat(RuntimeError): """Couldn't recognize the archive type""" def default_filter(src,dst): - """The default progress/filter callback; returns True for all files""" - - return True + """The default progress/filter callback; returns True for all files""" + return dst + @@ -46,10 +46,11 @@ def unpack_archive(filename, extract_dir, progress_filter=default_filter, `progress_filter` is a function taking two arguments: a source path internal to the archive ('/'-separated), and a filesystem path where it - will be extracted. The callback must return a true value, or else that - file or directory will be skipped. The callback can thus be used to - report on the progress of the extraction, as well as to filter the items - extracted. + will be extracted. The callback must return the desired extract path + (which may be the same as the one passed in), or else ``None`` to skip + that file or directory. The callback can thus be used to report on the + progress of the extraction, as well as to filter the items extracted or + alter their extraction paths. `drivers`, if supplied, must be a non-empty sequence of functions with the same signature as this function (minus the `drivers` argument), that raise @@ -79,7 +80,6 @@ def unpack_archive(filename, extract_dir, progress_filter=default_filter, - def unpack_zipfile(filename, extract_dir, progress_filter=default_filter): """Unpack zip `filename` to `extract_dir` @@ -100,8 +100,9 @@ def unpack_zipfile(filename, extract_dir, progress_filter=default_filter): if name.startswith('/') or '..' in name: continue - target = os.path.join(extract_dir,name) - if not progress_filter(name,target): + target = os.path.join(extract_dir, *name.split('/')) + target = progress_filter(name, target) + if not target: continue if name.endswith('/'): # directory @@ -120,7 +121,6 @@ def unpack_zipfile(filename, extract_dir, progress_filter=default_filter): z.close() - def unpack_tarfile(filename, extract_dir, progress_filter=default_filter): """Unpack tar/tar.gz/tar.bz2 `filename` to `extract_dir` @@ -144,8 +144,11 @@ def unpack_tarfile(filename, extract_dir, progress_filter=default_filter): # don't extract absolute paths or ones with .. in them if not name.startswith('/') and '..' not in name: dst = os.path.join(extract_dir, *name.split('/')) - if progress_filter(name, dst): - tarobj.extract(member,extract_dir) + dst = progress_filter(name, dst) + if dst: + if dst.endswith(os.sep): + dst = dst[:-1] + tarobj._extract_member(member,dst) # XXX Ugh return True finally: tarobj.close() @@ -159,6 +162,3 @@ extraction_drivers = unpack_zipfile, unpack_tarfile - - - -- cgit v1.2.1 From 451377d0e877fc610d1bdf8181ba70a90e4c14cc Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Sun, 10 Jul 2005 04:49:31 +0000 Subject: Detect and handle conflicts with "unmanaged" packages when installing packages managed by EasyInstall. Also, add an option to exclude source files from .egg distributions. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041109 --- setuptools/archive_util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index 4def0a70..d24c6c13 100755 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -8,8 +8,9 @@ __all__ = [ import zipfile, tarfile, os from pkg_resources import ensure_directory +from distutils.errors import DistutilsError -class UnrecognizedFormat(RuntimeError): +class UnrecognizedFormat(DistutilsError): """Couldn't recognize the archive type""" def default_filter(src,dst): @@ -34,7 +35,6 @@ def default_filter(src,dst): - -- cgit v1.2.1 From 45885095198f5ec1d7e80791a73c7385f2620b38 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Tue, 6 Dec 2005 03:12:48 +0000 Subject: Added an ``unpack_directory()`` driver to ``setuptools.archive_util``, so that you can process a directory tree through a processing filter as if it were a zipfile or tarfile. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4041617 --- setuptools/archive_util.py | 47 +++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 3 deletions(-) (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index d24c6c13..4cc80b60 100755 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -3,10 +3,10 @@ __all__ = [ "unpack_archive", "unpack_zipfile", "unpack_tarfile", "default_filter", - "UnrecognizedFormat", "extraction_drivers" + "UnrecognizedFormat", "extraction_drivers", "unpack_directory", ] -import zipfile, tarfile, os +import zipfile, tarfile, os, shutil from pkg_resources import ensure_directory from distutils.errors import DistutilsError @@ -80,6 +80,47 @@ def unpack_archive(filename, extract_dir, progress_filter=default_filter, +def unpack_directory(filename, extract_dir, progress_filter=default_filter): + """"Unpack" a directory, using the same interface as for archives + + Raises ``UnrecognizedFormat`` if `filename` is not a directory + """ + if not os.path.isdir(filename): + raise UnrecognizedFormat("%s is not a directory" % (filename,)) + + paths = {filename:('',extract_dir)} + for base, dirs, files in os.walk(filename): + src,dst = paths[base] + for d in dirs: + paths[os.path.join(base,d)] = src+d+'/', os.path.join(dst,d) + for f in files: + name = src+f + target = os.path.join(dst,f) + target = progress_filter(src+f, target) + if not target: + continue # skip non-files + ensure_directory(target) + shutil.copyfile(os.path.join(base,f), target) + + + + + + + + + + + + + + + + + + + + def unpack_zipfile(filename, extract_dir, progress_filter=default_filter): """Unpack zip `filename` to `extract_dir` @@ -156,7 +197,7 @@ def unpack_tarfile(filename, extract_dir, progress_filter=default_filter): -extraction_drivers = unpack_zipfile, unpack_tarfile +extraction_drivers = unpack_directory, unpack_zipfile, unpack_tarfile -- cgit v1.2.1 From 0b0d41cde67414ab19b030ea48b497e17134be19 Mon Sep 17 00:00:00 2001 From: PJ Eby Date: Fri, 17 Mar 2006 15:52:05 +0000 Subject: Preserve timestamps and permissions when "unpacking" (copying) a directory tree. --HG-- branch : setuptools extra : convert_revision : svn%3A6015fed2-1504-0410-9fe1-9d1591cc4771/sandbox/trunk/setuptools%4043114 --- setuptools/archive_util.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index 4cc80b60..511f05ad 100755 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -100,9 +100,9 @@ def unpack_directory(filename, extract_dir, progress_filter=default_filter): if not target: continue # skip non-files ensure_directory(target) - shutil.copyfile(os.path.join(base,f), target) - - + f = os.path.join(base,f) + shutil.copyfile(f, target) + shutil.copystat(f, target) -- cgit v1.2.1 From b83d5bd69240600575bc086100d9bab8ca854073 Mon Sep 17 00:00:00 2001 From: Hanno Schlichting Date: Thu, 16 Jul 2009 18:54:30 +0200 Subject: Protected against failures in tarfile extraction, as we use a private method which can raise those. Patch as per PJE's setuptools trunk commit r65946. This closes #10. --HG-- branch : distribute extra : rebase_source : c621c9d165d820c32bc08d9a9a168074842e3d66 --- setuptools/archive_util.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index 511f05ad..ab786f3d 100755 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -189,7 +189,10 @@ def unpack_tarfile(filename, extract_dir, progress_filter=default_filter): if dst: if dst.endswith(os.sep): dst = dst[:-1] - tarobj._extract_member(member,dst) # XXX Ugh + try: + tarobj._extract_member(member,dst) # XXX Ugh + except tarfile.ExtractError: + pass # chown/chmod/mkfifo/mknode/makedev failed return True finally: tarobj.close() -- cgit v1.2.1 From 39f910e2e4b68cf11ecd2cf12c697aacce451d3d Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Fri, 23 Mar 2012 13:50:04 -0400 Subject: Include symlinks when extracting source dist. Fixes #183. --HG-- branch : distribute extra : rebase_source : c0d44c559d3433e4e4ceddaff5467c2d82dd7688 --- setuptools/archive_util.py | 29 ++++++++++++++++------------- 1 file changed, 16 insertions(+), 13 deletions(-) (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index ab786f3d..5787753f 100755 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -180,19 +180,22 @@ def unpack_tarfile(filename, extract_dir, progress_filter=default_filter): try: tarobj.chown = lambda *args: None # don't do any chowning! for member in tarobj: - if member.isfile() or member.isdir(): - name = member.name - # don't extract absolute paths or ones with .. in them - if not name.startswith('/') and '..' not in name: - dst = os.path.join(extract_dir, *name.split('/')) - dst = progress_filter(name, dst) - if dst: - if dst.endswith(os.sep): - dst = dst[:-1] - try: - tarobj._extract_member(member,dst) # XXX Ugh - except tarfile.ExtractError: - pass # chown/chmod/mkfifo/mknode/makedev failed + name = member.name + # don't extract absolute paths or ones with .. in them + if not name.startswith('/') and '..' not in name: + prelim_dst = os.path.join(extract_dir, *name.split('/')) + final_dst = progress_filter(name, prelim_dst) + # If progress_filter returns None, then we do not extract + # this file + # TODO: Do we really need to limit to just these file types? + # tarobj.extract() will handle all files on all platforms, + # turning file types that aren't allowed on that platform into + # regular files. + if final_dst and (member.isfile() or member.isdir() or + member.islnk() or member.issym()): + tarobj.extract(member, extract_dir) + if final_dst != prelim_dst: + shutil.move(prelim_dst, final_dst) return True finally: tarobj.close() -- cgit v1.2.1 From ad445467dd7ddd5e69165fa38b83d303c096f132 Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Fri, 10 Aug 2012 12:49:20 +0100 Subject: Set permissions when extracting from zipfile. --HG-- branch : distribute extra : rebase_source : 68b7fd4d6cc28f77a7202c5e7f63121d721be624 --- setuptools/archive_util.py | 1 + 1 file changed, 1 insertion(+) (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index 5787753f..8ad14752 100755 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -158,6 +158,7 @@ def unpack_zipfile(filename, extract_dir, progress_filter=default_filter): finally: f.close() del data + os.chmod(target, info.external_attr >> 16) finally: z.close() -- cgit v1.2.1 From 6cfedcaaf8d8e65a0686b2d856d2eb95fd6c85bf Mon Sep 17 00:00:00 2001 From: Jonathan Lange Date: Wed, 19 Sep 2012 11:17:36 +0100 Subject: Guard the chmod in case external_attr is 0. --HG-- branch : distribute extra : rebase_source : 79d8c285c70c3cf44273439c5178cfa54b0c3b21 --- setuptools/archive_util.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index 8ad14752..e22b25c0 100755 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -158,7 +158,9 @@ def unpack_zipfile(filename, extract_dir, progress_filter=default_filter): finally: f.close() del data - os.chmod(target, info.external_attr >> 16) + unix_attributes = info.external_attr >> 16 + if unix_attributes: + os.chmod(target, unix_attributes) finally: z.close() -- cgit v1.2.1 From b03a9e293b94b338027c4983c2ac3764692dd297 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 3 May 2013 07:59:45 -0400 Subject: Copy changes to setuptools/archive_util.py setuptools/depends.py setuptools/extension.py and setuptools/__init__.py from 1aae1efe5733 --HG-- branch : Setuptools-Distribute merge extra : source : a3f8891a67625e751eacbf027d66feff19779da4 --- setuptools/archive_util.py | 36 ++++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index 511f05ad..d44264f8 100755 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -6,7 +6,7 @@ __all__ = [ "UnrecognizedFormat", "extraction_drivers", "unpack_directory", ] -import zipfile, tarfile, os, shutil +import zipfile, tarfile, os, shutil, posixpath from pkg_resources import ensure_directory from distutils.errors import DistutilsError @@ -138,7 +138,7 @@ def unpack_zipfile(filename, extract_dir, progress_filter=default_filter): name = info.filename # don't extract absolute paths or ones with .. in them - if name.startswith('/') or '..' in name: + if name.startswith('/') or '..' in name.split('/'): continue target = os.path.join(extract_dir, *name.split('/')) @@ -169,37 +169,37 @@ def unpack_tarfile(filename, extract_dir, progress_filter=default_filter): by ``tarfile.open()``). See ``unpack_archive()`` for an explanation of the `progress_filter` argument. """ - try: tarobj = tarfile.open(filename) except tarfile.TarError: raise UnrecognizedFormat( "%s is not a compressed or uncompressed tar file" % (filename,) ) - try: tarobj.chown = lambda *args: None # don't do any chowning! for member in tarobj: - if member.isfile() or member.isdir(): - name = member.name - # don't extract absolute paths or ones with .. in them - if not name.startswith('/') and '..' not in name: - dst = os.path.join(extract_dir, *name.split('/')) + name = member.name + # don't extract absolute paths or ones with .. in them + if not name.startswith('/') and '..' not in name.split('/'): + dst = os.path.join(extract_dir, *name.split('/')) + while member is not None and (member.islnk() or member.issym()): + linkpath = member.linkname + if member.issym(): + linkpath = posixpath.join(posixpath.dirname(member.name), linkpath) + linkpath = posixpath.normpath(linkpath) + member = tarobj._getmember(linkpath) + + if member is not None and (member.isfile() or member.isdir()): dst = progress_filter(name, dst) if dst: if dst.endswith(os.sep): dst = dst[:-1] - tarobj._extract_member(member,dst) # XXX Ugh + try: + tarobj._extract_member(member,dst) # XXX Ugh + except tarfile.ExtractError: + pass # chown/chmod/mkfifo/mknode/makedev failed return True finally: tarobj.close() - - - extraction_drivers = unpack_directory, unpack_zipfile, unpack_tarfile - - - - - -- cgit v1.2.1 From 14f168f249c86085dbf9f67fdc8d6a5b801946a0 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 17 May 2014 22:58:50 -0400 Subject: Clean up whitespace --- setuptools/archive_util.py | 54 ++++++---------------------------------------- 1 file changed, 7 insertions(+), 47 deletions(-) (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index 1109f346..a5be2f98 100755 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -6,7 +6,11 @@ __all__ = [ "UnrecognizedFormat", "extraction_drivers", "unpack_directory", ] -import zipfile, tarfile, os, shutil, posixpath +import zipfile +import tarfile +import os +import shutil +import posixpath from pkg_resources import ensure_directory from distutils.errors import DistutilsError @@ -14,34 +18,12 @@ class UnrecognizedFormat(DistutilsError): """Couldn't recognize the archive type""" def default_filter(src,dst): - """The default progress/filter callback; returns True for all files""" + """The default progress/filter callback; returns True for all files""" return dst - - - - - - - - - - - - - - - - - - - - - def unpack_archive(filename, extract_dir, progress_filter=default_filter, - drivers=None -): + drivers=None): """Unpack `filename` to `extract_dir`, or raise ``UnrecognizedFormat`` `progress_filter` is a function taking two arguments: a source path @@ -75,11 +57,6 @@ def unpack_archive(filename, extract_dir, progress_filter=default_filter, ) - - - - - def unpack_directory(filename, extract_dir, progress_filter=default_filter): """"Unpack" a directory, using the same interface as for archives @@ -94,7 +71,6 @@ def unpack_directory(filename, extract_dir, progress_filter=default_filter): for d in dirs: paths[os.path.join(base,d)] = src+d+'/', os.path.join(dst,d) for f in files: - name = src+f target = os.path.join(dst,f) target = progress_filter(src+f, target) if not target: @@ -105,22 +81,6 @@ def unpack_directory(filename, extract_dir, progress_filter=default_filter): shutil.copystat(f, target) - - - - - - - - - - - - - - - - def unpack_zipfile(filename, extract_dir, progress_filter=default_filter): """Unpack zip `filename` to `extract_dir` -- cgit v1.2.1 From 0f7f8d71dc1c8eda869c423a324064d4bc419879 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 17 May 2014 23:09:56 -0400 Subject: Use ContextualZipFile and contextlib.closing for archiveutil --- setuptools/archive_util.py | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index a5be2f98..67a67e23 100755 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -11,7 +11,8 @@ import tarfile import os import shutil import posixpath -from pkg_resources import ensure_directory +import contextlib +from pkg_resources import ensure_directory, ContextualZipFile from distutils.errors import DistutilsError class UnrecognizedFormat(DistutilsError): @@ -92,8 +93,7 @@ def unpack_zipfile(filename, extract_dir, progress_filter=default_filter): if not zipfile.is_zipfile(filename): raise UnrecognizedFormat("%s is not a zip file" % (filename,)) - z = zipfile.ZipFile(filename) - try: + with ContextualZipFile(filename) as z: for info in z.infolist(): name = info.filename @@ -121,8 +121,6 @@ def unpack_zipfile(filename, extract_dir, progress_filter=default_filter): unix_attributes = info.external_attr >> 16 if unix_attributes: os.chmod(target, unix_attributes) - finally: - z.close() def unpack_tarfile(filename, extract_dir, progress_filter=default_filter): @@ -138,7 +136,7 @@ def unpack_tarfile(filename, extract_dir, progress_filter=default_filter): raise UnrecognizedFormat( "%s is not a compressed or uncompressed tar file" % (filename,) ) - try: + with contextlib.closing(tarobj): tarobj.chown = lambda *args: None # don't do any chowning! for member in tarobj: name = member.name @@ -164,7 +162,5 @@ def unpack_tarfile(filename, extract_dir, progress_filter=default_filter): except tarfile.ExtractError: pass # chown/chmod/mkfifo/mknode/makedev failed return True - finally: - tarobj.close() extraction_drivers = unpack_directory, unpack_zipfile, unpack_tarfile -- cgit v1.2.1 From ec6e6fb1d2078c619e85fa2eb6290f0cee813e2a Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 31 Dec 2014 14:22:53 -0500 Subject: Normalize indentation and whitespace --- setuptools/archive_util.py | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index 67a67e23..2cd5b34e 100755 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -64,20 +64,23 @@ def unpack_directory(filename, extract_dir, progress_filter=default_filter): Raises ``UnrecognizedFormat`` if `filename` is not a directory """ if not os.path.isdir(filename): - raise UnrecognizedFormat("%s is not a directory" % (filename,)) + raise UnrecognizedFormat("%s is not a directory" % filename) - paths = {filename:('',extract_dir)} + paths = { + filename: ('', extract_dir), + } for base, dirs, files in os.walk(filename): - src,dst = paths[base] + src, dst = paths[base] for d in dirs: - paths[os.path.join(base,d)] = src+d+'/', os.path.join(dst,d) + paths[os.path.join(base, d)] = src + d + '/', os.path.join(dst, d) for f in files: - target = os.path.join(dst,f) - target = progress_filter(src+f, target) + target = os.path.join(dst, f) + target = progress_filter(src + f, target) if not target: - continue # skip non-files + # skip non-files + continue ensure_directory(target) - f = os.path.join(base,f) + f = os.path.join(base, f) shutil.copyfile(f, target) shutil.copystat(f, target) @@ -112,7 +115,7 @@ def unpack_zipfile(filename, extract_dir, progress_filter=default_filter): # file ensure_directory(target) data = z.read(info.filename) - f = open(target,'wb') + f = open(target, 'wb') try: f.write(data) finally: @@ -137,18 +140,21 @@ def unpack_tarfile(filename, extract_dir, progress_filter=default_filter): "%s is not a compressed or uncompressed tar file" % (filename,) ) with contextlib.closing(tarobj): - tarobj.chown = lambda *args: None # don't do any chowning! + # don't do any chowning! + tarobj.chown = lambda *args: None for member in tarobj: name = member.name # don't extract absolute paths or ones with .. in them if not name.startswith('/') and '..' not in name.split('/'): prelim_dst = os.path.join(extract_dir, *name.split('/')) - # resolve any links and to extract the link targets as normal files + # resolve any links and to extract the link targets as normal + # files while member is not None and (member.islnk() or member.issym()): linkpath = member.linkname if member.issym(): - linkpath = posixpath.join(posixpath.dirname(member.name), linkpath) + base = posixpath.dirname(member.name) + linkpath = posixpath.join(base, linkpath) linkpath = posixpath.normpath(linkpath) member = tarobj._getmember(linkpath) @@ -158,9 +164,11 @@ def unpack_tarfile(filename, extract_dir, progress_filter=default_filter): if final_dst.endswith(os.sep): final_dst = final_dst[:-1] try: - tarobj._extract_member(member, final_dst) # XXX Ugh + # XXX Ugh + tarobj._extract_member(member, final_dst) except tarfile.ExtractError: - pass # chown/chmod/mkfifo/mknode/makedev failed + # chown/chmod/mkfifo/mknode/makedev failed + pass return True extraction_drivers = unpack_directory, unpack_zipfile, unpack_tarfile -- cgit v1.2.1 From 86a0dc8419a3649389bd7d58a1f7c79bae0baad6 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Wed, 31 Dec 2014 14:29:26 -0500 Subject: Use simple context manager; don't bother deleting the variable. --- setuptools/archive_util.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index 2cd5b34e..b3c9fa56 100755 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -115,12 +115,8 @@ def unpack_zipfile(filename, extract_dir, progress_filter=default_filter): # file ensure_directory(target) data = z.read(info.filename) - f = open(target, 'wb') - try: + with open(target, 'wb') as f: f.write(data) - finally: - f.close() - del data unix_attributes = info.external_attr >> 16 if unix_attributes: os.chmod(target, unix_attributes) -- cgit v1.2.1 From 6d11e88f938f09ef16db4c6064b6e74acba4db1d Mon Sep 17 00:00:00 2001 From: stepshal Date: Tue, 12 Jul 2016 22:00:43 +0700 Subject: Fix quantity of blank lines after code object. --- setuptools/archive_util.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index b3c9fa56..4da24ec2 100755 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -15,9 +15,11 @@ import contextlib from pkg_resources import ensure_directory, ContextualZipFile from distutils.errors import DistutilsError + class UnrecognizedFormat(DistutilsError): """Couldn't recognize the archive type""" + def default_filter(src,dst): """The default progress/filter callback; returns True for all files""" return dst -- cgit v1.2.1 From f749ccab1e55723848946c9aba5c3eddebe0b24e Mon Sep 17 00:00:00 2001 From: stepshal Date: Thu, 14 Jul 2016 09:26:06 +0700 Subject: Add missing whitespace. --- setuptools/archive_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index 4da24ec2..d1950638 100755 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -20,7 +20,7 @@ class UnrecognizedFormat(DistutilsError): """Couldn't recognize the archive type""" -def default_filter(src,dst): +def default_filter(src, dst): """The default progress/filter callback; returns True for all files""" return dst -- cgit v1.2.1 From 01de794bc829cc9eb0c1512b3570acec970e1acf Mon Sep 17 00:00:00 2001 From: stepshal Date: Thu, 14 Jul 2016 21:45:22 +0700 Subject: Put imports in same block alphabeticaly. --- setuptools/archive_util.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index d1950638..3b41db15 100755 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -12,8 +12,8 @@ import os import shutil import posixpath import contextlib -from pkg_resources import ensure_directory, ContextualZipFile from distutils.errors import DistutilsError +from pkg_resources import ensure_directory, ContextualZipFile class UnrecognizedFormat(DistutilsError): -- cgit v1.2.1 From 39bf3155d47c0024240be414a611dcb6d549f53c Mon Sep 17 00:00:00 2001 From: stepshal Date: Thu, 21 Jul 2016 09:37:34 +0700 Subject: Add missing blank lines after class or function definition. --- setuptools/archive_util.py | 1 + 1 file changed, 1 insertion(+) (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index 3b41db15..24d4e7bb 100755 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -169,4 +169,5 @@ def unpack_tarfile(filename, extract_dir, progress_filter=default_filter): pass return True + extraction_drivers = unpack_directory, unpack_zipfile, unpack_tarfile -- cgit v1.2.1 From 315cd94d90349748016c842a3d7f851c631b76b7 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 1 Aug 2016 21:23:21 -0400 Subject: Reorganize imports --- setuptools/archive_util.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index 24d4e7bb..b6411cc5 100755 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -1,11 +1,5 @@ """Utilities for extracting common archive formats""" - -__all__ = [ - "unpack_archive", "unpack_zipfile", "unpack_tarfile", "default_filter", - "UnrecognizedFormat", "extraction_drivers", "unpack_directory", -] - import zipfile import tarfile import os @@ -13,9 +7,16 @@ import shutil import posixpath import contextlib from distutils.errors import DistutilsError + from pkg_resources import ensure_directory, ContextualZipFile +__all__ = [ + "unpack_archive", "unpack_zipfile", "unpack_tarfile", "default_filter", + "UnrecognizedFormat", "extraction_drivers", "unpack_directory", +] + + class UnrecognizedFormat(DistutilsError): """Couldn't recognize the archive type""" -- cgit v1.2.1 From 1b8dc8c7ac45662f7719b1bd531d0575e7fb1f02 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 1 Aug 2016 22:30:09 -0400 Subject: Fix UnicodeDecodeError when tarfile names have non-ascii characters. Fixes #709. --- setuptools/archive_util.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index b6411cc5..a1960be8 100755 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -143,6 +143,8 @@ def unpack_tarfile(filename, extract_dir, progress_filter=default_filter): tarobj.chown = lambda *args: None for member in tarobj: name = member.name + if isinstance(name, bytes): + name = name.decode(tarfile.ENCODING) # don't extract absolute paths or ones with .. in them if not name.startswith('/') and '..' not in name.split('/'): prelim_dst = os.path.join(extract_dir, *name.split('/')) -- cgit v1.2.1 From f9b2902f312834ca6676f0b79bedf845c2bc42c3 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Mon, 1 Aug 2016 23:02:27 -0400 Subject: _extract_member needs final_dst to be a native string. Ref #710. --- setuptools/archive_util.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index a1960be8..6493b448 100755 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -8,6 +8,8 @@ import posixpath import contextlib from distutils.errors import DistutilsError +from setuptools.extern import six + from pkg_resources import ensure_directory, ContextualZipFile @@ -164,6 +166,8 @@ def unpack_tarfile(filename, extract_dir, progress_filter=default_filter): if final_dst: if final_dst.endswith(os.sep): final_dst = final_dst[:-1] + if six.PY2: + final_dst = final_dst.encode(tarfile.ENCODING) try: # XXX Ugh tarobj._extract_member(member, final_dst) -- cgit v1.2.1 From d12b01679cb0e2e6a3ea1832891614c6ee86095c Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 2 Aug 2016 08:54:51 -0400 Subject: Backout 1b8dc8c7ac. Reopens #709. --- setuptools/archive_util.py | 2 -- 1 file changed, 2 deletions(-) (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index 6493b448..16355092 100755 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -145,8 +145,6 @@ def unpack_tarfile(filename, extract_dir, progress_filter=default_filter): tarobj.chown = lambda *args: None for member in tarobj: name = member.name - if isinstance(name, bytes): - name = name.decode(tarfile.ENCODING) # don't extract absolute paths or ones with .. in them if not name.startswith('/') and '..' not in name.split('/'): prelim_dst = os.path.join(extract_dir, *name.split('/')) -- cgit v1.2.1 From 27e455c506748ad119813f2f8a23d1cc13726100 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Tue, 2 Aug 2016 08:55:32 -0400 Subject: Backout f9b2902f312. Reopens #710. --- setuptools/archive_util.py | 4 ---- 1 file changed, 4 deletions(-) (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index 16355092..b6411cc5 100755 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -8,8 +8,6 @@ import posixpath import contextlib from distutils.errors import DistutilsError -from setuptools.extern import six - from pkg_resources import ensure_directory, ContextualZipFile @@ -164,8 +162,6 @@ def unpack_tarfile(filename, extract_dir, progress_filter=default_filter): if final_dst: if final_dst.endswith(os.sep): final_dst = final_dst[:-1] - if six.PY2: - final_dst = final_dst.encode(tarfile.ENCODING) try: # XXX Ugh tarobj._extract_member(member, final_dst) -- cgit v1.2.1 From 31bd37c6ac8de9e8c1bacebc2d8e1215df91eb96 Mon Sep 17 00:00:00 2001 From: stepshal Date: Tue, 18 Oct 2016 20:24:35 +0700 Subject: Fix quantity of blank lines. --- setuptools/archive_util.py | 1 - 1 file changed, 1 deletion(-) (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index b6411cc5..cc82b3da 100755 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -10,7 +10,6 @@ from distutils.errors import DistutilsError from pkg_resources import ensure_directory, ContextualZipFile - __all__ = [ "unpack_archive", "unpack_zipfile", "unpack_tarfile", "default_filter", "UnrecognizedFormat", "extraction_drivers", "unpack_directory", -- cgit v1.2.1 From f14930e66601b462699c44384c482cd966f53b8f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Fri, 9 Dec 2016 08:16:33 -0500 Subject: Drop support for Python 2.6, removing lots of compatibility code for a leaner, cleaner codebase. Fixes #878. --- setuptools/archive_util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index cc82b3da..81436044 100755 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -8,7 +8,7 @@ import posixpath import contextlib from distutils.errors import DistutilsError -from pkg_resources import ensure_directory, ContextualZipFile +from pkg_resources import ensure_directory __all__ = [ "unpack_archive", "unpack_zipfile", "unpack_tarfile", "default_filter", @@ -98,7 +98,7 @@ def unpack_zipfile(filename, extract_dir, progress_filter=default_filter): if not zipfile.is_zipfile(filename): raise UnrecognizedFormat("%s is not a zip file" % (filename,)) - with ContextualZipFile(filename) as z: + with zipfile.ZipFile(filename) as z: for info in z.infolist(): name = info.filename -- cgit v1.2.1 From 760e2e1df9c9c9d1fc072e7b6ad9df4c32bfc835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Miro=20Hron=C4=8Dok?= Date: Fri, 27 Jul 2018 14:36:34 +0200 Subject: Remove spurious executable permissions --- setuptools/archive_util.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 setuptools/archive_util.py (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py old mode 100755 new mode 100644 -- cgit v1.2.1 From 3d4d8b9dde61b87271861b8c7ebeb168ac4fa72b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 19 Jan 2020 12:46:30 -0500 Subject: =?UTF-8?q?=F0=9F=91=B9=20Feed=20the=20hobgoblins=20(delint).?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- setuptools/archive_util.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index 81436044..64528ca7 100644 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -25,7 +25,8 @@ def default_filter(src, dst): return dst -def unpack_archive(filename, extract_dir, progress_filter=default_filter, +def unpack_archive( + filename, extract_dir, progress_filter=default_filter, drivers=None): """Unpack `filename` to `extract_dir`, or raise ``UnrecognizedFormat`` @@ -148,7 +149,8 @@ def unpack_tarfile(filename, extract_dir, progress_filter=default_filter): # resolve any links and to extract the link targets as normal # files - while member is not None and (member.islnk() or member.issym()): + while member is not None and ( + member.islnk() or member.issym()): linkpath = member.linkname if member.issym(): base = posixpath.dirname(member.name) -- cgit v1.2.1 From a9eb9e73def8ca6c469e59f1b008746e368ad4c1 Mon Sep 17 00:00:00 2001 From: Ram Rachum Date: Tue, 16 Jun 2020 13:31:12 +0300 Subject: Fix exception causes all over the codebase --- setuptools/archive_util.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index 64528ca7..0ce190b8 100644 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -134,10 +134,10 @@ def unpack_tarfile(filename, extract_dir, progress_filter=default_filter): """ try: tarobj = tarfile.open(filename) - except tarfile.TarError: + except tarfile.TarError as e: raise UnrecognizedFormat( "%s is not a compressed or uncompressed tar file" % (filename,) - ) + ) from e with contextlib.closing(tarobj): # don't do any chowning! tarobj.chown = lambda *args: None -- cgit v1.2.1 From 02038f6272bd2fc04066480f7ba7d564ddb58769 Mon Sep 17 00:00:00 2001 From: Sviatoslav Sydorenko Date: Thu, 31 Dec 2020 13:09:59 +0100 Subject: Simplify `setuptools.archive_util.unpack_tarfile` --- setuptools/archive_util.py | 94 ++++++++++++++++++++++++++++++---------------- 1 file changed, 62 insertions(+), 32 deletions(-) (limited to 'setuptools/archive_util.py') diff --git a/setuptools/archive_util.py b/setuptools/archive_util.py index 0ce190b8..0f702848 100644 --- a/setuptools/archive_util.py +++ b/setuptools/archive_util.py @@ -125,6 +125,56 @@ def unpack_zipfile(filename, extract_dir, progress_filter=default_filter): os.chmod(target, unix_attributes) +def _resolve_tar_file_or_dir(tar_obj, tar_member_obj): + """Resolve any links and extract link targets as normal files.""" + while tar_member_obj is not None and ( + tar_member_obj.islnk() or tar_member_obj.issym()): + linkpath = tar_member_obj.linkname + if tar_member_obj.issym(): + base = posixpath.dirname(tar_member_obj.name) + linkpath = posixpath.join(base, linkpath) + linkpath = posixpath.normpath(linkpath) + tar_member_obj = tar_obj._getmember(linkpath) + + is_file_or_dir = ( + tar_member_obj is not None and + (tar_member_obj.isfile() or tar_member_obj.isdir()) + ) + if is_file_or_dir: + return tar_member_obj + + raise LookupError('Got unknown file type') + + +def _iter_open_tar(tar_obj, extract_dir, progress_filter): + """Emit member-destination pairs from a tar archive.""" + # don't do any chowning! + tar_obj.chown = lambda *args: None + + with contextlib.closing(tar_obj): + for member in tar_obj: + name = member.name + # don't extract absolute paths or ones with .. in them + if name.startswith('/') or '..' in name.split('/'): + continue + + prelim_dst = os.path.join(extract_dir, *name.split('/')) + + try: + member = _resolve_tar_file_or_dir(tar_obj, member) + except LookupError: + continue + + final_dst = progress_filter(name, prelim_dst) + if not final_dst: + continue + + if final_dst.endswith(os.sep): + final_dst = final_dst[:-1] + + yield member, final_dst + + def unpack_tarfile(filename, extract_dir, progress_filter=default_filter): """Unpack tar/tar.gz/tar.bz2 `filename` to `extract_dir` @@ -138,38 +188,18 @@ def unpack_tarfile(filename, extract_dir, progress_filter=default_filter): raise UnrecognizedFormat( "%s is not a compressed or uncompressed tar file" % (filename,) ) from e - with contextlib.closing(tarobj): - # don't do any chowning! - tarobj.chown = lambda *args: None - for member in tarobj: - name = member.name - # don't extract absolute paths or ones with .. in them - if not name.startswith('/') and '..' not in name.split('/'): - prelim_dst = os.path.join(extract_dir, *name.split('/')) - - # resolve any links and to extract the link targets as normal - # files - while member is not None and ( - member.islnk() or member.issym()): - linkpath = member.linkname - if member.issym(): - base = posixpath.dirname(member.name) - linkpath = posixpath.join(base, linkpath) - linkpath = posixpath.normpath(linkpath) - member = tarobj._getmember(linkpath) - - if member is not None and (member.isfile() or member.isdir()): - final_dst = progress_filter(name, prelim_dst) - if final_dst: - if final_dst.endswith(os.sep): - final_dst = final_dst[:-1] - try: - # XXX Ugh - tarobj._extract_member(member, final_dst) - except tarfile.ExtractError: - # chown/chmod/mkfifo/mknode/makedev failed - pass - return True + + for member, final_dst in _iter_open_tar( + tarobj, extract_dir, progress_filter, + ): + try: + # XXX Ugh + tarobj._extract_member(member, final_dst) + except tarfile.ExtractError: + # chown/chmod/mkfifo/mknode/makedev failed + pass + + return True extraction_drivers = unpack_directory, unpack_zipfile, unpack_tarfile -- cgit v1.2.1