From 065e8eb834d65b4e0572611c16f7f2950db4512b Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 11 May 2013 10:59:30 +0100 Subject: Verified that not isinstance(IOError(), os.error), so there's no value in this statement being in the try block. --HG-- branch : distribute extra : rebase_source : 79dc86f2251503336feaa3d3f99c744d0ea45887 --- pkg_resources.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg_resources.py b/pkg_resources.py index 69601480..9d406e38 100644 --- a/pkg_resources.py +++ b/pkg_resources.py @@ -1409,10 +1409,10 @@ class ZipProvider(EggProvider): ) timestamp = time.mktime(date_time) + if not WRITE_SUPPORT: + raise IOError('"os.rename" and "os.unlink" are not supported ' + 'on this platform') try: - if not WRITE_SUPPORT: - raise IOError('"os.rename" and "os.unlink" are not supported ' - 'on this platform') real_path = manager.get_cache_path( self.egg_name, self._parts(zip_path) -- cgit v1.2.1 From 9027ce5419a346e068927df72dbfcd650cdee33f Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 11 May 2013 12:43:23 +0100 Subject: Extract static method for calculating size and timestamp on a zip resource --HG-- branch : distribute extra : rebase_source : 3acd099bb90d9abf4d3b265946e0c59b8edcae6e --- pkg_resources.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/pkg_resources.py b/pkg_resources.py index 9d406e38..579e3b46 100644 --- a/pkg_resources.py +++ b/pkg_resources.py @@ -1392,6 +1392,16 @@ class ZipProvider(EggProvider): self._extract_resource(manager, self._eager_to_zip(name)) return self._extract_resource(manager, zip_path) + @staticmethod + def _get_date_and_size(zip_stat): + t,d,size = zip_stat[5], zip_stat[6], zip_stat[3] + date_time = ( + (d>>9)+1980, (d>>5)&0xF, d&0x1F, # ymd + (t&0xFFFF)>>11, (t>>5)&0x3F, (t&0x1F) * 2, 0, 0, -1 # hms, etc. + ) + timestamp = time.mktime(date_time) + return timestamp, size + def _extract_resource(self, manager, zip_path): if zip_path in self._index(): @@ -1401,13 +1411,7 @@ class ZipProvider(EggProvider): ) return os.path.dirname(last) # return the extracted directory name - zip_stat = self.zipinfo[zip_path] - t,d,size = zip_stat[5], zip_stat[6], zip_stat[3] - date_time = ( - (d>>9)+1980, (d>>5)&0xF, d&0x1F, # ymd - (t&0xFFFF)>>11, (t>>5)&0x3F, (t&0x1F) * 2, 0, 0, -1 # hms, etc. - ) - timestamp = time.mktime(date_time) + timestamp, size = self._get_date_and_size(self.zipinfo[zip_path]) if not WRITE_SUPPORT: raise IOError('"os.rename" and "os.unlink" are not supported ' -- cgit v1.2.1 From 7adfcead12a21b2ed6b13f5a75477914b524cb12 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 11 May 2013 12:54:26 +0100 Subject: Extract method to determine if a temporary file is current. --HG-- branch : distribute extra : rebase_source : ec2c1860f0ce9abbc3ea999aa54304ea9cc6ecd7 --- pkg_resources.py | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/pkg_resources.py b/pkg_resources.py index 579e3b46..b59f1703 100644 --- a/pkg_resources.py +++ b/pkg_resources.py @@ -1422,11 +1422,8 @@ class ZipProvider(EggProvider): self.egg_name, self._parts(zip_path) ) - if os.path.isfile(real_path): - stat = os.stat(real_path) - if stat.st_size==size and stat.st_mtime==timestamp: - # size and stamp match, don't bother extracting - return real_path + if self.is_current(real_path, zip_path): + return real_path outf, tmpnam = _mkstemp(".$extract", dir=os.path.dirname(real_path)) os.write(outf, self.loader.get_data(zip_path)) @@ -1439,11 +1436,9 @@ class ZipProvider(EggProvider): except os.error: if os.path.isfile(real_path): - stat = os.stat(real_path) - - if stat.st_size==size and stat.st_mtime==timestamp: - # size and stamp match, somebody did it just ahead of - # us, so we're done + if self._is_current(real_path, zip_path): + # the file became current since it was checked above, + # so proceed. return real_path elif os.name=='nt': # Windows, del old file and retry unlink(real_path) @@ -1456,6 +1451,16 @@ class ZipProvider(EggProvider): return real_path + def _is_current(self, file_path, zip_path): + """ + Return True if the file_path is current for this zip_path + """ + timestamp, size = self._get_date_and_size(self.zipinfo[zip_path]) + if not os.path.isfile(file_path): + return False + stat = os.stat(file_path) + return stat.st_size==size and stat.st_mtime==timestamp + def _get_eager_resources(self): if self.eagers is None: eagers = [] -- cgit v1.2.1 From f0a3905357e596c9a2a85358fda0f41384ff3d39 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sat, 11 May 2013 13:05:32 +0100 Subject: Fix for yet unpublished issue to ensure that get_resource_filename always re-extracts the content of a temporary filename if it does not match that of the source content. --HG-- branch : distribute extra : rebase_source : 5605ee258010cde1237db058b770c62264c215e2 --- CHANGES.txt | 7 +++++++ pkg_resources.py | 9 ++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 6e2b3b28..5d4c13bc 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -3,6 +3,13 @@ CHANGES ======= ------ +0.6.39 +------ + +* Issue ####: Resources extracted from a zip egg to the file system now also + check the contents of the file against the zip contents during each + invocation of get_resource_filename. + 0.6.38 ------ diff --git a/pkg_resources.py b/pkg_resources.py index b59f1703..2ec645d3 100644 --- a/pkg_resources.py +++ b/pkg_resources.py @@ -1459,7 +1459,14 @@ class ZipProvider(EggProvider): if not os.path.isfile(file_path): return False stat = os.stat(file_path) - return stat.st_size==size and stat.st_mtime==timestamp + if stat.st_size!=size or stat.st_mtime!=timestamp: + return False + # check that the contents match + zip_contents = self.loader.get_data(zip_path) + f = open(file_path, 'rb') + file_contents = f.read() + f.close() + return zip_contents == file_contents def _get_eager_resources(self): if self.eagers is None: -- cgit v1.2.1 From b9661afd506c7c5f43d01c092ce313f33e730892 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 12 May 2013 13:34:15 -0400 Subject: Fix AttributeError on underscore-prefixed method name. --HG-- branch : distribute extra : rebase_source : 8ceb2c2f067a60225bb83817a8584d93f489a3d0 --- pkg_resources.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg_resources.py b/pkg_resources.py index 2ec645d3..f8de449e 100644 --- a/pkg_resources.py +++ b/pkg_resources.py @@ -1422,7 +1422,7 @@ class ZipProvider(EggProvider): self.egg_name, self._parts(zip_path) ) - if self.is_current(real_path, zip_path): + if self._is_current(real_path, zip_path): return real_path outf, tmpnam = _mkstemp(".$extract", dir=os.path.dirname(real_path)) -- cgit v1.2.1 From e73268755f8d4889f5491907a9b366f8c2b0770d Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 12 May 2013 13:46:50 -0400 Subject: Adding test that captures the new requirement. --HG-- branch : distribute extra : rebase_source : fcf8db4d0becf51a1e192ec438c13f81d391e342 --- tests/test_pkg_resources.py | 61 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 tests/test_pkg_resources.py diff --git a/tests/test_pkg_resources.py b/tests/test_pkg_resources.py new file mode 100644 index 00000000..7009b4ab --- /dev/null +++ b/tests/test_pkg_resources.py @@ -0,0 +1,61 @@ +import sys +import tempfile +import os +import zipfile + +import pkg_resources + +class EggRemover(unicode): + def __call__(self): + if self in sys.path: + sys.path.remove(self) + if os.path.exists(self): + os.remove(self) + +class TestZipProvider(object): + finalizers = [] + + @classmethod + def setup_class(cls): + "create a zip egg and add it to sys.path" + egg = tempfile.NamedTemporaryFile(suffix='.egg', delete=False) + zip_egg = zipfile.ZipFile(egg, 'w') + zip_info = zipfile.ZipInfo() + zip_info.filename = 'mod.py' + zip_info.date_time = 2013, 5, 12, 13, 25, 0 + zip_egg.writestr(zip_info, 'x = 3\n') + zip_info = zipfile.ZipInfo() + zip_info.filename = 'data.dat' + zip_info.date_time = 2013, 5, 12, 13, 25, 0 + zip_egg.writestr(zip_info, 'hello, world!') + zip_egg.close() + egg.close() + + sys.path.append(egg.name) + cls.finalizers.append(EggRemover(egg.name)) + + @classmethod + def teardown_class(cls): + for finalizer in cls.finalizers: + finalizer() + + def test_resource_filename_rewrites_on_change(self): + """ + If a previous call to get_resource_filename has saved the file, but + the file has been subsequently mutated with different file of the + same size and modification time, it should not be overwritten on a + subsequent call to get_resource_filename. + """ + import mod + manager = pkg_resources.ResourceManager() + zp = pkg_resources.ZipProvider(mod) + filename = zp.get_resource_filename(manager, 'data.dat') + assert os.stat(filename).st_mtime == 1368379500 + f = open(filename, 'wb') + f.write('hello, world?') + f.close() + os.utime(filename, (1368379500, 1368379500)) + filename = zp.get_resource_filename(manager, 'data.dat') + f = open(filename) + assert f.read() == 'hello, world!' + manager.cleanup_resources() -- cgit v1.2.1 From f3dce599fca8023d441decd8c60896c44151b803 Mon Sep 17 00:00:00 2001 From: "Jason R. Coombs" Date: Sun, 12 May 2013 13:52:43 -0400 Subject: Update CHANGELOG to reflect relevant issue. --HG-- branch : distribute extra : rebase_source : 785fddb50677fe7779949089e3b090891c3c5bd1 --- CHANGES.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGES.txt b/CHANGES.txt index 5d4c13bc..377dc8a5 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -6,7 +6,7 @@ CHANGES 0.6.39 ------ -* Issue ####: Resources extracted from a zip egg to the file system now also +* Issue #375: Resources extracted from a zip egg to the file system now also check the contents of the file against the zip contents during each invocation of get_resource_filename. -- cgit v1.2.1