From 932997c3be7a8f9b0186965c91e7a06cbaa32b3b Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Tue, 3 Mar 2015 12:55:41 +0000 Subject: bins: We no longer want chunks to be tarballs Change create_chunk to put the contents of an artifact into a directory rather than storing them in a tarball, as we want to store chunks as directory trees in OSTree rather than tarballs now. Change-Id: Ic8aab162fdbbc03ec92538759fef06f35ff0696e --- morphlib/bins.py | 59 ++++++++++++++++-------------- morphlib/bins_tests.py | 98 +++----------------------------------------------- 2 files changed, 37 insertions(+), 120 deletions(-) diff --git a/morphlib/bins.py b/morphlib/bins.py index 2e8ba0b3..c5bacc26 100644 --- a/morphlib/bins.py +++ b/morphlib/bins.py @@ -78,12 +78,8 @@ if sys.version_info < (2, 7, 3): # pragma: no cover raise ExtractError("could not change owner") tarfile.TarFile.chown = fixed_chown -def create_chunk(rootdir, f, include, dump_memory_profile=None): - '''Create a chunk from the contents of a directory. - - ``f`` is an open file handle, to which the tar file is written. - - ''' +def create_chunk(rootdir, chunkdir, include, dump_memory_profile=None): + '''Create a chunk from the contents of a directory.''' dump_memory_profile = dump_memory_profile or (lambda msg: None) @@ -91,31 +87,42 @@ def create_chunk(rootdir, f, include, dump_memory_profile=None): # chunk artifact. This is useful to avoid problems from smallish # clock skew. It needs to be recent enough, however, that GNU tar # does not complain about an implausibly old timestamp. - normalized_timestamp = 683074800 + normalized_timestamp = (683074800, 683074800) dump_memory_profile('at beginning of create_chunk') - - path_pairs = [(relname, os.path.join(rootdir, relname)) - for relname in include] - tar = tarfile.open(fileobj=f, mode='w') - for relname, filename in path_pairs: + + def check_parent(name, paths): + parent = os.path.dirname(name) + if parent: + path = os.path.join(rootdir, parent) + if parent != rootdir and path not in paths: + paths.append(path) + check_parent(parent, paths) + + def filter_contents(dirname, filenames): + paths = [os.path.join(rootdir, relname) for relname in include] + for name in include: + check_parent(name, paths) + + return [f for f in filenames if os.path.join(dirname, f) not in paths] + + logging.debug('Copying artifact into %s.' % chunkdir) + shutil.copytree(rootdir, chunkdir, + symlinks=True, ignore=filter_contents) + + path_triplets = [(relname, os.path.join(chunkdir, relname), + os.path.join(rootdir, relname)) + for relname in include] + for relname, filename, orig in path_triplets: # Normalize mtime for everything. - tarinfo = tar.gettarinfo(filename, - arcname=relname) - tarinfo.ctime = normalized_timestamp - tarinfo.mtime = normalized_timestamp - if tarinfo.isreg(): - with open(filename, 'rb') as f: - tar.addfile(tarinfo, fileobj=f) - else: - tar.addfile(tarinfo) - tar.close() + if not os.path.islink(filename): + os.utime(filename, normalized_timestamp) - for relname, filename in reversed(path_pairs): - if os.path.isdir(filename) and not os.path.islink(filename): + for relname, filename, orig in reversed(path_triplets): + if os.path.isdir(orig) and not os.path.islink(orig): continue else: - os.remove(filename) + os.remove(orig) dump_memory_profile('after removing in create_chunks') @@ -209,7 +216,7 @@ def unpack_binary_from_file(f, dirname): # pragma: no cover tf.close() -def unpack_binary(filename, dirname): +def unpack_binary(filename, dirname): # pragma: no cover with open(filename, "rb") as f: unpack_binary_from_file(f, dirname) diff --git a/morphlib/bins_tests.py b/morphlib/bins_tests.py index 3895680f..879aada4 100644 --- a/morphlib/bins_tests.py +++ b/morphlib/bins_tests.py @@ -78,11 +78,9 @@ class ChunkTests(BinsTest): self.tempdir = tempfile.mkdtemp() self.instdir = os.path.join(self.tempdir, 'inst') self.chunk_file = os.path.join(self.tempdir, 'chunk') - self.chunk_f = open(self.chunk_file, 'wb') self.unpacked = os.path.join(self.tempdir, 'unpacked') def tearDown(self): - self.chunk_f.close() shutil.rmtree(self.tempdir) def populate_instdir(self): @@ -108,109 +106,21 @@ class ChunkTests(BinsTest): def create_chunk(self, includes): self.populate_instdir() - morphlib.bins.create_chunk(self.instdir, self.chunk_f, includes) - self.chunk_f.flush() - - def unpack_chunk(self): - os.mkdir(self.unpacked) - morphlib.bins.unpack_binary(self.chunk_file, self.unpacked) + morphlib.bins.create_chunk(self.instdir, self.chunk_file, includes) def test_empties_files(self): self.create_chunk(['bin/foo', 'lib/libfoo.so']) self.assertEqual([x for x, y in self.recursive_lstat(self.instdir)], ['.', 'bin', 'lib']) - def test_creates_and_unpacks_chunk_exactly(self): + def test_creates_chunk_exactly(self): self.create_chunk(['bin', 'bin/foo', 'lib', 'lib/libfoo.so']) - self.unpack_chunk() self.assertEqual(self.instdir_orig_files, - self.recursive_lstat(self.unpacked)) + self.recursive_lstat(self.chunk_file)) def test_uses_only_matching_names(self): self.create_chunk(['bin/foo']) - self.unpack_chunk() - self.assertEqual([x for x, y in self.recursive_lstat(self.unpacked)], + self.assertEqual([x for x, y in self.recursive_lstat(self.chunk_file)], ['.', 'bin', 'bin/foo']) self.assertEqual([x for x, y in self.recursive_lstat(self.instdir)], ['.', 'bin', 'lib', 'lib/libfoo.so']) - - def test_does_not_compress_artifact(self): - self.create_chunk(['bin']) - f = gzip.open(self.chunk_file) - self.assertRaises(IOError, f.read) - f.close() - - -class ExtractTests(unittest.TestCase): - - def setUp(self): - self.tempdir = tempfile.mkdtemp() - self.instdir = os.path.join(self.tempdir, 'inst') - self.unpacked = os.path.join(self.tempdir, 'unpacked') - - def tearDown(self): - shutil.rmtree(self.tempdir) - - def create_chunk(self, callback): - fh = StringIO.StringIO() - os.mkdir(self.instdir) - patterns = callback(self.instdir) - morphlib.bins.create_chunk(self.instdir, fh, patterns) - shutil.rmtree(self.instdir) - fh.flush() - fh.seek(0) - return fh - - def test_extracted_files_replace_links(self): - def make_linkfile(basedir): - with open(os.path.join(basedir, 'babar'), 'w') as f: - pass - os.symlink('babar', os.path.join(basedir, 'bar')) - return ['babar'] - linktar = self.create_chunk(make_linkfile) - - def make_file(basedir): - with open(os.path.join(basedir, 'bar'), 'w') as f: - pass - return ['bar'] - filetar = self.create_chunk(make_file) - - os.mkdir(self.unpacked) - morphlib.bins.unpack_binary_from_file(linktar, self.unpacked) - morphlib.bins.unpack_binary_from_file(filetar, self.unpacked) - mode = os.lstat(os.path.join(self.unpacked, 'bar')).st_mode - self.assertTrue(stat.S_ISREG(mode)) - - def test_extracted_dirs_keep_links(self): - def make_usrlink(basedir): - os.symlink('.', os.path.join(basedir, 'usr')) - return ['usr'] - linktar = self.create_chunk(make_usrlink) - - def make_usrdir(basedir): - os.mkdir(os.path.join(basedir, 'usr')) - return ['usr'] - dirtar = self.create_chunk(make_usrdir) - - morphlib.bins.unpack_binary_from_file(linktar, self.unpacked) - morphlib.bins.unpack_binary_from_file(dirtar, self.unpacked) - mode = os.lstat(os.path.join(self.unpacked, 'usr')).st_mode - self.assertTrue(stat.S_ISLNK(mode)) - - def test_extracted_files_follow_links(self): - def make_usrlink(basedir): - os.symlink('.', os.path.join(basedir, 'usr')) - return ['usr'] - linktar = self.create_chunk(make_usrlink) - - def make_usrdir(basedir): - os.mkdir(os.path.join(basedir, 'usr')) - with open(os.path.join(basedir, 'usr', 'foo'), 'w') as f: - pass - return ['usr', 'usr/foo'] - dirtar = self.create_chunk(make_usrdir) - - morphlib.bins.unpack_binary_from_file(linktar, self.unpacked) - morphlib.bins.unpack_binary_from_file(dirtar, self.unpacked) - mode = os.lstat(os.path.join(self.unpacked, 'foo')).st_mode - self.assertTrue(stat.S_ISREG(mode)) -- cgit v1.2.1