bins: We no longer want chunks to be tarballs

Change create_chunk to put the contents of an artifact into a directory
rather than storing them in a tarball, as we want to store chunks as
directory trees in OSTree rather than tarballs now.

2 files changed, 37 insertions, 120 deletions

diff --git a/morphlib/bins.py b/morphlib/bins.py
index 2e8ba0b3..c5bacc26 100644
--- a/morphlib/bins.py
+++ b/morphlib/bins.py
@@ -78,12 +78,8 @@ if sys.version_info < (2, 7, 3): # pragma: no cover
                 raise ExtractError("could not change owner")
     tarfile.TarFile.chown = fixed_chown
 
-def create_chunk(rootdir, f, include, dump_memory_profile=None):
-    '''Create a chunk from the contents of a directory.
-    
-    ``f`` is an open file handle, to which the tar file is written.
-
-    '''
+def create_chunk(rootdir, chunkdir, include, dump_memory_profile=None):
+    '''Create a chunk from the contents of a directory.'''
 
     dump_memory_profile = dump_memory_profile or (lambda msg: None)
 
@@ -91,31 +87,42 @@ def create_chunk(rootdir, f, include, dump_memory_profile=None):
     # chunk artifact. This is useful to avoid problems from smallish
     # clock skew. It needs to be recent enough, however, that GNU tar
     # does not complain about an implausibly old timestamp.
-    normalized_timestamp = 683074800
+    normalized_timestamp = (683074800, 683074800)
 
     dump_memory_profile('at beginning of create_chunk')
-    
-    path_pairs = [(relname, os.path.join(rootdir, relname))
-                  for relname in include]
-    tar = tarfile.open(fileobj=f, mode='w')
-    for relname, filename in path_pairs:
+
+    def check_parent(name, paths):
+        parent = os.path.dirname(name)
+        if parent:
+            path = os.path.join(rootdir, parent)
+            if parent != rootdir and path not in paths:
+                paths.append(path)
+            check_parent(parent, paths)
+
+    def filter_contents(dirname, filenames):
+        paths = [os.path.join(rootdir, relname) for relname in include]
+        for name in include:
+            check_parent(name, paths)
+
+        return [f for f in filenames if os.path.join(dirname, f) not in paths]
+
+    logging.debug('Copying artifact into %s.' % chunkdir)
+    shutil.copytree(rootdir, chunkdir,
+                    symlinks=True, ignore=filter_contents)
+
+    path_triplets = [(relname, os.path.join(chunkdir, relname),
+                      os.path.join(rootdir, relname))
+                     for relname in include]
+    for relname, filename, orig in path_triplets:
         # Normalize mtime for everything.
-        tarinfo = tar.gettarinfo(filename,
-                                 arcname=relname)
-        tarinfo.ctime = normalized_timestamp
-        tarinfo.mtime = normalized_timestamp
-        if tarinfo.isreg():
-            with open(filename, 'rb') as f:
-                tar.addfile(tarinfo, fileobj=f)
-        else:
-            tar.addfile(tarinfo)
-    tar.close()
+        if not os.path.islink(filename):
+            os.utime(filename, normalized_timestamp)
 
-    for relname, filename in reversed(path_pairs):
-        if os.path.isdir(filename) and not os.path.islink(filename):
+    for relname, filename, orig in reversed(path_triplets):
+        if os.path.isdir(orig) and not os.path.islink(orig):
             continue
         else:
-            os.remove(filename)
+            os.remove(orig)
     dump_memory_profile('after removing in create_chunks')
 
 
@@ -209,7 +216,7 @@ def unpack_binary_from_file(f, dirname):  # pragma: no cover
         tf.close()
 
 
-def unpack_binary(filename, dirname):
+def unpack_binary(filename, dirname): # pragma: no cover
     with open(filename, "rb") as f:
         unpack_binary_from_file(f, dirname)
 
diff --git a/morphlib/bins_tests.py b/morphlib/bins_tests.py
index 3895680f..879aada4 100644
--- a/morphlib/bins_tests.py
+++ b/morphlib/bins_tests.py
@@ -78,11 +78,9 @@ class ChunkTests(BinsTest):
         self.tempdir = tempfile.mkdtemp()
         self.instdir = os.path.join(self.tempdir, 'inst')
         self.chunk_file = os.path.join(self.tempdir, 'chunk')
-        self.chunk_f = open(self.chunk_file, 'wb')
         self.unpacked = os.path.join(self.tempdir, 'unpacked')
 
     def tearDown(self):
-        self.chunk_f.close()
         shutil.rmtree(self.tempdir)
 
     def populate_instdir(self):
@@ -108,109 +106,21 @@ class ChunkTests(BinsTest):
 
     def create_chunk(self, includes):
         self.populate_instdir()
-        morphlib.bins.create_chunk(self.instdir, self.chunk_f, includes)
-        self.chunk_f.flush()
-
-    def unpack_chunk(self):
-        os.mkdir(self.unpacked)
-        morphlib.bins.unpack_binary(self.chunk_file, self.unpacked)
+        morphlib.bins.create_chunk(self.instdir, self.chunk_file, includes)
 
     def test_empties_files(self):
         self.create_chunk(['bin/foo', 'lib/libfoo.so'])
         self.assertEqual([x for x, y in self.recursive_lstat(self.instdir)],
                          ['.', 'bin', 'lib'])
 
-    def test_creates_and_unpacks_chunk_exactly(self):
+    def test_creates_chunk_exactly(self):
         self.create_chunk(['bin', 'bin/foo', 'lib', 'lib/libfoo.so'])
-        self.unpack_chunk()
         self.assertEqual(self.instdir_orig_files,
-                         self.recursive_lstat(self.unpacked))
+                         self.recursive_lstat(self.chunk_file))
 
     def test_uses_only_matching_names(self):
         self.create_chunk(['bin/foo'])
-        self.unpack_chunk()
-        self.assertEqual([x for x, y in self.recursive_lstat(self.unpacked)],
+        self.assertEqual([x for x, y in self.recursive_lstat(self.chunk_file)],
                          ['.', 'bin', 'bin/foo'])
         self.assertEqual([x for x, y in self.recursive_lstat(self.instdir)],
                          ['.', 'bin', 'lib', 'lib/libfoo.so'])
-
-    def test_does_not_compress_artifact(self):
-        self.create_chunk(['bin'])
-        f = gzip.open(self.chunk_file)
-        self.assertRaises(IOError, f.read)
-        f.close()
-
-
-class ExtractTests(unittest.TestCase):
-
-    def setUp(self):
-        self.tempdir = tempfile.mkdtemp()
-        self.instdir = os.path.join(self.tempdir, 'inst')
-        self.unpacked = os.path.join(self.tempdir, 'unpacked')
-
-    def tearDown(self):
-        shutil.rmtree(self.tempdir)
-
-    def create_chunk(self, callback):
-        fh = StringIO.StringIO()
-        os.mkdir(self.instdir)
-        patterns = callback(self.instdir)
-        morphlib.bins.create_chunk(self.instdir, fh, patterns)
-        shutil.rmtree(self.instdir)
-        fh.flush()
-        fh.seek(0)
-        return fh
-
-    def test_extracted_files_replace_links(self):
-        def make_linkfile(basedir):
-            with open(os.path.join(basedir, 'babar'), 'w') as f:
-                pass
-            os.symlink('babar', os.path.join(basedir, 'bar'))
-            return ['babar']
-        linktar = self.create_chunk(make_linkfile)
-
-        def make_file(basedir):
-            with open(os.path.join(basedir, 'bar'), 'w') as f:
-                pass
-            return ['bar']
-        filetar = self.create_chunk(make_file)
-
-        os.mkdir(self.unpacked)
-        morphlib.bins.unpack_binary_from_file(linktar, self.unpacked)
-        morphlib.bins.unpack_binary_from_file(filetar, self.unpacked)
-        mode = os.lstat(os.path.join(self.unpacked, 'bar')).st_mode
-        self.assertTrue(stat.S_ISREG(mode))
-
-    def test_extracted_dirs_keep_links(self):
-        def make_usrlink(basedir):
-            os.symlink('.', os.path.join(basedir, 'usr'))
-            return ['usr']
-        linktar = self.create_chunk(make_usrlink)
-
-        def make_usrdir(basedir):
-            os.mkdir(os.path.join(basedir, 'usr'))
-            return ['usr']
-        dirtar = self.create_chunk(make_usrdir)
-
-        morphlib.bins.unpack_binary_from_file(linktar, self.unpacked)
-        morphlib.bins.unpack_binary_from_file(dirtar, self.unpacked)
-        mode = os.lstat(os.path.join(self.unpacked, 'usr')).st_mode
-        self.assertTrue(stat.S_ISLNK(mode))
-
-    def test_extracted_files_follow_links(self):
-        def make_usrlink(basedir):
-            os.symlink('.', os.path.join(basedir, 'usr'))
-            return ['usr']
-        linktar = self.create_chunk(make_usrlink)
-
-        def make_usrdir(basedir):
-            os.mkdir(os.path.join(basedir, 'usr'))
-            with open(os.path.join(basedir, 'usr', 'foo'), 'w') as f:
-                pass
-            return ['usr', 'usr/foo']
-        dirtar = self.create_chunk(make_usrdir)
-
-        morphlib.bins.unpack_binary_from_file(linktar, self.unpacked)
-        morphlib.bins.unpack_binary_from_file(dirtar, self.unpacked)
-        mode = os.lstat(os.path.join(self.unpacked, 'foo')).st_mode
-        self.assertTrue(stat.S_ISREG(mode))