Morph build c2397bf4873742a3ba9202cbdcfffea8

System branch: master

1 files changed, 57 insertions, 61 deletions

diff --git a/morphlib/builder.py b/morphlib/builder.py
index 04ebd149..9b01f983 100644
--- a/morphlib/builder.py
+++ b/morphlib/builder.py
@@ -125,11 +125,7 @@ def ldconfig(runcmd, rootdir):  # pragma: no cover
 def download_depends(constituents, lac, rac, metadatas=None):
     for constituent in constituents:
         if not lac.has(constituent):
-            source = rac.get(constituent)
-            target = lac.put(constituent)
-            shutil.copyfileobj(source, target)
-            target.close()
-            source.close()
+            lac.copy_from_remote(constituent, rac)
         if metadatas is not None:
             for metadata in metadatas:
                 if not lac.has_artifact_metadata(constituent, metadata):
@@ -246,28 +242,6 @@ class ChunkBuilder(BuilderBase):
 
     '''Build chunk artifacts.'''
 
-    def create_devices(self, destdir): # pragma: no cover
-        '''Creates device nodes if the morphology specifies them'''
-        morphology = self.source.morphology
-        perms_mask = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO
-        if 'devices' in morphology and morphology['devices'] is not None:
-            for dev in morphology['devices']:
-                destfile = os.path.join(destdir, './' + dev['filename'])
-                mode = int(dev['permissions'], 8) & perms_mask
-                if dev['type'] == 'c':
-                    mode = mode | stat.S_IFCHR
-                elif dev['type'] == 'b':
-                    mode = mode | stat.S_IFBLK
-                else:
-                    raise IOError('Cannot create device node %s,'
-                                  'unrecognized device type "%s"'
-                                  % (destfile, dev['type']))
-                self.app.status(msg="Creating device node %s"
-                                    % destfile)
-                os.mknod(destfile, mode,
-                         os.makedev(dev['major'], dev['minor']))
-                os.chown(destfile, dev['uid'], dev['gid'])
-
     def build_and_cache(self):  # pragma: no cover
         with self.build_watch('overall-build'):
 
@@ -286,7 +260,6 @@ class ChunkBuilder(BuilderBase):
             try:
                 self.get_sources(builddir)
                 self.run_commands(builddir, destdir, temppath, stdout)
-                self.create_devices(destdir)
 
                 os.rename(temppath, logpath)
             except BaseException as e:
@@ -459,13 +432,23 @@ class ChunkBuilder(BuilderBase):
                 extra_files += ['baserock/%s.meta' % chunk_artifact_name]
                 parented_paths = parentify(file_paths + extra_files)
 
-                with self.local_artifact_cache.put(chunk_artifact) as f:
-                    self.write_metadata(destdir, chunk_artifact_name,
-                                        parented_paths)
+                self.write_metadata(destdir, chunk_artifact_name,
+                                    parented_paths)
 
-                    self.app.status(msg='Creating chunk artifact %(name)s',
-                                    name=chunk_artifact_name)
-                    morphlib.bins.create_chunk(destdir, f, parented_paths)
+                self.app.status(msg='Creating chunk artifact %(name)s',
+                                name=chunk_artifact_name)
+                # TODO: This is not concurrency safe, bins.create_chunk will
+                #       fail if tempdir already exists (eg if another build
+                #       has created it).
+                tempdir = os.path.join(self.app.settings['tempdir'],
+                                       chunk_artifact.basename())
+                try:
+                    morphlib.bins.create_chunk(destdir, tempdir,
+                                               parented_paths)
+                    self.local_artifact_cache.put(tempdir, chunk_artifact)
+                finally:
+                    if os.path.isdir(tempdir):
+                        shutil.rmtree(tempdir)
                 built_artifacts.append(chunk_artifact)
 
         for dirname, subdirs, files in os.walk(destdir):
@@ -509,8 +492,13 @@ class StratumBuilder(BuilderBase):
                         [x.name for x in constituents])
                     with lac.put_artifact_metadata(a, 'meta')  as f:
                         json.dump(meta, f, indent=4, sort_keys=True)
-                    with self.local_artifact_cache.put(a) as f:
+                    # TODO: This is not concurrency safe, put_stratum_artifact
+                    #       deletes temp which could be in use by another
+                    #       build.
+                    temp = os.path.join(self.app.settings['tempdir'], a.name)
+                    with open(temp, 'w+') as f:
                         json.dump([c.basename() for c in constituents], f)
+                    self.local_artifact_cache.put_non_ostree_artifact(a, temp)
         self.save_build_times()
         return self.source.artifacts.values()
 
@@ -532,33 +520,40 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
             arch = self.source.morphology['arch']
 
             for a_name, artifact in self.source.artifacts.iteritems():
-                handle = self.local_artifact_cache.put(artifact)
-
                 try:
                     fs_root = self.staging_area.destdir(self.source)
                     self.unpack_strata(fs_root)
-                    self.write_metadata(fs_root, a_name)
-                    self.run_system_integration_commands(fs_root)
-                    unslashy_root = fs_root[1:]
-                    def uproot_info(info):
-                        info.name = relpath(info.name, unslashy_root)
-                        if info.islnk():
-                            info.linkname = relpath(info.linkname,
-                                                    unslashy_root)
-                        return info
-                    tar = tarfile.open(fileobj=handle, mode="w", name=a_name)
-                    self.app.status(msg='Constructing tarball of rootfs',
-                                    chatty=True)
-                    tar.add(fs_root, recursive=True, filter=uproot_info)
-                    tar.close()
+                    upperdir = self.staging_area.overlay_upperdir(
+                        self.source)
+                    editable_root = self.staging_area.overlaydir(self.source)
+                    workdir = os.path.join(self.staging_area.dirname,
+                                           'overlayfs-workdir')
+                    if not os.path.exists(workdir):
+                        os.makedirs(workdir)
+                    union_filesystem = self.app.settings['union-filesystem']
+                    morphlib.fsutils.overlay_mount(self.app.runcmd,
+                                                   'overlay-%s' % a_name,
+                                                   editable_root, fs_root,
+                                                   upperdir, workdir,
+                                                   union_filesystem)
+                    self.write_metadata(editable_root, a_name)
+                    self.run_system_integration_commands(editable_root)
+                    # Put the contents of upperdir into the local artifact
+                    # cache. Don't use editable root as we only want to
+                    # store the modified files.
+                    self.local_artifact_cache.put(upperdir, artifact)
                 except BaseException as e:
                     logging.error(traceback.format_exc())
                     self.app.status(msg='Error while building system',
                                     error=True)
-                    handle.abort()
+                    if editable_root and os.path.exists(editable_root):
+                        morphlib.fsutils.unmount(self.app.runcmd,
+                                                 editable_root)
                     raise
                 else:
-                    handle.close()
+                    if editable_root and os.path.exists(editable_root):
+                        morphlib.fsutils.unmount(self.app.runcmd,
+                                                 editable_root)
 
         self.save_build_times()
         return self.source.artifacts.itervalues()
@@ -567,13 +562,12 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
         '''Unpack a single stratum into a target directory'''
 
         cache = self.local_artifact_cache
-        with cache.get(stratum_artifact) as stratum_file:
+        with open(cache.get(stratum_artifact), 'r') as stratum_file:
             artifact_list = json.load(stratum_file, encoding='unicode-escape')
             for chunk in (ArtifactCacheReference(a) for a in artifact_list):
-                self.app.status(msg='Unpacking chunk %(basename)s',
+                self.app.status(msg='Checkout chunk %(basename)s',
                                 basename=chunk.basename(), chatty=True)
-                with cache.get(chunk) as chunk_file:
-                    morphlib.bins.unpack_binary_from_file(chunk_file, target)
+                cache.get(chunk, target)
 
         target_metadata = os.path.join(
                 target, 'baserock', '%s.meta' % stratum_artifact.name)
@@ -584,7 +578,7 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
     def unpack_strata(self, path):
         '''Unpack strata into a directory.'''
 
-        self.app.status(msg='Unpacking strata to %(path)s',
+        self.app.status(msg='Checking out strata to %(path)s',
                         path=path, chatty=True)
         with self.build_watch('unpack-strata'):
             for a_name, a in self.source.artifacts.iteritems():
@@ -596,12 +590,14 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
 
                 # download the chunk artifacts if necessary
                 for stratum_artifact in self.source.dependencies:
-                    f = self.local_artifact_cache.get(stratum_artifact)
-                    chunks = [ArtifactCacheReference(c) for c in json.load(f)]
+                    stratum_path = self.local_artifact_cache.get(
+                        stratum_artifact)
+                    with open(stratum_path, 'r') as stratum:
+                        chunks = [ArtifactCacheReference(c)
+                                  for c in json.load(stratum)]
                     download_depends(chunks,
                                      self.local_artifact_cache,
                                      self.remote_artifact_cache)
-                    f.close()
 
                 # unpack it from the local artifact cache
                 for stratum_artifact in self.source.dependencies: