Morph build 271da1e1d62c40748b586dc0345d0f7d

System branch: master

1 files changed, 61 insertions, 57 deletions

diff --git a/morphlib/builder.py b/morphlib/builder.py
index 9b01f983..04ebd149 100644
--- a/morphlib/builder.py
+++ b/morphlib/builder.py
@@ -125,7 +125,11 @@ def ldconfig(runcmd, rootdir):  # pragma: no cover
 def download_depends(constituents, lac, rac, metadatas=None):
     for constituent in constituents:
         if not lac.has(constituent):
-            lac.copy_from_remote(constituent, rac)
+            source = rac.get(constituent)
+            target = lac.put(constituent)
+            shutil.copyfileobj(source, target)
+            target.close()
+            source.close()
         if metadatas is not None:
             for metadata in metadatas:
                 if not lac.has_artifact_metadata(constituent, metadata):
@@ -242,6 +246,28 @@ class ChunkBuilder(BuilderBase):
 
     '''Build chunk artifacts.'''
 
+    def create_devices(self, destdir): # pragma: no cover
+        '''Creates device nodes if the morphology specifies them'''
+        morphology = self.source.morphology
+        perms_mask = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO
+        if 'devices' in morphology and morphology['devices'] is not None:
+            for dev in morphology['devices']:
+                destfile = os.path.join(destdir, './' + dev['filename'])
+                mode = int(dev['permissions'], 8) & perms_mask
+                if dev['type'] == 'c':
+                    mode = mode | stat.S_IFCHR
+                elif dev['type'] == 'b':
+                    mode = mode | stat.S_IFBLK
+                else:
+                    raise IOError('Cannot create device node %s,'
+                                  'unrecognized device type "%s"'
+                                  % (destfile, dev['type']))
+                self.app.status(msg="Creating device node %s"
+                                    % destfile)
+                os.mknod(destfile, mode,
+                         os.makedev(dev['major'], dev['minor']))
+                os.chown(destfile, dev['uid'], dev['gid'])
+
     def build_and_cache(self):  # pragma: no cover
         with self.build_watch('overall-build'):
 
@@ -260,6 +286,7 @@ class ChunkBuilder(BuilderBase):
             try:
                 self.get_sources(builddir)
                 self.run_commands(builddir, destdir, temppath, stdout)
+                self.create_devices(destdir)
 
                 os.rename(temppath, logpath)
             except BaseException as e:
@@ -432,23 +459,13 @@ class ChunkBuilder(BuilderBase):
                 extra_files += ['baserock/%s.meta' % chunk_artifact_name]
                 parented_paths = parentify(file_paths + extra_files)
 
-                self.write_metadata(destdir, chunk_artifact_name,
-                                    parented_paths)
+                with self.local_artifact_cache.put(chunk_artifact) as f:
+                    self.write_metadata(destdir, chunk_artifact_name,
+                                        parented_paths)
 
-                self.app.status(msg='Creating chunk artifact %(name)s',
-                                name=chunk_artifact_name)
-                # TODO: This is not concurrency safe, bins.create_chunk will
-                #       fail if tempdir already exists (eg if another build
-                #       has created it).
-                tempdir = os.path.join(self.app.settings['tempdir'],
-                                       chunk_artifact.basename())
-                try:
-                    morphlib.bins.create_chunk(destdir, tempdir,
-                                               parented_paths)
-                    self.local_artifact_cache.put(tempdir, chunk_artifact)
-                finally:
-                    if os.path.isdir(tempdir):
-                        shutil.rmtree(tempdir)
+                    self.app.status(msg='Creating chunk artifact %(name)s',
+                                    name=chunk_artifact_name)
+                    morphlib.bins.create_chunk(destdir, f, parented_paths)
                 built_artifacts.append(chunk_artifact)
 
         for dirname, subdirs, files in os.walk(destdir):
@@ -492,13 +509,8 @@ class StratumBuilder(BuilderBase):
                         [x.name for x in constituents])
                     with lac.put_artifact_metadata(a, 'meta')  as f:
                         json.dump(meta, f, indent=4, sort_keys=True)
-                    # TODO: This is not concurrency safe, put_stratum_artifact
-                    #       deletes temp which could be in use by another
-                    #       build.
-                    temp = os.path.join(self.app.settings['tempdir'], a.name)
-                    with open(temp, 'w+') as f:
+                    with self.local_artifact_cache.put(a) as f:
                         json.dump([c.basename() for c in constituents], f)
-                    self.local_artifact_cache.put_non_ostree_artifact(a, temp)
         self.save_build_times()
         return self.source.artifacts.values()
 
@@ -520,40 +532,33 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
             arch = self.source.morphology['arch']
 
             for a_name, artifact in self.source.artifacts.iteritems():
+                handle = self.local_artifact_cache.put(artifact)
+
                 try:
                     fs_root = self.staging_area.destdir(self.source)
                     self.unpack_strata(fs_root)
-                    upperdir = self.staging_area.overlay_upperdir(
-                        self.source)
-                    editable_root = self.staging_area.overlaydir(self.source)
-                    workdir = os.path.join(self.staging_area.dirname,
-                                           'overlayfs-workdir')
-                    if not os.path.exists(workdir):
-                        os.makedirs(workdir)
-                    union_filesystem = self.app.settings['union-filesystem']
-                    morphlib.fsutils.overlay_mount(self.app.runcmd,
-                                                   'overlay-%s' % a_name,
-                                                   editable_root, fs_root,
-                                                   upperdir, workdir,
-                                                   union_filesystem)
-                    self.write_metadata(editable_root, a_name)
-                    self.run_system_integration_commands(editable_root)
-                    # Put the contents of upperdir into the local artifact
-                    # cache. Don't use editable root as we only want to
-                    # store the modified files.
-                    self.local_artifact_cache.put(upperdir, artifact)
+                    self.write_metadata(fs_root, a_name)
+                    self.run_system_integration_commands(fs_root)
+                    unslashy_root = fs_root[1:]
+                    def uproot_info(info):
+                        info.name = relpath(info.name, unslashy_root)
+                        if info.islnk():
+                            info.linkname = relpath(info.linkname,
+                                                    unslashy_root)
+                        return info
+                    tar = tarfile.open(fileobj=handle, mode="w", name=a_name)
+                    self.app.status(msg='Constructing tarball of rootfs',
+                                    chatty=True)
+                    tar.add(fs_root, recursive=True, filter=uproot_info)
+                    tar.close()
                 except BaseException as e:
                     logging.error(traceback.format_exc())
                     self.app.status(msg='Error while building system',
                                     error=True)
-                    if editable_root and os.path.exists(editable_root):
-                        morphlib.fsutils.unmount(self.app.runcmd,
-                                                 editable_root)
+                    handle.abort()
                     raise
                 else:
-                    if editable_root and os.path.exists(editable_root):
-                        morphlib.fsutils.unmount(self.app.runcmd,
-                                                 editable_root)
+                    handle.close()
 
         self.save_build_times()
         return self.source.artifacts.itervalues()
@@ -562,12 +567,13 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
         '''Unpack a single stratum into a target directory'''
 
         cache = self.local_artifact_cache
-        with open(cache.get(stratum_artifact), 'r') as stratum_file:
+        with cache.get(stratum_artifact) as stratum_file:
             artifact_list = json.load(stratum_file, encoding='unicode-escape')
             for chunk in (ArtifactCacheReference(a) for a in artifact_list):
-                self.app.status(msg='Checkout chunk %(basename)s',
+                self.app.status(msg='Unpacking chunk %(basename)s',
                                 basename=chunk.basename(), chatty=True)
-                cache.get(chunk, target)
+                with cache.get(chunk) as chunk_file:
+                    morphlib.bins.unpack_binary_from_file(chunk_file, target)
 
         target_metadata = os.path.join(
                 target, 'baserock', '%s.meta' % stratum_artifact.name)
@@ -578,7 +584,7 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
     def unpack_strata(self, path):
         '''Unpack strata into a directory.'''
 
-        self.app.status(msg='Checking out strata to %(path)s',
+        self.app.status(msg='Unpacking strata to %(path)s',
                         path=path, chatty=True)
         with self.build_watch('unpack-strata'):
             for a_name, a in self.source.artifacts.iteritems():
@@ -590,14 +596,12 @@ class SystemBuilder(BuilderBase):  # pragma: no cover
 
                 # download the chunk artifacts if necessary
                 for stratum_artifact in self.source.dependencies:
-                    stratum_path = self.local_artifact_cache.get(
-                        stratum_artifact)
-                    with open(stratum_path, 'r') as stratum:
-                        chunks = [ArtifactCacheReference(c)
-                                  for c in json.load(stratum)]
+                    f = self.local_artifact_cache.get(stratum_artifact)
+                    chunks = [ArtifactCacheReference(c) for c in json.load(f)]
                     download_depends(chunks,
                                      self.local_artifact_cache,
                                      self.remote_artifact_cache)
+                    f.close()
 
                 # unpack it from the local artifact cache
                 for stratum_artifact in self.source.dependencies: