From 65ee2f821c8c642aae60f462ff8f209ebdc93e20 Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Thu, 19 Mar 2015 12:38:11 +0000 Subject: Morph build 11ac5b237b3640718da94ad8e252d330 System branch: master --- morphlib/builder.py | 118 +++++++++++++++++++++++++--------------------------- 1 file changed, 57 insertions(+), 61 deletions(-) (limited to 'morphlib/builder.py') diff --git a/morphlib/builder.py b/morphlib/builder.py index 04ebd149..9b01f983 100644 --- a/morphlib/builder.py +++ b/morphlib/builder.py @@ -125,11 +125,7 @@ def ldconfig(runcmd, rootdir): # pragma: no cover def download_depends(constituents, lac, rac, metadatas=None): for constituent in constituents: if not lac.has(constituent): - source = rac.get(constituent) - target = lac.put(constituent) - shutil.copyfileobj(source, target) - target.close() - source.close() + lac.copy_from_remote(constituent, rac) if metadatas is not None: for metadata in metadatas: if not lac.has_artifact_metadata(constituent, metadata): @@ -246,28 +242,6 @@ class ChunkBuilder(BuilderBase): '''Build chunk artifacts.''' - def create_devices(self, destdir): # pragma: no cover - '''Creates device nodes if the morphology specifies them''' - morphology = self.source.morphology - perms_mask = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO - if 'devices' in morphology and morphology['devices'] is not None: - for dev in morphology['devices']: - destfile = os.path.join(destdir, './' + dev['filename']) - mode = int(dev['permissions'], 8) & perms_mask - if dev['type'] == 'c': - mode = mode | stat.S_IFCHR - elif dev['type'] == 'b': - mode = mode | stat.S_IFBLK - else: - raise IOError('Cannot create device node %s,' - 'unrecognized device type "%s"' - % (destfile, dev['type'])) - self.app.status(msg="Creating device node %s" - % destfile) - os.mknod(destfile, mode, - os.makedev(dev['major'], dev['minor'])) - os.chown(destfile, dev['uid'], dev['gid']) - def build_and_cache(self): # pragma: no cover with self.build_watch('overall-build'): @@ -286,7 +260,6 @@ class ChunkBuilder(BuilderBase): try: self.get_sources(builddir) self.run_commands(builddir, destdir, temppath, stdout) - self.create_devices(destdir) os.rename(temppath, logpath) except BaseException as e: @@ -459,13 +432,23 @@ class ChunkBuilder(BuilderBase): extra_files += ['baserock/%s.meta' % chunk_artifact_name] parented_paths = parentify(file_paths + extra_files) - with self.local_artifact_cache.put(chunk_artifact) as f: - self.write_metadata(destdir, chunk_artifact_name, - parented_paths) + self.write_metadata(destdir, chunk_artifact_name, + parented_paths) - self.app.status(msg='Creating chunk artifact %(name)s', - name=chunk_artifact_name) - morphlib.bins.create_chunk(destdir, f, parented_paths) + self.app.status(msg='Creating chunk artifact %(name)s', + name=chunk_artifact_name) + # TODO: This is not concurrency safe, bins.create_chunk will + # fail if tempdir already exists (eg if another build + # has created it). + tempdir = os.path.join(self.app.settings['tempdir'], + chunk_artifact.basename()) + try: + morphlib.bins.create_chunk(destdir, tempdir, + parented_paths) + self.local_artifact_cache.put(tempdir, chunk_artifact) + finally: + if os.path.isdir(tempdir): + shutil.rmtree(tempdir) built_artifacts.append(chunk_artifact) for dirname, subdirs, files in os.walk(destdir): @@ -509,8 +492,13 @@ class StratumBuilder(BuilderBase): [x.name for x in constituents]) with lac.put_artifact_metadata(a, 'meta') as f: json.dump(meta, f, indent=4, sort_keys=True) - with self.local_artifact_cache.put(a) as f: + # TODO: This is not concurrency safe, put_stratum_artifact + # deletes temp which could be in use by another + # build. + temp = os.path.join(self.app.settings['tempdir'], a.name) + with open(temp, 'w+') as f: json.dump([c.basename() for c in constituents], f) + self.local_artifact_cache.put_non_ostree_artifact(a, temp) self.save_build_times() return self.source.artifacts.values() @@ -532,33 +520,40 @@ class SystemBuilder(BuilderBase): # pragma: no cover arch = self.source.morphology['arch'] for a_name, artifact in self.source.artifacts.iteritems(): - handle = self.local_artifact_cache.put(artifact) - try: fs_root = self.staging_area.destdir(self.source) self.unpack_strata(fs_root) - self.write_metadata(fs_root, a_name) - self.run_system_integration_commands(fs_root) - unslashy_root = fs_root[1:] - def uproot_info(info): - info.name = relpath(info.name, unslashy_root) - if info.islnk(): - info.linkname = relpath(info.linkname, - unslashy_root) - return info - tar = tarfile.open(fileobj=handle, mode="w", name=a_name) - self.app.status(msg='Constructing tarball of rootfs', - chatty=True) - tar.add(fs_root, recursive=True, filter=uproot_info) - tar.close() + upperdir = self.staging_area.overlay_upperdir( + self.source) + editable_root = self.staging_area.overlaydir(self.source) + workdir = os.path.join(self.staging_area.dirname, + 'overlayfs-workdir') + if not os.path.exists(workdir): + os.makedirs(workdir) + union_filesystem = self.app.settings['union-filesystem'] + morphlib.fsutils.overlay_mount(self.app.runcmd, + 'overlay-%s' % a_name, + editable_root, fs_root, + upperdir, workdir, + union_filesystem) + self.write_metadata(editable_root, a_name) + self.run_system_integration_commands(editable_root) + # Put the contents of upperdir into the local artifact + # cache. Don't use editable root as we only want to + # store the modified files. + self.local_artifact_cache.put(upperdir, artifact) except BaseException as e: logging.error(traceback.format_exc()) self.app.status(msg='Error while building system', error=True) - handle.abort() + if editable_root and os.path.exists(editable_root): + morphlib.fsutils.unmount(self.app.runcmd, + editable_root) raise else: - handle.close() + if editable_root and os.path.exists(editable_root): + morphlib.fsutils.unmount(self.app.runcmd, + editable_root) self.save_build_times() return self.source.artifacts.itervalues() @@ -567,13 +562,12 @@ class SystemBuilder(BuilderBase): # pragma: no cover '''Unpack a single stratum into a target directory''' cache = self.local_artifact_cache - with cache.get(stratum_artifact) as stratum_file: + with open(cache.get(stratum_artifact), 'r') as stratum_file: artifact_list = json.load(stratum_file, encoding='unicode-escape') for chunk in (ArtifactCacheReference(a) for a in artifact_list): - self.app.status(msg='Unpacking chunk %(basename)s', + self.app.status(msg='Checkout chunk %(basename)s', basename=chunk.basename(), chatty=True) - with cache.get(chunk) as chunk_file: - morphlib.bins.unpack_binary_from_file(chunk_file, target) + cache.get(chunk, target) target_metadata = os.path.join( target, 'baserock', '%s.meta' % stratum_artifact.name) @@ -584,7 +578,7 @@ class SystemBuilder(BuilderBase): # pragma: no cover def unpack_strata(self, path): '''Unpack strata into a directory.''' - self.app.status(msg='Unpacking strata to %(path)s', + self.app.status(msg='Checking out strata to %(path)s', path=path, chatty=True) with self.build_watch('unpack-strata'): for a_name, a in self.source.artifacts.iteritems(): @@ -596,12 +590,14 @@ class SystemBuilder(BuilderBase): # pragma: no cover # download the chunk artifacts if necessary for stratum_artifact in self.source.dependencies: - f = self.local_artifact_cache.get(stratum_artifact) - chunks = [ArtifactCacheReference(c) for c in json.load(f)] + stratum_path = self.local_artifact_cache.get( + stratum_artifact) + with open(stratum_path, 'r') as stratum: + chunks = [ArtifactCacheReference(c) + for c in json.load(stratum)] download_depends(chunks, self.local_artifact_cache, self.remote_artifact_cache) - f.close() # unpack it from the local artifact cache for stratum_artifact in self.source.dependencies: -- cgit v1.2.1