diff options
Diffstat (limited to 'morphlib/builder.py')
| -rw-r--r-- | morphlib/builder.py | 67 |
1 files changed, 36 insertions, 31 deletions
diff --git a/morphlib/builder.py b/morphlib/builder.py index 0c681353..9b01f983 100644 --- a/morphlib/builder.py +++ b/morphlib/builder.py @@ -432,13 +432,23 @@ class ChunkBuilder(BuilderBase): extra_files += ['baserock/%s.meta' % chunk_artifact_name] parented_paths = parentify(file_paths + extra_files) - with self.local_artifact_cache.put(chunk_artifact) as f: - self.write_metadata(destdir, chunk_artifact_name, - parented_paths) - - self.app.status(msg='Creating chunk artifact %(name)s', - name=chunk_artifact_name) - morphlib.bins.create_chunk(destdir, f, parented_paths) + self.write_metadata(destdir, chunk_artifact_name, + parented_paths) + + self.app.status(msg='Creating chunk artifact %(name)s', + name=chunk_artifact_name) + # TODO: This is not concurrency safe, bins.create_chunk will + # fail if tempdir already exists (eg if another build + # has created it). + tempdir = os.path.join(self.app.settings['tempdir'], + chunk_artifact.basename()) + try: + morphlib.bins.create_chunk(destdir, tempdir, + parented_paths) + self.local_artifact_cache.put(tempdir, chunk_artifact) + finally: + if os.path.isdir(tempdir): + shutil.rmtree(tempdir) built_artifacts.append(chunk_artifact) for dirname, subdirs, files in os.walk(destdir): @@ -482,8 +492,13 @@ class StratumBuilder(BuilderBase): [x.name for x in constituents]) with lac.put_artifact_metadata(a, 'meta') as f: json.dump(meta, f, indent=4, sort_keys=True) - with self.local_artifact_cache.put(a) as f: + # TODO: This is not concurrency safe, put_stratum_artifact + # deletes temp which could be in use by another + # build. + temp = os.path.join(self.app.settings['tempdir'], a.name) + with open(temp, 'w+') as f: json.dump([c.basename() for c in constituents], f) + self.local_artifact_cache.put_non_ostree_artifact(a, temp) self.save_build_times() return self.source.artifacts.values() @@ -505,7 +520,6 @@ class SystemBuilder(BuilderBase): # pragma: no cover arch = self.source.morphology['arch'] for a_name, artifact in self.source.artifacts.iteritems(): - handle = self.local_artifact_cache.put(artifact) try: fs_root = self.staging_area.destdir(self.source) self.unpack_strata(fs_root) @@ -524,18 +538,10 @@ class SystemBuilder(BuilderBase): # pragma: no cover union_filesystem) self.write_metadata(editable_root, a_name) self.run_system_integration_commands(editable_root) - unslashy_root = editable_root[1:] - def uproot_info(info): - info.name = relpath(info.name, unslashy_root) - if info.islnk(): - info.linkname = relpath(info.linkname, - unslashy_root) - return info - tar = tarfile.open(fileobj=handle, mode="w", name=a_name) - self.app.status(msg='Constructing tarball of rootfs', - chatty=True) - tar.add(editable_root, recursive=True, filter=uproot_info) - tar.close() + # Put the contents of upperdir into the local artifact + # cache. Don't use editable root as we only want to + # store the modified files. + self.local_artifact_cache.put(upperdir, artifact) except BaseException as e: logging.error(traceback.format_exc()) self.app.status(msg='Error while building system', @@ -543,13 +549,11 @@ class SystemBuilder(BuilderBase): # pragma: no cover if editable_root and os.path.exists(editable_root): morphlib.fsutils.unmount(self.app.runcmd, editable_root) - handle.abort() raise else: if editable_root and os.path.exists(editable_root): morphlib.fsutils.unmount(self.app.runcmd, editable_root) - handle.close() self.save_build_times() return self.source.artifacts.itervalues() @@ -558,13 +562,12 @@ class SystemBuilder(BuilderBase): # pragma: no cover '''Unpack a single stratum into a target directory''' cache = self.local_artifact_cache - with cache.get(stratum_artifact) as stratum_file: + with open(cache.get(stratum_artifact), 'r') as stratum_file: artifact_list = json.load(stratum_file, encoding='unicode-escape') for chunk in (ArtifactCacheReference(a) for a in artifact_list): - self.app.status(msg='Unpacking chunk %(basename)s', + self.app.status(msg='Checkout chunk %(basename)s', basename=chunk.basename(), chatty=True) - with cache.get(chunk) as chunk_file: - morphlib.bins.unpack_binary_from_file(chunk_file, target) + cache.get(chunk, target) target_metadata = os.path.join( target, 'baserock', '%s.meta' % stratum_artifact.name) @@ -575,7 +578,7 @@ class SystemBuilder(BuilderBase): # pragma: no cover def unpack_strata(self, path): '''Unpack strata into a directory.''' - self.app.status(msg='Unpacking strata to %(path)s', + self.app.status(msg='Checking out strata to %(path)s', path=path, chatty=True) with self.build_watch('unpack-strata'): for a_name, a in self.source.artifacts.iteritems(): @@ -587,12 +590,14 @@ class SystemBuilder(BuilderBase): # pragma: no cover # download the chunk artifacts if necessary for stratum_artifact in self.source.dependencies: - f = self.local_artifact_cache.get(stratum_artifact) - chunks = [ArtifactCacheReference(c) for c in json.load(f)] + stratum_path = self.local_artifact_cache.get( + stratum_artifact) + with open(stratum_path, 'r') as stratum: + chunks = [ArtifactCacheReference(c) + for c in json.load(stratum)] download_depends(chunks, self.local_artifact_cache, self.remote_artifact_cache) - f.close() # unpack it from the local artifact cache for stratum_artifact in self.source.dependencies: |