diff options
| -rw-r--r-- | morphlib/buildcommand.py | 9 | ||||
| -rw-r--r-- | morphlib/builder.py | 67 | ||||
| -rw-r--r-- | morphlib/builder_tests.py | 18 |
3 files changed, 49 insertions, 45 deletions
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index d7c445df..c4d7a6f5 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -422,8 +422,10 @@ class BuildCommand(object): # module into morphlib.remoteartififactcache first. to_fetch = [] if not self.lac.has(artifact): - to_fetch.append((self.rac.get(artifact), - self.lac.put(artifact))) + self.app.status( + msg='Fetching to local cache: artifact %(name)s', + name=artifact.name) + self.lac.copy_from_remote(artifact, self.rac) if artifact.source.morphology.needs_artifact_metadata_cached: if not self.lac.has_artifact_metadata(artifact, 'meta'): @@ -432,9 +434,6 @@ class BuildCommand(object): self.lac.put_artifact_metadata(artifact, 'meta'))) if len(to_fetch) > 0: - self.app.status( - msg='Fetching to local cache: artifact %(name)s', - name=artifact.name) fetch_files(to_fetch) def create_staging_area(self, build_env, use_chroot=True, extra_env={}, diff --git a/morphlib/builder.py b/morphlib/builder.py index 0c681353..9b01f983 100644 --- a/morphlib/builder.py +++ b/morphlib/builder.py @@ -432,13 +432,23 @@ class ChunkBuilder(BuilderBase): extra_files += ['baserock/%s.meta' % chunk_artifact_name] parented_paths = parentify(file_paths + extra_files) - with self.local_artifact_cache.put(chunk_artifact) as f: - self.write_metadata(destdir, chunk_artifact_name, - parented_paths) - - self.app.status(msg='Creating chunk artifact %(name)s', - name=chunk_artifact_name) - morphlib.bins.create_chunk(destdir, f, parented_paths) + self.write_metadata(destdir, chunk_artifact_name, + parented_paths) + + self.app.status(msg='Creating chunk artifact %(name)s', + name=chunk_artifact_name) + # TODO: This is not concurrency safe, bins.create_chunk will + # fail if tempdir already exists (eg if another build + # has created it). + tempdir = os.path.join(self.app.settings['tempdir'], + chunk_artifact.basename()) + try: + morphlib.bins.create_chunk(destdir, tempdir, + parented_paths) + self.local_artifact_cache.put(tempdir, chunk_artifact) + finally: + if os.path.isdir(tempdir): + shutil.rmtree(tempdir) built_artifacts.append(chunk_artifact) for dirname, subdirs, files in os.walk(destdir): @@ -482,8 +492,13 @@ class StratumBuilder(BuilderBase): [x.name for x in constituents]) with lac.put_artifact_metadata(a, 'meta') as f: json.dump(meta, f, indent=4, sort_keys=True) - with self.local_artifact_cache.put(a) as f: + # TODO: This is not concurrency safe, put_stratum_artifact + # deletes temp which could be in use by another + # build. + temp = os.path.join(self.app.settings['tempdir'], a.name) + with open(temp, 'w+') as f: json.dump([c.basename() for c in constituents], f) + self.local_artifact_cache.put_non_ostree_artifact(a, temp) self.save_build_times() return self.source.artifacts.values() @@ -505,7 +520,6 @@ class SystemBuilder(BuilderBase): # pragma: no cover arch = self.source.morphology['arch'] for a_name, artifact in self.source.artifacts.iteritems(): - handle = self.local_artifact_cache.put(artifact) try: fs_root = self.staging_area.destdir(self.source) self.unpack_strata(fs_root) @@ -524,18 +538,10 @@ class SystemBuilder(BuilderBase): # pragma: no cover union_filesystem) self.write_metadata(editable_root, a_name) self.run_system_integration_commands(editable_root) - unslashy_root = editable_root[1:] - def uproot_info(info): - info.name = relpath(info.name, unslashy_root) - if info.islnk(): - info.linkname = relpath(info.linkname, - unslashy_root) - return info - tar = tarfile.open(fileobj=handle, mode="w", name=a_name) - self.app.status(msg='Constructing tarball of rootfs', - chatty=True) - tar.add(editable_root, recursive=True, filter=uproot_info) - tar.close() + # Put the contents of upperdir into the local artifact + # cache. Don't use editable root as we only want to + # store the modified files. + self.local_artifact_cache.put(upperdir, artifact) except BaseException as e: logging.error(traceback.format_exc()) self.app.status(msg='Error while building system', @@ -543,13 +549,11 @@ class SystemBuilder(BuilderBase): # pragma: no cover if editable_root and os.path.exists(editable_root): morphlib.fsutils.unmount(self.app.runcmd, editable_root) - handle.abort() raise else: if editable_root and os.path.exists(editable_root): morphlib.fsutils.unmount(self.app.runcmd, editable_root) - handle.close() self.save_build_times() return self.source.artifacts.itervalues() @@ -558,13 +562,12 @@ class SystemBuilder(BuilderBase): # pragma: no cover '''Unpack a single stratum into a target directory''' cache = self.local_artifact_cache - with cache.get(stratum_artifact) as stratum_file: + with open(cache.get(stratum_artifact), 'r') as stratum_file: artifact_list = json.load(stratum_file, encoding='unicode-escape') for chunk in (ArtifactCacheReference(a) for a in artifact_list): - self.app.status(msg='Unpacking chunk %(basename)s', + self.app.status(msg='Checkout chunk %(basename)s', basename=chunk.basename(), chatty=True) - with cache.get(chunk) as chunk_file: - morphlib.bins.unpack_binary_from_file(chunk_file, target) + cache.get(chunk, target) target_metadata = os.path.join( target, 'baserock', '%s.meta' % stratum_artifact.name) @@ -575,7 +578,7 @@ class SystemBuilder(BuilderBase): # pragma: no cover def unpack_strata(self, path): '''Unpack strata into a directory.''' - self.app.status(msg='Unpacking strata to %(path)s', + self.app.status(msg='Checking out strata to %(path)s', path=path, chatty=True) with self.build_watch('unpack-strata'): for a_name, a in self.source.artifacts.iteritems(): @@ -587,12 +590,14 @@ class SystemBuilder(BuilderBase): # pragma: no cover # download the chunk artifacts if necessary for stratum_artifact in self.source.dependencies: - f = self.local_artifact_cache.get(stratum_artifact) - chunks = [ArtifactCacheReference(c) for c in json.load(f)] + stratum_path = self.local_artifact_cache.get( + stratum_artifact) + with open(stratum_path, 'r') as stratum: + chunks = [ArtifactCacheReference(c) + for c in json.load(stratum)] download_depends(chunks, self.local_artifact_cache, self.remote_artifact_cache) - f.close() # unpack it from the local artifact cache for stratum_artifact in self.source.dependencies: diff --git a/morphlib/builder_tests.py b/morphlib/builder_tests.py index a571e3d0..b5e66521 100644 --- a/morphlib/builder_tests.py +++ b/morphlib/builder_tests.py @@ -105,8 +105,8 @@ class FakeArtifactCache(object): def __init__(self): self._cached = {} - def put(self, artifact): - return FakeFileHandle(self, (artifact.cache_key, artifact.name)) + def put(self, artifact, directory): + self._cached[(artifact.cache_key, artifact.name)] = artifact.name def put_artifact_metadata(self, artifact, name): return FakeFileHandle(self, (artifact.cache_key, artifact.name, name)) @@ -114,7 +114,7 @@ class FakeArtifactCache(object): def put_source_metadata(self, source, cachekey, name): return FakeFileHandle(self, (cachekey, name)) - def get(self, artifact): + def get(self, artifact, directory=None): return StringIO.StringIO( self._cached[(artifact.cache_key, artifact.name)]) @@ -134,6 +134,10 @@ class FakeArtifactCache(object): def has_source_metadata(self, source, cachekey, name): return (cachekey, name) in self._cached + def copy_from_remote(self, artifact, remote): + self._cached[(artifact.cache_key, artifact.name)] = \ + remote._cached[(artifact.cache_key, artifact.name)] + class BuilderBaseTests(unittest.TestCase): @@ -191,9 +195,7 @@ class BuilderBaseTests(unittest.TestCase): rac = FakeArtifactCache() afacts = [FakeArtifact(name) for name in ('a', 'b', 'c')] for a in afacts: - fh = rac.put(a) - fh.write(a.name) - fh.close() + rac.put(a, 'not-a-dir') morphlib.builder.download_depends(afacts, lac, rac) self.assertTrue(all(lac.has(a) for a in afacts)) @@ -202,9 +204,7 @@ class BuilderBaseTests(unittest.TestCase): rac = FakeArtifactCache() afacts = [FakeArtifact(name) for name in ('a', 'b', 'c')] for a in afacts: - fh = rac.put(a) - fh.write(a.name) - fh.close() + rac.put(a, 'not-a-dir') fh = rac.put_artifact_metadata(a, 'meta') fh.write('metadata') fh.close() |