diff options
author | Jannis Pohlmann <jannis.pohlmann@codethink.co.uk> | 2012-05-03 10:46:23 +0000 |
---|---|---|
committer | Jannis Pohlmann <jannis.pohlmann@codethink.co.uk> | 2012-05-03 12:53:53 +0000 |
commit | b857a4ff09d3ec69211a655888e7bb64105303bd (patch) | |
tree | a60709519a7e80dcbdfcb25e64f7285098ad031d /morphlib | |
parent | c349db9577380caa2df17d7c08f7ea9d4bfe17c2 (diff) | |
download | morph-b857a4ff09d3ec69211a655888e7bb64105303bd.tar.gz |
Integrate RemoteArtifactCache into the building process.
For now we do not provide an option to avoid downloading artifacts
during build. Unsetting --cache-server is an option but it's not
user-friendly. Of course if there is no network connection, then
downloading will fail and things will be built locally. We may
want to add a generic --no-network switch to replace/extend
--no-git-update later.
Diffstat (limited to 'morphlib')
-rw-r--r-- | morphlib/builder2.py | 52 |
1 files changed, 37 insertions, 15 deletions
diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 1d0cbd98..3ff4e811 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -64,10 +64,12 @@ class BuilderBase(object): '''Base class for building artifacts.''' - def __init__(self, staging_area, artifact_cache, artifact, repo_cache, + def __init__(self, staging_area, local_artifact_cache, + remote_artifact_cache, artifact, repo_cache, build_env, max_jobs, setup_proc): self.staging_area = staging_area - self.artifact_cache = artifact_cache + self.local_artifact_cache = local_artifact_cache + self.remote_artifact_cache = remote_artifact_cache self.artifact = artifact self.repo_cache = repo_cache self.build_env = build_env @@ -88,7 +90,7 @@ class BuilderBase(object): } logging.debug('Writing metadata to the cache') - with self.artifact_cache.put_source_metadata( + with self.local_artifact_cache.put_source_metadata( self.artifact.source, self.artifact.cache_key, 'meta') as f: json.dump(meta, f, indent=4, sort_keys=True) @@ -295,7 +297,7 @@ class ChunkBuilder(BuilderBase): patterns += [r'baserock/%s\.' % artifact_name] artifact = self.new_artifact(artifact_name) - with self.artifact_cache.put(artifact) as f: + with self.local_artifact_cache.put(artifact) as f: logging.debug('assembling chunk %s' % artifact_name) logging.debug('assembling into %s' % f.name) morphlib.bins.create_chunk(destdir, f, patterns, ex) @@ -319,14 +321,23 @@ class StratumBuilder(BuilderBase): if dependency.source.morphology['kind'] == 'chunk'] with self.build_watch('unpack-chunks'): for chunk_artifact in constituents: - with self.artifact_cache.get(chunk_artifact) as f: - morphlib.bins.unpack_binary_from_file(f, destdir) + # download the chunk artifact if necessary + if not self.local_artifact_cache.has(chunk_artifact): + source = self.remote_artifact_cache.get(chunk_artifact) + target = self.local_artifact_cache.put(chunk_artifact) + shutil.copyfileobj(source, target) + target.close() + source.close() + # unpack it from the local artifact cache + f = self.local_artifact_cache.get(chunk_artifact) + morphlib.bins.unpack_binary_from_file(f, destdir) + f.close() with self.build_watch('create-binary'): artifact_name = self.artifact.source.morphology['name'] self.write_metadata(destdir, artifact_name) artifact = self.new_artifact(artifact_name) - with self.artifact_cache.put(artifact) as f: + with self.local_artifact_cache.put(artifact) as f: morphlib.bins.create_stratum(destdir, f, None) self.save_build_times() @@ -416,8 +427,17 @@ class SystemBuilder(BuilderBase): # pragma: no cover logging.debug('Unpacking strata to %s' % path) with self.build_watch('unpack-strata'): for stratum_artifact in self.artifact.dependencies: - with self.artifact_cache.get(stratum_artifact) as f: - morphlib.bins.unpack_binary_from_file(f, path) + # download the stratum artifact if necessary + if not self.local_artifact_cache.has(stratum_artifact): + source = self.remote_artifact_cache.get(stratum_artifact) + target = self.local_artifact_cache.put(stratum_artifact) + shutil.copyfileobj(source, target) + target.close() + source.close() + # unpack it from the local artifact cache + f = self.local_artifact_cache.get(stratum_artifact) + morphlib.bins.unpack_binary_from_file(f, path) + f.close() ldconfig(self.ex, path) def _create_fstab(self, path): @@ -486,7 +506,7 @@ class SystemBuilder(BuilderBase): # pragma: no cover # FIXME: Need to create file directly in cache to avoid costly # copying here. with self.build_watch('cache-image'): - with self.artifact_cache.put(self.artifact) as outf: + with self.local_artifact_cache.put(self.artifact) as outf: with open(image_name) as inf: while True: data = inf.read(1024**2) @@ -505,10 +525,11 @@ class Builder(object): # pragma: no cover 'system': SystemBuilder, } - def __init__(self, staging_area, artifact_cache, repo_cache, build_env, - max_jobs): + def __init__(self, staging_area, local_artifact_cache, + remote_artifact_cache, repo_cache, build_env, max_jobs): self.staging_area = staging_area - self.artifact_cache = artifact_cache + self.local_artifact_cache = local_artifact_cache + self.remote_artifact_cache = remote_artifact_cache self.repo_cache = repo_cache self.build_env = build_env self.max_jobs = max_jobs @@ -516,8 +537,9 @@ class Builder(object): # pragma: no cover def build_and_cache(self, artifact): kind = artifact.source.morphology['kind'] - o = self.classes[kind](self.staging_area, self.artifact_cache, - artifact, self.repo_cache, self.build_env, + o = self.classes[kind](self.staging_area, self.local_artifact_cache, + self.remote_artifact_cache, artifact, + self.repo_cache, self.build_env, self.max_jobs, self.setup_proc) logging.debug('Builder.build: artifact %s with %s' % (artifact.name, repr(o))) |