diff options
| -rwxr-xr-x | morph | 12 | ||||
| -rw-r--r-- | morphlib/builder2.py | 52 |
2 files changed, 46 insertions, 18 deletions
@@ -185,6 +185,11 @@ class Morph(cliapp.Application): ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) lac = morphlib.localartifactcache.LocalArtifactCache( os.path.join(cachedir, 'artifacts')) + if self.settings['cache-server']: + rac = morphlib.remoteartifactcache.RemoteArtifactCache( + self.settings['cache-server']) + else: + rac = None lrc = morphlib.localrepocache.LocalRepoCache( os.path.join(cachedir, 'gits'), self.settings['git-base-url'], @@ -220,7 +225,8 @@ class Morph(cliapp.Application): for group in order.groups: for artifact in group: if not lac.has(artifact): - needed.append(artifact) + if not rac or not rac.has(artifact): + needed.append(artifact) logging.debug('cmd_build: cloning/updating cached repos') done = set() @@ -258,8 +264,8 @@ class Morph(cliapp.Application): self._install_initial_staging(staging_area) builder = morphlib.builder2.Builder( - staging_area, lac, lrc, - build_env, self.settings['max-jobs']) + staging_area, lac, rac, lrc, build_env, + self.settings['max-jobs']) if setup_proc: builder.setup_proc = True diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 1d0cbd98..3ff4e811 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -64,10 +64,12 @@ class BuilderBase(object): '''Base class for building artifacts.''' - def __init__(self, staging_area, artifact_cache, artifact, repo_cache, + def __init__(self, staging_area, local_artifact_cache, + remote_artifact_cache, artifact, repo_cache, build_env, max_jobs, setup_proc): self.staging_area = staging_area - self.artifact_cache = artifact_cache + self.local_artifact_cache = local_artifact_cache + self.remote_artifact_cache = remote_artifact_cache self.artifact = artifact self.repo_cache = repo_cache self.build_env = build_env @@ -88,7 +90,7 @@ class BuilderBase(object): } logging.debug('Writing metadata to the cache') - with self.artifact_cache.put_source_metadata( + with self.local_artifact_cache.put_source_metadata( self.artifact.source, self.artifact.cache_key, 'meta') as f: json.dump(meta, f, indent=4, sort_keys=True) @@ -295,7 +297,7 @@ class ChunkBuilder(BuilderBase): patterns += [r'baserock/%s\.' % artifact_name] artifact = self.new_artifact(artifact_name) - with self.artifact_cache.put(artifact) as f: + with self.local_artifact_cache.put(artifact) as f: logging.debug('assembling chunk %s' % artifact_name) logging.debug('assembling into %s' % f.name) morphlib.bins.create_chunk(destdir, f, patterns, ex) @@ -319,14 +321,23 @@ class StratumBuilder(BuilderBase): if dependency.source.morphology['kind'] == 'chunk'] with self.build_watch('unpack-chunks'): for chunk_artifact in constituents: - with self.artifact_cache.get(chunk_artifact) as f: - morphlib.bins.unpack_binary_from_file(f, destdir) + # download the chunk artifact if necessary + if not self.local_artifact_cache.has(chunk_artifact): + source = self.remote_artifact_cache.get(chunk_artifact) + target = self.local_artifact_cache.put(chunk_artifact) + shutil.copyfileobj(source, target) + target.close() + source.close() + # unpack it from the local artifact cache + f = self.local_artifact_cache.get(chunk_artifact) + morphlib.bins.unpack_binary_from_file(f, destdir) + f.close() with self.build_watch('create-binary'): artifact_name = self.artifact.source.morphology['name'] self.write_metadata(destdir, artifact_name) artifact = self.new_artifact(artifact_name) - with self.artifact_cache.put(artifact) as f: + with self.local_artifact_cache.put(artifact) as f: morphlib.bins.create_stratum(destdir, f, None) self.save_build_times() @@ -416,8 +427,17 @@ class SystemBuilder(BuilderBase): # pragma: no cover logging.debug('Unpacking strata to %s' % path) with self.build_watch('unpack-strata'): for stratum_artifact in self.artifact.dependencies: - with self.artifact_cache.get(stratum_artifact) as f: - morphlib.bins.unpack_binary_from_file(f, path) + # download the stratum artifact if necessary + if not self.local_artifact_cache.has(stratum_artifact): + source = self.remote_artifact_cache.get(stratum_artifact) + target = self.local_artifact_cache.put(stratum_artifact) + shutil.copyfileobj(source, target) + target.close() + source.close() + # unpack it from the local artifact cache + f = self.local_artifact_cache.get(stratum_artifact) + morphlib.bins.unpack_binary_from_file(f, path) + f.close() ldconfig(self.ex, path) def _create_fstab(self, path): @@ -486,7 +506,7 @@ class SystemBuilder(BuilderBase): # pragma: no cover # FIXME: Need to create file directly in cache to avoid costly # copying here. with self.build_watch('cache-image'): - with self.artifact_cache.put(self.artifact) as outf: + with self.local_artifact_cache.put(self.artifact) as outf: with open(image_name) as inf: while True: data = inf.read(1024**2) @@ -505,10 +525,11 @@ class Builder(object): # pragma: no cover 'system': SystemBuilder, } - def __init__(self, staging_area, artifact_cache, repo_cache, build_env, - max_jobs): + def __init__(self, staging_area, local_artifact_cache, + remote_artifact_cache, repo_cache, build_env, max_jobs): self.staging_area = staging_area - self.artifact_cache = artifact_cache + self.local_artifact_cache = local_artifact_cache + self.remote_artifact_cache = remote_artifact_cache self.repo_cache = repo_cache self.build_env = build_env self.max_jobs = max_jobs @@ -516,8 +537,9 @@ class Builder(object): # pragma: no cover def build_and_cache(self, artifact): kind = artifact.source.morphology['kind'] - o = self.classes[kind](self.staging_area, self.artifact_cache, - artifact, self.repo_cache, self.build_env, + o = self.classes[kind](self.staging_area, self.local_artifact_cache, + self.remote_artifact_cache, artifact, + self.repo_cache, self.build_env, self.max_jobs, self.setup_proc) logging.debug('Builder.build: artifact %s with %s' % (artifact.name, repr(o))) |