summaryrefslogtreecommitdiff
path: root/morphlib/builder.py
diff options
context:
space:
mode:
Diffstat (limited to 'morphlib/builder.py')
-rw-r--r--morphlib/builder.py131
1 files changed, 51 insertions, 80 deletions
diff --git a/morphlib/builder.py b/morphlib/builder.py
index 1c016674..426c0ed0 100644
--- a/morphlib/builder.py
+++ b/morphlib/builder.py
@@ -28,7 +28,6 @@ import tempfile
import cliapp
import morphlib
-from morphlib.artifactcachereference import ArtifactCacheReference
from morphlib.util import error_message_for_containerised_commandline
import morphlib.gitversion
@@ -125,11 +124,7 @@ def ldconfig(runcmd, rootdir): # pragma: no cover
def download_depends(constituents, lac, rac, metadatas=None):
for constituent in constituents:
if not lac.has(constituent):
- source = rac.get(constituent)
- target = lac.put(constituent)
- shutil.copyfileobj(source, target)
- target.close()
- source.close()
+ lac.copy_from_remote(constituent, rac)
if metadatas is not None:
for metadata in metadatas:
if not lac.has_artifact_metadata(constituent, metadata):
@@ -246,28 +241,6 @@ class ChunkBuilder(BuilderBase):
'''Build chunk artifacts.'''
- def create_devices(self, destdir): # pragma: no cover
- '''Creates device nodes if the morphology specifies them'''
- morphology = self.source.morphology
- perms_mask = stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO
- if 'devices' in morphology and morphology['devices'] is not None:
- for dev in morphology['devices']:
- destfile = os.path.join(destdir, './' + dev['filename'])
- mode = int(dev['permissions'], 8) & perms_mask
- if dev['type'] == 'c':
- mode = mode | stat.S_IFCHR
- elif dev['type'] == 'b':
- mode = mode | stat.S_IFBLK
- else:
- raise IOError('Cannot create device node %s,'
- 'unrecognized device type "%s"'
- % (destfile, dev['type']))
- self.app.status(msg="Creating device node %s"
- % destfile)
- os.mknod(destfile, mode,
- os.makedev(dev['major'], dev['minor']))
- os.chown(destfile, dev['uid'], dev['gid'])
-
def build_and_cache(self): # pragma: no cover
with self.build_watch('overall-build'):
@@ -286,7 +259,6 @@ class ChunkBuilder(BuilderBase):
try:
self.get_sources(builddir)
self.run_commands(builddir, destdir, temppath, stdout)
- self.create_devices(destdir)
os.rename(temppath, logpath)
except BaseException as e:
@@ -459,13 +431,23 @@ class ChunkBuilder(BuilderBase):
extra_files += ['baserock/%s.meta' % chunk_artifact_name]
parented_paths = parentify(file_paths + extra_files)
- with self.local_artifact_cache.put(chunk_artifact) as f:
- self.write_metadata(destdir, chunk_artifact_name,
- parented_paths)
+ self.write_metadata(destdir, chunk_artifact_name,
+ parented_paths)
- self.app.status(msg='Creating chunk artifact %(name)s',
- name=chunk_artifact_name)
- morphlib.bins.create_chunk(destdir, f, parented_paths)
+ self.app.status(msg='Creating chunk artifact %(name)s',
+ name=chunk_artifact_name)
+ # TODO: This is not concurrency safe, bins.create_chunk will
+ # fail if tempdir already exists (eg if another build
+ # has created it).
+ tempdir = os.path.join(self.app.settings['tempdir'],
+ chunk_artifact.basename())
+ try:
+ morphlib.bins.create_chunk(destdir, tempdir,
+ parented_paths)
+ self.local_artifact_cache.put(tempdir, chunk_artifact)
+ finally:
+ if os.path.isdir(tempdir):
+ shutil.rmtree(tempdir)
built_artifacts.append(chunk_artifact)
for dirname, subdirs, files in os.walk(destdir):
@@ -509,8 +491,13 @@ class StratumBuilder(BuilderBase):
[x.name for x in constituents])
with lac.put_artifact_metadata(a, 'meta') as f:
json.dump(meta, f, indent=4, sort_keys=True)
- with self.local_artifact_cache.put(a) as f:
+ # TODO: This is not concurrency safe, put_stratum_artifact
+ # deletes temp which could be in use by another
+ # build.
+ temp = os.path.join(self.app.settings['tempdir'], a.name)
+ with open(temp, 'w+') as f:
json.dump([c.basename() for c in constituents], f)
+ self.local_artifact_cache.put_non_ostree_artifact(a, temp)
self.save_build_times()
return self.source.artifacts.values()
@@ -532,64 +519,47 @@ class SystemBuilder(BuilderBase): # pragma: no cover
arch = self.source.morphology['arch']
for a_name, artifact in self.source.artifacts.iteritems():
- handle = self.local_artifact_cache.put(artifact)
-
try:
fs_root = self.staging_area.destdir(self.source)
- self.unpack_strata(fs_root)
- self.write_metadata(fs_root, a_name)
- self.run_system_integration_commands(fs_root)
- unslashy_root = fs_root[1:]
- def uproot_info(info):
- info.name = relpath(info.name, unslashy_root)
- if info.islnk():
- info.linkname = relpath(info.linkname,
- unslashy_root)
- return info
- tar = tarfile.open(fileobj=handle, mode="w", name=a_name)
- self.app.status(msg='Constructing tarball of rootfs',
- chatty=True)
- tar.add(fs_root, recursive=True, filter=uproot_info)
- tar.close()
+ upperdir = self.staging_area.overlay_upperdir(
+ self.source)
+ editable_root = self.staging_area.overlaydir(self.source)
+ workdir = os.path.join(self.staging_area.dirname,
+ 'overlayfs-workdir')
+ if not os.path.exists(workdir):
+ os.makedirs(workdir)
+ union_filesystem = self.app.settings['union-filesystem']
+ morphlib.fsutils.overlay_mount(self.app.runcmd,
+ 'overlay-%s' % a_name,
+ editable_root, fs_root,
+ upperdir, workdir,
+ union_filesystem)
+ try:
+ self.unpack_strata(fs_root)
+ self.write_metadata(editable_root, a_name)
+ self.run_system_integration_commands(editable_root)
+ self.local_artifact_cache.put(editable_root, artifact)
+ finally:
+ morphlib.fsutils.unmount(self.app.runcmd,
+ editable_root)
except BaseException as e:
logging.error(traceback.format_exc())
self.app.status(msg='Error while building system',
error=True)
- handle.abort()
raise
- else:
- handle.close()
self.save_build_times()
return self.source.artifacts.itervalues()
- def load_stratum(self, stratum_artifact):
- '''Load a stratum from the local artifact cache.
-
- Returns a list of ArtifactCacheReference instances for the chunks
- contained in the stratum.
-
- '''
- cache = self.local_artifact_cache
- with cache.get(stratum_artifact) as stratum_file:
- try:
- artifact_list = json.load(stratum_file,
- encoding='unicode-escape')
- except ValueError as e:
- raise cliapp.AppException(
- 'Corruption detected: %s while loading %s' %
- (e, cache.artifact_filename(stratum_artifact)))
- return [ArtifactCacheReference(a) for a in artifact_list]
-
def unpack_one_stratum(self, stratum_artifact, target):
'''Unpack a single stratum into a target directory'''
cache = self.local_artifact_cache
- for chunk in self.load_stratum(stratum_artifact):
- self.app.status(msg='Unpacking chunk %(basename)s',
+ chunks = morphlib.util.get_stratum_contents(cache, stratum_artifact)
+ for chunk in chunks:
+ self.app.status(msg='Checkout chunk %(basename)s',
basename=chunk.basename(), chatty=True)
- with cache.get(chunk) as chunk_file:
- morphlib.bins.unpack_binary_from_file(chunk_file, target)
+ cache.get(chunk, target)
target_metadata = os.path.join(
target, 'baserock', '%s.meta' % stratum_artifact.name)
@@ -600,7 +570,7 @@ class SystemBuilder(BuilderBase): # pragma: no cover
def unpack_strata(self, path):
'''Unpack strata into a directory.'''
- self.app.status(msg='Unpacking strata to %(path)s',
+ self.app.status(msg='Checking out strata to %(path)s',
path=path, chatty=True)
with self.build_watch('unpack-strata'):
for a_name, a in self.source.artifacts.iteritems():
@@ -612,7 +582,8 @@ class SystemBuilder(BuilderBase): # pragma: no cover
# download the chunk artifacts if necessary
for stratum_artifact in self.source.dependencies:
- chunks = self.load_stratum(stratum_artifact)
+ chunks = morphlib.util.get_stratum_contents(
+ self.local_artifact_cache, stratum_artifact)
download_depends(chunks,
self.local_artifact_cache,
self.remote_artifact_cache)
View Lorry Controller Status