From d7e9e092aee7d607be48dc99b0fe2275575df4fb Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Tue, 18 Mar 2014 17:13:59 +0000 Subject: Extend morphlib.fsutils.mount to take a fstype --- morphlib/fsutils.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/morphlib/fsutils.py b/morphlib/fsutils.py index 0212b987..751f73f6 100644 --- a/morphlib/fsutils.py +++ b/morphlib/fsutils.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -47,10 +47,14 @@ def create_fs(runcmd, partition): # pragma: no cover runcmd(['mkfs.btrfs', '-L', 'baserock', partition]) -def mount(runcmd, partition, mount_point): # pragma: no cover +def mount(runcmd, partition, mount_point, fstype=None): # pragma: no cover if not os.path.exists(mount_point): os.mkdir(mount_point) - runcmd(['mount', partition, mount_point]) + if not fstype: + fstype = [] + else: + fstype = ['-t', fstype] + runcmd(['mount', partition, mount_point] + fstype) def unmount(runcmd, mount_point): # pragma: no cover -- cgit v1.2.1 From 9b5e7c2efe1abb2cc981df085a849afd5a27d355 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Tue, 18 Mar 2014 17:14:52 +0000 Subject: Use morphlib.fsutils.mount when possible --- morphlib/stagingarea.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index dca19c23..61f9e660 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -236,7 +236,7 @@ class StagingArea(object): path = os.path.join(self.dirname, mount_point) if not os.path.exists(path): os.makedirs(path) - self._app.runcmd(['mount', '-t', mount_type, source, path]) + morphlib.fsutils.mount(self._app.runcmd, source, path, mount_type) self.mounted.append(path) return -- cgit v1.2.1 From d29557989a2542bf8c34202eeec9b45390121de7 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Mon, 17 Mar 2014 15:00:14 +0000 Subject: Generate scripts with the system-integration-commands of chunk morphologies. The scripts will be created into the folder: /baserock/sytem-integration/ The system integration commands have the following syntax: name: linux kind: chunk ... install-commands: ... system-integration: linux-libs: 00-depmod: - depmod -a 70-more-integration: - touch /baserock/FILE - | for FOLDER in $(ls /) do echo "$FOLDER" done linux-misc: 70-more-integration: - echo "Hello world" In this concrete example, the following files will be created: $DESTDIR/baserock/system-integration/ 00-depmod-linux-libs-0000: #!/bin/sh set -xev depmod -a 70-more-integration-linux-libs-0000 #!/bin/sh set -xev touch /baserock/FILE 70-more-integration-linux-libs-0001 #!/bin/sh set -xev for FOLDER in $(ls /) do echo "$FOLDER" done 70-more-integration-linux-misc-0000 #!/bin/sh set -xev echo "Hello world" --- morphlib/builder2.py | 45 ++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 42 insertions(+), 3 deletions(-) diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 2c99c6f6..caaad0d0 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -36,6 +36,8 @@ import morphlib from morphlib.artifactcachereference import ArtifactCacheReference import morphlib.gitversion +SYSTEM_INTEGRATION_PATH = os.path.join('baserock', 'system-integration') + def extract_sources(app, repo_cache, repo, sha1, srcdir): #pragma: no cover '''Get sources from git to a source directory, including submodules''' @@ -419,11 +421,44 @@ class ChunkBuilder(BuilderBase): shutil.copyfileobj(readlog, self.app.output) raise e + def write_system_integration_commands(self, destdir, + integration_commands, artifact_name): # pragma: no cover + + rel_path = SYSTEM_INTEGRATION_PATH + dest_path = os.path.join(destdir, SYSTEM_INTEGRATION_PATH) + + scripts_created = [] + + if not os.path.exists(dest_path): + os.makedirs(dest_path) + + if artifact_name in integration_commands: + prefixes_per_artifact = integration_commands[artifact_name] + for prefix, commands in prefixes_per_artifact.iteritems(): + for index, script in enumerate(commands): + script_name = "%s-%s-%04d" % (prefix, + artifact_name, + index) + script_path = os.path.join(dest_path, script_name) + + with morphlib.savefile.SaveFile(script_path, 'w') as f: + f.write("#!/bin/sh\nset -xeu\n") + f.write(script) + os.chmod(script_path, 0555) + + rel_script_path = os.path.join(SYSTEM_INTEGRATION_PATH, + script_name) + scripts_created += [rel_script_path] + + return scripts_created + def assemble_chunk_artifacts(self, destdir): # pragma: no cover built_artifacts = [] filenames = [] source = self.artifact.source split_rules = source.split_rules + morphology = source.morphology + sys_tag = 'system-integration' def filepaths(destdir): for dirname, subdirs, basenames in os.walk(destdir): @@ -438,6 +473,8 @@ class ChunkBuilder(BuilderBase): matches, overlaps, unmatched = \ split_rules.partition(filepaths(destdir)) + system_integration = morphology.get(sys_tag) or {} + with self.build_watch('create-chunks'): for chunk_artifact_name, chunk_artifact \ in source.artifacts.iteritems(): @@ -455,9 +492,11 @@ class ChunkBuilder(BuilderBase): names.update(all_parents(name)) return sorted(names) - parented_paths = \ - parentify(file_paths + - ['baserock/%s.meta' % chunk_artifact_name]) + extra_files = self.write_system_integration_commands( + destdir, system_integration, + chunk_artifact_name) + extra_files += ['baserock/%s.meta' % chunk_artifact_name] + parented_paths = parentify(file_paths + extra_files) with self.local_artifact_cache.put(chunk_artifact) as f: self.write_metadata(destdir, chunk_artifact_name, -- cgit v1.2.1 From 69e48f2cca2e70224562efc75a946dde70a4cfc8 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Thu, 20 Mar 2014 11:11:11 +0000 Subject: Run the system integration commands when building --- morphlib/builder2.py | 46 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) diff --git a/morphlib/builder2.py b/morphlib/builder2.py index caaad0d0..02e8b485 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -588,6 +588,7 @@ class SystemBuilder(BuilderBase): # pragma: no cover fs_root = self.staging_area.destdir(self.artifact.source) self.unpack_strata(fs_root) self.write_metadata(fs_root, rootfs_name) + self.run_system_integration_commands(fs_root) self.copy_kernel_into_artifact_cache(fs_root) unslashy_root = fs_root[1:] def uproot_info(info): @@ -687,6 +688,51 @@ class SystemBuilder(BuilderBase): # pragma: no cover os.chmod(os_release_file, 0644) + def run_system_integration_commands(self, rootdir): # pragma: no cover + ''' Run the system integration commands ''' + + sys_integration_dir = os.path.join(rootdir, SYSTEM_INTEGRATION_PATH) + if not os.path.isdir(sys_integration_dir): + return + + env = { + 'PATH': '/bin:/usr/bin:/sbin:/usr/sbin' + } + + self.app.status(msg='Running the system integration commands', + error=True) + + mounted = [] + to_mount = ( + ('proc', 'proc', 'none'), + ('dev/shm', 'tmpfs', 'none'), + ) + + try: + for mount_point, mount_type, source in to_mount: + logging.debug('Mounting %s in system root filesystem' + % mount_point) + path = os.path.join(rootdir, mount_point) + if not os.path.exists(path): + os.makedirs(path) + morphlib.fsutils.mount(self.app.runcmd, source, path, + mount_type) + mounted.append(path) + + self.app.runcmd(['chroot', rootdir, 'sh', '-c', + 'cd / && run-parts "$1"', '-', SYSTEM_INTEGRATION_PATH], + env=env) + except BaseException, e: + self.app.status( + msg='Error while running system integration commands', + error=True) + raise + finally: + for mount_path in reversed(mounted): + logging.debug('Unmounting %s in system root filesystem' + % mount_path) + morphlib.fsutils.unmount(self.app.runcmd, mount_path) + def copy_kernel_into_artifact_cache(self, path): '''Copy the installed kernel image into the local artifact cache. -- cgit v1.2.1