diff options
-rwxr-xr-x | distbuild-helper | 2 | ||||
-rw-r--r-- | distbuild/build_controller.py | 2 | ||||
-rw-r--r-- | morphlib/__init__.py | 1 | ||||
-rw-r--r-- | morphlib/app.py | 77 | ||||
-rw-r--r-- | morphlib/cachekeycomputer.py | 33 | ||||
-rw-r--r-- | morphlib/cachekeycomputer_tests.py | 13 | ||||
-rw-r--r-- | morphlib/extensions.py | 159 | ||||
-rw-r--r-- | morphlib/exts/nfsboot.write.help | 12 | ||||
-rw-r--r-- | morphlib/exts/rawdisk.write.help | 7 | ||||
-rw-r--r-- | morphlib/exts/tar.write.help | 5 | ||||
-rw-r--r-- | morphlib/plugins/deploy_plugin.py | 59 | ||||
-rw-r--r-- | without-test-modules | 1 |
12 files changed, 284 insertions, 87 deletions
diff --git a/distbuild-helper b/distbuild-helper index 9d70642e..7399fb59 100755 --- a/distbuild-helper +++ b/distbuild-helper @@ -212,7 +212,7 @@ class HelperMachine(distbuild.StateMachine): def _relay_exec_output(self, event_source, event): distbuild.crash_point() - buf_size = 64 + buf_size = 16 * 1024 fd = event.file.fileno() data = os.read(fd, buf_size) if data: diff --git a/distbuild/build_controller.py b/distbuild/build_controller.py index 7849db17..b00344f9 100644 --- a/distbuild/build_controller.py +++ b/distbuild/build_controller.py @@ -213,7 +213,7 @@ class BuildController(distbuild.StateMachine): self.mainloop.queue_event(self, _Start()) def _maybe_abort(self, event_source, event): - if disconnect.id == self._request['id']: + if event.id == self._request['id']: self.mainloop.queue_event(self, _Abort()) def _start_graphing(self, event_source, event): diff --git a/morphlib/__init__.py b/morphlib/__init__.py index f416ae0c..0c928fd3 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -59,6 +59,7 @@ import buildsystem import builder2 import cachedrepo import cachekeycomputer +import extensions import extractedtarball import fsutils import git diff --git a/morphlib/app.py b/morphlib/app.py index 409e0a12..91647a32 100644 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -22,6 +22,7 @@ import sys import time import urlparse import warnings +import extensions import morphlib @@ -60,10 +61,9 @@ class Morph(cliapp.Application): 'show no output unless there is an error') self.settings.boolean(['help', 'h'], - 'show this help message and exit') - + 'show this help message and exit') self.settings.boolean(['help-all'], - 'show help message including hidden subcommands') + 'show help message including hidden subcommands') self.settings.string(['build-ref-prefix'], 'Prefix to use for temporary build refs', @@ -198,6 +198,8 @@ class Morph(cliapp.Application): def setup(self): self.status_prefix = '' + self.add_subcommand('help-extensions', self.help_extensions) + def process_args(self, args): self.check_time() @@ -476,41 +478,52 @@ class Morph(cliapp.Application): compute_setting_values=self.compute_setting_values, add_help_option=False) - class IdentityFormat(): - def format(self, text): - return text - - def _help_helper(self, args, show_all): - try: - width = int(os.environ.get('COLUMNS', '78')) - except ValueError: - width = 78 - - if args: - cmd = args[0] - if cmd not in self.subcommands: - raise cliapp.AppException('Unknown subcommand %s' % cmd) - # TODO Search for other things we might want help on - # such as write or configuration extensions. - usage = self._format_usage_for(cmd) - fmt = self.IdentityFormat() - description = fmt.format(self._format_subcommand_help(cmd)) + def _help(self, show_all): + pp = self.settings.build_parser( + configs_only=True, + arg_synopsis=self.arg_synopsis, + cmd_synopsis=self.cmd_synopsis, + all_options=show_all, + add_help_option=False) + text = pp.format_help() + self.output.write(text) + + def _help_topic(self, topic): + build_ref_prefix = self.settings['build-ref-prefix'] + if topic in self.subcommands: + usage = self._format_usage_for(topic) + description = self._format_subcommand_help(topic) text = '%s\n\n%s' % (usage, description) self.output.write(text) + elif topic in extensions.list_extensions(build_ref_prefix): + name, kind = os.path.splitext(topic) + try: + with extensions.get_extension_filename(build_ref_prefix, + name, + kind + '.help', executable=False) as fname: + with open(fname, 'r') as f: + help_data = morphlib.yamlparse.load(f.read()) + print help_data['help'] + except extensions.ExtensionError: + raise cliapp.AppException( + 'Help not available for extension %s' % topic) else: - pp = self.settings.build_parser( - configs_only=True, - arg_synopsis=self.arg_synopsis, - cmd_synopsis=self.cmd_synopsis, - all_options=show_all, - add_help_option=False) - text = pp.format_help() - self.output.write(text) + raise cliapp.AppException( + 'Unknown subcommand or extension %s' % topic) def help(self, args): # pragma: no cover '''Print help.''' - self._help_helper(args, False) + if args: + self._help_topic(args[0]) + else: + self._help(False) def help_all(self, args): # pragma: no cover '''Print help, including hidden subcommands.''' - self._help_helper(args, True) + self._help(True) + + def help_extensions(self, args): + exts = extensions.list_extensions(self.settings['build-ref-prefix']) + template = "Extensions:\n %s\n" + ext_string = '\n '.join(exts) + self.output.write(template % (ext_string)) diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index 3efe1cbb..ca374436 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -25,6 +25,7 @@ class CacheKeyComputer(object): def __init__(self, build_env): self._build_env = build_env self._calculated = {} + self._hashed = {} def _filterenv(self, env): keys = ["LOGNAME", "MORPH_ARCH", "TARGET", "TARGET_STAGE1", @@ -32,11 +33,18 @@ class CacheKeyComputer(object): return dict([(k, env[k]) for k in keys]) def compute_key(self, artifact): - logging.debug('computing cache key for artifact %s from source ' - 'repo %s, sha1 %s, filename %s' % - (artifact.name, artifact.source.repo_name, - artifact.source.sha1, artifact.source.filename)) - return self._hash_id(self.get_cache_id(artifact)) + try: + ret = self._hashed[artifact] + logging.debug('returning cached key for artifact %s from source ', + (artifact.name, artifact.source.repo_name, + artifact.source.sha1, artifact.source.filename)) + return ret + except KeyError: + logging.debug('computing cache key for artifact %s from source ', + (artifact.name, artifact.source.repo_name, + artifact.source.sha1, artifact.source.filename)) + self._hashed[artifact] = self._hash_id(self.get_cache_id(artifact)) + return self._hashed[artifact] def _hash_id(self, cache_id): sha = hashlib.sha256() @@ -66,13 +74,18 @@ class CacheKeyComputer(object): self._hash_thing(sha, item) def get_cache_id(self, artifact): - logging.debug('computing cache id for artifact %s from source ' - 'repo %s, sha1 %s, filename %s' % - (artifact.name, artifact.source.repo_name, - artifact.source.sha1, artifact.source.filename)) try: - return self._calculated[artifact] + ret = self._calculated[artifact] + logging.debug('returning cached id for artifact %s from source ' + 'repo %s, sha1 %s, filename %s' % + (artifact.name, artifact.source.repo_name, + artifact.source.sha1, artifact.source.filename)) + return ret except KeyError: + logging.debug('computing cache id for artifact %s from source ' + 'repo %s, sha1 %s, filename %s' % + (artifact.name, artifact.source.repo_name, + artifact.source.sha1, artifact.source.filename)) cacheid = self._calculate(artifact) self._calculated[artifact] = cacheid return cacheid diff --git a/morphlib/cachekeycomputer_tests.py b/morphlib/cachekeycomputer_tests.py index 4e73e905..dd10307f 100644 --- a/morphlib/cachekeycomputer_tests.py +++ b/morphlib/cachekeycomputer_tests.py @@ -148,6 +148,19 @@ class CacheKeyComputerTests(unittest.TestCase): validchars = '0123456789abcdef' return len(s) == 64 and all([c in validchars for c in s]) + def test_compute_twice_same_key(self): + artifact = self._find_artifact('system-rootfs') + self.assertEqual(self.ckc.compute_key(artifact), + self.ckc.compute_key(artifact)) + + def test_compute_twice_same_id(self): + artifact = self._find_artifact('system-rootfs') + id1 = self.ckc.get_cache_id(artifact) + id2 = self.ckc.get_cache_id(artifact) + hash1 = self.ckc._hash_id(id1) + hash2 = self.ckc._hash_id(id2) + self.assertEqual(hash1, hash2) + def test_compute_key_returns_sha256(self): artifact = self._find_artifact('system-rootfs') self.assertTrue(self._valid_sha256( diff --git a/morphlib/extensions.py b/morphlib/extensions.py new file mode 100644 index 00000000..be551fdd --- /dev/null +++ b/morphlib/extensions.py @@ -0,0 +1,159 @@ +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import cliapp +import morphlib +import glob +import os +import sysbranchdir +import stat +import tempfile + +class ExtensionError(morphlib.Error): + pass + +class ExtensionNotFoundError(ExtensionError): + pass + +class ExtensionNotExecutableError(ExtensionError): + pass + +def _get_root_repo(build_ref_prefix): + system_branch = morphlib.sysbranchdir.open_from_within('.') + root_repo_dir = morphlib.gitdir.GitDirectory( + system_branch.get_git_directory_name( + system_branch.root_repository_url)) + build_branch = morphlib.buildbranch.BuildBranch(system_branch, + build_ref_prefix, + push_temporary=False) + ref = build_branch.root_ref + + return (build_branch.root_ref, root_repo_dir) + +def _get_morph_extension_directory(): + code_dir = os.path.dirname(morphlib.__file__) + return os.path.join(code_dir, 'exts') + +def _list_repo_extension_filenames(build_ref_prefix, + kind): #pragma: no cover + (ref, repo_dir) = _get_root_repo(build_ref_prefix) + files = repo_dir.list_files(ref) + return (f for f in files if os.path.splitext(f)[1] == kind) + +def _list_morph_extension_filenames(kind): + return glob.glob(os.path.join(_get_morph_extension_directory(), + '*' + kind)) + +def _get_extension_name(filename): + return os.path.basename(filename) + +def _get_repo_extension_contents(build_ref_prefix, name, kind): + (ref, repo_dir) = _get_root_repo(build_ref_prefix) + return repo_dir.get_file_from_ref(ref, name + kind) + +def _get_morph_extension_filename(name, kind): + return os.path.join(_get_morph_extension_directory(), name + kind) + +def _is_executable(filename): + st = os.stat(filename) + mask = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH + return (stat.S_IMODE(st.st_mode) & mask) != 0 + +def _list_extensions(build_ref_prefix, kind): + repo_extension_filenames = [] + try: + repo_extension_filenames = \ + _list_repo_extension_filenames(build_ref_prefix, kind) + except (sysbranchdir.NotInSystemBranch): + # Squash this and just return no system branch extensions + pass + morph_extension_filenames = _list_morph_extension_filenames(kind) + + repo_extension_names = \ + (_get_extension_name(f) for f in repo_extension_filenames) + morph_extension_names = \ + (_get_extension_name(f) for f in morph_extension_filenames) + + extension_names = set(repo_extension_names) + extension_names.update(set(morph_extension_names)) + return list(extension_names) + +def list_extensions(build_ref_prefix, kind=None): + """ + List all available extensions by 'kind'. + + 'kind' should be one of '.write' or '.configure'. + If 'kind' is not provided available extensions of both + types will be returned. + + '.check' extensions are not listed here as they should + be associated with a '.write' extension of the same name. + """ + if kind: + return _list_extensions(build_ref_prefix, kind) + else: + configure_extensions = _list_extensions(build_ref_prefix, '.configure') + write_extensions = _list_extensions(build_ref_prefix, '.write') + + return configure_extensions + write_extensions + +class get_extension_filename(): + """ + Find the filename of an extension by its 'name' and 'kind'. + + 'kind' should be one of '.configure', '.write' or '.check'. + + '.help' files for the extensions may also be retrieved by + passing the kind as '.write.help' or '.configure.help'. + + If the extension is in the build repository then a temporary + file will be created, which will be deleted on exting the with block. + """ + def __init__(self, build_ref_prefix, name, kind, executable=True): + self.build_ref_prefix = build_ref_prefix + self.name = name + self.kind = kind + self.executable = executable + self.delete = False + + def __enter__(self): + ext_filename = None + try: + ext_contents = _get_repo_extension_contents(self.build_ref_prefix, + self.name, + self.kind) + except cliapp.AppException, sysbranchdir.NotInSystemBranch: + # Not found: look for it in the Morph code. + ext_filename = _get_morph_extension_filename(self.name, self.kind) + if not os.path.exists(ext_filename): + raise ExtensionNotFoundError( + 'Could not find extension %s%s' % (self.name, self.kind)) + if self.executable and not _is_executable(ext_filename): + raise ExtensionNotExecutableError( + 'Extension not executable: %s' % ext_filename) + else: + # Found it in the system morphology's repository. + fd, ext_filename = tempfile.mkstemp() + os.write(fd, ext_contents) + os.close(fd) + os.chmod(ext_filename, 0700) + self.delete = True + + self.ext_filename = ext_filename + return ext_filename + + def __exit__(self, type, value, trace): + if self.delete: + os.remove(self.ext_filename) diff --git a/morphlib/exts/nfsboot.write.help b/morphlib/exts/nfsboot.write.help new file mode 100644 index 00000000..598b1b23 --- /dev/null +++ b/morphlib/exts/nfsboot.write.help @@ -0,0 +1,12 @@ +help: | + Deploy a system image and kernel to an nfsboot server. + + An nfsboot server is defined as a baserock system that has + tftp and nfs servers running, the tftp server is exporting + the contents of /srv/nfsboot/tftp/ and the user has sufficient + permissions to create nfs roots in /srv/nfsboot/nfs/. + + The `location` argument is the hostname of the nfsboot server. + + The extension will connect to root@HOST via ssh to copy the + kernel and rootfs, and configure the nfs server. diff --git a/morphlib/exts/rawdisk.write.help b/morphlib/exts/rawdisk.write.help new file mode 100644 index 00000000..a514a4e8 --- /dev/null +++ b/morphlib/exts/rawdisk.write.help @@ -0,0 +1,7 @@ +help: | + Create a raw disk image during Morph's deployment. + + If the image already exists, it is upgraded. + + The `location` argument is a pathname to the image to be + created or upgraded. diff --git a/morphlib/exts/tar.write.help b/morphlib/exts/tar.write.help new file mode 100644 index 00000000..f052ac03 --- /dev/null +++ b/morphlib/exts/tar.write.help @@ -0,0 +1,5 @@ +help: | + Create a .tar file of the deployed system. + + The `location` argument is a pathname to the .tar file to be + created. diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index ae62b75d..1d582949 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -26,11 +26,6 @@ import uuid import morphlib - -class ExtensionNotFoundError(morphlib.Error): - pass - - class DeployPlugin(cliapp.Plugin): def enable(self): @@ -425,9 +420,9 @@ class DeployPlugin(cliapp.Plugin): # extension itself has the chance to raise an error. try: self._run_extension( - root_repo_dir, ref, deployment_type, '.check', + root_repo_dir, deployment_type, '.check', [location], env) - except ExtensionNotFoundError: + except morphlib.extensions.ExtensionNotFoundError: pass def setup_deploy(self, build_command, deploy_tempdir, root_repo_dir, ref, @@ -485,7 +480,6 @@ class DeployPlugin(cliapp.Plugin): for name in names: self._run_extension( root_repo_dir, - ref, name, '.configure', [system_tree], @@ -495,7 +489,6 @@ class DeployPlugin(cliapp.Plugin): self.app.status(msg='Writing to device') self._run_extension( root_repo_dir, - ref, deployment_type, '.write', [system_tree, location], @@ -506,7 +499,7 @@ class DeployPlugin(cliapp.Plugin): self.app.status(msg='Cleaning up') shutil.rmtree(deploy_private_tempdir) - def _run_extension(self, gd, ref, name, kind, args, env): + def _run_extension(self, gd, name, kind, args, env): '''Run an extension. The ``kind`` should be either ``.configure`` or ``.write``, @@ -516,39 +509,19 @@ class DeployPlugin(cliapp.Plugin): system morphology (repo, ref), or with the Morph code. ''' - - # Look for extension in the system morphology's repository. - try: - ext = gd.get_file_from_ref(ref, name + kind) - except cliapp.AppException: - # Not found: look for it in the Morph code. - code_dir = os.path.dirname(morphlib.__file__) - ext_filename = os.path.join(code_dir, 'exts', name + kind) - if not os.path.exists(ext_filename): - raise ExtensionNotFoundError( - 'Could not find extension %s%s' % (name, kind)) - if not self._is_executable(ext_filename): - raise morphlib.Error( - 'Extension not executable: %s' % ext_filename) - delete_ext = False - else: - # Found it in the system morphology's repository. - fd, ext_filename = tempfile.mkstemp() - os.write(fd, ext) - os.close(fd) - os.chmod(ext_filename, 0700) - delete_ext = True - - self.app.status(msg='Running extension %(name)s%(kind)s', - name=name, kind=kind) - self.app.runcmd( - [ext_filename] + args, - ['sh', '-c', 'while read l; do echo `date "+%F %T"` "$1$l"; done', - '-', '%s[%s]' % (self.app.status_prefix, name + kind)], - cwd=gd.dirname, env=env, stdout=None, stderr=None) - - if delete_ext: - os.remove(ext_filename) + build_ref_prefix = self.app.settings['build-ref-prefix'] + with morphlib.extensions.get_extension_filename( + build_ref_prefix, name, kind) as ext_filename: + self.app.status(msg='Running extension %(name)s%(kind)s', + name=name, kind=kind) + self.app.runcmd( + [ext_filename] + args, + ['sh', + '-c', + 'while read l; do echo `date "+%F %T"` "$1$l"; done', + '-', + '%s[%s]' % (self.app.status_prefix, name + kind)], + cwd=gd.dirname, env=env, stdout=None, stderr=None) def _is_executable(self, filename): st = os.stat(filename) diff --git a/without-test-modules b/without-test-modules index 9b893300..a42cce97 100644 --- a/without-test-modules +++ b/without-test-modules @@ -6,6 +6,7 @@ morphlib/tester.py morphlib/git.py morphlib/app.py morphlib/mountableimage.py +morphlib/extensions.py morphlib/extractedtarball.py morphlib/plugins/artifact_inspection_plugin.py morphlib/plugins/cross-bootstrap_plugin.py |