Merge remote-tracking branch 'origin/baserock/markdoffman/s10617/add-yaml-help-option'

8 files changed, 246 insertions, 75 deletions

diff --git a/morphlib/__init__.py b/morphlib/__init__.py
index f416ae0c..0c928fd3 100644
--- a/morphlib/__init__.py
+++ b/morphlib/__init__.py
@@ -59,6 +59,7 @@ import buildsystem
 import builder2
 import cachedrepo
 import cachekeycomputer
+import extensions
 import extractedtarball
 import fsutils
 import git
diff --git a/morphlib/app.py b/morphlib/app.py
index 409e0a12..91647a32 100644
--- a/morphlib/app.py
+++ b/morphlib/app.py
@@ -22,6 +22,7 @@ import sys
 import time
 import urlparse
 import warnings
+import extensions
 
 import morphlib
 
@@ -60,10 +61,9 @@ class Morph(cliapp.Application):
                               'show no output unless there is an error')
 
         self.settings.boolean(['help', 'h'],
-                             'show this help message and exit')
-
+                              'show this help message and exit') 
         self.settings.boolean(['help-all'],
-                             'show help message including hidden subcommands')
+                              'show help message including hidden subcommands')
 
         self.settings.string(['build-ref-prefix'],
                              'Prefix to use for temporary build refs',
@@ -198,6 +198,8 @@ class Morph(cliapp.Application):
     def setup(self):
         self.status_prefix = ''
 
+        self.add_subcommand('help-extensions', self.help_extensions)
+
     def process_args(self, args):
         self.check_time()
 
@@ -476,41 +478,52 @@ class Morph(cliapp.Application):
                          compute_setting_values=self.compute_setting_values,
                          add_help_option=False)
 
-    class IdentityFormat():
-        def format(self, text):
-            return text
-
-    def _help_helper(self, args, show_all):
-        try:
-            width = int(os.environ.get('COLUMNS', '78'))
-        except ValueError:
-            width = 78
-
-        if args:
-            cmd = args[0]
-            if cmd not in self.subcommands:
-                raise cliapp.AppException('Unknown subcommand %s' % cmd)
-                # TODO Search for other things we might want help on
-                # such as write or configuration extensions.
-            usage = self._format_usage_for(cmd)
-            fmt = self.IdentityFormat()
-            description = fmt.format(self._format_subcommand_help(cmd))
+    def _help(self, show_all):
+        pp = self.settings.build_parser(
+            configs_only=True,
+            arg_synopsis=self.arg_synopsis,
+            cmd_synopsis=self.cmd_synopsis,
+            all_options=show_all,
+            add_help_option=False)
+        text = pp.format_help()
+        self.output.write(text)
+
+    def _help_topic(self, topic):
+        build_ref_prefix = self.settings['build-ref-prefix']
+        if topic in self.subcommands:
+            usage = self._format_usage_for(topic)
+            description = self._format_subcommand_help(topic)
             text = '%s\n\n%s' % (usage, description)
             self.output.write(text)
+        elif topic in extensions.list_extensions(build_ref_prefix):
+            name, kind = os.path.splitext(topic)
+            try:
+                with extensions.get_extension_filename(build_ref_prefix,
+                        name,
+                        kind + '.help', executable=False) as fname:
+                    with open(fname, 'r') as f:
+                        help_data = morphlib.yamlparse.load(f.read())
+                        print help_data['help']
+            except extensions.ExtensionError:
+                    raise cliapp.AppException(
+                            'Help not available for extension %s' % topic)
         else:
-            pp = self.settings.build_parser(
-                configs_only=True,
-                arg_synopsis=self.arg_synopsis,
-                cmd_synopsis=self.cmd_synopsis,
-                all_options=show_all,
-                add_help_option=False)
-            text = pp.format_help()
-            self.output.write(text)
+            raise cliapp.AppException(
+                    'Unknown subcommand or extension %s' % topic)
 
     def help(self, args): # pragma: no cover
         '''Print help.'''
-        self._help_helper(args, False)
+        if args:
+            self._help_topic(args[0])
+        else:
+            self._help(False)
 
     def help_all(self, args): # pragma: no cover
         '''Print help, including hidden subcommands.'''
-        self._help_helper(args, True)
+        self._help(True)
+
+    def help_extensions(self, args):
+        exts = extensions.list_extensions(self.settings['build-ref-prefix'])
+        template = "Extensions:\n    %s\n"
+        ext_string = '\n    '.join(exts)
+        self.output.write(template % (ext_string))
diff --git a/morphlib/extensions.py b/morphlib/extensions.py
new file mode 100644
index 00000000..be551fdd
--- /dev/null
+++ b/morphlib/extensions.py
@@ -0,0 +1,159 @@
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import cliapp
+import morphlib
+import glob
+import os
+import sysbranchdir
+import stat
+import tempfile
+
+class ExtensionError(morphlib.Error):
+    pass
+
+class ExtensionNotFoundError(ExtensionError):
+    pass
+
+class ExtensionNotExecutableError(ExtensionError):
+    pass
+
+def _get_root_repo(build_ref_prefix):
+    system_branch = morphlib.sysbranchdir.open_from_within('.')
+    root_repo_dir = morphlib.gitdir.GitDirectory(
+            system_branch.get_git_directory_name(
+                system_branch.root_repository_url))
+    build_branch = morphlib.buildbranch.BuildBranch(system_branch,
+                                                    build_ref_prefix,
+                                                    push_temporary=False)
+    ref = build_branch.root_ref
+
+    return (build_branch.root_ref, root_repo_dir)
+
+def _get_morph_extension_directory():
+    code_dir = os.path.dirname(morphlib.__file__)
+    return os.path.join(code_dir, 'exts')
+
+def _list_repo_extension_filenames(build_ref_prefix,
+                                   kind): #pragma: no cover
+    (ref, repo_dir) = _get_root_repo(build_ref_prefix)
+    files = repo_dir.list_files(ref)
+    return (f for f in files if os.path.splitext(f)[1] == kind)
+
+def _list_morph_extension_filenames(kind):
+    return glob.glob(os.path.join(_get_morph_extension_directory(),
+                                  '*' + kind))
+
+def _get_extension_name(filename):
+    return os.path.basename(filename)
+
+def _get_repo_extension_contents(build_ref_prefix, name, kind):
+    (ref, repo_dir) = _get_root_repo(build_ref_prefix)
+    return repo_dir.get_file_from_ref(ref, name + kind)
+
+def _get_morph_extension_filename(name, kind):
+    return os.path.join(_get_morph_extension_directory(), name + kind)
+
+def _is_executable(filename):
+    st = os.stat(filename)
+    mask = stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH
+    return (stat.S_IMODE(st.st_mode) & mask) != 0
+
+def _list_extensions(build_ref_prefix, kind):
+    repo_extension_filenames = []
+    try:
+        repo_extension_filenames = \
+                _list_repo_extension_filenames(build_ref_prefix, kind)
+    except (sysbranchdir.NotInSystemBranch):
+        # Squash this and just return no system branch extensions
+        pass
+    morph_extension_filenames = _list_morph_extension_filenames(kind)
+
+    repo_extension_names = \
+            (_get_extension_name(f) for f in repo_extension_filenames)
+    morph_extension_names = \
+            (_get_extension_name(f) for f in morph_extension_filenames)
+
+    extension_names = set(repo_extension_names)
+    extension_names.update(set(morph_extension_names))
+    return list(extension_names)
+
+def list_extensions(build_ref_prefix, kind=None):
+    """
+    List all available extensions by 'kind'.
+
+    'kind' should be one of '.write' or '.configure'.
+    If 'kind' is not provided available extensions of both
+    types will be returned.
+
+    '.check' extensions are not listed here as they should
+    be associated with a '.write' extension of the same name.
+    """
+    if kind:
+        return _list_extensions(build_ref_prefix, kind)
+    else:
+        configure_extensions = _list_extensions(build_ref_prefix, '.configure')
+        write_extensions = _list_extensions(build_ref_prefix, '.write')
+
+        return configure_extensions + write_extensions
+
+class get_extension_filename():
+    """
+    Find the filename of an extension by its 'name' and 'kind'.
+
+    'kind' should be one of '.configure', '.write' or '.check'.
+
+    '.help' files for the extensions may also be retrieved by
+    passing the kind as '.write.help' or '.configure.help'.
+
+    If the extension is in the build repository then a temporary
+    file will be created, which will be deleted on exting the with block.
+    """
+    def __init__(self, build_ref_prefix, name, kind, executable=True):
+        self.build_ref_prefix = build_ref_prefix
+        self.name = name
+        self.kind = kind
+        self.executable = executable
+        self.delete = False
+
+    def __enter__(self):
+        ext_filename = None
+        try:
+            ext_contents = _get_repo_extension_contents(self.build_ref_prefix,
+                                                        self.name,
+                                                        self.kind)
+        except cliapp.AppException, sysbranchdir.NotInSystemBranch:
+            # Not found: look for it in the Morph code.
+            ext_filename = _get_morph_extension_filename(self.name, self.kind)
+            if not os.path.exists(ext_filename):
+                raise ExtensionNotFoundError(
+                    'Could not find extension %s%s' % (self.name, self.kind))
+            if self.executable and not _is_executable(ext_filename):
+                raise ExtensionNotExecutableError(
+                    'Extension not executable: %s' % ext_filename)
+        else:
+            # Found it in the system morphology's repository.
+            fd, ext_filename = tempfile.mkstemp()
+            os.write(fd, ext_contents)
+            os.close(fd)
+            os.chmod(ext_filename, 0700)
+            self.delete = True
+
+        self.ext_filename = ext_filename
+        return ext_filename
+
+    def __exit__(self, type, value, trace):
+        if self.delete:
+            os.remove(self.ext_filename)
diff --git a/morphlib/exts/nfsboot.write.help b/morphlib/exts/nfsboot.write.help
new file mode 100644
index 00000000..598b1b23
--- /dev/null
+++ b/morphlib/exts/nfsboot.write.help
@@ -0,0 +1,12 @@
+help: |
+    Deploy a system image and kernel to an nfsboot server.
+    
+    An nfsboot server is defined as a baserock system that has
+    tftp and nfs servers running, the tftp server is exporting
+    the contents of /srv/nfsboot/tftp/ and the user has sufficient
+    permissions to create nfs roots in /srv/nfsboot/nfs/.
+
+    The `location` argument is the hostname of the nfsboot server.
+
+    The extension will connect to root@HOST via ssh to copy the
+    kernel and rootfs, and configure the nfs server.
diff --git a/morphlib/exts/rawdisk.write.help b/morphlib/exts/rawdisk.write.help
new file mode 100644
index 00000000..a514a4e8
--- /dev/null
+++ b/morphlib/exts/rawdisk.write.help
@@ -0,0 +1,7 @@
+help: |
+    Create a raw disk image during Morph's deployment.
+    
+    If the image already exists, it is upgraded.
+
+    The `location` argument is a pathname to the image to be
+    created or upgraded.
diff --git a/morphlib/exts/tar.write.help b/morphlib/exts/tar.write.help
new file mode 100644
index 00000000..f052ac03
--- /dev/null
+++ b/morphlib/exts/tar.write.help
@@ -0,0 +1,5 @@
+help: |
+    Create a .tar file of the deployed system.
+    
+    The `location` argument is a pathname to the .tar file to be
+    created.
diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py
index ae62b75d..1d582949 100644
--- a/morphlib/plugins/deploy_plugin.py
+++ b/morphlib/plugins/deploy_plugin.py
@@ -26,11 +26,6 @@ import uuid
 
 import morphlib
 
-
-class ExtensionNotFoundError(morphlib.Error):
-    pass
-
-
 class DeployPlugin(cliapp.Plugin):
 
     def enable(self):
@@ -425,9 +420,9 @@ class DeployPlugin(cliapp.Plugin):
         # extension itself has the chance to raise an error.
         try:
             self._run_extension(
-                root_repo_dir, ref, deployment_type, '.check',
+                root_repo_dir, deployment_type, '.check',
                 [location], env)
-        except ExtensionNotFoundError:
+        except morphlib.extensions.ExtensionNotFoundError:
             pass
 
     def setup_deploy(self, build_command, deploy_tempdir, root_repo_dir, ref,
@@ -485,7 +480,6 @@ class DeployPlugin(cliapp.Plugin):
             for name in names:
                 self._run_extension(
                     root_repo_dir,
-                    ref,
                     name,
                     '.configure',
                     [system_tree],
@@ -495,7 +489,6 @@ class DeployPlugin(cliapp.Plugin):
             self.app.status(msg='Writing to device')
             self._run_extension(
                 root_repo_dir,
-                ref,
                 deployment_type,
                 '.write',
                 [system_tree, location],
@@ -506,7 +499,7 @@ class DeployPlugin(cliapp.Plugin):
             self.app.status(msg='Cleaning up')
             shutil.rmtree(deploy_private_tempdir)
 
-    def _run_extension(self, gd, ref, name, kind, args, env):
+    def _run_extension(self, gd, name, kind, args, env):
         '''Run an extension.
         
         The ``kind`` should be either ``.configure`` or ``.write``,
@@ -516,39 +509,19 @@ class DeployPlugin(cliapp.Plugin):
         system morphology (repo, ref), or with the Morph code.
         
         '''
-        
-        # Look for extension in the system morphology's repository.
-        try:
-            ext = gd.get_file_from_ref(ref, name + kind)
-        except cliapp.AppException:
-            # Not found: look for it in the Morph code.
-            code_dir = os.path.dirname(morphlib.__file__)
-            ext_filename = os.path.join(code_dir, 'exts', name + kind)
-            if not os.path.exists(ext_filename):
-                raise ExtensionNotFoundError(
-                    'Could not find extension %s%s' % (name, kind))
-            if not self._is_executable(ext_filename):
-                raise morphlib.Error(
-                    'Extension not executable: %s' % ext_filename)
-            delete_ext = False
-        else:
-            # Found it in the system morphology's repository.
-            fd, ext_filename = tempfile.mkstemp()
-            os.write(fd, ext)
-            os.close(fd)
-            os.chmod(ext_filename, 0700)
-            delete_ext = True
-
-        self.app.status(msg='Running extension %(name)s%(kind)s',
-                        name=name, kind=kind)
-        self.app.runcmd(
-            [ext_filename] + args,
-            ['sh', '-c', 'while read l; do echo `date "+%F %T"` "$1$l"; done',
-             '-', '%s[%s]' % (self.app.status_prefix, name + kind)],
-            cwd=gd.dirname, env=env, stdout=None, stderr=None)
-        
-        if delete_ext:
-            os.remove(ext_filename)
+        build_ref_prefix = self.app.settings['build-ref-prefix']
+        with morphlib.extensions.get_extension_filename(
+                build_ref_prefix, name, kind) as ext_filename:
+            self.app.status(msg='Running extension %(name)s%(kind)s',
+                            name=name, kind=kind)
+            self.app.runcmd(
+                [ext_filename] + args,
+                ['sh',
+                 '-c',
+                 'while read l; do echo `date "+%F %T"` "$1$l"; done',
+                 '-',
+                 '%s[%s]' % (self.app.status_prefix, name + kind)],
+                cwd=gd.dirname, env=env, stdout=None, stderr=None)
 
     def _is_executable(self, filename):
         st = os.stat(filename)
diff --git a/without-test-modules b/without-test-modules
index 9b893300..a42cce97 100644
--- a/without-test-modules
+++ b/without-test-modules
@@ -6,6 +6,7 @@ morphlib/tester.py
 morphlib/git.py
 morphlib/app.py
 morphlib/mountableimage.py
+morphlib/extensions.py
 morphlib/extractedtarball.py
 morphlib/plugins/artifact_inspection_plugin.py
 morphlib/plugins/cross-bootstrap_plugin.py