Add common code for test plugins to use

5 files changed, 767 insertions, 1 deletions

diff --git a/mason/__init__.py b/mason/__init__.py
index 237344b..5f91a06 100644
--- a/mason/__init__.py
+++ b/mason/__init__.py
@@ -1,2 +1,5 @@
-import tests
+import deployment
+import publishers
+import runners
 import util
+import tests
diff --git a/mason/deployment.py b/mason/deployment.py
new file mode 100644
index 0000000..a23b0ea
--- /dev/null
+++ b/mason/deployment.py
@@ -0,0 +1,270 @@
+# Copyright 2014 Codethink Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import cliapp
+import os
+import pipes
+import shlex
+import shutil
+import socket
+import tempfile
+import time
+import uuid
+
+import mason
+
+
+class DeployedSystemInstance(object):
+
+    def __init__(self, deployment, config, host_machine, vm_id, rootfs_path,
+            ip_addr=None):
+        self.deployment = deployment
+        self.config = config
+        self.ip_address = ip_addr or self.config['HOSTNAME']
+        self.host_machine = host_machine
+        self.vm_id = vm_id
+        self.rootfs_path = rootfs_path
+        self.hostname = self.config['HOSTNAME']
+        self.log = None
+
+    @property
+    def ssh_host(self):
+        # TODO: Stop assuming we ssh into test instances as root
+        return 'root@{host}'.format(host=self.ip_address)
+
+    def runcmd(self, argv, chdir='.', **kwargs):
+        ssh_cmd = ['ssh', '-o', 'StrictHostKeyChecking=no',
+                   '-o', 'UserKnownHostsFile=/dev/null', self.ssh_host]
+        cmd = ['sh', '-c', 'cd "$1" && shift && exec "$@"', '-', chdir]
+        cmd += argv
+        ssh_cmd.append(' '.join(map(pipes.quote, cmd)))
+        return cliapp.runcmd(ssh_cmd, **kwargs)
+
+    def _wait_for_dhcp(self, timeout):
+        '''Block until given hostname resolves successfully.
+
+        Raises AppException if the hostname has not appeared in 'timeout'
+        seconds.
+
+        '''
+        start_time = time.time()
+        while True:
+            try:
+                socket.gethostbyname(self.ip_address)
+                return
+            except socket.gaierror:
+                pass
+            if time.time() > start_time + timeout:
+                raise cliapp.AppException("Host %s did not appear "
+                                          "after %i seconds" %
+                                          (self.ip_address, timeout))
+            time.sleep(0.5)
+
+    def _wait_for_ssh(self, timeout):
+        """Wait until the deployed VM is responding via SSH"""
+        start_time = time.time()
+        while True:
+            try:
+                self.runcmd(['true'], stdin=None, stdout=self.log, stderr=self.log)
+                return
+            except cliapp.AppException:
+                # TODO: Stop assuming the ssh part of the command is what failed
+                if time.time() > start_time + timeout:
+                    raise cliapp.AppException("%s sshd did not start after "
+                                              "%i seconds" %
+                                              (self.ip_address, timeout))
+                time.sleep(0.5)
+
+    def _wait_for_cloud_init(self, timeout):
+        """Wait until cloud init has resized the disc"""
+        start_time = time.time()
+        while True:
+            try:
+                out = self.runcmd(['sh', '-c',
+                              'test -e "$1" && echo exists || echo does not exist',
+                              '-', '/root/cloud-init-finished'])
+            except:
+                import traceback
+                traceback.print_exc()
+                raise
+            if out.strip() == 'exists':
+                return
+            if time.time() > start_time + timeout:
+                raise cliapp.AppException("Disc size not increased after "
+                                          "%i seconds" % (timeout))
+            time.sleep(3)
+
+    def wait_until_online(self, timeout=120):
+        self._wait_for_dhcp(timeout)
+        self._wait_for_ssh(timeout)
+        if self.config['type'] == 'openstack':
+            self._wait_for_cloud_init(timeout)
+        if self.log:
+            self.log.write("Test system %s ready to run tests.\n" %
+                           (self.hostname))
+            self.log.flush()
+
+    def delete(self):
+        # Stop and remove VM
+        if self.log:
+            self.log.write("Deleting %s test instance\n" % (self.hostname))
+        try:
+            if self.config['type'] == 'openstack':
+                cliapp.runcmd(['nova', 'delete', self.hostname])
+            elif self.config['type'] == 'kvm':
+                self.host_machine.virsh('destroy', self.vm_id)
+        except cliapp.AppException as e:
+            # TODO: Stop assuming that delete failed because the instance
+            #       wasn't running
+            if self.log:
+                self.log.write("- Failed\n")
+            pass
+        if self.log:
+            self.log.write("Deleting %s test disc image\n" % (self.hostname))
+        try:
+            if self.config['type'] == 'openstack':
+                cliapp.runcmd(['nova', 'image-delete', self.hostname])
+            elif self.config['type'] == 'kvm':
+                self.host_machine.virsh('undefine', self.vm_id,
+                                        '--remove-all-storage')
+        except cliapp.AppException as e:
+            # TODO: Stop assuming that image-delete failed because it was
+            #       already removed
+            if self.log:
+                self.log.write("- Failed\n")
+            pass
+
+
+class Deployment(object):
+
+    def __init__(self, cluster_path, name, deployment_config,
+                 host_machine, net_id, log_path):
+        self.cluster_path = cluster_path
+        self.name = name
+        self.deployment_config = deployment_config
+        self.host_machine = host_machine
+        self.net_id = net_id
+        self.logfile = open(log_path, 'w+')
+
+    @staticmethod
+    def _ssh_host_key_exists(hostname):
+        """Check if an ssh host key exists in known_hosts"""
+        if not os.path.exists('/root/.ssh/known_hosts'):
+            return False
+        with open('/root/.ssh/known_hosts', 'r') as known_hosts:
+            return any(line.startswith(hostname) for line in known_hosts)
+
+    def _update_known_hosts(self):
+        if not self._ssh_host_key_exists(self.host_machine.address):
+            with open('/root/.ssh/known_hosts', 'a') as known_hosts:
+                cliapp.runcmd(['ssh-keyscan', self.host_machine.address],
+                              stdout=known_hosts)
+
+    @staticmethod
+    def _generate_sshkey_config(tempdir, config):
+        manifest = os.path.join(tempdir, 'manifest')
+        with open(manifest, 'w') as f:
+            f.write('0040700 0 0 /root/.ssh\n')
+            f.write('overwrite 0100600 0 0 /root/.ssh/authorized_keys\n')
+        authkeys = os.path.join(tempdir, 'root', '.ssh', 'authorized_keys')
+        os.makedirs(os.path.dirname(authkeys))
+        with open(authkeys, 'w') as auth_f:
+            with open('/root/.ssh/id_rsa.pub', 'r') as key_f:
+                shutil.copyfileobj(key_f, auth_f)
+
+        install_files = shlex.split(config.get('INSTALL_FILES', ''))
+        install_files.append(manifest)
+        yield 'INSTALL_FILES', ' '.join(pipes.quote(f) for f in install_files)
+
+    def deploy(self, deployment_type):
+        self._update_known_hosts()
+
+        hostname = str(uuid.uuid4())
+        vm_id = hostname
+        image_base = self.host_machine.disk_path
+        rootpath = '{image_base}/{hostname}.img'.format(image_base=image_base,
+                                                        hostname=hostname)
+        loc = 'http://{ssh_host}:5000/v2.0'.format(
+            ssh_host=self.host_machine.ssh_host, id=vm_id, path=rootpath)
+
+        options = {
+            'type': deployment_type,
+            'location': loc,
+            'HOSTNAME': hostname,
+            'DISK_SIZE': '5G',
+            'RAM_SIZE': '2G',
+            'VERSION_LABEL': 'release-test'
+        }
+        if deployment_type == 'openstack':
+            options.update({
+                'OPENSTACK_USER': os.environ['OS_USERNAME'],
+                'OPENSTACK_TENANT': os.environ['OS_TENANT_NAME'],
+                'OPENSTACK_PASSWORD': os.environ['OS_PASSWORD'],
+                'OPENSTACK_IMAGENAME': hostname,
+                'CLOUD_INIT': 'yes',
+                'KERNEL_ARGS': 'console=tty0 console=ttyS0',
+            })
+        elif deployment_type == 'kvm':
+            options['AUTOSTART'] = 'True'
+
+        tempdir = tempfile.mkdtemp()
+        try:
+            options.update(
+                self._generate_sshkey_config(tempdir,
+                                            self.deployment_config))
+            # Deploy the image to openstack
+            args = ['morph', 'deploy', self.cluster_path, self.name]
+            for k, v in options.iteritems():
+                args.append('%s.%s=%s' % (self.name, k, v))
+            cliapp.runcmd(args, stdin=None,
+                          stdout=self.logfile,
+                          stderr=self.logfile)
+            config = dict(self.deployment_config)
+            config.update(options)
+
+            ip_addr = None
+            if deployment_type == 'openstack':
+                # Boot an instance from the image
+                args = ['nova', 'boot',
+                    '--flavor', 'm1.medium',
+                    '--image', hostname,
+                    '--user-data', '/usr/lib/mason/os-init-script',
+                    '--nic', "net-id=%s" % (self.net_id),
+                    hostname]
+                output = cliapp.runcmd(args)
+
+                # Print nova boot output, with adminPass line removed
+                output_lines = output.split('\n')
+                for line in output_lines:
+                    if line.find('adminPass') != -1:
+                        password_line = line
+                output_lines.remove(password_line)
+                output = '\n'.join(output_lines)
+                self.logfile.write(output)
+
+                # Get ip address from nova list
+                nl = mason.util.NovaList()
+                ip_addr = nl.get_nova_ip_for_instance_timeout(hostname)
+                self.logfile.write("IP address for instance %s: %s\n" %
+                                   (hostname, ip_addr))
+
+                return DeployedSystemInstance(self, config, self.host_machine,
+                                              vm_id, rootpath, ip_addr)
+            elif deployment_type == 'kvm':
+                return DeployedSystemInstance(self, config, self.host_machine,
+                                              vm_id, rootpath)
+        finally:
+            shutil.rmtree(tempdir)
+            self.logfile.close()
diff --git a/mason/publishers.py b/mason/publishers.py
new file mode 100644
index 0000000..7c81310
--- /dev/null
+++ b/mason/publishers.py
@@ -0,0 +1,239 @@
+# Copyright 2014 Codethink Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import cliapp
+import json
+import logging
+
+import mason
+
+
+class BuildArtifactPublisher(object):
+
+    '''Publish build artifacts related to the release.'''
+
+    logging.getLogger('mason.publishers.BuildArtifactPublisher')
+
+    def __init__(self, config, defs_repo):
+        self.config = config
+        self.morph_helper = mason.util.MorphologyHelper(defs_repo)
+
+    def publish_build_artifacts(self):
+        artifact_basenames = self.list_build_artifacts_for_release(
+            self.config['cluster-morphology'])
+        logging.info(
+            'Found %s build artifact files in release',
+            len(artifact_basenames))
+
+        to_be_uploaded = self.filter_away_build_artifacts_on_public_trove(
+            artifact_basenames)
+
+        logging.debug('List of artifacts (basenames) to upload (without already uploaded):')
+        for i, basename in enumerate(to_be_uploaded):
+            logging.debug('  {0}: {1}'.format(i, basename))
+        logging.debug('End of artifact list (to_be_uploaded)')
+
+        logging.info(
+            'Need to fetch locally, then upload %s build artifacts',
+            len(to_be_uploaded))
+
+        self.upload_build_artifacts_to_public_trove(to_be_uploaded)
+
+    def list_build_artifacts_for_release(self, cluster_morphology_path):
+        logging.info('Find build artifacts included in release')
+
+        # FIXME: These are hardcoded for simplicity. They would be
+        # possible to deduce automatically from the workspace, but
+        # that can happen later.
+        repo = 'file://%s' % \
+            os.path.abspath(self.morph_helper.defs_repo.dirname)
+        ref = 'HEAD'
+
+        argv = [self.config['morph-cmd'],
+                'list-artifacts', '--quiet',
+                repo, ref]
+        argv += self.morph_helper.find_systems_by_arch(
+            cluster_morphology_path, self.config['architecture'])
+
+        output = cliapp.runcmd(argv)
+        basenames = output.splitlines()
+        logging.debug('List of build artifacts in release:')
+        for basename in basenames:
+            logging.debug('  {0}'.format(basename))
+        logging.debug('End of list of build artifacts in release')
+
+        return basenames
+
+    def filter_away_build_artifacts_on_public_trove(self, basenames):
+        result = []
+        logging.debug('Filtering away already existing artifacts:')
+        for basename, exists in self.query_public_trove_for_artifacts(basenames):
+            logging.debug('  {0}: {1}'.format(basename, exists))
+            if not exists:
+                result.append(basename)
+        logging.debug('End of filtering away')
+        return result
+
+    def query_public_trove_for_artifacts(self, basenames):
+        host = self.config['public-trove-host']
+
+        # FIXME: This could use
+        # contextlib.closing(urllib2.urlopen(url, data=data) instead
+        # of explicit closing.
+        url = 'http://{host}:8080/1.0/artifacts'.format(host=host)
+        data = json.dumps(basenames)
+        f = urllib2.urlopen(url, data=data)
+        obj = json.load(f)
+        return obj.items()
+
+    def upload_build_artifacts_to_public_trove(self, basenames):
+        self.download_artifacts_locally(basenames)
+        self.upload_artifacts_to_public_trove(basenames)
+
+    def download_artifacts_locally(self, basenames):
+        dirname = self.config['local-build-artifacts-dir']
+        self.create_directory_if_missing(dirname)
+        for i, basename in enumerate(basenames):
+            pathname = os.path.join(dirname, basename)
+            if not os.path.exists(pathname):
+                logging.info(
+                    'Downloading %s/%s %s',
+                    i, len(basenames), repr(basename))
+                orig = '/srv/distbuild/artifacts/%s' % basename
+                #TODO: download the artifacts from a shared cache
+                shutil.copy2(orig, pathname)
+
+    def create_directory_if_missing(self, dirname):
+        if not os.path.exists(dirname):
+            os.makedirs(dirname)
+
+    def upload_artifacts_to_public_trove(self, basenames):
+        logging.info(
+            'Upload build artifacts to %s',
+            self.config['public-trove-host'])
+        rsync_files_to_server(
+            self.config['local-build-artifacts-dir'],
+            basenames,
+            self.config['public-trove-username'],
+            self.config['public-trove-host'],
+            self.config['public-trove-artifact-dir'])
+        set_permissions_on_server(
+            self.config['public-trove-username'],
+            self.config['public-trove-host'],
+            self.config['public-trove-artifact-dir'],
+            basenames)
+
+
+class ReleaseArtifactPublisher(object):
+
+    '''Publish release artifacts for a release.'''
+
+    logging.getLogger('mason.publishers.ReleaseArtifactPublisher')
+
+    def __init__(self, config):
+        self.config = config
+
+    def publish_release_artifacts(self):
+        files = self.list_release_artifacts()
+        if files:
+            self.upload_release_artifacts_to_private_dir(files)
+            self.move_release_artifacts_to_public_dir(files)
+            self.create_symlinks_to_new_release_artifacts(files)
+
+    def list_release_artifacts(self):
+        logging.info('Find release artifacts to publish')
+        return os.listdir(self.config['release-artifact-dir'])
+
+    def upload_release_artifacts_to_private_dir(self, files):
+        logging.info('Upload release artifacts to private directory')
+        path = self.config['download-server-private-dir']
+        self.create_directory_on_download_server(path)
+        self.rsync_files_to_download_server(files, path)
+
+    def create_directory_on_download_server(self, path):
+        user = self.config['download-server-username']
+        host = self.config['download-server-address']
+        logging.info(msg='Create {host}:{path}', host=host, path=path)
+        target = '{user}@{host}'.format(user=user, host=host)
+        cliapp.ssh_runcmd(target, ['mkdir', '-p', path])
+
+    def rsync_files_to_download_server(self, files, path):
+        logging.info('Upload release artifacts to download server')
+        rsync_files_to_server(
+            self.config['release-artifact-dir'],
+            files,
+            self.config['download-server-username'],
+            self.config['download-server-address'],
+            path)
+        set_permissions_on_server(
+            self.config['download-server-username'],
+            self.config['download-server-address'],
+            path,
+            files)
+
+    def move_release_artifacts_to_public_dir(self, files):
+        logging.info('Move release artifacts to public directory')
+        private_dir = self.config['download-server-private-dir']
+        public_dir = self.config['download-server-public-dir']
+        self.create_directory_on_download_server(public_dir)
+
+        # Move just the contents of the private dir, not the dir
+        # itself (-mindepth). Avoid overwriting existing files (mv
+        # -n).
+        argv = ['find', private_dir, '-mindepth', '1',
+                '-exec', 'mv', '-n', '{}', public_dir + '/.', ';']
+
+        target = '{user}@{host}'.format(
+            user=self.config['download-server-username'],
+            host=self.config['download-server-address'])
+        cliapp.ssh_runcmd(target, argv)
+
+    def create_symlinks_to_new_release_artifacts(self, files):
+        logging.info('FIXME: Create symlinks to new release artifacts')
+
+
+def rsync_files_to_server(
+    source_dir, source_filenames, user, host, target_dir):
+
+    if not source_filenames:
+        return
+
+    argv = [
+        'rsync',
+        '-a',
+        '--progress',
+        '--partial',
+        '--human-readable',
+        '--sparse',
+        '--protect-args',
+        '-0',
+        '--files-from=-',
+        source_dir,
+        '{user}@{host}:{path}'.format(user=user, host=host, path=target_dir),
+        ]
+
+    files_list = '\0'.join(filename for filename in source_filenames)
+    cliapp.runcmd(argv, feed_stdin=files_list, stdout=None, stderr=None)
+
+
+def set_permissions_on_server(user, host, target_dir, filenames):
+    # If we have no files, we can't form a valid command to run on the server
+    if not filenames:
+        return
+    target = '{user}@{host}'.format(user=user, host=host)
+    argv = ['xargs', '-0', 'chmod', '0644']
+    files_list = ''.join(
+        '{0}\0'.format(os.path.join(target_dir, filename)) for filename in filenames)
+    cliapp.ssh_runcmd(target, argv, feed_stdin=files_list, stdout=None, stderr=None)
diff --git a/mason/runners.py b/mason/runners.py
new file mode 100644
index 0000000..71711b0
--- /dev/null
+++ b/mason/runners.py
@@ -0,0 +1,109 @@
+# Copyright 2014 Codethink Ltd.
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import json
+import logging
+import os
+
+class JobRunner(object):
+
+    """A generic Zuul job runner."""
+
+    logging.getLogger('mason.runners.JobRunner')
+
+    def __init__(self, worker_server, config, function):
+        self._reset()
+
+        self.worker_server = worker_server
+        self.plugin_config = config
+        self.function = function
+
+        self.total_steps = 0
+
+    def _reset(self):
+        self.success = True
+        self.cancelled = False
+        self.job = None
+        self.job_arguments = None
+        self.current_step = 0
+
+
+    def start_job(self, job):
+        self._reset()
+        self.job = job
+
+        if self.job is not None:
+            print "starting job"
+            try:
+                self.job_arguments = json.loads(self.job.arguments.decode('utf-8'))
+                self.send_work_data()
+                self.run_job()
+                self.send_work_complete()
+            except Exception as e:
+                self.success = False
+                self.send_work_data(result='Exception: %s' % e)
+                self.job.sendWorkException(str(e).encode('utf-8'))
+
+    def _get_work_data(self):
+        work_data = {'name': self.function,
+                     'location': os.uname()[1],
+                     'number': self.job.unique,
+        }
+        return work_data
+
+    def send_work_data(self, data=None, result=''):
+        if data is None:
+            data = self._get_work_data()
+        if self.success:
+            data['result'] = 'SUCCESS'
+        else:
+            data['result'] = result
+        logging.debug(json.dumps(data))
+        print data
+        self.job.sendWorkData(json.dumps(data))
+
+    def send_work_complete(self, data=None):
+        if data is None:
+            data = self._get_work_data()
+        self.send_work_data(data)
+        if self.success:
+            print 'sending work-complete'
+            self.job.sendWorkComplete(json.dumps(data))
+        else:
+            print 'sending work-fail'
+            self.job.sendWorkFail()
+
+    def stop_working(self, number=None):
+        if number is None or number == self.job.unique:
+            self.cancelled = True
+
+    def run_job(self):
+        # This function is to be implemented by subclasses
+        pass
+
+    def _do_next_step(self):
+        self._handle_cancellation()
+        self.current_step += 1
+        data = self._get_work_data()
+        data['current-step'] = self.current_step
+        self.send_work_data(data)
+
+    def _handle_cancellation(self):
+        if self.cancelled:
+            data = self._get_work_data()
+            data['result'] = 'Cancelled'
+            self.send_work_data(data)
+            self.job.sendWorkFail()
+            raise Exception('Job cancelled')
diff --git a/mason/util.py b/mason/util.py
new file mode 100644
index 0000000..90713ed
--- /dev/null
+++ b/mason/util.py
@@ -0,0 +1,145 @@
+# Copyright 2014 Codethink Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import cliapp
+import morphlib
+import time
+
+
+class NovaList:
+    def __init__(self):
+        self.output = []
+        self.lines = []
+        self.instance = []
+
+    def update(self):
+        self.output = cliapp.runcmd(['nova', 'list'])
+        self.lines = self.output.split('\n')
+        self.lines = self.lines[3:-2]
+
+    def get_nova_details_for_instance(self, name):
+        self.update()
+
+        for line in self.lines:
+            entries = line.split('|')
+            stripped_line = [entry.strip() for entry in entries]
+            if stripped_line.count(name) == 1:
+                self.instance = stripped_line
+
+    def get_nova_state_for_instance(self, name):
+        self.get_nova_details_for_instance(name)
+        if not self.instance:
+            return
+        return self.instance[3]
+
+    def get_nova_ip_for_instance(self, name):
+        self.get_nova_details_for_instance(name)
+        if not self.instance:
+            return
+
+        if self.get_nova_state_for_instance(name) != 'ACTIVE':
+            return
+
+        return self.instance[6]
+
+    def get_nova_ip_for_instance_timeout(self, name, timeout=120):
+        start_time = time.time()
+
+        while self.get_nova_state_for_instance(name) != 'ACTIVE':
+
+            if time.time() > start_time + timeout:
+                print "%s not ACTIVE after %i seconds" % (name, timeout)
+                return
+
+            time.sleep(1)
+
+        ip_addr = self.get_nova_ip_for_instance(name)
+        if not ip_addr:
+            return
+
+        if ip_addr.count('=') == 0:
+            return
+
+        ip_addr = ip_addr[ip_addr.find('=') + 1:]
+
+        if ip_addr.count(',') == 0:
+            return ip_addr
+
+        return ip_addr[:ip_addr.find(',')]
+
+
+class MorphologyHelper(object):
+
+    def __init__(self, path):
+        self.defs_repo = morphlib.gitdir.GitDirectory(path)
+        self.loader = morphlib.morphloader.MorphologyLoader()
+        self.finder = morphlib.morphologyfinder.MorphologyFinder(self.defs_repo)
+
+    def load_morphology(self, path):
+        text = self.finder.read_morphology(path)
+        return self.loader.load_from_string(text)
+
+    @classmethod
+    def iterate_systems(cls, systems_list):
+        for system in systems_list:
+            yield morphlib.util.sanitise_morphology_path(system['morph'])
+            if 'subsystems' in system:
+                for subsystem in cls.iterate_systems(system['subsystems']):
+                    yield subsystem
+
+    def iterate_cluster_deployments(cls, cluster_morph):
+        for system in cluster_morph['systems']:
+            path = morphlib.util.sanitise_morphology_path(system['morph'])
+            defaults = system.get('deploy-defaults', {})
+            for name, options in system['deploy'].iteritems():
+                config = dict(defaults)
+                config.update(options)
+                yield path, name, config
+
+    def load_cluster_systems(self, cluster_morph):
+        for system_path in set(self.iterate_systems(cluster_morph['systems'])):
+            system_morph = self.load_morphology(system_path)
+            yield system_path, system_morph
+
+    def find_systems_by_arch(self, cluster_morph, archs):
+        deployments = self.iterate_cluster_deployments(cluster_morph)
+        return [deployment[0]
+                for deployment in deployments
+                if self.load_morphology(deployments[0])['arch'] in archs]
+
+
+class VMHost(object):
+
+    def __init__(self, user, address, disk_path):
+        self.user = user
+        self.address = address
+        self.disk_path = disk_path
+
+    @property
+    def ssh_host(self):
+        return '{user}@{address}'.format(user=self.user, address=self.address)
+
+    def runcmd(self, *args, **kwargs):
+        cliapp.ssh_runcmd(self.ssh_host, *args, **kwargs)
+
+    def virsh(self, *args, **kwargs):
+        self.runcmd(['virsh', '-c', 'qemu:///system'] + list(args), **kwargs)
+
+
+def job_step(function):
+    def next_step(*args, **kwargs):
+        args[0]._do_next_step()
+        return function(*args, **kwargs)
+    return next_step