From ad4b9927c9f2d6896bedaf8de586fa29682da332 Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Tue, 9 Dec 2014 13:58:28 +0000 Subject: Add common code for test plugins to use --- mason/__init__.py | 5 +- mason/deployment.py | 270 ++++++++++++++++++++++++++++++++++++++++++++++++++++ mason/publishers.py | 239 ++++++++++++++++++++++++++++++++++++++++++++++ mason/runners.py | 109 +++++++++++++++++++++ mason/util.py | 145 ++++++++++++++++++++++++++++ 5 files changed, 767 insertions(+), 1 deletion(-) create mode 100644 mason/deployment.py create mode 100644 mason/publishers.py create mode 100644 mason/runners.py create mode 100644 mason/util.py diff --git a/mason/__init__.py b/mason/__init__.py index 237344b..5f91a06 100644 --- a/mason/__init__.py +++ b/mason/__init__.py @@ -1,2 +1,5 @@ -import tests +import deployment +import publishers +import runners import util +import tests diff --git a/mason/deployment.py b/mason/deployment.py new file mode 100644 index 0000000..a23b0ea --- /dev/null +++ b/mason/deployment.py @@ -0,0 +1,270 @@ +# Copyright 2014 Codethink Ltd +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import cliapp +import os +import pipes +import shlex +import shutil +import socket +import tempfile +import time +import uuid + +import mason + + +class DeployedSystemInstance(object): + + def __init__(self, deployment, config, host_machine, vm_id, rootfs_path, + ip_addr=None): + self.deployment = deployment + self.config = config + self.ip_address = ip_addr or self.config['HOSTNAME'] + self.host_machine = host_machine + self.vm_id = vm_id + self.rootfs_path = rootfs_path + self.hostname = self.config['HOSTNAME'] + self.log = None + + @property + def ssh_host(self): + # TODO: Stop assuming we ssh into test instances as root + return 'root@{host}'.format(host=self.ip_address) + + def runcmd(self, argv, chdir='.', **kwargs): + ssh_cmd = ['ssh', '-o', 'StrictHostKeyChecking=no', + '-o', 'UserKnownHostsFile=/dev/null', self.ssh_host] + cmd = ['sh', '-c', 'cd "$1" && shift && exec "$@"', '-', chdir] + cmd += argv + ssh_cmd.append(' '.join(map(pipes.quote, cmd))) + return cliapp.runcmd(ssh_cmd, **kwargs) + + def _wait_for_dhcp(self, timeout): + '''Block until given hostname resolves successfully. + + Raises AppException if the hostname has not appeared in 'timeout' + seconds. + + ''' + start_time = time.time() + while True: + try: + socket.gethostbyname(self.ip_address) + return + except socket.gaierror: + pass + if time.time() > start_time + timeout: + raise cliapp.AppException("Host %s did not appear " + "after %i seconds" % + (self.ip_address, timeout)) + time.sleep(0.5) + + def _wait_for_ssh(self, timeout): + """Wait until the deployed VM is responding via SSH""" + start_time = time.time() + while True: + try: + self.runcmd(['true'], stdin=None, stdout=self.log, stderr=self.log) + return + except cliapp.AppException: + # TODO: Stop assuming the ssh part of the command is what failed + if time.time() > start_time + timeout: + raise cliapp.AppException("%s sshd did not start after " + "%i seconds" % + (self.ip_address, timeout)) + time.sleep(0.5) + + def _wait_for_cloud_init(self, timeout): + """Wait until cloud init has resized the disc""" + start_time = time.time() + while True: + try: + out = self.runcmd(['sh', '-c', + 'test -e "$1" && echo exists || echo does not exist', + '-', '/root/cloud-init-finished']) + except: + import traceback + traceback.print_exc() + raise + if out.strip() == 'exists': + return + if time.time() > start_time + timeout: + raise cliapp.AppException("Disc size not increased after " + "%i seconds" % (timeout)) + time.sleep(3) + + def wait_until_online(self, timeout=120): + self._wait_for_dhcp(timeout) + self._wait_for_ssh(timeout) + if self.config['type'] == 'openstack': + self._wait_for_cloud_init(timeout) + if self.log: + self.log.write("Test system %s ready to run tests.\n" % + (self.hostname)) + self.log.flush() + + def delete(self): + # Stop and remove VM + if self.log: + self.log.write("Deleting %s test instance\n" % (self.hostname)) + try: + if self.config['type'] == 'openstack': + cliapp.runcmd(['nova', 'delete', self.hostname]) + elif self.config['type'] == 'kvm': + self.host_machine.virsh('destroy', self.vm_id) + except cliapp.AppException as e: + # TODO: Stop assuming that delete failed because the instance + # wasn't running + if self.log: + self.log.write("- Failed\n") + pass + if self.log: + self.log.write("Deleting %s test disc image\n" % (self.hostname)) + try: + if self.config['type'] == 'openstack': + cliapp.runcmd(['nova', 'image-delete', self.hostname]) + elif self.config['type'] == 'kvm': + self.host_machine.virsh('undefine', self.vm_id, + '--remove-all-storage') + except cliapp.AppException as e: + # TODO: Stop assuming that image-delete failed because it was + # already removed + if self.log: + self.log.write("- Failed\n") + pass + + +class Deployment(object): + + def __init__(self, cluster_path, name, deployment_config, + host_machine, net_id, log_path): + self.cluster_path = cluster_path + self.name = name + self.deployment_config = deployment_config + self.host_machine = host_machine + self.net_id = net_id + self.logfile = open(log_path, 'w+') + + @staticmethod + def _ssh_host_key_exists(hostname): + """Check if an ssh host key exists in known_hosts""" + if not os.path.exists('/root/.ssh/known_hosts'): + return False + with open('/root/.ssh/known_hosts', 'r') as known_hosts: + return any(line.startswith(hostname) for line in known_hosts) + + def _update_known_hosts(self): + if not self._ssh_host_key_exists(self.host_machine.address): + with open('/root/.ssh/known_hosts', 'a') as known_hosts: + cliapp.runcmd(['ssh-keyscan', self.host_machine.address], + stdout=known_hosts) + + @staticmethod + def _generate_sshkey_config(tempdir, config): + manifest = os.path.join(tempdir, 'manifest') + with open(manifest, 'w') as f: + f.write('0040700 0 0 /root/.ssh\n') + f.write('overwrite 0100600 0 0 /root/.ssh/authorized_keys\n') + authkeys = os.path.join(tempdir, 'root', '.ssh', 'authorized_keys') + os.makedirs(os.path.dirname(authkeys)) + with open(authkeys, 'w') as auth_f: + with open('/root/.ssh/id_rsa.pub', 'r') as key_f: + shutil.copyfileobj(key_f, auth_f) + + install_files = shlex.split(config.get('INSTALL_FILES', '')) + install_files.append(manifest) + yield 'INSTALL_FILES', ' '.join(pipes.quote(f) for f in install_files) + + def deploy(self, deployment_type): + self._update_known_hosts() + + hostname = str(uuid.uuid4()) + vm_id = hostname + image_base = self.host_machine.disk_path + rootpath = '{image_base}/{hostname}.img'.format(image_base=image_base, + hostname=hostname) + loc = 'http://{ssh_host}:5000/v2.0'.format( + ssh_host=self.host_machine.ssh_host, id=vm_id, path=rootpath) + + options = { + 'type': deployment_type, + 'location': loc, + 'HOSTNAME': hostname, + 'DISK_SIZE': '5G', + 'RAM_SIZE': '2G', + 'VERSION_LABEL': 'release-test' + } + if deployment_type == 'openstack': + options.update({ + 'OPENSTACK_USER': os.environ['OS_USERNAME'], + 'OPENSTACK_TENANT': os.environ['OS_TENANT_NAME'], + 'OPENSTACK_PASSWORD': os.environ['OS_PASSWORD'], + 'OPENSTACK_IMAGENAME': hostname, + 'CLOUD_INIT': 'yes', + 'KERNEL_ARGS': 'console=tty0 console=ttyS0', + }) + elif deployment_type == 'kvm': + options['AUTOSTART'] = 'True' + + tempdir = tempfile.mkdtemp() + try: + options.update( + self._generate_sshkey_config(tempdir, + self.deployment_config)) + # Deploy the image to openstack + args = ['morph', 'deploy', self.cluster_path, self.name] + for k, v in options.iteritems(): + args.append('%s.%s=%s' % (self.name, k, v)) + cliapp.runcmd(args, stdin=None, + stdout=self.logfile, + stderr=self.logfile) + config = dict(self.deployment_config) + config.update(options) + + ip_addr = None + if deployment_type == 'openstack': + # Boot an instance from the image + args = ['nova', 'boot', + '--flavor', 'm1.medium', + '--image', hostname, + '--user-data', '/usr/lib/mason/os-init-script', + '--nic', "net-id=%s" % (self.net_id), + hostname] + output = cliapp.runcmd(args) + + # Print nova boot output, with adminPass line removed + output_lines = output.split('\n') + for line in output_lines: + if line.find('adminPass') != -1: + password_line = line + output_lines.remove(password_line) + output = '\n'.join(output_lines) + self.logfile.write(output) + + # Get ip address from nova list + nl = mason.util.NovaList() + ip_addr = nl.get_nova_ip_for_instance_timeout(hostname) + self.logfile.write("IP address for instance %s: %s\n" % + (hostname, ip_addr)) + + return DeployedSystemInstance(self, config, self.host_machine, + vm_id, rootpath, ip_addr) + elif deployment_type == 'kvm': + return DeployedSystemInstance(self, config, self.host_machine, + vm_id, rootpath) + finally: + shutil.rmtree(tempdir) + self.logfile.close() diff --git a/mason/publishers.py b/mason/publishers.py new file mode 100644 index 0000000..7c81310 --- /dev/null +++ b/mason/publishers.py @@ -0,0 +1,239 @@ +# Copyright 2014 Codethink Ltd +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import cliapp +import json +import logging + +import mason + + +class BuildArtifactPublisher(object): + + '''Publish build artifacts related to the release.''' + + logging.getLogger('mason.publishers.BuildArtifactPublisher') + + def __init__(self, config, defs_repo): + self.config = config + self.morph_helper = mason.util.MorphologyHelper(defs_repo) + + def publish_build_artifacts(self): + artifact_basenames = self.list_build_artifacts_for_release( + self.config['cluster-morphology']) + logging.info( + 'Found %s build artifact files in release', + len(artifact_basenames)) + + to_be_uploaded = self.filter_away_build_artifacts_on_public_trove( + artifact_basenames) + + logging.debug('List of artifacts (basenames) to upload (without already uploaded):') + for i, basename in enumerate(to_be_uploaded): + logging.debug(' {0}: {1}'.format(i, basename)) + logging.debug('End of artifact list (to_be_uploaded)') + + logging.info( + 'Need to fetch locally, then upload %s build artifacts', + len(to_be_uploaded)) + + self.upload_build_artifacts_to_public_trove(to_be_uploaded) + + def list_build_artifacts_for_release(self, cluster_morphology_path): + logging.info('Find build artifacts included in release') + + # FIXME: These are hardcoded for simplicity. They would be + # possible to deduce automatically from the workspace, but + # that can happen later. + repo = 'file://%s' % \ + os.path.abspath(self.morph_helper.defs_repo.dirname) + ref = 'HEAD' + + argv = [self.config['morph-cmd'], + 'list-artifacts', '--quiet', + repo, ref] + argv += self.morph_helper.find_systems_by_arch( + cluster_morphology_path, self.config['architecture']) + + output = cliapp.runcmd(argv) + basenames = output.splitlines() + logging.debug('List of build artifacts in release:') + for basename in basenames: + logging.debug(' {0}'.format(basename)) + logging.debug('End of list of build artifacts in release') + + return basenames + + def filter_away_build_artifacts_on_public_trove(self, basenames): + result = [] + logging.debug('Filtering away already existing artifacts:') + for basename, exists in self.query_public_trove_for_artifacts(basenames): + logging.debug(' {0}: {1}'.format(basename, exists)) + if not exists: + result.append(basename) + logging.debug('End of filtering away') + return result + + def query_public_trove_for_artifacts(self, basenames): + host = self.config['public-trove-host'] + + # FIXME: This could use + # contextlib.closing(urllib2.urlopen(url, data=data) instead + # of explicit closing. + url = 'http://{host}:8080/1.0/artifacts'.format(host=host) + data = json.dumps(basenames) + f = urllib2.urlopen(url, data=data) + obj = json.load(f) + return obj.items() + + def upload_build_artifacts_to_public_trove(self, basenames): + self.download_artifacts_locally(basenames) + self.upload_artifacts_to_public_trove(basenames) + + def download_artifacts_locally(self, basenames): + dirname = self.config['local-build-artifacts-dir'] + self.create_directory_if_missing(dirname) + for i, basename in enumerate(basenames): + pathname = os.path.join(dirname, basename) + if not os.path.exists(pathname): + logging.info( + 'Downloading %s/%s %s', + i, len(basenames), repr(basename)) + orig = '/srv/distbuild/artifacts/%s' % basename + #TODO: download the artifacts from a shared cache + shutil.copy2(orig, pathname) + + def create_directory_if_missing(self, dirname): + if not os.path.exists(dirname): + os.makedirs(dirname) + + def upload_artifacts_to_public_trove(self, basenames): + logging.info( + 'Upload build artifacts to %s', + self.config['public-trove-host']) + rsync_files_to_server( + self.config['local-build-artifacts-dir'], + basenames, + self.config['public-trove-username'], + self.config['public-trove-host'], + self.config['public-trove-artifact-dir']) + set_permissions_on_server( + self.config['public-trove-username'], + self.config['public-trove-host'], + self.config['public-trove-artifact-dir'], + basenames) + + +class ReleaseArtifactPublisher(object): + + '''Publish release artifacts for a release.''' + + logging.getLogger('mason.publishers.ReleaseArtifactPublisher') + + def __init__(self, config): + self.config = config + + def publish_release_artifacts(self): + files = self.list_release_artifacts() + if files: + self.upload_release_artifacts_to_private_dir(files) + self.move_release_artifacts_to_public_dir(files) + self.create_symlinks_to_new_release_artifacts(files) + + def list_release_artifacts(self): + logging.info('Find release artifacts to publish') + return os.listdir(self.config['release-artifact-dir']) + + def upload_release_artifacts_to_private_dir(self, files): + logging.info('Upload release artifacts to private directory') + path = self.config['download-server-private-dir'] + self.create_directory_on_download_server(path) + self.rsync_files_to_download_server(files, path) + + def create_directory_on_download_server(self, path): + user = self.config['download-server-username'] + host = self.config['download-server-address'] + logging.info(msg='Create {host}:{path}', host=host, path=path) + target = '{user}@{host}'.format(user=user, host=host) + cliapp.ssh_runcmd(target, ['mkdir', '-p', path]) + + def rsync_files_to_download_server(self, files, path): + logging.info('Upload release artifacts to download server') + rsync_files_to_server( + self.config['release-artifact-dir'], + files, + self.config['download-server-username'], + self.config['download-server-address'], + path) + set_permissions_on_server( + self.config['download-server-username'], + self.config['download-server-address'], + path, + files) + + def move_release_artifacts_to_public_dir(self, files): + logging.info('Move release artifacts to public directory') + private_dir = self.config['download-server-private-dir'] + public_dir = self.config['download-server-public-dir'] + self.create_directory_on_download_server(public_dir) + + # Move just the contents of the private dir, not the dir + # itself (-mindepth). Avoid overwriting existing files (mv + # -n). + argv = ['find', private_dir, '-mindepth', '1', + '-exec', 'mv', '-n', '{}', public_dir + '/.', ';'] + + target = '{user}@{host}'.format( + user=self.config['download-server-username'], + host=self.config['download-server-address']) + cliapp.ssh_runcmd(target, argv) + + def create_symlinks_to_new_release_artifacts(self, files): + logging.info('FIXME: Create symlinks to new release artifacts') + + +def rsync_files_to_server( + source_dir, source_filenames, user, host, target_dir): + + if not source_filenames: + return + + argv = [ + 'rsync', + '-a', + '--progress', + '--partial', + '--human-readable', + '--sparse', + '--protect-args', + '-0', + '--files-from=-', + source_dir, + '{user}@{host}:{path}'.format(user=user, host=host, path=target_dir), + ] + + files_list = '\0'.join(filename for filename in source_filenames) + cliapp.runcmd(argv, feed_stdin=files_list, stdout=None, stderr=None) + + +def set_permissions_on_server(user, host, target_dir, filenames): + # If we have no files, we can't form a valid command to run on the server + if not filenames: + return + target = '{user}@{host}'.format(user=user, host=host) + argv = ['xargs', '-0', 'chmod', '0644'] + files_list = ''.join( + '{0}\0'.format(os.path.join(target_dir, filename)) for filename in filenames) + cliapp.ssh_runcmd(target, argv, feed_stdin=files_list, stdout=None, stderr=None) diff --git a/mason/runners.py b/mason/runners.py new file mode 100644 index 0000000..71711b0 --- /dev/null +++ b/mason/runners.py @@ -0,0 +1,109 @@ +# Copyright 2014 Codethink Ltd. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import json +import logging +import os + +class JobRunner(object): + + """A generic Zuul job runner.""" + + logging.getLogger('mason.runners.JobRunner') + + def __init__(self, worker_server, config, function): + self._reset() + + self.worker_server = worker_server + self.plugin_config = config + self.function = function + + self.total_steps = 0 + + def _reset(self): + self.success = True + self.cancelled = False + self.job = None + self.job_arguments = None + self.current_step = 0 + + + def start_job(self, job): + self._reset() + self.job = job + + if self.job is not None: + print "starting job" + try: + self.job_arguments = json.loads(self.job.arguments.decode('utf-8')) + self.send_work_data() + self.run_job() + self.send_work_complete() + except Exception as e: + self.success = False + self.send_work_data(result='Exception: %s' % e) + self.job.sendWorkException(str(e).encode('utf-8')) + + def _get_work_data(self): + work_data = {'name': self.function, + 'location': os.uname()[1], + 'number': self.job.unique, + } + return work_data + + def send_work_data(self, data=None, result=''): + if data is None: + data = self._get_work_data() + if self.success: + data['result'] = 'SUCCESS' + else: + data['result'] = result + logging.debug(json.dumps(data)) + print data + self.job.sendWorkData(json.dumps(data)) + + def send_work_complete(self, data=None): + if data is None: + data = self._get_work_data() + self.send_work_data(data) + if self.success: + print 'sending work-complete' + self.job.sendWorkComplete(json.dumps(data)) + else: + print 'sending work-fail' + self.job.sendWorkFail() + + def stop_working(self, number=None): + if number is None or number == self.job.unique: + self.cancelled = True + + def run_job(self): + # This function is to be implemented by subclasses + pass + + def _do_next_step(self): + self._handle_cancellation() + self.current_step += 1 + data = self._get_work_data() + data['current-step'] = self.current_step + self.send_work_data(data) + + def _handle_cancellation(self): + if self.cancelled: + data = self._get_work_data() + data['result'] = 'Cancelled' + self.send_work_data(data) + self.job.sendWorkFail() + raise Exception('Job cancelled') diff --git a/mason/util.py b/mason/util.py new file mode 100644 index 0000000..90713ed --- /dev/null +++ b/mason/util.py @@ -0,0 +1,145 @@ +# Copyright 2014 Codethink Ltd +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +import cliapp +import morphlib +import time + + +class NovaList: + def __init__(self): + self.output = [] + self.lines = [] + self.instance = [] + + def update(self): + self.output = cliapp.runcmd(['nova', 'list']) + self.lines = self.output.split('\n') + self.lines = self.lines[3:-2] + + def get_nova_details_for_instance(self, name): + self.update() + + for line in self.lines: + entries = line.split('|') + stripped_line = [entry.strip() for entry in entries] + if stripped_line.count(name) == 1: + self.instance = stripped_line + + def get_nova_state_for_instance(self, name): + self.get_nova_details_for_instance(name) + if not self.instance: + return + return self.instance[3] + + def get_nova_ip_for_instance(self, name): + self.get_nova_details_for_instance(name) + if not self.instance: + return + + if self.get_nova_state_for_instance(name) != 'ACTIVE': + return + + return self.instance[6] + + def get_nova_ip_for_instance_timeout(self, name, timeout=120): + start_time = time.time() + + while self.get_nova_state_for_instance(name) != 'ACTIVE': + + if time.time() > start_time + timeout: + print "%s not ACTIVE after %i seconds" % (name, timeout) + return + + time.sleep(1) + + ip_addr = self.get_nova_ip_for_instance(name) + if not ip_addr: + return + + if ip_addr.count('=') == 0: + return + + ip_addr = ip_addr[ip_addr.find('=') + 1:] + + if ip_addr.count(',') == 0: + return ip_addr + + return ip_addr[:ip_addr.find(',')] + + +class MorphologyHelper(object): + + def __init__(self, path): + self.defs_repo = morphlib.gitdir.GitDirectory(path) + self.loader = morphlib.morphloader.MorphologyLoader() + self.finder = morphlib.morphologyfinder.MorphologyFinder(self.defs_repo) + + def load_morphology(self, path): + text = self.finder.read_morphology(path) + return self.loader.load_from_string(text) + + @classmethod + def iterate_systems(cls, systems_list): + for system in systems_list: + yield morphlib.util.sanitise_morphology_path(system['morph']) + if 'subsystems' in system: + for subsystem in cls.iterate_systems(system['subsystems']): + yield subsystem + + def iterate_cluster_deployments(cls, cluster_morph): + for system in cluster_morph['systems']: + path = morphlib.util.sanitise_morphology_path(system['morph']) + defaults = system.get('deploy-defaults', {}) + for name, options in system['deploy'].iteritems(): + config = dict(defaults) + config.update(options) + yield path, name, config + + def load_cluster_systems(self, cluster_morph): + for system_path in set(self.iterate_systems(cluster_morph['systems'])): + system_morph = self.load_morphology(system_path) + yield system_path, system_morph + + def find_systems_by_arch(self, cluster_morph, archs): + deployments = self.iterate_cluster_deployments(cluster_morph) + return [deployment[0] + for deployment in deployments + if self.load_morphology(deployments[0])['arch'] in archs] + + +class VMHost(object): + + def __init__(self, user, address, disk_path): + self.user = user + self.address = address + self.disk_path = disk_path + + @property + def ssh_host(self): + return '{user}@{address}'.format(user=self.user, address=self.address) + + def runcmd(self, *args, **kwargs): + cliapp.ssh_runcmd(self.ssh_host, *args, **kwargs) + + def virsh(self, *args, **kwargs): + self.runcmd(['virsh', '-c', 'qemu:///system'] + list(args), **kwargs) + + +def job_step(function): + def next_step(*args, **kwargs): + args[0]._do_next_step() + return function(*args, **kwargs) + return next_step -- cgit v1.2.1