# Copyright 2014 Codethink Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. import cliapp import json import logging import os import urlparse import mason class Runner(mason.runners.JobRunner): """This thread handles running the build test, which is used to ensure that Baserock can build Baserock.""" log = logging.getLogger("mason.tests.build_test.Runner") def __init__(self, worker_server, plugin_config, job_name): super(Runner, self).__init__(worker_server, plugin_config, job_name) self.total_steps = 4 def run_job(self): self.log.info('Step 1: Creating a workspace') self._create_workspace() self.log.info('Step 2: Create the log directory') self._prepare_log_dir() self.log.info('Step 3: Deploy and test the systems') self._deploy_and_test_systems() self.log.info('Step 4: Clean up') self._clean_up() def _do_git_config(self, name='Mason Test Runner', email='mason@runner'): cliapp.runcmd(['git', 'config', '--global', 'user.name', name]) cliapp.runcmd(['git', 'config', '--global', 'user.email', email]) @staticmethod def _run_tests(instance, system_path, system_morph, (trove_host, trove_id, build_ref_prefix), systems, logfile): instance.log = logfile instance.wait_until_online() tests = [] def baserock_build_test(instance): instance.runcmd(['git', 'config', '--global', 'user.name', 'Test Instance of %s' % instance.deployment.name], stdin=None, stdout=logfile, stderr=logfile) instance.runcmd(['git', 'config', '--global', 'user.email', 'ci-test@%s' % instance.config['HOSTNAME']], stdin=None, stdout=logfile, stderr=logfile) instance.runcmd(['mkdir', '-p', '/src/ws', '/src/cache', '/src/tmp'], stdin=None, stdout=logfile, stderr=logfile) def morph_cmd(*args, **kwargs): # TODO: decide whether to use cached artifacts or not by # adding --artifact-cache-server= --cache-server= argv = ['morph', '--log=/src/morph.log', '--cachedir=/src/cache', '--tempdir=/src/tmp', '--log-max=100M', '--trove-host', trove_host, '--trove-id', trove_id, '--build-ref-prefix', build_ref_prefix] argv.extend(args) instance.runcmd(argv, **kwargs) repo = self.morph_helper.sb.root_repository_url ref = self.morph_helper.defs_repo.HEAD sha1 = self.morph_helper.defs_repo.resolve_ref_to_commit(ref) morph_cmd('init', '/src/ws', stdin=None, stdout=logfile, stderr=logfile) chdir = '/src/ws' morph_cmd('checkout', repo, ref, chdir=chdir, stdin=None, stdout=logfile, stderr=logfile) # TODO: Add a morph subcommand that gives the path to the root repository. repo_path = os.path.relpath( self.morph_helper.sb.get_git_directory_name(repo), self.morph_helper.sb.root_directory) chdir = os.path.join(chdir, ref, repo_path) instance.runcmd(['git', 'reset', '--hard', sha1], chdir=chdir, stdin=None, stdout=logfile, stderr=logfile) logfile.write('Building test systems for %s\n' % system_path) for to_build_path, to_build_morph in systems.iteritems(): if to_build_morph['arch'] == system_morph['arch']: logfile.write('Test building %s\n' % to_build_path) logfile.flush() morph_cmd('build', to_build_path, chdir=chdir, stdin=None, stdout=logfile, stderr=logfile) logfile.write('Finished Building test systems\n') logfile.flush() # TODO: Match the systems with a regex in config? if 'devel' in system_path: tests.append(baserock_build_test) for test in tests: test(instance) @mason.util.job_step def _create_workspace(self): self.commit = self.job_arguments['ZUUL_COMMIT'] self.project = self.job_arguments['ZUUL_PROJECT'] self.ref = self.job_arguments['ZUUL_REF'] self.workspace = '/root/mason-workspace' self.zuul_url = self.job_arguments['ZUUL_URL'] url = urlparse.urlparse(self.zuul_url) self.defs_checkout = os.path.join(self.workspace, self.commit, url.hostname, '8080', self.project) self._do_git_config() cliapp.runcmd(['morph', 'init', self.workspace]) repo = 'http://%s:8080/%s' % (url.hostname, self.project) cliapp.runcmd(['morph', 'checkout', repo, self.commit], cwd=self.workspace) self.morph_helper = mason.util.MorphologyHelper(self.defs_checkout) @mason.util.job_step def _prepare_log_dir(self): self.logdir = '/var/www/logs/%s-%s/build_test' % \ (self.project, self.commit[:7]) if not os.path.exists(self.logdir): os.makedirs(self.logdir) @mason.util.job_step def _deploy_and_test_systems(self): config = self.plugin_config['config'] infrastructure = config['test-infrastructure-type'] build_test_config = (config['trove-host'], config['trove-id'], config.get('build-ref-prefix')) cluster_path = config['cluster-morphology'] cluster = self.morph_helper.load_morphology(cluster_path) systems = dict(self.morph_helper.load_cluster_systems(cluster)) deployment_hosts = {} for host_config in config['deployment-host']: arch, address = host_config.split(':', 1) user, address = address.split('@', 1) address, disk_path = address.split(':', 1) if user == '': user = 'root' deployment_hosts[arch] = mason.util.VMHost( user, address, disk_path) for system_path, deployment_name, deployment_config in \ self.morph_helper.iterate_cluster_deployments(cluster): system = systems[system_path] # We can only test systems in KVM that have a BSP if not any('bsp' in si['morph'] for si in system['strata']): continue # We can only test systems in KVM that we have a host for if system['arch'] not in deployment_hosts: continue host = deployment_hosts[system['arch']] log_path = os.path.join(self.logdir, '%s-deploy.log' % system['name']) net_id = config['openstack-network-id'] deployment = mason.deployment.Deployment(cluster_path, deployment_name, deployment_config, host, net_id, log_path) os.chdir(self.defs_checkout) instance = deployment.deploy(infrastructure) log_path = os.path.join(self.logdir, '%s-test.log' % system['name']) logfile = open(log_path, 'w+') try: self._run_tests(instance, system_path, system, build_test_config, systems, logfile) finally: instance.delete() logfile.close() # If we cancel, we don't want to have to wait for all # the systems to be tested before we stop. self._handle_cancellation() @mason.util.job_step def _clean_up(self): cliapp.runcmd(['rm', '-rf', self.workspace])