Add mason 'build', 'test' and 'upload' as Zuul pluginsbaserock/mason-v2

Much of this code comes from the release-* scripts in
Baserock's defintions repo.

At the moment, only build, build_test and artifact_upload plugins
are implemented. Also note that the test build run on a deployed
instance in the build_test plugin currently builds master rather
than the ref of the proposed change being tested. This is still
adequate as that system is itself built from the ref of the
proposed change.

5 files changed, 432 insertions, 0 deletions

diff --git a/mason/__init__.py b/mason/__init__.py
index dc5a195..20b3ee2 100644
--- a/mason/__init__.py
+++ b/mason/__init__.py
@@ -2,3 +2,4 @@ from . import deployment
 from . import publishers
 from . import runners
 from . import util
+from . import tests
diff --git a/mason/tests/__init__.py b/mason/tests/__init__.py
new file mode 100644
index 0000000..309ce19
--- /dev/null
+++ b/mason/tests/__init__.py
@@ -0,0 +1,3 @@
+from . import build
+from . import artifact_upload
+from . import build_test
diff --git a/mason/tests/artifact_upload.py b/mason/tests/artifact_upload.py
new file mode 100644
index 0000000..1167167
--- /dev/null
+++ b/mason/tests/artifact_upload.py
@@ -0,0 +1,112 @@
+# Copyright 2014-2015 Codethink Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import json
+import logging
+import os
+import urlparse
+
+import cliapp
+
+import mason
+
+class Runner(mason.runners.JobRunner):
+
+    """Uploads the artifacts produced during a build
+
+    This handles running the build-deploy-build test, which
+    is used to ensure that Baserock can build Baserock.
+    """
+
+    log = logging.getLogger("mason.tests.artifact_upload.Runner")
+
+    def __init__(self, worker_server, plugin_config, job_name):
+        super(Runner, self).__init__(worker_server, plugin_config, job_name)
+        self.config = self.plugin_config['config']
+        self._set_defaults()
+
+        self.total_steps = 2
+        if self.config['upload-build-artifacts']:
+            self.total_steps += 1
+        if self.config['upload-release-artifacts']:
+            self.total_steps += 1
+
+    def _set_defaults(self):
+        self.config['public-trove-host'] = \
+            self.config.get('public-trove-host') or 'git.baserock.org'
+        self.config['public-trove-username'] = \
+            self.config.get('public-trove-username') or 'root'
+        self.config['public-trove-artifact-dir'] = \
+            self.config.get('public-trove-artifact-dir') \
+            or '/home/cache/artifacts'
+
+        self.config['download-server-address'] = \
+            self.config.get('download-server-address') \
+            or 'download.baserock.org'
+        self.config['download-server-username'] = \
+            self.config.get('download-server-username') or 'root'
+        self.config['download-server-private-dir'] = \
+            self.config.get('download-server-private-dir') \
+            or '/srv/download.baserock.org/baserock/.publish-temp'
+        self.config['download-server-public-dir'] = \
+            self.config.get('download-server-public-dir') \
+            or '/srv/download.baserock.org/baserock'
+
+        self.config['release-artifact-dir'] = \
+            self.config.get('release-artifact-dir') or '.'
+        self.config['local-build-artifacts-dir'] = \
+            self.config.get('local-build-artifacts-dir') or 'build-artifacts'
+        self.config['architecture'] = \
+            self.config.get('architecture') or []
+
+        if 'upload-release-artifacts' not in self.config:
+            self.config['upload-release-artifacts'] = True
+        if 'upload-build-artifacts' not in self.config:
+            self.config['upload-build-artifacts'] = True
+
+    def run_job(self):
+        self.log.info('Step 1 of %d: Creating a workspace' %
+            (self.total_steps))
+        self._create_workspace()
+
+        try:
+            if self.config['upload-build-artifacts']:
+                self.log.info('Step 2 of %d: Publish the build artifacts' %
+                    (self.total_steps))
+                self._publish_build_artifacts()
+
+            if self.config['upload-release-artifacts']:
+                self.log.info('Step %d of %d: Publish the release artifacts' %
+                              (self.current_step + 1, self.total_steps))
+                self._publish_release_artifacts()
+        except Exception as e:
+            self.log.info('Exception: %s' % (e))
+            self._remove_workspace()
+            raise e
+
+        self.log.info('Step %d of %d: Clean up' %
+            (self.current_step + 1, self.total_steps))
+        self._clean_up()
+
+    @mason.util.job_step
+    def _publish_build_artifacts(self):
+        publisher = mason.publishers.BuildArtifactPublisher(
+            self.config, self.defs_checkout)
+        publisher.publish_build_artifacts()
+
+    @mason.util.job_step
+    def _publish_release_artifacts(self):
+        publisher = mason.publishers.ReleaseArtifactPublisher(self.config)
+        publisher.publish_release_artifacts()
diff --git a/mason/tests/build.py b/mason/tests/build.py
new file mode 100644
index 0000000..625252d
--- /dev/null
+++ b/mason/tests/build.py
@@ -0,0 +1,128 @@
+# Copyright 2014 Codethink Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import cliapp
+import json
+import logging
+import morphlib
+import os
+import subprocess
+import time
+import urlparse
+
+import mason
+
+
+class Build(object):
+
+    """A single build instance."""
+
+    def __init__(self, name, controller, logfile):
+        self.system_name = name
+        self.controller = controller
+        self.log_path = logfile
+        self.logfile = open(logfile, 'w+')
+        #TODO: use distbuild not local build
+        self.command = [
+            'morph', 'build', self.system_name]
+
+    def start(self):
+        self.process = subprocess.Popen(self.command, stdout=self.logfile, stderr=self.logfile)
+
+    def completed(self):
+        return (self.process.poll() is not None)
+
+    def close_log(self):
+        self.logfile.close()
+
+
+class Runner(mason.runners.JobRunner):
+
+    """Test that the built system will deploy and run
+
+    This handles running the build-deploy-build test, which
+    is used to ensure that Baserock can build Baserock.
+    """
+
+    log = logging.getLogger("mason.tests.build.Runner")
+
+    def __init__(self, worker_server, plugin_config, job_name):
+        super(Runner, self).__init__(worker_server, plugin_config, job_name)
+
+        self.total_steps = 4
+
+    def run_job(self):
+        self.log.debug('Got job: %s', self.job_arguments)
+
+        self.log.info('Step 1: Creating a workspace')
+        self._create_workspace()
+
+        self.log.info('Step 2: Prepare build log directory')
+        self._prepare_build_log_dir()
+
+        self.log.info('Step 3: Building the systems')
+        try:
+            self._build_systems()
+        except Exception as e:
+            self._remove_workspace()
+            raise e
+
+        self.log.info('Step 4: Clean up')
+        self._clean_up()
+
+    @staticmethod
+    def _parse_controllers(conf):
+        return dict(item.split(':', 1) for item in conf['controllers'])
+
+    def _prepare_builds(self, conf):
+        cluster = self.morph_helper.load_morphology(conf['cluster-morphology'])
+        systems = set(self.morph_helper.iterate_systems(cluster['systems']))
+        controllers = self._parse_controllers(conf)
+        builds = []
+        for system_name in systems:
+            system = self.morph_helper.load_morphology(system_name)
+            print 'loaded %s' % system_name
+            if system['arch'] in controllers:
+                logfile = os.path.join(self.logdir, '%s.log' % system['name'])
+                builds.append(Build(system_name, controllers[system['arch']], logfile))
+        print 'prepared builds'
+        return builds
+
+    @mason.util.job_step
+    def _prepare_build_log_dir(self):
+        self.logdir = '/var/www/logs/%s-%s/build' % \
+                (self.project, self.commit[:7])
+        if not os.path.exists(self.logdir):
+            os.makedirs(self.logdir)
+
+    @mason.util.job_step
+    def _build_systems(self):
+        builds = self._prepare_builds(self.plugin_config['config'])
+        os.chdir(self.defs_checkout)
+        for build in builds:
+            build.start()
+            # TODO: Don't force serialisation when we change to distbuild.
+            while not build.completed():
+                time.sleep(1)
+
+        fail = False
+        for build in builds:
+            build.close_log()
+            if build.process.returncode != 0:
+                fail = True
+                logging.error('Building failed for %s. Log is at %s.' % 
+                              (build.system_name, build.log_path))
+        if fail:
+            raise cliapp.AppException('Building of systems failed.')
diff --git a/mason/tests/build_test.py b/mason/tests/build_test.py
new file mode 100644
index 0000000..cdf98c3
--- /dev/null
+++ b/mason/tests/build_test.py
@@ -0,0 +1,188 @@
+# Copyright 2014 Codethink Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import cliapp
+import json
+import logging
+import os
+import urlparse
+
+import mason
+
+
+class Runner(mason.runners.JobRunner):
+
+    """Run the Build test
+
+    This handles running the Build test, which is used to ensure
+    that Baserock can build Baserock.
+    """
+
+    log = logging.getLogger("mason.tests.build_test.Runner")
+
+    def __init__(self, worker_server, plugin_config, job_name):
+        super(Runner, self).__init__(worker_server, plugin_config, job_name)
+
+        self.total_steps = 4
+
+    def run_job(self):
+        self.log.debug('Got job: %s', self.job_arguments)
+
+        self.log.info('Step 1: Creating a workspace')
+        self._create_workspace()
+
+        self.log.info('Step 2: Create the log directory')
+        self._prepare_log_dir()
+
+        self.log.info('Step 3: Deploy and test the systems')
+        try:
+            self._deploy_and_test_systems()
+        except Exception as e:
+            self.log.info('Failed to deploy and test systems: Exception %s', e)
+            self._remove_workspace()
+            raise e
+
+        self.log.info('Step 4: Clean up')
+        self._clean_up()
+
+    @staticmethod
+    def _run_tests(instance, system_path, system_morph,
+                   (trove_host, trove_id), systems, logfile):
+        instance.logfile = logfile
+        has_cloudinit = any('cloudinit-support' in si['morph'] for si in system_morph['strata'])
+        instance.wait_until_online(has_cloudinit)
+
+        tests = []
+        def baserock_build_test(instance):
+            instance.runcmd(['git', 'config', '--global', 'user.name',
+                             'Test Instance of %s' % instance.deployment.name],
+                             stdin=None, stdout=logfile, stderr=logfile)
+            instance.runcmd(['git', 'config', '--global', 'user.email',
+                             'ci-test@%s' % instance.config['HOSTNAME']],
+                             stdin=None, stdout=logfile, stderr=logfile)
+            instance.runcmd(['mkdir', '-p', '/src/ws', '/src/cache',
+                             '/src/tmp'],
+                             stdin=None, stdout=logfile, stderr=logfile)
+            def morph_cmd(*args, **kwargs):
+                # TODO: decide whether to use cached artifacts or not by
+                # adding --artifact-cache-server= --cache-server=
+                argv = ['morph', '--log=/src/morph.log', '--cachedir=/src/cache',
+                        '--tempdir=/src/tmp', '--log-max=100M',
+                        '--trove-host', trove_host, '--trove-id', trove_id,
+                        '--local-changes=ignore']
+                argv.extend(args)
+                instance.runcmd(argv, **kwargs)
+
+            repo = 'baserock:baserock/definitions'
+            ref = 'master'
+
+            morph_cmd('init', '/src/ws',
+                      stdin=None, stdout=logfile, stderr=logfile)
+            chdir = '/src/ws'
+
+            morph_cmd('checkout', repo, ref, chdir=chdir,
+                      stdin=None, stdout=logfile, stderr=logfile)
+            # TODO: Add a morph subcommand that gives the path to the root repository.
+            chdir = os.path.join(chdir, ref, 'baserock', 'baserock',
+                                 'definitions')
+
+            instance.runcmd(['git', 'reset', '--hard', 'HEAD'], chdir=chdir,
+                            stdin=None, stdout=logfile, stderr=logfile)
+            logfile.write('Building test systems for %s\n' % system_path)
+            for to_build_path, to_build_morph in systems.iteritems():
+                if to_build_morph['arch'] == system_morph['arch']:
+                    logfile.write('Test building %s\n' % to_build_path)
+                    logfile.flush()
+                    morph_cmd('build', to_build_path, chdir=chdir,
+                              stdin=None, stdout=logfile, stderr=logfile)
+            logfile.write('Finished Building test systems\n')
+            logfile.flush()
+
+        # TODO: Match the systems with a regex in config?
+        if 'devel' in system_path:
+            tests.append(baserock_build_test)
+
+        for test in tests:
+            test(instance)
+
+    @mason.util.job_step
+    def _prepare_log_dir(self):
+        self.logdir = '/var/www/logs/%s-%s/build_test' % \
+                (self.project, self.commit[:7])
+        if not os.path.exists(self.logdir):
+            os.makedirs(self.logdir)
+
+    @mason.util.job_step
+    def _deploy_and_test_systems(self):
+        config = self.plugin_config['config']
+        infrastructure = config['test-infrastructure-type']
+        build_test_config = (config['trove-host'],
+                             config['trove-id'])
+        cluster_path = config['cluster-morphology']
+        cluster = self.morph_helper.load_morphology(cluster_path)
+        systems = dict(self.morph_helper.load_cluster_systems(cluster))
+
+        deployment_hosts = {}
+        for host_config in config['deployment-host']:
+            arch, address = host_config.split(':', 1)
+            user, address = address.split('@', 1)
+            address, disk_path = address.split(':', 1)
+            if user == '':
+                user = 'root'
+            deployment_hosts[arch] = mason.util.VMHost(
+                user, address, disk_path)
+
+        for system_path, deployment_name, deployment_config in \
+            self.morph_helper.iterate_cluster_deployments(cluster):
+
+            system = systems[system_path]
+            # We can only test systems in KVM that have a BSP
+            if not any('bsp' in si['morph'] for si in system['strata']):
+                continue
+
+            # We can only test systems in KVM that we have a host for
+            if system['arch'] not in deployment_hosts:
+                continue
+
+            host = deployment_hosts[system['arch']]
+            log_path = os.path.join(self.logdir,
+                                    '%s-deploy.log' % system['name'])
+            net_id = config['openstack-network-id']
+            deployment = mason.deployment.Deployment(cluster_path,
+                                                     deployment_name,
+                                                     deployment_config, host,
+                                                     net_id, log_path)
+
+            os.chdir(self.defs_checkout)
+            try:
+                instance = deployment.deploy(infrastructure)
+            except Exception as e:
+                raise e
+            log_path = os.path.join(self.logdir,
+                                    '%s-test.log' % system['name'])
+            logfile = open(log_path, 'w+')
+            try:
+                self._run_tests(instance, system_path, system,
+                                build_test_config, systems, logfile)
+            except Exception as e:
+                logfile.write("Exception while running tests: %s\n" % (e))
+                raise e
+            finally:
+                instance.delete()
+                logfile.close()
+
+            # If we cancel, we don't want to have to wait for all
+            # the systems to be tested before we stop.
+            self._handle_cancellation()