diff options
| -rwxr-xr-x | mason/mason.sh | 22 | ||||
| -rw-r--r-- | mason/os-init-script | 6 | ||||
| -rw-r--r-- | mason/os.conf | 23 | ||||
| -rwxr-xr-x | scripts/release-test-os | 524 |
4 files changed, 570 insertions, 5 deletions
diff --git a/mason/mason.sh b/mason/mason.sh index 13ea21db..cf1341e3 100755 --- a/mason/mason.sh +++ b/mason/mason.sh @@ -1,5 +1,8 @@ #!/bin/sh +# Load OpenStack credentials +. /root/os.conf + set -e set -x @@ -59,11 +62,20 @@ else echo INFO: Created "$releases_made" release images fi -"scripts/release-test" \ - --deployment-host "$DISTBUILD_ARCH":"$TEST_VM_HOST_SSH_URL" \ - --trove-host "$UPSTREAM_TROVE_ADDRESS" \ - --trove-id "$TROVE_ID" \ - "$BUILD_CLUSTER_MORPHOLOGY" +if [ "$TEST_INFRA" = "openstack" ]; then + "/root/release-test-os" \ + --deployment-host "$DISTBUILD_ARCH":"$TEST_VM_HOST_SSH_URL" \ + --trove-host "$UPSTREAM_TROVE_ADDRESS" \ + --trove-id "$TROVE_ID" \ + --net-id "$OPENSTACK_NET_ID" \ + "$BUILD_CLUSTER_MORPHOLOGY" +elif [ "$TEST_INFRA" = "kvmhost" ]; then + "scripts/release-test" \ + --deployment-host "$DISTBUILD_ARCH":"$TEST_VM_HOST_SSH_URL" \ + --trove-host "$UPSTREAM_TROVE_ADDRESS" \ + --trove-id "$TROVE_ID" \ + "$BUILD_CLUSTER_MORPHOLOGY" +fi "scripts/release-upload" --build-trove-host "$ARTIFACT_CACHE_SERVER" \ --arch "$DISTBUILD_ARCH" \ diff --git a/mason/os-init-script b/mason/os-init-script new file mode 100644 index 00000000..77afb926 --- /dev/null +++ b/mason/os-init-script @@ -0,0 +1,6 @@ +#!/bin/bash + +# This allows the test runner to know that cloud-init has completed the +# disc resizing, and there is enough free space to continue. +touch /root/cloud-init-finished + diff --git a/mason/os.conf b/mason/os.conf new file mode 100644 index 00000000..d9f69fda --- /dev/null +++ b/mason/os.conf @@ -0,0 +1,23 @@ +#!/bin/bash + +# With the addition of Keystone, to use an openstack cloud you should +# authenticate against keystone, which returns a **Token** and **Service +# Catalog**. The catalog contains the endpoint for all services the +# user/tenant has access to - including nova, glance, keystone, swift. +# +# *NOTE*: Using the 2.0 *auth api* does not mean that compute api is 2.0. We +# will use the 1.1 *compute api* +export OS_AUTH_URL=<os-auth-url> + +# With the addition of Keystone we have standardized on the term **tenant** +# as the entity that owns the resources. +export OS_TENANT_ID=<os-tenant-id> +export OS_TENANT_NAME="<os-tenant>" + +# In addition to the owning entity (tenant), openstack stores the entity +# performing the action as the **user**. +export OS_USERNAME="<os-user>" + +# With Keystone you pass the keystone password. +export OS_PASSWORD="<os-pass>" + diff --git a/scripts/release-test-os b/scripts/release-test-os new file mode 100755 index 00000000..ca4bd485 --- /dev/null +++ b/scripts/release-test-os @@ -0,0 +1,524 @@ +#!/usr/bin/env python +# +# Copyright 2014 Codethink Ltd +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''release-test + +This script deploys the set of systems in the cluster morphology it is +instructed to read, to test that they work correctly. + +''' + +import cliapp +import os +import pipes +import shlex +import shutil +import socket +import tempfile +import time +import uuid + +import morphlib + + +class NovaList: + def __init__(self): + self.output = [] + self.lines = [] + self.instance = [] + + def update(self): + self.output = cliapp.runcmd(['nova', 'list']) + self.lines = self.output.split('\n') + self.lines = self.lines[3:-2] + + def get_nova_details_for_instance(self, name): + self.update() + + for line in self.lines: + entries = line.split('|') + stripped_line = [entry.strip() for entry in entries] + if stripped_line.count(name) == 1: + self.instance = stripped_line + + def get_nova_state_for_instance(self, name): + self.get_nova_details_for_instance(name) + if not self.instance: + return + return self.instance[3] + + def get_nova_ip_for_instance(self, name): + self.get_nova_details_for_instance(name) + if not self.instance: + return + + if self.get_nova_state_for_instance(name) != 'ACTIVE': + return + + return self.instance[6] + + def get_nova_ip_for_instance_timeout(self, name, timeout=120): + start_time = time.time() + + while self.get_nova_state_for_instance(name) != 'ACTIVE': + + if time.time() > start_time + timeout: + print "%s not ACTIVE after %i seconds" % (name, timeout) + return + + time.sleep(1) + + ip_addr = self.get_nova_ip_for_instance(name) + if not ip_addr: + return + + if ip_addr.count('=') == 0: + return + + ip_addr = ip_addr[ip_addr.find('=') + 1:] + + if ip_addr.count(',') == 0: + return ip_addr + + return ip_addr[:ip_addr.find(',')] + + + +class MorphologyHelper(object): + + def __init__(self): + self.sb = sb = morphlib.sysbranchdir.open_from_within('.') + defs_repo_path = sb.get_git_directory_name(sb.root_repository_url) + self.defs_repo = morphlib.gitdir.GitDirectory(defs_repo_path) + self.loader = morphlib.morphloader.MorphologyLoader() + self.finder = morphlib.morphologyfinder.MorphologyFinder(self.defs_repo) + + def load_morphology(self, path): + text = self.finder.read_morphology(path) + return self.loader.load_from_string(text) + + @classmethod + def iterate_systems(cls, systems_list): + for system in systems_list: + yield morphlib.util.sanitise_morphology_path(system['morph']) + if 'subsystems' in system: + for subsystem in cls.iterate_systems(system['subsystems']): + yield subsystem + + def iterate_cluster_deployments(cls, cluster_morph): + for system in cluster_morph['systems']: + path = morphlib.util.sanitise_morphology_path(system['morph']) + defaults = system.get('deploy-defaults', {}) + for name, options in system['deploy'].iteritems(): + config = dict(defaults) + config.update(options) + yield path, name, config + + def load_cluster_systems(self, cluster_morph): + for system_path in set(self.iterate_systems(cluster_morph['systems'])): + system_morph = self.load_morphology(system_path) + yield system_path, system_morph + + +class TimeoutError(cliapp.AppException): + + """Error to be raised when a connection waits too long""" + + def __init__(self, msg): + super(TimeoutError, self).__init__(msg) + + +class VMHost(object): + + def __init__(self, user, address, disk_path): + self.user = user + self.address = address + self.disk_path = disk_path + + @property + def ssh_host(self): + return '{user}@{address}'.format(user=self.user, address=self.address) + + def runcmd(self, *args, **kwargs): + cliapp.ssh_runcmd(self.ssh_host, *args, **kwargs) + + +class DeployedSystemInstance(object): + + def __init__(self, deployment, config, host_machine, vm_id, rootfs_path, + ip_addr, hostname): + self.deployment = deployment + self.config = config + self.ip_address = ip_addr + self.host_machine = host_machine + self.vm_id = vm_id + self.rootfs_path = rootfs_path + self.hostname = hostname + + @property + def ssh_host(self): + # TODO: Stop assuming we ssh into test instances as root + return 'root@{host}'.format(host=self.ip_address) + + def runcmd(self, argv, chdir='.', **kwargs): + ssh_cmd = ['ssh', '-o', 'StrictHostKeyChecking=no', + '-o', 'UserKnownHostsFile=/dev/null', self.ssh_host] + cmd = ['sh', '-c', 'cd "$1" && shift && exec "$@"', '-', chdir] + cmd += argv + ssh_cmd.append(' '.join(map(pipes.quote, cmd))) + return cliapp.runcmd(ssh_cmd, **kwargs) + + def _wait_for_dhcp(self, timeout): + '''Block until given hostname resolves successfully. + + Raises TimeoutError if the hostname has not appeared in 'timeout' + seconds. + + ''' + start_time = time.time() + while True: + try: + socket.gethostbyname(self.ip_address) + return + except socket.gaierror: + pass + if time.time() > start_time + timeout: + raise TimeoutError("Host %s did not appear after %i seconds" % + (self.ip_address, timeout)) + time.sleep(0.5) + + def _wait_for_ssh(self, timeout): + """Wait until the deployed VM is responding via SSH""" + start_time = time.time() + while True: + try: + self.runcmd(['true'], stdin=None, stdout=None, stderr=None) + return + except cliapp.AppException: + # TODO: Stop assuming the ssh part of the command is what failed + if time.time() > start_time + timeout: + raise TimeoutError("%s sshd did not start after %i seconds" + % (self.ip_address, timeout)) + time.sleep(0.5) + + def _wait_for_cloud_init(self, timeout): + """Wait until cloud init has resized the disc""" + start_time = time.time() + while True: + try: + out = self.runcmd(['sh', '-c', + 'test -e "$1" && echo exists || echo does not exist', + '-', + '/root/cloud-init-finished']) + except: + import traceback + traceback.print_exc() + raise + if out.strip() == 'exists': + return + if time.time() > start_time + timeout: + raise TimeoutError("Disc size not increased after %i seconds" + % (timeout)) + time.sleep(3) + + def wait_until_online(self, timeout=120): + self._wait_for_dhcp(timeout) + self._wait_for_ssh(timeout) + self._wait_for_cloud_init(timeout) + print "Test system %s ready to run tests." % (self.hostname) + + def delete(self): + # Stop and remove VM + print "Deleting %s test instance" % (self.hostname) + try: + cliapp.runcmd(['nova', 'delete', self.hostname]) + except cliapp.AppException as e: + # TODO: Stop assuming that delete failed because the instance + # wasn't running + print "- Failed" + pass + print "Deleting %s test disc image" % (self.hostname) + try: + cliapp.runcmd(['nova', 'image-delete', self.hostname]) + except cliapp.AppException as e: + # TODO: Stop assuming that image-delete failed because it was + # already removed + print "- Failed" + pass + + +class Deployment(object): + + def __init__(self, cluster_path, name, deployment_config, + host_machine, net_id): + self.cluster_path = cluster_path + self.name = name + self.deployment_config = deployment_config + self.host_machine = host_machine + self.net_id = net_id + + @staticmethod + def _ssh_host_key_exists(hostname): + """Check if an ssh host key exists in known_hosts""" + with open('/root/.ssh/known_hosts', 'r') as known_hosts: + return any(line.startswith(hostname) for line in known_hosts) + + def _update_known_hosts(self): + if not self._ssh_host_key_exists(self.host_machine.address): + with open('/root/.ssh/known_hosts', 'a') as known_hosts: + cliapp.runcmd(['ssh-keyscan', self.host_machine.address], + stdout=known_hosts) + + @staticmethod + def _generate_sshkey_config(tempdir, config): + manifest = os.path.join(tempdir, 'manifest') + with open(manifest, 'w') as f: + f.write('0040700 0 0 /root/.ssh\n') + f.write('overwrite 0100600 0 0 /root/.ssh/authorized_keys\n') + authkeys = os.path.join(tempdir, 'root', '.ssh', 'authorized_keys') + os.makedirs(os.path.dirname(authkeys)) + with open(authkeys, 'w') as auth_f: + with open('/root/.ssh/id_rsa.pub', 'r') as key_f: + shutil.copyfileobj(key_f, auth_f) + + install_files = shlex.split(config.get('INSTALL_FILES', '')) + install_files.append(manifest) + yield 'INSTALL_FILES', ' '.join(pipes.quote(f) for f in install_files) + + def deploy(self): + self._update_known_hosts() + + hostname = str(uuid.uuid4()) + vm_id = hostname + image_base = self.host_machine.disk_path + rootpath = '{image_base}/{hostname}.img'.format(image_base=image_base, + hostname=hostname) + loc = 'http://{ssh_host}:5000/v2.0'.format( + ssh_host=self.host_machine.ssh_host, id=vm_id, path=rootpath) + + options = { + 'type': 'openstack', + 'location': loc, + 'HOSTNAME': hostname, + 'DISK_SIZE': '5G', + 'RAM_SIZE': '2G', + 'VERSION_LABEL': 'release-test', + 'OPENSTACK_USER': os.environ['OS_USERNAME'], + 'OPENSTACK_TENANT': os.environ['OS_TENANT_NAME'], + 'OPENSTACK_PASSWORD': os.environ['OS_PASSWORD'], + 'OPENSTACK_IMAGENAME': hostname, + 'CLOUD_INIT': 'yes', + 'KERNEL_ARGS': 'console=tty0 console=ttyS0', + } + + tempdir = tempfile.mkdtemp() + try: + options.update( + self._generate_sshkey_config(tempdir, + self.deployment_config)) + + # Deploy the image to openstack + args = ['morph', 'deploy', self.cluster_path, self.name] + for k, v in options.iteritems(): + args.append('%s.%s=%s' % (self.name, k, v)) + cliapp.runcmd(args, stdin=None, stdout=None, stderr=None) + + config = dict(self.deployment_config) + config.update(options) + + # Boot an instance from the image + args = ['nova', 'boot', + '--flavor', 'm1.medium', + '--image', hostname, + '--user-data', '/root/os-init-script', + '--nic', "net-id=%s" % (self.net_id), + hostname] + output = cliapp.runcmd(args) + + # Print nova boot output, with adminPass line removed + output_lines = output.split('\n') + for line in output_lines: + if line.find('adminPass') != -1: + password_line = line + output_lines.remove(password_line) + output = '\n'.join(output_lines) + print output + + # Get ip address from nova list + nl = NovaList() + ip_addr = nl.get_nova_ip_for_instance_timeout(hostname) + print "IP address for inastance %s: %s" % (hostname, ip_addr) + + return DeployedSystemInstance(self, config, self.host_machine, + vm_id, rootpath, ip_addr, hostname) + finally: + shutil.rmtree(tempdir) + + +class ReleaseApp(cliapp.Application): + + """Cliapp application which handles automatic builds and tests""" + + def add_settings(self): + """Add the command line options needed""" + group_main = 'Program Options' + self.settings.string_list(['deployment-host'], + 'ARCH:HOST:PATH that VMs can be deployed to', + default=None, + group=group_main) + self.settings.string(['trove-host'], + 'Address of Trove for test systems to build from', + default=None, + group=group_main) + self.settings.string(['trove-id'], + 'ID of Trove for test systems to build from', + default=None, + group=group_main) + self.settings.string(['build-ref-prefix'], + 'Prefix of build branches for test systems', + default=None, + group=group_main) + self.settings.string(['net-id'], + 'Openstack network ID', + default=None, + group=group_main) + + @staticmethod + def _run_tests(instance, system_path, system_morph, + (trove_host, trove_id, build_ref_prefix), + morph_helper, systems): + instance.wait_until_online() + + tests = [] + def baserock_build_test(instance): + instance.runcmd(['git', 'config', '--global', 'user.name', + 'Test Instance of %s' % instance.deployment.name]) + instance.runcmd(['git', 'config', '--global', 'user.email', + 'ci-test@%s' % instance.config['HOSTNAME']]) + instance.runcmd(['mkdir', '-p', '/src/ws', '/src/cache', + '/src/tmp']) + def morph_cmd(*args, **kwargs): + # TODO: decide whether to use cached artifacts or not by + # adding --artifact-cache-server= --cache-server= + argv = ['morph', '--log=/src/morph.log', '--cachedir=/src/cache', + '--tempdir=/src/tmp', '--log-max=100M', + '--trove-host', trove_host, '--trove-id', trove_id, + '--build-ref-prefix', build_ref_prefix] + argv.extend(args) + instance.runcmd(argv, **kwargs) + + repo = morph_helper.sb.root_repository_url + ref = morph_helper.defs_repo.HEAD + sha1 = morph_helper.defs_repo.resolve_ref_to_commit(ref) + morph_cmd('init', '/src/ws') + chdir = '/src/ws' + + morph_cmd('checkout', repo, ref, chdir=chdir) + # TODO: Add a morph subcommand that gives the path to the root repository. + repo_path = os.path.relpath( + morph_helper.sb.get_git_directory_name(repo), + morph_helper.sb.root_directory) + chdir = os.path.join(chdir, ref, repo_path) + + instance.runcmd(['git', 'reset', '--hard', sha1], chdir=chdir) + print 'Building test systems for {sys}'.format(sys=system_path) + for to_build_path, to_build_morph in systems.iteritems(): + if to_build_morph['arch'] == system_morph['arch']: + print 'Test building {path}'.format(path=to_build_path) + morph_cmd('build', to_build_path, chdir=chdir, + stdin=None, stdout=None, stderr=None) + print 'Finished Building test systems' + + def python_smoke_test(instance): + instance.runcmd(['python', '-c', 'print "Hello World"']) + + # TODO: Come up with a better way of determining which tests to run + if 'devel' in system_path: + tests.append(baserock_build_test) + else: + tests.append(python_smoke_test) + + for test in tests: + test(instance) + + def deploy_and_test_systems(self, cluster_path, + deployment_hosts, build_test_config, + net_id): + """Run the deployments and tests""" + + version = 'release-test' + + morph_helper = MorphologyHelper() + cluster_morph = morph_helper.load_morphology(cluster_path) + systems = dict(morph_helper.load_cluster_systems(cluster_morph)) + + for system_path, deployment_name, deployment_config in \ + morph_helper.iterate_cluster_deployments(cluster_morph): + + system_morph = systems[system_path] + # We can only test systems in KVM that have a BSP + if not any('bsp' in si['morph'] for si in system_morph['strata']): + continue + + # We can only test systems in KVM that we have a host for + if system_morph['arch'] not in deployment_hosts: + continue + host_machine = deployment_hosts[system_morph['arch']] + deployment = Deployment(cluster_path, deployment_name, + deployment_config, host_machine, + net_id) + + instance = deployment.deploy() + try: + self._run_tests(instance, system_path, system_morph, + build_test_config, morph_helper, systems) + finally: + instance.delete() + + def process_args(self, args): + """Process the command line args and kick off the builds/tests""" + if self.settings['build-ref-prefix'] is None: + self.settings['build-ref-prefix'] = ( + os.path.join(self.settings['trove-id'], 'builds')) + for setting in ('deployment-host', 'trove-host', + 'trove-id', 'build-ref-prefix', 'net-id'): + self.settings.require(setting) + + deployment_hosts = {} + for host_config in self.settings['deployment-host']: + arch, address = host_config.split(':', 1) + user, address = address.split('@', 1) + address, disk_path = address.split(':', 1) + if user == '': + user = 'root' + # TODO: Don't assume root is the user with deploy access + deployment_hosts[arch] = VMHost(user, address, disk_path) + + build_test_config = (self.settings['trove-host'], + self.settings['trove-id'], + self.settings['build-ref-prefix']) + + if len(args) != 1: + raise cliapp.AppException('Usage: release-test CLUSTER') + cluster_path = morphlib.util.sanitise_morphology_path(args[0]) + self.deploy_and_test_systems(cluster_path, deployment_hosts, + build_test_config, + self.settings['net-id']) + + +if __name__ == '__main__': + ReleaseApp().run() |