summaryrefslogtreecommitdiff
path: root/scripts
diff options
context:
space:
mode:
authorMichael Drake <michael.drake@codethink.co.uk>2014-09-11 14:50:52 (GMT)
committerAdam Coldrick <adam.coldrick@codethink.co.uk>2014-10-27 09:08:52 (GMT)
commit1e0771efd2313d6a07f0f088e0bf4660154054fd (patch)
tree108061e04e71e41a8dba94fe676edb1709f6d4a2 /scripts
parent6b9c0c1ba8b23741349c535adf426a9e8a41d3c1 (diff)
downloaddefinitions-1e0771efd2313d6a07f0f088e0bf4660154054fd.tar.gz
Mason: Provide moving parts needed for testing on OpenStack
Testing currently involves building a Baserock devel system, deploying it as a VM and testing that that deployed VM can build Baserock successfully. Originally Mason used scripts/release-test to do testing on a kvm host. In this patch the bits needed to do testing on an OpenStack host are provided. The new scripts/release-test-os script is based on the old scripts/release-test, and it uses `nova` to boot/delete/etc images and instances on OpenStack. The mason script, `mason.sh` is updated to optionally run either scripts/release-test-os or scripts/release-test, depending on whether TEST_INFRASTRUCTURE_TYPE is set to 'openstack' or 'kvmhost' in the Mason's mason.conf. The `os.conf` file is sourced by `mason.sh`, and should be updated to contain the relevant credentials and details for the OpenStack tenancy to be used for test deployments. When Mason creates a test OpenStack instance, there is potential for a race condition depending on whether ssh comes up before the cloud-init has finished resizing the instance's disc. If morph running on the test instance tries to build before the disc size is increased, it will fail complaining of insufficient free space. To eliminate this race, the cloud init script `os-init-script` is passed to `nova boot`. This touches a file after the disc is resized, which Mason checks for before it runs a `morph build`. The `os.conf` and `os-init-script` files must both be placed in the Mason system's `/root/` directory before the system is deployed. This should happen in the `mason.configure` configuration extension. The `mason.configure` configuration extension should also be updated to handle adding two extra variables to the `mason.conf` file. These are the aforementioned TEST_INFRASTRUCTURE_TYPE and OPENSTACK_NETWORK_ID, which is the ID for the configured OpenStack network that test instances should use.
Diffstat (limited to 'scripts')
-rwxr-xr-xscripts/release-test-os524
1 files changed, 524 insertions, 0 deletions
diff --git a/scripts/release-test-os b/scripts/release-test-os
new file mode 100755
index 0000000..ca4bd48
--- /dev/null
+++ b/scripts/release-test-os
@@ -0,0 +1,524 @@
+#!/usr/bin/env python
+#
+# Copyright 2014 Codethink Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+'''release-test
+
+This script deploys the set of systems in the cluster morphology it is
+instructed to read, to test that they work correctly.
+
+'''
+
+import cliapp
+import os
+import pipes
+import shlex
+import shutil
+import socket
+import tempfile
+import time
+import uuid
+
+import morphlib
+
+
+class NovaList:
+ def __init__(self):
+ self.output = []
+ self.lines = []
+ self.instance = []
+
+ def update(self):
+ self.output = cliapp.runcmd(['nova', 'list'])
+ self.lines = self.output.split('\n')
+ self.lines = self.lines[3:-2]
+
+ def get_nova_details_for_instance(self, name):
+ self.update()
+
+ for line in self.lines:
+ entries = line.split('|')
+ stripped_line = [entry.strip() for entry in entries]
+ if stripped_line.count(name) == 1:
+ self.instance = stripped_line
+
+ def get_nova_state_for_instance(self, name):
+ self.get_nova_details_for_instance(name)
+ if not self.instance:
+ return
+ return self.instance[3]
+
+ def get_nova_ip_for_instance(self, name):
+ self.get_nova_details_for_instance(name)
+ if not self.instance:
+ return
+
+ if self.get_nova_state_for_instance(name) != 'ACTIVE':
+ return
+
+ return self.instance[6]
+
+ def get_nova_ip_for_instance_timeout(self, name, timeout=120):
+ start_time = time.time()
+
+ while self.get_nova_state_for_instance(name) != 'ACTIVE':
+
+ if time.time() > start_time + timeout:
+ print "%s not ACTIVE after %i seconds" % (name, timeout)
+ return
+
+ time.sleep(1)
+
+ ip_addr = self.get_nova_ip_for_instance(name)
+ if not ip_addr:
+ return
+
+ if ip_addr.count('=') == 0:
+ return
+
+ ip_addr = ip_addr[ip_addr.find('=') + 1:]
+
+ if ip_addr.count(',') == 0:
+ return ip_addr
+
+ return ip_addr[:ip_addr.find(',')]
+
+
+
+class MorphologyHelper(object):
+
+ def __init__(self):
+ self.sb = sb = morphlib.sysbranchdir.open_from_within('.')
+ defs_repo_path = sb.get_git_directory_name(sb.root_repository_url)
+ self.defs_repo = morphlib.gitdir.GitDirectory(defs_repo_path)
+ self.loader = morphlib.morphloader.MorphologyLoader()
+ self.finder = morphlib.morphologyfinder.MorphologyFinder(self.defs_repo)
+
+ def load_morphology(self, path):
+ text = self.finder.read_morphology(path)
+ return self.loader.load_from_string(text)
+
+ @classmethod
+ def iterate_systems(cls, systems_list):
+ for system in systems_list:
+ yield morphlib.util.sanitise_morphology_path(system['morph'])
+ if 'subsystems' in system:
+ for subsystem in cls.iterate_systems(system['subsystems']):
+ yield subsystem
+
+ def iterate_cluster_deployments(cls, cluster_morph):
+ for system in cluster_morph['systems']:
+ path = morphlib.util.sanitise_morphology_path(system['morph'])
+ defaults = system.get('deploy-defaults', {})
+ for name, options in system['deploy'].iteritems():
+ config = dict(defaults)
+ config.update(options)
+ yield path, name, config
+
+ def load_cluster_systems(self, cluster_morph):
+ for system_path in set(self.iterate_systems(cluster_morph['systems'])):
+ system_morph = self.load_morphology(system_path)
+ yield system_path, system_morph
+
+
+class TimeoutError(cliapp.AppException):
+
+ """Error to be raised when a connection waits too long"""
+
+ def __init__(self, msg):
+ super(TimeoutError, self).__init__(msg)
+
+
+class VMHost(object):
+
+ def __init__(self, user, address, disk_path):
+ self.user = user
+ self.address = address
+ self.disk_path = disk_path
+
+ @property
+ def ssh_host(self):
+ return '{user}@{address}'.format(user=self.user, address=self.address)
+
+ def runcmd(self, *args, **kwargs):
+ cliapp.ssh_runcmd(self.ssh_host, *args, **kwargs)
+
+
+class DeployedSystemInstance(object):
+
+ def __init__(self, deployment, config, host_machine, vm_id, rootfs_path,
+ ip_addr, hostname):
+ self.deployment = deployment
+ self.config = config
+ self.ip_address = ip_addr
+ self.host_machine = host_machine
+ self.vm_id = vm_id
+ self.rootfs_path = rootfs_path
+ self.hostname = hostname
+
+ @property
+ def ssh_host(self):
+ # TODO: Stop assuming we ssh into test instances as root
+ return 'root@{host}'.format(host=self.ip_address)
+
+ def runcmd(self, argv, chdir='.', **kwargs):
+ ssh_cmd = ['ssh', '-o', 'StrictHostKeyChecking=no',
+ '-o', 'UserKnownHostsFile=/dev/null', self.ssh_host]
+ cmd = ['sh', '-c', 'cd "$1" && shift && exec "$@"', '-', chdir]
+ cmd += argv
+ ssh_cmd.append(' '.join(map(pipes.quote, cmd)))
+ return cliapp.runcmd(ssh_cmd, **kwargs)
+
+ def _wait_for_dhcp(self, timeout):
+ '''Block until given hostname resolves successfully.
+
+ Raises TimeoutError if the hostname has not appeared in 'timeout'
+ seconds.
+
+ '''
+ start_time = time.time()
+ while True:
+ try:
+ socket.gethostbyname(self.ip_address)
+ return
+ except socket.gaierror:
+ pass
+ if time.time() > start_time + timeout:
+ raise TimeoutError("Host %s did not appear after %i seconds" %
+ (self.ip_address, timeout))
+ time.sleep(0.5)
+
+ def _wait_for_ssh(self, timeout):
+ """Wait until the deployed VM is responding via SSH"""
+ start_time = time.time()
+ while True:
+ try:
+ self.runcmd(['true'], stdin=None, stdout=None, stderr=None)
+ return
+ except cliapp.AppException:
+ # TODO: Stop assuming the ssh part of the command is what failed
+ if time.time() > start_time + timeout:
+ raise TimeoutError("%s sshd did not start after %i seconds"
+ % (self.ip_address, timeout))
+ time.sleep(0.5)
+
+ def _wait_for_cloud_init(self, timeout):
+ """Wait until cloud init has resized the disc"""
+ start_time = time.time()
+ while True:
+ try:
+ out = self.runcmd(['sh', '-c',
+ 'test -e "$1" && echo exists || echo does not exist',
+ '-',
+ '/root/cloud-init-finished'])
+ except:
+ import traceback
+ traceback.print_exc()
+ raise
+ if out.strip() == 'exists':
+ return
+ if time.time() > start_time + timeout:
+ raise TimeoutError("Disc size not increased after %i seconds"
+ % (timeout))
+ time.sleep(3)
+
+ def wait_until_online(self, timeout=120):
+ self._wait_for_dhcp(timeout)
+ self._wait_for_ssh(timeout)
+ self._wait_for_cloud_init(timeout)
+ print "Test system %s ready to run tests." % (self.hostname)
+
+ def delete(self):
+ # Stop and remove VM
+ print "Deleting %s test instance" % (self.hostname)
+ try:
+ cliapp.runcmd(['nova', 'delete', self.hostname])
+ except cliapp.AppException as e:
+ # TODO: Stop assuming that delete failed because the instance
+ # wasn't running
+ print "- Failed"
+ pass
+ print "Deleting %s test disc image" % (self.hostname)
+ try:
+ cliapp.runcmd(['nova', 'image-delete', self.hostname])
+ except cliapp.AppException as e:
+ # TODO: Stop assuming that image-delete failed because it was
+ # already removed
+ print "- Failed"
+ pass
+
+
+class Deployment(object):
+
+ def __init__(self, cluster_path, name, deployment_config,
+ host_machine, net_id):
+ self.cluster_path = cluster_path
+ self.name = name
+ self.deployment_config = deployment_config
+ self.host_machine = host_machine
+ self.net_id = net_id
+
+ @staticmethod
+ def _ssh_host_key_exists(hostname):
+ """Check if an ssh host key exists in known_hosts"""
+ with open('/root/.ssh/known_hosts', 'r') as known_hosts:
+ return any(line.startswith(hostname) for line in known_hosts)
+
+ def _update_known_hosts(self):
+ if not self._ssh_host_key_exists(self.host_machine.address):
+ with open('/root/.ssh/known_hosts', 'a') as known_hosts:
+ cliapp.runcmd(['ssh-keyscan', self.host_machine.address],
+ stdout=known_hosts)
+
+ @staticmethod
+ def _generate_sshkey_config(tempdir, config):
+ manifest = os.path.join(tempdir, 'manifest')
+ with open(manifest, 'w') as f:
+ f.write('0040700 0 0 /root/.ssh\n')
+ f.write('overwrite 0100600 0 0 /root/.ssh/authorized_keys\n')
+ authkeys = os.path.join(tempdir, 'root', '.ssh', 'authorized_keys')
+ os.makedirs(os.path.dirname(authkeys))
+ with open(authkeys, 'w') as auth_f:
+ with open('/root/.ssh/id_rsa.pub', 'r') as key_f:
+ shutil.copyfileobj(key_f, auth_f)
+
+ install_files = shlex.split(config.get('INSTALL_FILES', ''))
+ install_files.append(manifest)
+ yield 'INSTALL_FILES', ' '.join(pipes.quote(f) for f in install_files)
+
+ def deploy(self):
+ self._update_known_hosts()
+
+ hostname = str(uuid.uuid4())
+ vm_id = hostname
+ image_base = self.host_machine.disk_path
+ rootpath = '{image_base}/{hostname}.img'.format(image_base=image_base,
+ hostname=hostname)
+ loc = 'http://{ssh_host}:5000/v2.0'.format(
+ ssh_host=self.host_machine.ssh_host, id=vm_id, path=rootpath)
+
+ options = {
+ 'type': 'openstack',
+ 'location': loc,
+ 'HOSTNAME': hostname,
+ 'DISK_SIZE': '5G',
+ 'RAM_SIZE': '2G',
+ 'VERSION_LABEL': 'release-test',
+ 'OPENSTACK_USER': os.environ['OS_USERNAME'],
+ 'OPENSTACK_TENANT': os.environ['OS_TENANT_NAME'],
+ 'OPENSTACK_PASSWORD': os.environ['OS_PASSWORD'],
+ 'OPENSTACK_IMAGENAME': hostname,
+ 'CLOUD_INIT': 'yes',
+ 'KERNEL_ARGS': 'console=tty0 console=ttyS0',
+ }
+
+ tempdir = tempfile.mkdtemp()
+ try:
+ options.update(
+ self._generate_sshkey_config(tempdir,
+ self.deployment_config))
+
+ # Deploy the image to openstack
+ args = ['morph', 'deploy', self.cluster_path, self.name]
+ for k, v in options.iteritems():
+ args.append('%s.%s=%s' % (self.name, k, v))
+ cliapp.runcmd(args, stdin=None, stdout=None, stderr=None)
+
+ config = dict(self.deployment_config)
+ config.update(options)
+
+ # Boot an instance from the image
+ args = ['nova', 'boot',
+ '--flavor', 'm1.medium',
+ '--image', hostname,
+ '--user-data', '/root/os-init-script',
+ '--nic', "net-id=%s" % (self.net_id),
+ hostname]
+ output = cliapp.runcmd(args)
+
+ # Print nova boot output, with adminPass line removed
+ output_lines = output.split('\n')
+ for line in output_lines:
+ if line.find('adminPass') != -1:
+ password_line = line
+ output_lines.remove(password_line)
+ output = '\n'.join(output_lines)
+ print output
+
+ # Get ip address from nova list
+ nl = NovaList()
+ ip_addr = nl.get_nova_ip_for_instance_timeout(hostname)
+ print "IP address for inastance %s: %s" % (hostname, ip_addr)
+
+ return DeployedSystemInstance(self, config, self.host_machine,
+ vm_id, rootpath, ip_addr, hostname)
+ finally:
+ shutil.rmtree(tempdir)
+
+
+class ReleaseApp(cliapp.Application):
+
+ """Cliapp application which handles automatic builds and tests"""
+
+ def add_settings(self):
+ """Add the command line options needed"""
+ group_main = 'Program Options'
+ self.settings.string_list(['deployment-host'],
+ 'ARCH:HOST:PATH that VMs can be deployed to',
+ default=None,
+ group=group_main)
+ self.settings.string(['trove-host'],
+ 'Address of Trove for test systems to build from',
+ default=None,
+ group=group_main)
+ self.settings.string(['trove-id'],
+ 'ID of Trove for test systems to build from',
+ default=None,
+ group=group_main)
+ self.settings.string(['build-ref-prefix'],
+ 'Prefix of build branches for test systems',
+ default=None,
+ group=group_main)
+ self.settings.string(['net-id'],
+ 'Openstack network ID',
+ default=None,
+ group=group_main)
+
+ @staticmethod
+ def _run_tests(instance, system_path, system_morph,
+ (trove_host, trove_id, build_ref_prefix),
+ morph_helper, systems):
+ instance.wait_until_online()
+
+ tests = []
+ def baserock_build_test(instance):
+ instance.runcmd(['git', 'config', '--global', 'user.name',
+ 'Test Instance of %s' % instance.deployment.name])
+ instance.runcmd(['git', 'config', '--global', 'user.email',
+ 'ci-test@%s' % instance.config['HOSTNAME']])
+ instance.runcmd(['mkdir', '-p', '/src/ws', '/src/cache',
+ '/src/tmp'])
+ def morph_cmd(*args, **kwargs):
+ # TODO: decide whether to use cached artifacts or not by
+ # adding --artifact-cache-server= --cache-server=
+ argv = ['morph', '--log=/src/morph.log', '--cachedir=/src/cache',
+ '--tempdir=/src/tmp', '--log-max=100M',
+ '--trove-host', trove_host, '--trove-id', trove_id,
+ '--build-ref-prefix', build_ref_prefix]
+ argv.extend(args)
+ instance.runcmd(argv, **kwargs)
+
+ repo = morph_helper.sb.root_repository_url
+ ref = morph_helper.defs_repo.HEAD
+ sha1 = morph_helper.defs_repo.resolve_ref_to_commit(ref)
+ morph_cmd('init', '/src/ws')
+ chdir = '/src/ws'
+
+ morph_cmd('checkout', repo, ref, chdir=chdir)
+ # TODO: Add a morph subcommand that gives the path to the root repository.
+ repo_path = os.path.relpath(
+ morph_helper.sb.get_git_directory_name(repo),
+ morph_helper.sb.root_directory)
+ chdir = os.path.join(chdir, ref, repo_path)
+
+ instance.runcmd(['git', 'reset', '--hard', sha1], chdir=chdir)
+ print 'Building test systems for {sys}'.format(sys=system_path)
+ for to_build_path, to_build_morph in systems.iteritems():
+ if to_build_morph['arch'] == system_morph['arch']:
+ print 'Test building {path}'.format(path=to_build_path)
+ morph_cmd('build', to_build_path, chdir=chdir,
+ stdin=None, stdout=None, stderr=None)
+ print 'Finished Building test systems'
+
+ def python_smoke_test(instance):
+ instance.runcmd(['python', '-c', 'print "Hello World"'])
+
+ # TODO: Come up with a better way of determining which tests to run
+ if 'devel' in system_path:
+ tests.append(baserock_build_test)
+ else:
+ tests.append(python_smoke_test)
+
+ for test in tests:
+ test(instance)
+
+ def deploy_and_test_systems(self, cluster_path,
+ deployment_hosts, build_test_config,
+ net_id):
+ """Run the deployments and tests"""
+
+ version = 'release-test'
+
+ morph_helper = MorphologyHelper()
+ cluster_morph = morph_helper.load_morphology(cluster_path)
+ systems = dict(morph_helper.load_cluster_systems(cluster_morph))
+
+ for system_path, deployment_name, deployment_config in \
+ morph_helper.iterate_cluster_deployments(cluster_morph):
+
+ system_morph = systems[system_path]
+ # We can only test systems in KVM that have a BSP
+ if not any('bsp' in si['morph'] for si in system_morph['strata']):
+ continue
+
+ # We can only test systems in KVM that we have a host for
+ if system_morph['arch'] not in deployment_hosts:
+ continue
+ host_machine = deployment_hosts[system_morph['arch']]
+ deployment = Deployment(cluster_path, deployment_name,
+ deployment_config, host_machine,
+ net_id)
+
+ instance = deployment.deploy()
+ try:
+ self._run_tests(instance, system_path, system_morph,
+ build_test_config, morph_helper, systems)
+ finally:
+ instance.delete()
+
+ def process_args(self, args):
+ """Process the command line args and kick off the builds/tests"""
+ if self.settings['build-ref-prefix'] is None:
+ self.settings['build-ref-prefix'] = (
+ os.path.join(self.settings['trove-id'], 'builds'))
+ for setting in ('deployment-host', 'trove-host',
+ 'trove-id', 'build-ref-prefix', 'net-id'):
+ self.settings.require(setting)
+
+ deployment_hosts = {}
+ for host_config in self.settings['deployment-host']:
+ arch, address = host_config.split(':', 1)
+ user, address = address.split('@', 1)
+ address, disk_path = address.split(':', 1)
+ if user == '':
+ user = 'root'
+ # TODO: Don't assume root is the user with deploy access
+ deployment_hosts[arch] = VMHost(user, address, disk_path)
+
+ build_test_config = (self.settings['trove-host'],
+ self.settings['trove-id'],
+ self.settings['build-ref-prefix'])
+
+ if len(args) != 1:
+ raise cliapp.AppException('Usage: release-test CLUSTER')
+ cluster_path = morphlib.util.sanitise_morphology_path(args[0])
+ self.deploy_and_test_systems(cluster_path, deployment_hosts,
+ build_test_config,
+ self.settings['net-id'])
+
+
+if __name__ == '__main__':
+ ReleaseApp().run()