summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorMichael Drake <michael.drake@codethink.co.uk>2014-09-11 15:50:52 +0100
committerMichael Drake <michael.drake@codethink.co.uk>2014-09-11 15:50:52 +0100
commitbdd6a8d9cd36b215b85cee507bc8919a689e0fe5 (patch)
treeaa37f85086d99610355d2657df463f6ccd75949c
parent6523c60a6e918d982ac378beef48e0d3dfa0603e (diff)
downloaddefinitions-baserock/michaeldrake/mason-openstack.tar.gz
Mason: Provide moving parts needed for testing on OpenStackbaserock/michaeldrake/mason-openstack
Testing currently involves building a Baserock devel system, deploying it as a VM and testing that that deployed VM can build Baserock successfully. Originally Mason used scripts/release-test to do testing on a kvm host. In this patch the bits needed to do testing on an OpenStack host are provided. The new scripts/release-test-os script is based on the old scripts/release-test, and it uses `nova` to boot/delete/etc images and instances on OpenStack. The mason script, `mason.sh` is updated to optionally run either scripts/release-test-os or scripts/release-test, depending on whether TEST_INFRA is set to 'openstack' or 'kvmhost' in the Mason's mason.conf. The `os.conf` file is sourced by `mason.sh`, and should be updated to contain the relevant credentials and details for the OpenStack tenancy to be used for test deployments. When Mason creates a test OpenStack instance, there is potential for a race condition depending on whether ssh comes up before the cloud-init has finished resizing the instance's disc. If morph running on the test instance tries to build before the disc size is increased, it will fail complaining of insufficient free space. To eliminate this race, the cloud init script `os-init-script` is passed to `nova boot`. This touches a file after the disc is resized, which Mason checks for before it runs a `morph build`. The `os.conf` and `os-init-script` files must both be placed in the Mason system's `/root/` directory before the system is deployed. This should happen in the `mason.configure` configuration extension. The `mason.configure` configuration extension should also be updated to handle adding two extra variables to the `mason.conf` file. These are the aforementioned TEST_INFRA and OPENSTACK_NET_ID, which is the ID for the configured OpenStack network that test instances should use.
-rwxr-xr-xmason/mason.sh22
-rw-r--r--mason/os-init-script6
-rw-r--r--mason/os.conf23
-rwxr-xr-xscripts/release-test-os524
4 files changed, 570 insertions, 5 deletions
diff --git a/mason/mason.sh b/mason/mason.sh
index 13ea21db..cf1341e3 100755
--- a/mason/mason.sh
+++ b/mason/mason.sh
@@ -1,5 +1,8 @@
#!/bin/sh
+# Load OpenStack credentials
+. /root/os.conf
+
set -e
set -x
@@ -59,11 +62,20 @@ else
echo INFO: Created "$releases_made" release images
fi
-"scripts/release-test" \
- --deployment-host "$DISTBUILD_ARCH":"$TEST_VM_HOST_SSH_URL" \
- --trove-host "$UPSTREAM_TROVE_ADDRESS" \
- --trove-id "$TROVE_ID" \
- "$BUILD_CLUSTER_MORPHOLOGY"
+if [ "$TEST_INFRA" = "openstack" ]; then
+ "/root/release-test-os" \
+ --deployment-host "$DISTBUILD_ARCH":"$TEST_VM_HOST_SSH_URL" \
+ --trove-host "$UPSTREAM_TROVE_ADDRESS" \
+ --trove-id "$TROVE_ID" \
+ --net-id "$OPENSTACK_NET_ID" \
+ "$BUILD_CLUSTER_MORPHOLOGY"
+elif [ "$TEST_INFRA" = "kvmhost" ]; then
+ "scripts/release-test" \
+ --deployment-host "$DISTBUILD_ARCH":"$TEST_VM_HOST_SSH_URL" \
+ --trove-host "$UPSTREAM_TROVE_ADDRESS" \
+ --trove-id "$TROVE_ID" \
+ "$BUILD_CLUSTER_MORPHOLOGY"
+fi
"scripts/release-upload" --build-trove-host "$ARTIFACT_CACHE_SERVER" \
--arch "$DISTBUILD_ARCH" \
diff --git a/mason/os-init-script b/mason/os-init-script
new file mode 100644
index 00000000..77afb926
--- /dev/null
+++ b/mason/os-init-script
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# This allows the test runner to know that cloud-init has completed the
+# disc resizing, and there is enough free space to continue.
+touch /root/cloud-init-finished
+
diff --git a/mason/os.conf b/mason/os.conf
new file mode 100644
index 00000000..d9f69fda
--- /dev/null
+++ b/mason/os.conf
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# With the addition of Keystone, to use an openstack cloud you should
+# authenticate against keystone, which returns a **Token** and **Service
+# Catalog**. The catalog contains the endpoint for all services the
+# user/tenant has access to - including nova, glance, keystone, swift.
+#
+# *NOTE*: Using the 2.0 *auth api* does not mean that compute api is 2.0. We
+# will use the 1.1 *compute api*
+export OS_AUTH_URL=<os-auth-url>
+
+# With the addition of Keystone we have standardized on the term **tenant**
+# as the entity that owns the resources.
+export OS_TENANT_ID=<os-tenant-id>
+export OS_TENANT_NAME="<os-tenant>"
+
+# In addition to the owning entity (tenant), openstack stores the entity
+# performing the action as the **user**.
+export OS_USERNAME="<os-user>"
+
+# With Keystone you pass the keystone password.
+export OS_PASSWORD="<os-pass>"
+
diff --git a/scripts/release-test-os b/scripts/release-test-os
new file mode 100755
index 00000000..ca4bd485
--- /dev/null
+++ b/scripts/release-test-os
@@ -0,0 +1,524 @@
+#!/usr/bin/env python
+#
+# Copyright 2014 Codethink Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+'''release-test
+
+This script deploys the set of systems in the cluster morphology it is
+instructed to read, to test that they work correctly.
+
+'''
+
+import cliapp
+import os
+import pipes
+import shlex
+import shutil
+import socket
+import tempfile
+import time
+import uuid
+
+import morphlib
+
+
+class NovaList:
+ def __init__(self):
+ self.output = []
+ self.lines = []
+ self.instance = []
+
+ def update(self):
+ self.output = cliapp.runcmd(['nova', 'list'])
+ self.lines = self.output.split('\n')
+ self.lines = self.lines[3:-2]
+
+ def get_nova_details_for_instance(self, name):
+ self.update()
+
+ for line in self.lines:
+ entries = line.split('|')
+ stripped_line = [entry.strip() for entry in entries]
+ if stripped_line.count(name) == 1:
+ self.instance = stripped_line
+
+ def get_nova_state_for_instance(self, name):
+ self.get_nova_details_for_instance(name)
+ if not self.instance:
+ return
+ return self.instance[3]
+
+ def get_nova_ip_for_instance(self, name):
+ self.get_nova_details_for_instance(name)
+ if not self.instance:
+ return
+
+ if self.get_nova_state_for_instance(name) != 'ACTIVE':
+ return
+
+ return self.instance[6]
+
+ def get_nova_ip_for_instance_timeout(self, name, timeout=120):
+ start_time = time.time()
+
+ while self.get_nova_state_for_instance(name) != 'ACTIVE':
+
+ if time.time() > start_time + timeout:
+ print "%s not ACTIVE after %i seconds" % (name, timeout)
+ return
+
+ time.sleep(1)
+
+ ip_addr = self.get_nova_ip_for_instance(name)
+ if not ip_addr:
+ return
+
+ if ip_addr.count('=') == 0:
+ return
+
+ ip_addr = ip_addr[ip_addr.find('=') + 1:]
+
+ if ip_addr.count(',') == 0:
+ return ip_addr
+
+ return ip_addr[:ip_addr.find(',')]
+
+
+
+class MorphologyHelper(object):
+
+ def __init__(self):
+ self.sb = sb = morphlib.sysbranchdir.open_from_within('.')
+ defs_repo_path = sb.get_git_directory_name(sb.root_repository_url)
+ self.defs_repo = morphlib.gitdir.GitDirectory(defs_repo_path)
+ self.loader = morphlib.morphloader.MorphologyLoader()
+ self.finder = morphlib.morphologyfinder.MorphologyFinder(self.defs_repo)
+
+ def load_morphology(self, path):
+ text = self.finder.read_morphology(path)
+ return self.loader.load_from_string(text)
+
+ @classmethod
+ def iterate_systems(cls, systems_list):
+ for system in systems_list:
+ yield morphlib.util.sanitise_morphology_path(system['morph'])
+ if 'subsystems' in system:
+ for subsystem in cls.iterate_systems(system['subsystems']):
+ yield subsystem
+
+ def iterate_cluster_deployments(cls, cluster_morph):
+ for system in cluster_morph['systems']:
+ path = morphlib.util.sanitise_morphology_path(system['morph'])
+ defaults = system.get('deploy-defaults', {})
+ for name, options in system['deploy'].iteritems():
+ config = dict(defaults)
+ config.update(options)
+ yield path, name, config
+
+ def load_cluster_systems(self, cluster_morph):
+ for system_path in set(self.iterate_systems(cluster_morph['systems'])):
+ system_morph = self.load_morphology(system_path)
+ yield system_path, system_morph
+
+
+class TimeoutError(cliapp.AppException):
+
+ """Error to be raised when a connection waits too long"""
+
+ def __init__(self, msg):
+ super(TimeoutError, self).__init__(msg)
+
+
+class VMHost(object):
+
+ def __init__(self, user, address, disk_path):
+ self.user = user
+ self.address = address
+ self.disk_path = disk_path
+
+ @property
+ def ssh_host(self):
+ return '{user}@{address}'.format(user=self.user, address=self.address)
+
+ def runcmd(self, *args, **kwargs):
+ cliapp.ssh_runcmd(self.ssh_host, *args, **kwargs)
+
+
+class DeployedSystemInstance(object):
+
+ def __init__(self, deployment, config, host_machine, vm_id, rootfs_path,
+ ip_addr, hostname):
+ self.deployment = deployment
+ self.config = config
+ self.ip_address = ip_addr
+ self.host_machine = host_machine
+ self.vm_id = vm_id
+ self.rootfs_path = rootfs_path
+ self.hostname = hostname
+
+ @property
+ def ssh_host(self):
+ # TODO: Stop assuming we ssh into test instances as root
+ return 'root@{host}'.format(host=self.ip_address)
+
+ def runcmd(self, argv, chdir='.', **kwargs):
+ ssh_cmd = ['ssh', '-o', 'StrictHostKeyChecking=no',
+ '-o', 'UserKnownHostsFile=/dev/null', self.ssh_host]
+ cmd = ['sh', '-c', 'cd "$1" && shift && exec "$@"', '-', chdir]
+ cmd += argv
+ ssh_cmd.append(' '.join(map(pipes.quote, cmd)))
+ return cliapp.runcmd(ssh_cmd, **kwargs)
+
+ def _wait_for_dhcp(self, timeout):
+ '''Block until given hostname resolves successfully.
+
+ Raises TimeoutError if the hostname has not appeared in 'timeout'
+ seconds.
+
+ '''
+ start_time = time.time()
+ while True:
+ try:
+ socket.gethostbyname(self.ip_address)
+ return
+ except socket.gaierror:
+ pass
+ if time.time() > start_time + timeout:
+ raise TimeoutError("Host %s did not appear after %i seconds" %
+ (self.ip_address, timeout))
+ time.sleep(0.5)
+
+ def _wait_for_ssh(self, timeout):
+ """Wait until the deployed VM is responding via SSH"""
+ start_time = time.time()
+ while True:
+ try:
+ self.runcmd(['true'], stdin=None, stdout=None, stderr=None)
+ return
+ except cliapp.AppException:
+ # TODO: Stop assuming the ssh part of the command is what failed
+ if time.time() > start_time + timeout:
+ raise TimeoutError("%s sshd did not start after %i seconds"
+ % (self.ip_address, timeout))
+ time.sleep(0.5)
+
+ def _wait_for_cloud_init(self, timeout):
+ """Wait until cloud init has resized the disc"""
+ start_time = time.time()
+ while True:
+ try:
+ out = self.runcmd(['sh', '-c',
+ 'test -e "$1" && echo exists || echo does not exist',
+ '-',
+ '/root/cloud-init-finished'])
+ except:
+ import traceback
+ traceback.print_exc()
+ raise
+ if out.strip() == 'exists':
+ return
+ if time.time() > start_time + timeout:
+ raise TimeoutError("Disc size not increased after %i seconds"
+ % (timeout))
+ time.sleep(3)
+
+ def wait_until_online(self, timeout=120):
+ self._wait_for_dhcp(timeout)
+ self._wait_for_ssh(timeout)
+ self._wait_for_cloud_init(timeout)
+ print "Test system %s ready to run tests." % (self.hostname)
+
+ def delete(self):
+ # Stop and remove VM
+ print "Deleting %s test instance" % (self.hostname)
+ try:
+ cliapp.runcmd(['nova', 'delete', self.hostname])
+ except cliapp.AppException as e:
+ # TODO: Stop assuming that delete failed because the instance
+ # wasn't running
+ print "- Failed"
+ pass
+ print "Deleting %s test disc image" % (self.hostname)
+ try:
+ cliapp.runcmd(['nova', 'image-delete', self.hostname])
+ except cliapp.AppException as e:
+ # TODO: Stop assuming that image-delete failed because it was
+ # already removed
+ print "- Failed"
+ pass
+
+
+class Deployment(object):
+
+ def __init__(self, cluster_path, name, deployment_config,
+ host_machine, net_id):
+ self.cluster_path = cluster_path
+ self.name = name
+ self.deployment_config = deployment_config
+ self.host_machine = host_machine
+ self.net_id = net_id
+
+ @staticmethod
+ def _ssh_host_key_exists(hostname):
+ """Check if an ssh host key exists in known_hosts"""
+ with open('/root/.ssh/known_hosts', 'r') as known_hosts:
+ return any(line.startswith(hostname) for line in known_hosts)
+
+ def _update_known_hosts(self):
+ if not self._ssh_host_key_exists(self.host_machine.address):
+ with open('/root/.ssh/known_hosts', 'a') as known_hosts:
+ cliapp.runcmd(['ssh-keyscan', self.host_machine.address],
+ stdout=known_hosts)
+
+ @staticmethod
+ def _generate_sshkey_config(tempdir, config):
+ manifest = os.path.join(tempdir, 'manifest')
+ with open(manifest, 'w') as f:
+ f.write('0040700 0 0 /root/.ssh\n')
+ f.write('overwrite 0100600 0 0 /root/.ssh/authorized_keys\n')
+ authkeys = os.path.join(tempdir, 'root', '.ssh', 'authorized_keys')
+ os.makedirs(os.path.dirname(authkeys))
+ with open(authkeys, 'w') as auth_f:
+ with open('/root/.ssh/id_rsa.pub', 'r') as key_f:
+ shutil.copyfileobj(key_f, auth_f)
+
+ install_files = shlex.split(config.get('INSTALL_FILES', ''))
+ install_files.append(manifest)
+ yield 'INSTALL_FILES', ' '.join(pipes.quote(f) for f in install_files)
+
+ def deploy(self):
+ self._update_known_hosts()
+
+ hostname = str(uuid.uuid4())
+ vm_id = hostname
+ image_base = self.host_machine.disk_path
+ rootpath = '{image_base}/{hostname}.img'.format(image_base=image_base,
+ hostname=hostname)
+ loc = 'http://{ssh_host}:5000/v2.0'.format(
+ ssh_host=self.host_machine.ssh_host, id=vm_id, path=rootpath)
+
+ options = {
+ 'type': 'openstack',
+ 'location': loc,
+ 'HOSTNAME': hostname,
+ 'DISK_SIZE': '5G',
+ 'RAM_SIZE': '2G',
+ 'VERSION_LABEL': 'release-test',
+ 'OPENSTACK_USER': os.environ['OS_USERNAME'],
+ 'OPENSTACK_TENANT': os.environ['OS_TENANT_NAME'],
+ 'OPENSTACK_PASSWORD': os.environ['OS_PASSWORD'],
+ 'OPENSTACK_IMAGENAME': hostname,
+ 'CLOUD_INIT': 'yes',
+ 'KERNEL_ARGS': 'console=tty0 console=ttyS0',
+ }
+
+ tempdir = tempfile.mkdtemp()
+ try:
+ options.update(
+ self._generate_sshkey_config(tempdir,
+ self.deployment_config))
+
+ # Deploy the image to openstack
+ args = ['morph', 'deploy', self.cluster_path, self.name]
+ for k, v in options.iteritems():
+ args.append('%s.%s=%s' % (self.name, k, v))
+ cliapp.runcmd(args, stdin=None, stdout=None, stderr=None)
+
+ config = dict(self.deployment_config)
+ config.update(options)
+
+ # Boot an instance from the image
+ args = ['nova', 'boot',
+ '--flavor', 'm1.medium',
+ '--image', hostname,
+ '--user-data', '/root/os-init-script',
+ '--nic', "net-id=%s" % (self.net_id),
+ hostname]
+ output = cliapp.runcmd(args)
+
+ # Print nova boot output, with adminPass line removed
+ output_lines = output.split('\n')
+ for line in output_lines:
+ if line.find('adminPass') != -1:
+ password_line = line
+ output_lines.remove(password_line)
+ output = '\n'.join(output_lines)
+ print output
+
+ # Get ip address from nova list
+ nl = NovaList()
+ ip_addr = nl.get_nova_ip_for_instance_timeout(hostname)
+ print "IP address for inastance %s: %s" % (hostname, ip_addr)
+
+ return DeployedSystemInstance(self, config, self.host_machine,
+ vm_id, rootpath, ip_addr, hostname)
+ finally:
+ shutil.rmtree(tempdir)
+
+
+class ReleaseApp(cliapp.Application):
+
+ """Cliapp application which handles automatic builds and tests"""
+
+ def add_settings(self):
+ """Add the command line options needed"""
+ group_main = 'Program Options'
+ self.settings.string_list(['deployment-host'],
+ 'ARCH:HOST:PATH that VMs can be deployed to',
+ default=None,
+ group=group_main)
+ self.settings.string(['trove-host'],
+ 'Address of Trove for test systems to build from',
+ default=None,
+ group=group_main)
+ self.settings.string(['trove-id'],
+ 'ID of Trove for test systems to build from',
+ default=None,
+ group=group_main)
+ self.settings.string(['build-ref-prefix'],
+ 'Prefix of build branches for test systems',
+ default=None,
+ group=group_main)
+ self.settings.string(['net-id'],
+ 'Openstack network ID',
+ default=None,
+ group=group_main)
+
+ @staticmethod
+ def _run_tests(instance, system_path, system_morph,
+ (trove_host, trove_id, build_ref_prefix),
+ morph_helper, systems):
+ instance.wait_until_online()
+
+ tests = []
+ def baserock_build_test(instance):
+ instance.runcmd(['git', 'config', '--global', 'user.name',
+ 'Test Instance of %s' % instance.deployment.name])
+ instance.runcmd(['git', 'config', '--global', 'user.email',
+ 'ci-test@%s' % instance.config['HOSTNAME']])
+ instance.runcmd(['mkdir', '-p', '/src/ws', '/src/cache',
+ '/src/tmp'])
+ def morph_cmd(*args, **kwargs):
+ # TODO: decide whether to use cached artifacts or not by
+ # adding --artifact-cache-server= --cache-server=
+ argv = ['morph', '--log=/src/morph.log', '--cachedir=/src/cache',
+ '--tempdir=/src/tmp', '--log-max=100M',
+ '--trove-host', trove_host, '--trove-id', trove_id,
+ '--build-ref-prefix', build_ref_prefix]
+ argv.extend(args)
+ instance.runcmd(argv, **kwargs)
+
+ repo = morph_helper.sb.root_repository_url
+ ref = morph_helper.defs_repo.HEAD
+ sha1 = morph_helper.defs_repo.resolve_ref_to_commit(ref)
+ morph_cmd('init', '/src/ws')
+ chdir = '/src/ws'
+
+ morph_cmd('checkout', repo, ref, chdir=chdir)
+ # TODO: Add a morph subcommand that gives the path to the root repository.
+ repo_path = os.path.relpath(
+ morph_helper.sb.get_git_directory_name(repo),
+ morph_helper.sb.root_directory)
+ chdir = os.path.join(chdir, ref, repo_path)
+
+ instance.runcmd(['git', 'reset', '--hard', sha1], chdir=chdir)
+ print 'Building test systems for {sys}'.format(sys=system_path)
+ for to_build_path, to_build_morph in systems.iteritems():
+ if to_build_morph['arch'] == system_morph['arch']:
+ print 'Test building {path}'.format(path=to_build_path)
+ morph_cmd('build', to_build_path, chdir=chdir,
+ stdin=None, stdout=None, stderr=None)
+ print 'Finished Building test systems'
+
+ def python_smoke_test(instance):
+ instance.runcmd(['python', '-c', 'print "Hello World"'])
+
+ # TODO: Come up with a better way of determining which tests to run
+ if 'devel' in system_path:
+ tests.append(baserock_build_test)
+ else:
+ tests.append(python_smoke_test)
+
+ for test in tests:
+ test(instance)
+
+ def deploy_and_test_systems(self, cluster_path,
+ deployment_hosts, build_test_config,
+ net_id):
+ """Run the deployments and tests"""
+
+ version = 'release-test'
+
+ morph_helper = MorphologyHelper()
+ cluster_morph = morph_helper.load_morphology(cluster_path)
+ systems = dict(morph_helper.load_cluster_systems(cluster_morph))
+
+ for system_path, deployment_name, deployment_config in \
+ morph_helper.iterate_cluster_deployments(cluster_morph):
+
+ system_morph = systems[system_path]
+ # We can only test systems in KVM that have a BSP
+ if not any('bsp' in si['morph'] for si in system_morph['strata']):
+ continue
+
+ # We can only test systems in KVM that we have a host for
+ if system_morph['arch'] not in deployment_hosts:
+ continue
+ host_machine = deployment_hosts[system_morph['arch']]
+ deployment = Deployment(cluster_path, deployment_name,
+ deployment_config, host_machine,
+ net_id)
+
+ instance = deployment.deploy()
+ try:
+ self._run_tests(instance, system_path, system_morph,
+ build_test_config, morph_helper, systems)
+ finally:
+ instance.delete()
+
+ def process_args(self, args):
+ """Process the command line args and kick off the builds/tests"""
+ if self.settings['build-ref-prefix'] is None:
+ self.settings['build-ref-prefix'] = (
+ os.path.join(self.settings['trove-id'], 'builds'))
+ for setting in ('deployment-host', 'trove-host',
+ 'trove-id', 'build-ref-prefix', 'net-id'):
+ self.settings.require(setting)
+
+ deployment_hosts = {}
+ for host_config in self.settings['deployment-host']:
+ arch, address = host_config.split(':', 1)
+ user, address = address.split('@', 1)
+ address, disk_path = address.split(':', 1)
+ if user == '':
+ user = 'root'
+ # TODO: Don't assume root is the user with deploy access
+ deployment_hosts[arch] = VMHost(user, address, disk_path)
+
+ build_test_config = (self.settings['trove-host'],
+ self.settings['trove-id'],
+ self.settings['build-ref-prefix'])
+
+ if len(args) != 1:
+ raise cliapp.AppException('Usage: release-test CLUSTER')
+ cluster_path = morphlib.util.sanitise_morphology_path(args[0])
+ self.deploy_and_test_systems(cluster_path, deployment_hosts,
+ build_test_config,
+ self.settings['net-id'])
+
+
+if __name__ == '__main__':
+ ReleaseApp().run()