summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xmason/mason.sh22
-rw-r--r--mason/os-init-script6
-rw-r--r--mason/os.conf23
-rwxr-xr-xscripts/release-test-os524
4 files changed, 570 insertions, 5 deletions
diff --git a/mason/mason.sh b/mason/mason.sh
index 13ea21d..cf1341e 100755
--- a/mason/mason.sh
+++ b/mason/mason.sh
@@ -1,5 +1,8 @@
#!/bin/sh
+# Load OpenStack credentials
+. /root/os.conf
+
set -e
set -x
@@ -59,11 +62,20 @@ else
echo INFO: Created "$releases_made" release images
fi
-"scripts/release-test" \
- --deployment-host "$DISTBUILD_ARCH":"$TEST_VM_HOST_SSH_URL" \
- --trove-host "$UPSTREAM_TROVE_ADDRESS" \
- --trove-id "$TROVE_ID" \
- "$BUILD_CLUSTER_MORPHOLOGY"
+if [ "$TEST_INFRA" = "openstack" ]; then
+ "/root/release-test-os" \
+ --deployment-host "$DISTBUILD_ARCH":"$TEST_VM_HOST_SSH_URL" \
+ --trove-host "$UPSTREAM_TROVE_ADDRESS" \
+ --trove-id "$TROVE_ID" \
+ --net-id "$OPENSTACK_NET_ID" \
+ "$BUILD_CLUSTER_MORPHOLOGY"
+elif [ "$TEST_INFRA" = "kvmhost" ]; then
+ "scripts/release-test" \
+ --deployment-host "$DISTBUILD_ARCH":"$TEST_VM_HOST_SSH_URL" \
+ --trove-host "$UPSTREAM_TROVE_ADDRESS" \
+ --trove-id "$TROVE_ID" \
+ "$BUILD_CLUSTER_MORPHOLOGY"
+fi
"scripts/release-upload" --build-trove-host "$ARTIFACT_CACHE_SERVER" \
--arch "$DISTBUILD_ARCH" \
diff --git a/mason/os-init-script b/mason/os-init-script
new file mode 100644
index 0000000..77afb92
--- /dev/null
+++ b/mason/os-init-script
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# This allows the test runner to know that cloud-init has completed the
+# disc resizing, and there is enough free space to continue.
+touch /root/cloud-init-finished
+
diff --git a/mason/os.conf b/mason/os.conf
new file mode 100644
index 0000000..d9f69fd
--- /dev/null
+++ b/mason/os.conf
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# With the addition of Keystone, to use an openstack cloud you should
+# authenticate against keystone, which returns a **Token** and **Service
+# Catalog**. The catalog contains the endpoint for all services the
+# user/tenant has access to - including nova, glance, keystone, swift.
+#
+# *NOTE*: Using the 2.0 *auth api* does not mean that compute api is 2.0. We
+# will use the 1.1 *compute api*
+export OS_AUTH_URL=<os-auth-url>
+
+# With the addition of Keystone we have standardized on the term **tenant**
+# as the entity that owns the resources.
+export OS_TENANT_ID=<os-tenant-id>
+export OS_TENANT_NAME="<os-tenant>"
+
+# In addition to the owning entity (tenant), openstack stores the entity
+# performing the action as the **user**.
+export OS_USERNAME="<os-user>"
+
+# With Keystone you pass the keystone password.
+export OS_PASSWORD="<os-pass>"
+
diff --git a/scripts/release-test-os b/scripts/release-test-os
new file mode 100755
index 0000000..ca4bd48
--- /dev/null
+++ b/scripts/release-test-os
@@ -0,0 +1,524 @@
+#!/usr/bin/env python
+#
+# Copyright 2014 Codethink Ltd
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+'''release-test
+
+This script deploys the set of systems in the cluster morphology it is
+instructed to read, to test that they work correctly.
+
+'''
+
+import cliapp
+import os
+import pipes
+import shlex
+import shutil
+import socket
+import tempfile
+import time
+import uuid
+
+import morphlib
+
+
+class NovaList:
+ def __init__(self):
+ self.output = []
+ self.lines = []
+ self.instance = []
+
+ def update(self):
+ self.output = cliapp.runcmd(['nova', 'list'])
+ self.lines = self.output.split('\n')
+ self.lines = self.lines[3:-2]
+
+ def get_nova_details_for_instance(self, name):
+ self.update()
+
+ for line in self.lines:
+ entries = line.split('|')
+ stripped_line = [entry.strip() for entry in entries]
+ if stripped_line.count(name) == 1:
+ self.instance = stripped_line
+
+ def get_nova_state_for_instance(self, name):
+ self.get_nova_details_for_instance(name)
+ if not self.instance:
+ return
+ return self.instance[3]
+
+ def get_nova_ip_for_instance(self, name):
+ self.get_nova_details_for_instance(name)
+ if not self.instance:
+ return
+
+ if self.get_nova_state_for_instance(name) != 'ACTIVE':
+ return
+
+ return self.instance[6]
+
+ def get_nova_ip_for_instance_timeout(self, name, timeout=120):
+ start_time = time.time()
+
+ while self.get_nova_state_for_instance(name) != 'ACTIVE':
+
+ if time.time() > start_time + timeout:
+ print "%s not ACTIVE after %i seconds" % (name, timeout)
+ return
+
+ time.sleep(1)
+
+ ip_addr = self.get_nova_ip_for_instance(name)
+ if not ip_addr:
+ return
+
+ if ip_addr.count('=') == 0:
+ return
+
+ ip_addr = ip_addr[ip_addr.find('=') + 1:]
+
+ if ip_addr.count(',') == 0:
+ return ip_addr
+
+ return ip_addr[:ip_addr.find(',')]
+
+
+
+class MorphologyHelper(object):
+
+ def __init__(self):
+ self.sb = sb = morphlib.sysbranchdir.open_from_within('.')
+ defs_repo_path = sb.get_git_directory_name(sb.root_repository_url)
+ self.defs_repo = morphlib.gitdir.GitDirectory(defs_repo_path)
+ self.loader = morphlib.morphloader.MorphologyLoader()
+ self.finder = morphlib.morphologyfinder.MorphologyFinder(self.defs_repo)
+
+ def load_morphology(self, path):
+ text = self.finder.read_morphology(path)
+ return self.loader.load_from_string(text)
+
+ @classmethod
+ def iterate_systems(cls, systems_list):
+ for system in systems_list:
+ yield morphlib.util.sanitise_morphology_path(system['morph'])
+ if 'subsystems' in system:
+ for subsystem in cls.iterate_systems(system['subsystems']):
+ yield subsystem
+
+ def iterate_cluster_deployments(cls, cluster_morph):
+ for system in cluster_morph['systems']:
+ path = morphlib.util.sanitise_morphology_path(system['morph'])
+ defaults = system.get('deploy-defaults', {})
+ for name, options in system['deploy'].iteritems():
+ config = dict(defaults)
+ config.update(options)
+ yield path, name, config
+
+ def load_cluster_systems(self, cluster_morph):
+ for system_path in set(self.iterate_systems(cluster_morph['systems'])):
+ system_morph = self.load_morphology(system_path)
+ yield system_path, system_morph
+
+
+class TimeoutError(cliapp.AppException):
+
+ """Error to be raised when a connection waits too long"""
+
+ def __init__(self, msg):
+ super(TimeoutError, self).__init__(msg)
+
+
+class VMHost(object):
+
+ def __init__(self, user, address, disk_path):
+ self.user = user
+ self.address = address
+ self.disk_path = disk_path
+
+ @property
+ def ssh_host(self):
+ return '{user}@{address}'.format(user=self.user, address=self.address)
+
+ def runcmd(self, *args, **kwargs):
+ cliapp.ssh_runcmd(self.ssh_host, *args, **kwargs)
+
+
+class DeployedSystemInstance(object):
+
+ def __init__(self, deployment, config, host_machine, vm_id, rootfs_path,
+ ip_addr, hostname):
+ self.deployment = deployment
+ self.config = config
+ self.ip_address = ip_addr
+ self.host_machine = host_machine
+ self.vm_id = vm_id
+ self.rootfs_path = rootfs_path
+ self.hostname = hostname
+
+ @property
+ def ssh_host(self):
+ # TODO: Stop assuming we ssh into test instances as root
+ return 'root@{host}'.format(host=self.ip_address)
+
+ def runcmd(self, argv, chdir='.', **kwargs):
+ ssh_cmd = ['ssh', '-o', 'StrictHostKeyChecking=no',
+ '-o', 'UserKnownHostsFile=/dev/null', self.ssh_host]
+ cmd = ['sh', '-c', 'cd "$1" && shift && exec "$@"', '-', chdir]
+ cmd += argv
+ ssh_cmd.append(' '.join(map(pipes.quote, cmd)))
+ return cliapp.runcmd(ssh_cmd, **kwargs)
+
+ def _wait_for_dhcp(self, timeout):
+ '''Block until given hostname resolves successfully.
+
+ Raises TimeoutError if the hostname has not appeared in 'timeout'
+ seconds.
+
+ '''
+ start_time = time.time()
+ while True:
+ try:
+ socket.gethostbyname(self.ip_address)
+ return
+ except socket.gaierror:
+ pass
+ if time.time() > start_time + timeout:
+ raise TimeoutError("Host %s did not appear after %i seconds" %
+ (self.ip_address, timeout))
+ time.sleep(0.5)
+
+ def _wait_for_ssh(self, timeout):
+ """Wait until the deployed VM is responding via SSH"""
+ start_time = time.time()
+ while True:
+ try:
+ self.runcmd(['true'], stdin=None, stdout=None, stderr=None)
+ return
+ except cliapp.AppException:
+ # TODO: Stop assuming the ssh part of the command is what failed
+ if time.time() > start_time + timeout:
+ raise TimeoutError("%s sshd did not start after %i seconds"
+ % (self.ip_address, timeout))
+ time.sleep(0.5)
+
+ def _wait_for_cloud_init(self, timeout):
+ """Wait until cloud init has resized the disc"""
+ start_time = time.time()
+ while True:
+ try:
+ out = self.runcmd(['sh', '-c',
+ 'test -e "$1" && echo exists || echo does not exist',
+ '-',
+ '/root/cloud-init-finished'])
+ except:
+ import traceback
+ traceback.print_exc()
+ raise
+ if out.strip() == 'exists':
+ return
+ if time.time() > start_time + timeout:
+ raise TimeoutError("Disc size not increased after %i seconds"
+ % (timeout))
+ time.sleep(3)
+
+ def wait_until_online(self, timeout=120):
+ self._wait_for_dhcp(timeout)
+ self._wait_for_ssh(timeout)
+ self._wait_for_cloud_init(timeout)
+ print "Test system %s ready to run tests." % (self.hostname)
+
+ def delete(self):
+ # Stop and remove VM
+ print "Deleting %s test instance" % (self.hostname)
+ try:
+ cliapp.runcmd(['nova', 'delete', self.hostname])
+ except cliapp.AppException as e:
+ # TODO: Stop assuming that delete failed because the instance
+ # wasn't running
+ print "- Failed"
+ pass
+ print "Deleting %s test disc image" % (self.hostname)
+ try:
+ cliapp.runcmd(['nova', 'image-delete', self.hostname])
+ except cliapp.AppException as e:
+ # TODO: Stop assuming that image-delete failed because it was
+ # already removed
+ print "- Failed"
+ pass
+
+
+class Deployment(object):
+
+ def __init__(self, cluster_path, name, deployment_config,
+ host_machine, net_id):
+ self.cluster_path = cluster_path
+ self.name = name
+ self.deployment_config = deployment_config
+ self.host_machine = host_machine
+ self.net_id = net_id
+
+ @staticmethod
+ def _ssh_host_key_exists(hostname):
+ """Check if an ssh host key exists in known_hosts"""
+ with open('/root/.ssh/known_hosts', 'r') as known_hosts:
+ return any(line.startswith(hostname) for line in known_hosts)
+
+ def _update_known_hosts(self):
+ if not self._ssh_host_key_exists(self.host_machine.address):
+ with open('/root/.ssh/known_hosts', 'a') as known_hosts:
+ cliapp.runcmd(['ssh-keyscan', self.host_machine.address],
+ stdout=known_hosts)
+
+ @staticmethod
+ def _generate_sshkey_config(tempdir, config):
+ manifest = os.path.join(tempdir, 'manifest')
+ with open(manifest, 'w') as f:
+ f.write('0040700 0 0 /root/.ssh\n')
+ f.write('overwrite 0100600 0 0 /root/.ssh/authorized_keys\n')
+ authkeys = os.path.join(tempdir, 'root', '.ssh', 'authorized_keys')
+ os.makedirs(os.path.dirname(authkeys))
+ with open(authkeys, 'w') as auth_f:
+ with open('/root/.ssh/id_rsa.pub', 'r') as key_f:
+ shutil.copyfileobj(key_f, auth_f)
+
+ install_files = shlex.split(config.get('INSTALL_FILES', ''))
+ install_files.append(manifest)
+ yield 'INSTALL_FILES', ' '.join(pipes.quote(f) for f in install_files)
+
+ def deploy(self):
+ self._update_known_hosts()
+
+ hostname = str(uuid.uuid4())
+ vm_id = hostname
+ image_base = self.host_machine.disk_path
+ rootpath = '{image_base}/{hostname}.img'.format(image_base=image_base,
+ hostname=hostname)
+ loc = 'http://{ssh_host}:5000/v2.0'.format(
+ ssh_host=self.host_machine.ssh_host, id=vm_id, path=rootpath)
+
+ options = {
+ 'type': 'openstack',
+ 'location': loc,
+ 'HOSTNAME': hostname,
+ 'DISK_SIZE': '5G',
+ 'RAM_SIZE': '2G',
+ 'VERSION_LABEL': 'release-test',
+ 'OPENSTACK_USER': os.environ['OS_USERNAME'],
+ 'OPENSTACK_TENANT': os.environ['OS_TENANT_NAME'],
+ 'OPENSTACK_PASSWORD': os.environ['OS_PASSWORD'],
+ 'OPENSTACK_IMAGENAME': hostname,
+ 'CLOUD_INIT': 'yes',
+ 'KERNEL_ARGS': 'console=tty0 console=ttyS0',
+ }
+
+ tempdir = tempfile.mkdtemp()
+ try:
+ options.update(
+ self._generate_sshkey_config(tempdir,
+ self.deployment_config))
+
+ # Deploy the image to openstack
+ args = ['morph', 'deploy', self.cluster_path, self.name]
+ for k, v in options.iteritems():
+ args.append('%s.%s=%s' % (self.name, k, v))
+ cliapp.runcmd(args, stdin=None, stdout=None, stderr=None)
+
+ config = dict(self.deployment_config)
+ config.update(options)
+
+ # Boot an instance from the image
+ args = ['nova', 'boot',
+ '--flavor', 'm1.medium',
+ '--image', hostname,
+ '--user-data', '/root/os-init-script',
+ '--nic', "net-id=%s" % (self.net_id),
+ hostname]
+ output = cliapp.runcmd(args)
+
+ # Print nova boot output, with adminPass line removed
+ output_lines = output.split('\n')
+ for line in output_lines:
+ if line.find('adminPass') != -1:
+ password_line = line
+ output_lines.remove(password_line)
+ output = '\n'.join(output_lines)
+ print output
+
+ # Get ip address from nova list
+ nl = NovaList()
+ ip_addr = nl.get_nova_ip_for_instance_timeout(hostname)
+ print "IP address for inastance %s: %s" % (hostname, ip_addr)
+
+ return DeployedSystemInstance(self, config, self.host_machine,
+ vm_id, rootpath, ip_addr, hostname)
+ finally:
+ shutil.rmtree(tempdir)
+
+
+class ReleaseApp(cliapp.Application):
+
+ """Cliapp application which handles automatic builds and tests"""
+
+ def add_settings(self):
+ """Add the command line options needed"""
+ group_main = 'Program Options'
+ self.settings.string_list(['deployment-host'],
+ 'ARCH:HOST:PATH that VMs can be deployed to',
+ default=None,
+ group=group_main)
+ self.settings.string(['trove-host'],
+ 'Address of Trove for test systems to build from',
+ default=None,
+ group=group_main)
+ self.settings.string(['trove-id'],
+ 'ID of Trove for test systems to build from',
+ default=None,
+ group=group_main)
+ self.settings.string(['build-ref-prefix'],
+ 'Prefix of build branches for test systems',
+ default=None,
+ group=group_main)
+ self.settings.string(['net-id'],
+ 'Openstack network ID',
+ default=None,
+ group=group_main)
+
+ @staticmethod
+ def _run_tests(instance, system_path, system_morph,
+ (trove_host, trove_id, build_ref_prefix),
+ morph_helper, systems):
+ instance.wait_until_online()
+
+ tests = []
+ def baserock_build_test(instance):
+ instance.runcmd(['git', 'config', '--global', 'user.name',
+ 'Test Instance of %s' % instance.deployment.name])
+ instance.runcmd(['git', 'config', '--global', 'user.email',
+ 'ci-test@%s' % instance.config['HOSTNAME']])
+ instance.runcmd(['mkdir', '-p', '/src/ws', '/src/cache',
+ '/src/tmp'])
+ def morph_cmd(*args, **kwargs):
+ # TODO: decide whether to use cached artifacts or not by
+ # adding --artifact-cache-server= --cache-server=
+ argv = ['morph', '--log=/src/morph.log', '--cachedir=/src/cache',
+ '--tempdir=/src/tmp', '--log-max=100M',
+ '--trove-host', trove_host, '--trove-id', trove_id,
+ '--build-ref-prefix', build_ref_prefix]
+ argv.extend(args)
+ instance.runcmd(argv, **kwargs)
+
+ repo = morph_helper.sb.root_repository_url
+ ref = morph_helper.defs_repo.HEAD
+ sha1 = morph_helper.defs_repo.resolve_ref_to_commit(ref)
+ morph_cmd('init', '/src/ws')
+ chdir = '/src/ws'
+
+ morph_cmd('checkout', repo, ref, chdir=chdir)
+ # TODO: Add a morph subcommand that gives the path to the root repository.
+ repo_path = os.path.relpath(
+ morph_helper.sb.get_git_directory_name(repo),
+ morph_helper.sb.root_directory)
+ chdir = os.path.join(chdir, ref, repo_path)
+
+ instance.runcmd(['git', 'reset', '--hard', sha1], chdir=chdir)
+ print 'Building test systems for {sys}'.format(sys=system_path)
+ for to_build_path, to_build_morph in systems.iteritems():
+ if to_build_morph['arch'] == system_morph['arch']:
+ print 'Test building {path}'.format(path=to_build_path)
+ morph_cmd('build', to_build_path, chdir=chdir,
+ stdin=None, stdout=None, stderr=None)
+ print 'Finished Building test systems'
+
+ def python_smoke_test(instance):
+ instance.runcmd(['python', '-c', 'print "Hello World"'])
+
+ # TODO: Come up with a better way of determining which tests to run
+ if 'devel' in system_path:
+ tests.append(baserock_build_test)
+ else:
+ tests.append(python_smoke_test)
+
+ for test in tests:
+ test(instance)
+
+ def deploy_and_test_systems(self, cluster_path,
+ deployment_hosts, build_test_config,
+ net_id):
+ """Run the deployments and tests"""
+
+ version = 'release-test'
+
+ morph_helper = MorphologyHelper()
+ cluster_morph = morph_helper.load_morphology(cluster_path)
+ systems = dict(morph_helper.load_cluster_systems(cluster_morph))
+
+ for system_path, deployment_name, deployment_config in \
+ morph_helper.iterate_cluster_deployments(cluster_morph):
+
+ system_morph = systems[system_path]
+ # We can only test systems in KVM that have a BSP
+ if not any('bsp' in si['morph'] for si in system_morph['strata']):
+ continue
+
+ # We can only test systems in KVM that we have a host for
+ if system_morph['arch'] not in deployment_hosts:
+ continue
+ host_machine = deployment_hosts[system_morph['arch']]
+ deployment = Deployment(cluster_path, deployment_name,
+ deployment_config, host_machine,
+ net_id)
+
+ instance = deployment.deploy()
+ try:
+ self._run_tests(instance, system_path, system_morph,
+ build_test_config, morph_helper, systems)
+ finally:
+ instance.delete()
+
+ def process_args(self, args):
+ """Process the command line args and kick off the builds/tests"""
+ if self.settings['build-ref-prefix'] is None:
+ self.settings['build-ref-prefix'] = (
+ os.path.join(self.settings['trove-id'], 'builds'))
+ for setting in ('deployment-host', 'trove-host',
+ 'trove-id', 'build-ref-prefix', 'net-id'):
+ self.settings.require(setting)
+
+ deployment_hosts = {}
+ for host_config in self.settings['deployment-host']:
+ arch, address = host_config.split(':', 1)
+ user, address = address.split('@', 1)
+ address, disk_path = address.split(':', 1)
+ if user == '':
+ user = 'root'
+ # TODO: Don't assume root is the user with deploy access
+ deployment_hosts[arch] = VMHost(user, address, disk_path)
+
+ build_test_config = (self.settings['trove-host'],
+ self.settings['trove-id'],
+ self.settings['build-ref-prefix'])
+
+ if len(args) != 1:
+ raise cliapp.AppException('Usage: release-test CLUSTER')
+ cluster_path = morphlib.util.sanitise_morphology_path(args[0])
+ self.deploy_and_test_systems(cluster_path, deployment_hosts,
+ build_test_config,
+ self.settings['net-id'])
+
+
+if __name__ == '__main__':
+ ReleaseApp().run()