#!/usr/bin/env python # # Copyright 2014 Codethink Ltd # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. # # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. '''release-test This script deploys the set of systems in the cluster morphology it is instructed to read, to test that they work correctly. ''' import cliapp import os import pipes import shlex import shutil import socket import tempfile import time import uuid import yaml from novaclient import client class TimeoutError(cliapp.AppException): """Error to be raised when a connection waits too long""" def __init__(self, msg): super(TimeoutError, self).__init__(msg) class DeployedSystemInstance(object): def __init__(self, deployment, ip_addr, hostname): self.deployment = deployment self.ip_address = ip_addr self.hostname = hostname @property def ssh_host(self): # TODO: Stop assuming we ssh into test instances as root return 'root@{host}'.format(host=self.ip_address) def runcmd(self, argv, chdir='.', **kwargs): ssh_cmd = ['ssh', '-o', 'StrictHostKeyChecking=no', '-o', 'UserKnownHostsFile=/dev/null', self.ssh_host] cmd = ['sh', '-c', 'cd "$1" && shift && exec "$@"', '-', chdir] cmd += argv ssh_cmd.append(' '.join(map(pipes.quote, cmd))) return cliapp.runcmd(ssh_cmd, **kwargs) def _wait_for_dhcp(self, timeout): '''Block until given hostname resolves successfully. Raises TimeoutError if the hostname has not appeared in 'timeout' seconds. ''' start_time = time.time() while True: try: socket.gethostbyname(self.ip_address) return except socket.gaierror: pass if time.time() > start_time + timeout: raise TimeoutError("Host %s did not appear after %i seconds" % (self.ip_address, timeout)) time.sleep(0.5) def _wait_for_ssh(self, timeout): """Wait until the deployed VM is responding via SSH""" start_time = time.time() while True: try: self.runcmd(['true'], stdin=None, stdout=None, stderr=None) return except cliapp.AppException: # TODO: Stop assuming the ssh part of the command is what failed if time.time() > start_time + timeout: raise TimeoutError("%s sshd did not start after %i seconds" % (self.ip_address, timeout)) time.sleep(0.5) def _wait_for_cloud_init(self, timeout): """Wait until cloud init has resized the disc""" start_time = time.time() while True: try: out = self.runcmd(['sh', '-c', 'test -e "$1" && echo exists || echo does not exist', '-', '/root/cloud-init-finished']) except: import traceback traceback.print_exc() raise if out.strip() == 'exists': return if time.time() > start_time + timeout: raise TimeoutError("Disc size not increased after %i seconds" % (timeout)) time.sleep(3) def wait_until_online(self, timeout=180): self._wait_for_dhcp(timeout) self._wait_for_ssh(timeout) #self._wait_for_cloud_init(timeout) print "Test system %s ready to run tests." % (self.hostname) def delete(self): # Stop and remove VM print "Deleting %s test instance" % (self.hostname) try: cliapp.runcmd(['nova', 'delete', self.hostname]) except cliapp.AppException as e: # TODO: Stop assuming that delete failed because the instance # wasn't running print "- Failed" pass print "Deleting %s test disc image" % (self.hostname) try: cliapp.runcmd(['nova', 'image-delete', self.hostname]) except cliapp.AppException as e: # TODO: Stop assuming that image-delete failed because it was # already removed print "- Failed" pass class Deployment(object): def __init__(self, net_id, image_file, flavour): self.net_id = net_id self.image_file = image_file self.flavour = flavour def deploy(self): hostname = str(uuid.uuid4()) # Deploy the image to openstack # TODO: Ensure this is cleaned up when something raises an exception args = ['glance', 'image-create', '--name', hostname, '--disk-format', 'raw', '--container-format', 'bare', '--file', self.image_file] cliapp.runcmd(args, stdin=None, stdout=None, stderr=None) # Get a novaclient object nc = client.Client(2, os.environ['OS_USERNAME'], os.environ['OS_PASSWORD'], os.environ['OS_TENANT_NAME'], os.environ['OS_AUTH_URL']) # Boot an instance from the image # TODO: Ensure this is cleaned up when something raises an exception # TODO: use python-novaclient args = ['nova', 'boot', '--flavor', self.flavour, '--image', hostname, '--user-data', '/usr/lib/mason/os-init-script', '--nic', "net-id=%s" % (self.net_id), hostname] output = cliapp.runcmd(args) # Print nova boot output, with adminPass line removed output_lines = output.split('\n') for line in output_lines: if line.find('adminPass') != -1: password_line = line output_lines.remove(password_line) output = '\n'.join(output_lines) print output # Sleep for a bit, or nova explodes when trying to assign IP address time.sleep(20) # Assign a floating IP address for retries in range(5, 0, -1): ip_list = nc.floating_ips.list() free_ip = None for ip in ip_list: if ip.instance_id == None: free_ip = ip break if free_ip != None: instance = nc.servers.find(name=hostname) # TODO: switch back to cli tool, as python # approach gave error. instance.add_floating_ip(free_ip) ip_addr = free_ip.ip break else: raise cliapp.AppException('Could not get a floating IP') # Print the IP address print "IP address for instance %s: %s" % (hostname, ip_addr) return DeployedSystemInstance(self, ip_addr, hostname) class ReleaseApp(cliapp.Application): """Cliapp application which handles automatic builds and tests""" def add_settings(self): """Add the command line options needed""" group_main = 'Program Options' self.settings.string(['net-id'], 'Openstack network ID', default=None, group=group_main) self.settings.string(['image-file'], 'Path to system image to test', default=None, group=group_main) self.settings.string(['flavour'], 'Name of flavour to use for instance', default=None, group=group_main) def run_tests(self, instance, tests): instance.wait_until_online() for test in tests: with open(test, 'r') as stream: data=yaml.load(stream) if data == None: continue if 'name' not in data: print('Bad test: no name') continue if 'commands' not in data: print('Bad test: no commands') continue print('Running test: ' + data['name']) for cmd in data['commands']: print('$ ' + cmd) instance.runcmd(shlex.split(cmd), stdout=self.output) def deploy_and_test_systems(self, tests): """Run the deployments and tests""" deployment = Deployment(self.settings['net-id'], self.settings['image-file'], self.settings['flavour']) instance = deployment.deploy() try: self.run_tests(instance, tests) finally: instance.delete() def process_args(self, args): """Process the command line args and kick off the builds/tests""" for setting in ('net-id', 'image-file', 'flavour'): self.settings.require(setting) if len(args) == 0: raise cliapp.AppException( 'Usage: release-test-os --net-id --image-file --flavour ...') self.deploy_and_test_systems(args) if __name__ == '__main__': ReleaseApp().run()