From 644637928ab3f07a144a805728f64ab91c4bdd29 Mon Sep 17 00:00:00 2001 From: Michael Drake Date: Tue, 8 Sep 2015 13:35:49 +0000 Subject: Change to take image file as input, rather than cluster morphology. This changes the command line to: release-test-os --os-host --net-id --image-file The `morph deploy` is replaced with a `glance image-create`, and much of the internals are simplified. Also removes dep on morphlib. --- scripts/release-test-os | 233 ++++++++++++------------------------------------ 1 file changed, 55 insertions(+), 178 deletions(-) diff --git a/scripts/release-test-os b/scripts/release-test-os index a4877429..de2b656e 100755 --- a/scripts/release-test-os +++ b/scripts/release-test-os @@ -32,9 +32,6 @@ import tempfile import time import uuid -import morphlib - - class NovaList: def __init__(self): self.output = [] @@ -97,43 +94,6 @@ class NovaList: return ip_addr[:ip_addr.find(',')] - -class MorphologyHelper(object): - - def __init__(self): - self.sb = sb = morphlib.sysbranchdir.open_from_within('.') - defs_repo_path = sb.get_git_directory_name(sb.root_repository_url) - self.defs_repo = morphlib.gitdir.GitDirectory(defs_repo_path) - self.loader = morphlib.morphloader.MorphologyLoader() - self.finder = morphlib.morphologyfinder.MorphologyFinder(self.defs_repo) - - def load_morphology(self, path): - text = self.finder.read_morphology(path) - return self.loader.load_from_string(text) - - @classmethod - def iterate_systems(cls, systems_list): - for system in systems_list: - yield morphlib.util.sanitise_morphology_path(system['morph']) - if 'subsystems' in system: - for subsystem in cls.iterate_systems(system['subsystems']): - yield subsystem - - def iterate_cluster_deployments(cls, cluster_morph): - for system in cluster_morph['systems']: - path = morphlib.util.sanitise_morphology_path(system['morph']) - defaults = system.get('deploy-defaults', {}) - for name, options in system['deploy'].iteritems(): - config = dict(defaults) - config.update(options) - yield path, name, config - - def load_cluster_systems(self, cluster_morph): - for system_path in set(self.iterate_systems(cluster_morph['systems'])): - system_morph = self.load_morphology(system_path) - yield system_path, system_morph - - class TimeoutError(cliapp.AppException): """Error to be raised when a connection waits too long""" @@ -144,10 +104,9 @@ class TimeoutError(cliapp.AppException): class VMHost(object): - def __init__(self, user, address, disk_path): + def __init__(self, user, address): self.user = user self.address = address - self.disk_path = disk_path @property def ssh_host(self): @@ -159,14 +118,10 @@ class VMHost(object): class DeployedSystemInstance(object): - def __init__(self, deployment, config, host_machine, vm_id, rootfs_path, - ip_addr, hostname): + def __init__(self, deployment, host_machine, ip_addr, hostname): self.deployment = deployment - self.config = config self.ip_address = ip_addr self.host_machine = host_machine - self.vm_id = vm_id - self.rootfs_path = rootfs_path self.hostname = hostname @property @@ -263,13 +218,10 @@ class DeployedSystemInstance(object): class Deployment(object): - def __init__(self, cluster_path, name, deployment_config, - host_machine, net_id): - self.cluster_path = cluster_path - self.name = name - self.deployment_config = deployment_config + def __init__(self, host_machine, net_id, image_file): self.host_machine = host_machine self.net_id = net_id + self.image_file = image_file @staticmethod def _ssh_host_key_exists(hostname): @@ -290,96 +242,44 @@ class Deployment(object): cliapp.runcmd(['ssh-keyscan', self.host_machine.address], stdout=known_hosts) - @staticmethod - def _generate_sshkey_config(tempdir, config): - manifest = os.path.join(tempdir, 'manifest') - with open(manifest, 'w') as f: - f.write('0040700 0 0 /root/.ssh\n') - f.write('overwrite 0100600 0 0 /root/.ssh/authorized_keys\n') - authkeys = os.path.join(tempdir, 'root', '.ssh', 'authorized_keys') - os.makedirs(os.path.dirname(authkeys)) - with open(authkeys, 'w') as auth_f: - filename = '/root/.ssh/id_rsa'; - if not os.path.exists(filename): - print("Warning: No key found; generating key"); - cliapp.runcmd(['ssh-keygen', '-t', 'rsa', '-b', '2048', - '-f', filename, '-C', 'test@ciat', '-N', '']) - with open(filename + '.pub', 'r') as key_f: - shutil.copyfileobj(key_f, auth_f) - - install_files = shlex.split(config.get('INSTALL_FILES', '')) - install_files.append(manifest) - yield 'INSTALL_FILES', ' '.join(pipes.quote(f) for f in install_files) - def deploy(self): self._update_known_hosts() hostname = str(uuid.uuid4()) - vm_id = hostname - image_base = self.host_machine.disk_path - rootpath = '{image_base}/{hostname}.img'.format(image_base=image_base, - hostname=hostname) - loc = 'http://{ssh_host}:5000/v2.0'.format( - ssh_host=self.host_machine.ssh_host, id=vm_id, path=rootpath) - - options = { - 'type': 'extensions/openstack', - 'location': loc, - 'HOSTNAME': hostname, - 'DISK_SIZE': '5G', - 'RAM_SIZE': '2G', - 'VERSION_LABEL': 'release-test', - 'OPENSTACK_USER': os.environ['OS_USERNAME'], - 'OPENSTACK_TENANT': os.environ['OS_TENANT_NAME'], - 'OPENSTACK_PASSWORD': os.environ['OS_PASSWORD'], - 'OPENSTACK_IMAGENAME': hostname, - 'CLOUD_INIT': 'yes', - 'KERNEL_ARGS': 'console=tty0 console=ttyS0', - } - - tempdir = tempfile.mkdtemp() - try: - options.update( - self._generate_sshkey_config(tempdir, - self.deployment_config)) - - # Deploy the image to openstack - args = ['morph', 'deploy', self.cluster_path, self.name] - for k, v in options.iteritems(): - args.append('%s.%s=%s' % (self.name, k, v)) - cliapp.runcmd(args, stdin=None, stdout=None, stderr=None) - config = dict(self.deployment_config) - config.update(options) + # Deploy the image to openstack + args = ['glance', 'image-create', + '--name', hostname, + '--disk-format', 'raw', + '--container-format', 'bare', + '--file', self.image_file] + cliapp.runcmd(args, stdin=None, stdout=None, stderr=None) - # Boot an instance from the image - args = ['nova', 'boot', + # Boot an instance from the image + args = ['nova', 'boot', '--flavor', 'm1.medium', '--image', hostname, '--user-data', '/usr/lib/mason/os-init-script', '--nic', "net-id=%s" % (self.net_id), hostname] - output = cliapp.runcmd(args) - - # Print nova boot output, with adminPass line removed - output_lines = output.split('\n') - for line in output_lines: - if line.find('adminPass') != -1: - password_line = line - output_lines.remove(password_line) - output = '\n'.join(output_lines) - print output - - # Get ip address from nova list - nl = NovaList() - ip_addr = nl.get_nova_ip_for_instance_timeout(hostname) - print "IP address for instance %s: %s" % (hostname, ip_addr) - - return DeployedSystemInstance(self, config, self.host_machine, - vm_id, rootpath, ip_addr, hostname) - finally: - shutil.rmtree(tempdir) + output = cliapp.runcmd(args) + # Print nova boot output, with adminPass line removed + output_lines = output.split('\n') + for line in output_lines: + if line.find('adminPass') != -1: + password_line = line + output_lines.remove(password_line) + output = '\n'.join(output_lines) + print output + + # Get ip address from nova list + nl = NovaList() + ip_addr = nl.get_nova_ip_for_instance_timeout(hostname) + print "IP address for instance %s: %s" % (hostname, ip_addr) + + return DeployedSystemInstance(self, self.host_machine, + ip_addr, hostname) class ReleaseApp(cliapp.Application): @@ -388,14 +288,18 @@ class ReleaseApp(cliapp.Application): def add_settings(self): """Add the command line options needed""" group_main = 'Program Options' - self.settings.string_list(['deployment-host'], - 'ARCH:HOST:PATH that VMs can be deployed to', - default=None, - group=group_main) + self.settings.string(['os-host'], + 'HOST that VMs can be deployed to', + default=None, + group=group_main) self.settings.string(['net-id'], 'Openstack network ID', default=None, group=group_main) + self.settings.string(['image-file'], + 'Path to system image to test', + default=None, + group=group_main) def run_tests(self, instance): instance.wait_until_online() @@ -418,58 +322,31 @@ class ReleaseApp(cliapp.Application): for test in tests: test(instance) - def deploy_and_test_systems(self, cluster_path, - deployment_hosts, net_id): + def deploy_and_test_systems(self, host_machine, net_id, image_file): """Run the deployments and tests""" - version = 'release-test' - - morph_helper = MorphologyHelper() - cluster_morph = morph_helper.load_morphology(cluster_path) - systems = dict(morph_helper.load_cluster_systems(cluster_morph)) - - for system_path, deployment_name, deployment_config in \ - morph_helper.iterate_cluster_deployments(cluster_morph): + deployment = Deployment(host_machine, net_id, image_file) - system_morph = systems[system_path] - # We can only test systems that have a BSP - if not any('bsp' in si['morph'] for si in system_morph['strata']): - continue - - # We can only test systems that we have a host for - if system_morph['arch'] not in deployment_hosts: - continue - host_machine = deployment_hosts[system_morph['arch']] - deployment = Deployment(cluster_path, deployment_name, - deployment_config, host_machine, - net_id) - - instance = deployment.deploy() - try: - self.run_tests(instance) - finally: - instance.delete() + instance = deployment.deploy() + try: + self.run_tests(instance) + finally: + instance.delete() def process_args(self, args): """Process the command line args and kick off the builds/tests""" - for setting in ('deployment-host', 'net-id'): + for setting in ('os-host', 'net-id', 'image-file'): self.settings.require(setting) - deployment_hosts = {} - for host_config in self.settings['deployment-host']: - arch, address = host_config.split(':', 1) - user, address = address.split('@', 1) - address, disk_path = address.split(':', 1) - if user == '': - user = 'root' - # TODO: Don't assume root is the user with deploy access - deployment_hosts[arch] = VMHost(user, address, disk_path) - - if len(args) != 1: - raise cliapp.AppException('Usage: release-test CLUSTER') - cluster_path = morphlib.util.sanitise_morphology_path(args[0]) - self.deploy_and_test_systems(cluster_path, deployment_hosts, - self.settings['net-id']) + # TODO: Don't assume root is the user we ssh to for tests + host_machine = VMHost('root', self.settings['os-host']) + + if len(args) != 0: + raise cliapp.AppException( + 'Usage: release-test-os --os-host --net-id --image-file ') + self.deploy_and_test_systems(host_machine, + self.settings['net-id'], + self.settings['image-file']) if __name__ == '__main__': -- cgit v1.2.1