From bfd10c0db409161abd91d35c769ddc0d7b28f43d Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 23 Jan 2013 17:46:25 +0000 Subject: Add a configuration extension for setting hostname --- set-hostname.configure | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100755 set-hostname.configure diff --git a/set-hostname.configure b/set-hostname.configure new file mode 100755 index 00000000..e44c5d56 --- /dev/null +++ b/set-hostname.configure @@ -0,0 +1,27 @@ +#!/bin/sh +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +# Set hostname on system from HOSTNAME. + + +set -e + +if [ -n "$HOSTNAME" ] +then + echo "$HOSTNAME" > "$1/etc/hostname" +fi + -- cgit v1.2.1 From d64dc1e0b80d3bc09fcca05d15fe10cd60c332a6 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 23 Jan 2013 17:11:43 +0000 Subject: Add a write extension for raw disk images --- rawdisk.write | 70 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 70 insertions(+) create mode 100755 rawdisk.write diff --git a/rawdisk.write b/rawdisk.write new file mode 100755 index 00000000..c6f9c7f6 --- /dev/null +++ b/rawdisk.write @@ -0,0 +1,70 @@ +#!/usr/bin/python +# Copyright (C) 2012-2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +'''A Morph deployment write extension for raw disk images.''' + + +import os +import time +import tempfile + +import morphlib.writeexts + + +class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): + + '''Create a raw disk image during Morph's deployment. + + The location command line argument is the pathname of the disk image + to be created. + + ''' + + def process_args(self, args): + if len(args) != 2: + raise cliapp.AppException('Wrong number of command line args') + + temp_root, location = args + self.status( + msg='Temporary system root: %(temp_root)s', + temp_root=temp_root) + self.status( + msg='Disk image to create: %(location)s', + location=location) + + size = self.get_disk_size() + self.status(msg='Disk size is %(size)d bytes', size=size) + + self.create_raw_disk_image(location, size) + self.mkfs_btrfs(location) + mp = self.mount(location) + try: + self.create_factory(mp, temp_root) + self.create_fstab(mp) + self.install_extlinux(mp) + except BaseException, e: + self.status(msg='EEK') + self.unmount(mp) + raise + else: + self.unmount(mp) + + self.status(msg='Disk image has been created') + + +RawDiskWriteExtension().run() + -- cgit v1.2.1 From 2d40cac81a029f29c8e16593112a8c18e945f83b Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 24 Jan 2013 15:08:18 +0000 Subject: Add a write extension for VirtualBox --- virtualbox-ssh.write | 158 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100755 virtualbox-ssh.write diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write new file mode 100755 index 00000000..e2f592e7 --- /dev/null +++ b/virtualbox-ssh.write @@ -0,0 +1,158 @@ +#!/usr/bin/python +# Copyright (C) 2012-2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +'''A Morph deployment write extension for deploying to VirtualBox via ssh. + +VirtualBox is assumed to be running on a remote machine, which is +accessed over ssh. The machine gets created, but not started. + +''' + + +import os +import re +import time +import tempfile +import urlparse + +import morphlib.writeexts + + +class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): + + '''Create a VirtualBox virtual machine during Morph's deployment. + + The location command line argument is the pathname of the disk image + to be created. The user is expected to provide the location argument + using the following syntax: + + vbox+ssh://HOST/GUEST/PATH + + where: + + * HOST is the host on which VirtualBox is running + * GUEST is the name of the guest virtual machine on that host + * PATH is the path to the disk image that should be created, + on that host + + The extension will connect to HOST via ssh to run VirtualBox's + command line management tools. + + ''' + + def process_args(self, args): + if len(args) != 2: + raise cliapp.AppException('Wrong number of command line args') + + temp_root, location = args + ssh_host, vm_name, vdi_path = self.parse_location(location) + + self.status( + msg='Temporary system root: %(temp_root)s', + temp_root=temp_root) + self.status( + msg='VirtualBox server: %(ssh_host)s', + ssh_host=ssh_host) + self.status( + msg='VirtualBox guest: %(vm_name)s', + vm_name=vm_name) + + size = self.get_disk_size() + self.status(msg='Disk size is %(size)d bytes', size=size) + + raw_disk = tempfile.mkstemp() + self.create_raw_disk_image(raw_disk, size) + try: + self.mkfs_btrfs(raw_disk) + mp = self.mount(raw_disk) + except BaseException: + self.status('EEEK') + os.remove(raw_disk) + raise + try: + self.create_factory(mp, temp_root) + self.create_fstab(mp) + self.install_extlinux(mp) + except BaseException, e: + self.status(msg='EEK') + self.unmount(mp) + os.remove(raw_disk) + raise + else: + self.unmount(mp) + + try: + self.transfer_and_convert_to_vdi( + raw_disk, size, ssh_host, vdi_path) + self.create_virtualbox_guest(ssh_host, vm_name, vdi_path) + except BaseException: + self.status('EEEK') + os.remove(raw_disk) + raise + else: + os.remove(raw_disk) + + self.status( + msg='Virtual machine %(vm_name)s has been created', + vm_name=vm_name) + + def parse_location(self, location): + '''Parse the location argument to get relevant data.''' + + x = urlparse.urlparse(location) + if x.scheme != 'vbox+ssh': + raise cliapp.AppException( + 'URL schema must be vbox+ssh in %s' % location) + m = re.match('^/(?P[^/]+)(?P/.+)$', x.path) + if not m: + raise cliapp.AppException('Cannot parse location %s' % location) + return x.netloc, m.group('guest'), m.group('path') + + def transfer_and_convert_to_vdi(self, raw_disk, size, ssh_host, vdi_path): + '''Transfer raw disk image to VirtualBox host, and convert to VDI.''' + + self.status(msg='Transfer disk and convert to VDI') + with open(raw_disk, 'rb') as f: + cliapp.runcmd( + ['ssh', ssh_host, + 'VBoxManage', 'convertfromraw', 'stdin', vdi_path, str(size)], + stdin=f) + + def create_virtualbox_guest(self, ssh_host, vm_name, vdi_path): + '''Create the VirtualBox virtual machine.''' + + self.status(msg='Create VirtualBox virtual machine') + + commands = [ + ['createvm', '--name', vm_name, '--ostype', 'Linux26_64', + '--register'], + ['modifyvm', vm_name, '--ioapic', 'on', '--memory', '1024', + '--nic1', 'nat'], + ['storagectl', vm_name, '--name', '"SATA Controller"', + '--add', 'sata', '--bootable', 'on', '--sataportcount', '2'], + ['storageattach', vm_name, '--storagectl', '"SATA Controller"', + '--port', '0', '--device', '0', '--type', 'hdd', '--medium', + vdi_path], + ] + + for command in commands: + argv = ['ssh', ssh_host, 'VBoxManage'] + command + cliapp.runcmd(argv) + + +VirtualBoxPlusSshWriteExtension().run() + -- cgit v1.2.1 From 253b541bfbfb18f731f90348bae480f304297d36 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 24 Jan 2013 15:41:11 +0000 Subject: Add a write extension for kvm+libvirt --- kvm.write | 140 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100755 kvm.write diff --git a/kvm.write b/kvm.write new file mode 100755 index 00000000..ed85b17e --- /dev/null +++ b/kvm.write @@ -0,0 +1,140 @@ +#!/usr/bin/python +# Copyright (C) 2012-2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +'''A Morph deployment write extension for deploying to KVM+libvirt.''' + + +import os +import re +import urlparse + +import morphlib.writeexts + + +class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): + + '''Create a KVM/LibVirt virtual machine during Morph's deployment. + + The location command line argument is the pathname of the disk image + to be created. The user is expected to provide the location argument + using the following syntax: + + kvm+ssh://HOST/GUEST/PATH + + where: + + * HOST is the host on which KVM/LibVirt is running + * GUEST is the name of the guest virtual machine on that host + * PATH is the path to the disk image that should be created, + on that host + + The extension will connect to HOST via ssh to run libvirt's + command line management tools. + + ''' + + def process_args(self, args): + if len(args) != 2: + raise cliapp.AppException('Wrong number of command line args') + + temp_root, location = args + ssh_host, vm_name, vm_path = self.parse_location(location) + + self.status( + msg='Temporary system root: %(temp_root)s', + temp_root=temp_root) + self.status( + msg='libvirt server: %(ssh_host)s', + ssh_host=ssh_host) + self.status( + msg='libvirt guest: %(vm_name)s', + vm_name=vm_name) + self.status( + msg='libvirt disk image: %(vm_path)s', + vm_path=vm_path) + + size = self.get_disk_size() + self.status(msg='Disk size is %(size)d bytes', size=size) + + raw_disk = tempfile.mkstemp() + self.create_raw_disk_image(raw_disk, size) + try: + self.mkfs_btrfs(raw_disk) + mp = self.mount(raw_disk) + except BaseException: + self.status(msg='EEEK') + os.remove(raw_disk) + raise + try: + self.create_factory(mp, temp_root) + self.create_fstab(mp) + self.install_extlinux(mp) + except BaseException, e: + self.status(msg='EEK') + self.unmount(mp) + os.remove(raw_disk) + raise + else: + self.unmount(mp) + + try: + self.transfer(raw_disk, size, ssh_host, vm_path) + self.create_libvirt_guest(ssh_host, vm_name, vm_path) + except BaseException: + self.status(msg='EEEK') + os.remove(raw_disk) + raise + else: + os.remove(raw_disk) + + self.status( + msg='Virtual machine %(vm_name)s has been created', + vm_name=vm_name) + + def parse_location(self, location): + '''Parse the location argument to get relevant data.''' + + x = urlparse.urlparse(location) + if x.scheme != 'kvm+ssh': + raise cliapp.AppException( + 'URL schema must be vbox+ssh in %s' % location) + m = re.match('^/(?P[^/]+)(?P/.+)$', x.path) + if not m: + raise cliapp.AppException('Cannot parse location %s' % location) + return x.netloc, m.group('guest'), m.group('path') + + def transfer(self, raw_disk, size, ssh_host, vm_path): + '''Transfer raw disk image to libvirt host.''' + + self.status(msg='Transfer disk image') + target = '%s:%s' % (ssh_host, vm_path) + with open(raw_disk, 'rb') as f: + cliapp.runcmd(['rsync', '-zS', raw_disk, target]) + + def create_libvirt_guest(self, ssh_host, vm_name, vm_path): + '''Create the libvirt virtual machine.''' + + self.status(msg='Create libvirt/kvm virtual machine') + cliapp.runcmd( + ['ssh', ssh_host, + 'virt-install', '--connect qemu:///system', '--import', + '--name', vm_name, '--ram', '1024', '--vnc', '--noreboot', + '--disk path=%s,bus=ide' % vm_path]) + + +KvmPlusSshWriteExtension().run() + -- cgit v1.2.1 From 769c0ae2808a3c3e383c470795510b72c45c5a76 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 7 Feb 2013 11:51:06 +0000 Subject: Remove debugging output Suggested-By: Richard Maw --- kvm.write | 13 ------------- rawdisk.write | 6 ------ virtualbox-ssh.write | 10 ---------- 3 files changed, 29 deletions(-) diff --git a/kvm.write b/kvm.write index ed85b17e..e52e5553 100755 --- a/kvm.write +++ b/kvm.write @@ -54,19 +54,6 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): temp_root, location = args ssh_host, vm_name, vm_path = self.parse_location(location) - self.status( - msg='Temporary system root: %(temp_root)s', - temp_root=temp_root) - self.status( - msg='libvirt server: %(ssh_host)s', - ssh_host=ssh_host) - self.status( - msg='libvirt guest: %(vm_name)s', - vm_name=vm_name) - self.status( - msg='libvirt disk image: %(vm_path)s', - vm_path=vm_path) - size = self.get_disk_size() self.status(msg='Disk size is %(size)d bytes', size=size) diff --git a/rawdisk.write b/rawdisk.write index c6f9c7f6..89e9e82b 100755 --- a/rawdisk.write +++ b/rawdisk.write @@ -39,12 +39,6 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): raise cliapp.AppException('Wrong number of command line args') temp_root, location = args - self.status( - msg='Temporary system root: %(temp_root)s', - temp_root=temp_root) - self.status( - msg='Disk image to create: %(location)s', - location=location) size = self.get_disk_size() self.status(msg='Disk size is %(size)d bytes', size=size) diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index e2f592e7..dbfb90a7 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -61,16 +61,6 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): temp_root, location = args ssh_host, vm_name, vdi_path = self.parse_location(location) - self.status( - msg='Temporary system root: %(temp_root)s', - temp_root=temp_root) - self.status( - msg='VirtualBox server: %(ssh_host)s', - ssh_host=ssh_host) - self.status( - msg='VirtualBox guest: %(vm_name)s', - vm_name=vm_name) - size = self.get_disk_size() self.status(msg='Disk size is %(size)d bytes', size=size) -- cgit v1.2.1 From 099f2ef05cd52dd623b47901ecc361754aa729f7 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 7 Feb 2013 11:41:04 +0000 Subject: Refactor: Add WriteExtension.create_local_system method This allows code sharing amongst all the places that create a system in a raw disk image. This also adds the creation of a factory-run subvolume, and fixes error messages for errors that happen during a disk image creation. Suggested-By: Richard Maw Suggested-By: Sam Thursfield --- kvm.write | 35 +++++++++-------------------------- rawdisk.write | 21 +++------------------ virtualbox-ssh.write | 29 +++++------------------------ 3 files changed, 17 insertions(+), 68 deletions(-) diff --git a/kvm.write b/kvm.write index e52e5553..09a7d224 100755 --- a/kvm.write +++ b/kvm.write @@ -18,8 +18,11 @@ '''A Morph deployment write extension for deploying to KVM+libvirt.''' +import cliapp import os import re +import sys +import tempfile import urlparse import morphlib.writeexts @@ -54,35 +57,15 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): temp_root, location = args ssh_host, vm_name, vm_path = self.parse_location(location) - size = self.get_disk_size() - self.status(msg='Disk size is %(size)d bytes', size=size) + fd, raw_disk = tempfile.mkstemp() + os.close(fd) + self.create_local_system(temp_root, raw_disk) - raw_disk = tempfile.mkstemp() - self.create_raw_disk_image(raw_disk, size) try: - self.mkfs_btrfs(raw_disk) - mp = self.mount(raw_disk) - except BaseException: - self.status(msg='EEEK') - os.remove(raw_disk) - raise - try: - self.create_factory(mp, temp_root) - self.create_fstab(mp) - self.install_extlinux(mp) - except BaseException, e: - self.status(msg='EEK') - self.unmount(mp) - os.remove(raw_disk) - raise - else: - self.unmount(mp) - - try: - self.transfer(raw_disk, size, ssh_host, vm_path) + self.transfer(raw_disk, ssh_host, vm_path) self.create_libvirt_guest(ssh_host, vm_name, vm_path) except BaseException: - self.status(msg='EEEK') + sys.stderr.write('Error deploying to libvirt') os.remove(raw_disk) raise else: @@ -104,7 +87,7 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): raise cliapp.AppException('Cannot parse location %s' % location) return x.netloc, m.group('guest'), m.group('path') - def transfer(self, raw_disk, size, ssh_host, vm_path): + def transfer(self, raw_disk, ssh_host, vm_path): '''Transfer raw disk image to libvirt host.''' self.status(msg='Transfer disk image') diff --git a/rawdisk.write b/rawdisk.write index 89e9e82b..a55473f2 100755 --- a/rawdisk.write +++ b/rawdisk.write @@ -19,6 +19,7 @@ import os +import sys import time import tempfile @@ -40,24 +41,8 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): temp_root, location = args - size = self.get_disk_size() - self.status(msg='Disk size is %(size)d bytes', size=size) - - self.create_raw_disk_image(location, size) - self.mkfs_btrfs(location) - mp = self.mount(location) - try: - self.create_factory(mp, temp_root) - self.create_fstab(mp) - self.install_extlinux(mp) - except BaseException, e: - self.status(msg='EEK') - self.unmount(mp) - raise - else: - self.unmount(mp) - - self.status(msg='Disk image has been created') + self.create_local_system(temp_root, location) + self.status(msg='Disk image has been created at %s' % location) RawDiskWriteExtension().run() diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index dbfb90a7..5d118ec4 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -23,6 +23,7 @@ accessed over ssh. The machine gets created, but not started. ''' +import cliapp import os import re import time @@ -61,36 +62,16 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): temp_root, location = args ssh_host, vm_name, vdi_path = self.parse_location(location) - size = self.get_disk_size() - self.status(msg='Disk size is %(size)d bytes', size=size) - - raw_disk = tempfile.mkstemp() - self.create_raw_disk_image(raw_disk, size) - try: - self.mkfs_btrfs(raw_disk) - mp = self.mount(raw_disk) - except BaseException: - self.status('EEEK') - os.remove(raw_disk) - raise - try: - self.create_factory(mp, temp_root) - self.create_fstab(mp) - self.install_extlinux(mp) - except BaseException, e: - self.status(msg='EEK') - self.unmount(mp) - os.remove(raw_disk) - raise - else: - self.unmount(mp) + fd, raw_disk = tempfile.mkstemp() + os.close(fd) + self.create_local_system(temp_root) try: self.transfer_and_convert_to_vdi( raw_disk, size, ssh_host, vdi_path) self.create_virtualbox_guest(ssh_host, vm_name, vdi_path) except BaseException: - self.status('EEEK') + sys.stderr.write('Error deploying to VirtualBox') os.remove(raw_disk) raise else: -- cgit v1.2.1 From 311a41778532dc5d102f5c8e2cbe0d3dd58cadf1 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 7 Feb 2013 15:17:56 +0000 Subject: Add missing argument to create_local_system method Reported-By: Richard Maw --- virtualbox-ssh.write | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 5d118ec4..c21dcc57 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -64,7 +64,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): fd, raw_disk = tempfile.mkstemp() os.close(fd) - self.create_local_system(temp_root) + self.create_local_system(temp_root, raw_disk) try: self.transfer_and_convert_to_vdi( -- cgit v1.2.1 From 66b2c4cdb96fc20042d0ebb34f903c4692ae5bb5 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Tue, 12 Feb 2013 16:48:03 +0000 Subject: Improve message phrasing Suggested-By: Richard Maw --- kvm.write | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kvm.write b/kvm.write index 09a7d224..1579dc1f 100755 --- a/kvm.write +++ b/kvm.write @@ -90,7 +90,7 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): def transfer(self, raw_disk, ssh_host, vm_path): '''Transfer raw disk image to libvirt host.''' - self.status(msg='Transfer disk image') + self.status(msg='Transferring disk image') target = '%s:%s' % (ssh_host, vm_path) with open(raw_disk, 'rb') as f: cliapp.runcmd(['rsync', '-zS', raw_disk, target]) @@ -98,7 +98,7 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): def create_libvirt_guest(self, ssh_host, vm_name, vm_path): '''Create the libvirt virtual machine.''' - self.status(msg='Create libvirt/kvm virtual machine') + self.status(msg='Creating libvirt/kvm virtual machine') cliapp.runcmd( ['ssh', ssh_host, 'virt-install', '--connect qemu:///system', '--import', -- cgit v1.2.1 From c57547b87cdad636a28a3f378104c4279b03b45f Mon Sep 17 00:00:00 2001 From: Ric Holland Date: Tue, 5 Mar 2013 10:32:03 +0000 Subject: Fix virtualbox deployment It was missing an import for sys. It did not have the size of the disk either, this has also been fixed --- virtualbox-ssh.write | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index c21dcc57..9b99c7a1 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -26,6 +26,7 @@ accessed over ssh. The machine gets created, but not started. import cliapp import os import re +import sys import time import tempfile import urlparse @@ -68,7 +69,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): try: self.transfer_and_convert_to_vdi( - raw_disk, size, ssh_host, vdi_path) + raw_disk, ssh_host, vdi_path) self.create_virtualbox_guest(ssh_host, vm_name, vdi_path) except BaseException: sys.stderr.write('Error deploying to VirtualBox') @@ -93,14 +94,15 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): raise cliapp.AppException('Cannot parse location %s' % location) return x.netloc, m.group('guest'), m.group('path') - def transfer_and_convert_to_vdi(self, raw_disk, size, ssh_host, vdi_path): + def transfer_and_convert_to_vdi(self, raw_disk, ssh_host, vdi_path): '''Transfer raw disk image to VirtualBox host, and convert to VDI.''' self.status(msg='Transfer disk and convert to VDI') with open(raw_disk, 'rb') as f: cliapp.runcmd( ['ssh', ssh_host, - 'VBoxManage', 'convertfromraw', 'stdin', vdi_path, str(size)], + 'VBoxManage', 'convertfromraw', 'stdin', vdi_path, + str(os.path.getsize(raw_disk))], stdin=f) def create_virtualbox_guest(self, ssh_host, vm_name, vdi_path): -- cgit v1.2.1 From 5178028a2fe4f38a9f163a910926e1ae5e3eb517 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 13 Mar 2013 13:52:52 +0000 Subject: Add ATTACH_DISKS support to kvm --- kvm.write | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/kvm.write b/kvm.write index 1579dc1f..c491f206 100755 --- a/kvm.write +++ b/kvm.write @@ -99,11 +99,18 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): '''Create the libvirt virtual machine.''' self.status(msg='Creating libvirt/kvm virtual machine') + + attach_disks = self.parse_attach_disks() + attach_opts = [] + for disk in attach_disks: + attach_opts.extend(['--disk', 'path=%s' % disk]) + cliapp.runcmd( ['ssh', ssh_host, 'virt-install', '--connect qemu:///system', '--import', '--name', vm_name, '--ram', '1024', '--vnc', '--noreboot', - '--disk path=%s,bus=ide' % vm_path]) + '--disk path=%s,bus=ide' % vm_path] + + attach_opts) KvmPlusSshWriteExtension().run() -- cgit v1.2.1 From 717e3e91b3a606cca14f1a4c1d44a28cbbbe3c9e Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 13 Mar 2013 15:11:36 +0000 Subject: Implement ATTACH_DISKS in virtualbox-ssh.write --- virtualbox-ssh.write | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 9b99c7a1..862d4f02 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -121,7 +121,17 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): '--port', '0', '--device', '0', '--type', 'hdd', '--medium', vdi_path], ] - + + attach_disks = self.parse_attach_disks() + for device_no, disk in enumerate(attach_disks, 1): + cmd = ['storageattach', vm_name, + '--storagectl', '"SATA Controller"', + '--port', str(device_no), + '--device', '0', + '--type', 'hdd', + '--medium', disk] + commands.append(cmd) + for command in commands: argv = ['ssh', ssh_host, 'VBoxManage'] + command cliapp.runcmd(argv) -- cgit v1.2.1 From bc7b098eb300fee1cf7b5f09da49a5d23a82415c Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 28 Mar 2013 13:42:06 +0000 Subject: Set RAM size for kvm deployment --- kvm.write | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/kvm.write b/kvm.write index c491f206..630f6ae7 100755 --- a/kvm.write +++ b/kvm.write @@ -105,10 +105,13 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): for disk in attach_disks: attach_opts.extend(['--disk', 'path=%s' % disk]) + ram_mebibytes = str(self.get_ram_size() / (1024**2)) + cliapp.runcmd( ['ssh', ssh_host, 'virt-install', '--connect qemu:///system', '--import', - '--name', vm_name, '--ram', '1024', '--vnc', '--noreboot', + '--name', vm_name, '--vnc', '--noreboot', + '--ram=%s' % ram_mebibytes, '--disk path=%s,bus=ide' % vm_path] + attach_opts) -- cgit v1.2.1 From 02a5188ba2a89af403919f8635e146de003dd05f Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 28 Mar 2013 13:44:15 +0000 Subject: Set RAM size in VirtualBox deployments --- virtualbox-ssh.write | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 862d4f02..37f56524 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -110,10 +110,12 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Create VirtualBox virtual machine') + ram_mebibytes = str(self.get_ram_size() / (1024**2)) + commands = [ ['createvm', '--name', vm_name, '--ostype', 'Linux26_64', '--register'], - ['modifyvm', vm_name, '--ioapic', 'on', '--memory', '1024', + ['modifyvm', vm_name, '--ioapic', 'on', '--memory', ram_mebibytes, '--nic1', 'nat'], ['storagectl', vm_name, '--name', '"SATA Controller"', '--add', 'sata', '--bootable', 'on', '--sataportcount', '2'], -- cgit v1.2.1 From 9c9eb1cdd185145b272e32ed7327c2c91011f4e5 Mon Sep 17 00:00:00 2001 From: Richard Holland Date: Wed, 3 Apr 2013 13:56:46 +0000 Subject: SSH Configuration Extension Added a configuration extension that copies SSH keys across to the deployed system. --- ssh.configure | 141 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 141 insertions(+) create mode 100755 ssh.configure diff --git a/ssh.configure b/ssh.configure new file mode 100755 index 00000000..8650b4f5 --- /dev/null +++ b/ssh.configure @@ -0,0 +1,141 @@ +#!/usr/bin/python +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''A Morph deployment configuration to copy SSH keys. + +Keys are copied from the host to the new system. +''' + +import cliapp +import os +import sys +import shutil +import glob +import logging + +import morphlib + +class SshConfigurationExtension(cliapp.Application): + + '''Copy over SSH keys to new system from host. + + The extension requires SSH_KEY_DIR to be set at the command line as it + will otherwise pass with only a status update. SSH_KEY_DIR should be + set to the location of the SSH keys to be passed to the new system. + + ''' + + def process_args(self, args): + if 'SSH_KEY_DIR' in os.environ: + # Copies ssh_host keys. + key = 'ssh_host_*_key' + mode = 0755 + dest = os.path.join(args[0], 'etc/ssh/') + sshhost, sshhostpub = self.find_keys(key) + if sshhost or sshhostpub: + self.check_dir(dest, mode) + self.copy_keys(sshhost, sshhostpub, dest) + + # Copies root keys. + key = 'root_*_key' + mode = 0700 + dest = os.path.join(args[0], 'root/.ssh/') + roothost, roothostpub = self.find_keys(key) + key = 'root_authorized_key_*.pub' + authkey, bleh = self.find_keys(key) + if roothost or roothostpub: + self.check_dir(dest, mode) + self.copy_rename_keys(roothost, + roothostpub, dest, 'id_', [15, 4]) + if authkey: + self.check_dir(dest, mode) + self.comb_auth_key(authkey, dest) + else: + self.status(msg="No SSH key directory found.") + pass + + def find_keys(self, key_name): + '''Uses glob to find public and + private SSH keys and returns their path''' + + src = os.path.join(os.environ['SSH_KEY_DIR'], key_name) + keys = glob.glob(src) + pubkeys = glob.glob(src + '.pub') + if not (keys or pubkeys): + self.status(msg="No SSH keys of pattern %(src)s found.", src=src) + return keys, pubkeys + + def check_dir(self, dest, mode): + '''Checks if destination directory exists + and creates it if necessary''' + + if os.path.exists(dest) == False: + self.status(msg="Creating SSH key directory: %(dest)s", dest=dest) + os.mkdir(dest) + os.chmod(dest, mode) + else: + pass + + def copy_keys(self, keys, pubkeys, dest): + '''Copies SSH keys to new VM''' + + for key in keys: + shutil.copy(key, dest) + os.chmod(dest, 0600) + for key in pubkeys: + shutil.copy(key, dest) + os.chmod(dest, 0644) + + def copy_rename_keys(self, keys, pubkeys, dest, new, snip): + '''Copies SSH keys to new VM and renames them''' + + st, fi = snip + for key in keys: + s = len(key) + nw_dst = os.path.join(dest, new + key[st:s-fi]) + shutil.copy(key, nw_dst) + os.chmod(nw_dst, 0600) + for key in pubkeys: + s = len(key) + nw_dst = os.path.join(dest, new + key[st:s-fi-4]) + shutil.copy(key, nw_dst + '.pub') + os.chmod(nw_dst, 0644) + + def comb_auth_key(self, keys, dest): + '''Combines authorized_keys file in new VM''' + + dest = os.path.join(dest, 'authorized_keys') + fout = open(dest, 'a') + for key in keys: + fin = open(key, 'r') + data = fin.read() + fout.write(data) + fin.close() + fout.close() + os.chmod(dest, 0600) + + def status(self, **kwargs): + '''Provide status output. + + The ``msg`` keyword argument is the actual message, + the rest are values for fields in the message as interpolated + by %. + + ''' + + self.output.write('%s\n' % (kwargs['msg'] % kwargs)) + +SshConfigurationExtension().run() -- cgit v1.2.1 From a936783aeba65f7a0219523edc435bd988ae4062 Mon Sep 17 00:00:00 2001 From: Richard Holland Date: Thu, 4 Apr 2013 17:09:36 +0000 Subject: Fixed error SSH configuration extension Fixed error in function copy_rename_key that tried to place key in non existent directory. Required use of os.path.basename to leave name of key and remove rest of path. --- ssh.configure | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/ssh.configure b/ssh.configure index 8650b4f5..75b46b11 100755 --- a/ssh.configure +++ b/ssh.configure @@ -59,7 +59,7 @@ class SshConfigurationExtension(cliapp.Application): if roothost or roothostpub: self.check_dir(dest, mode) self.copy_rename_keys(roothost, - roothostpub, dest, 'id_', [15, 4]) + roothostpub, dest, 'id_', [5, 4]) if authkey: self.check_dir(dest, mode) self.comb_auth_key(authkey, dest) @@ -101,16 +101,18 @@ class SshConfigurationExtension(cliapp.Application): def copy_rename_keys(self, keys, pubkeys, dest, new, snip): '''Copies SSH keys to new VM and renames them''' - + st, fi = snip for key in keys: - s = len(key) - nw_dst = os.path.join(dest, new + key[st:s-fi]) + base = os.path.basename(key) + s = len(base) + nw_dst = os.path.join(dest, new + base[st:s-fi]) shutil.copy(key, nw_dst) os.chmod(nw_dst, 0600) for key in pubkeys: - s = len(key) - nw_dst = os.path.join(dest, new + key[st:s-fi-4]) + base = os.path.basename(key) + s = len(base) + nw_dst = os.path.join(dest, new + base[st:s-fi-4]) shutil.copy(key, nw_dst + '.pub') os.chmod(nw_dst, 0644) -- cgit v1.2.1 From 12f67e6a172fe611a014d65b753d3a51a27c7298 Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Tue, 26 Mar 2013 12:08:51 +0000 Subject: Add nfsboot configuration extension This configuration-extension removes every network interface but the loopback interface from /etc/network/interfaces and stops /etc/fstab from mounting "/". It will only do this if the environment variable NFSBOOT_CONFIGURE is set on the `morph deploy` command-line. --- nfsboot.configure | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100755 nfsboot.configure diff --git a/nfsboot.configure b/nfsboot.configure new file mode 100755 index 00000000..8dc6c67c --- /dev/null +++ b/nfsboot.configure @@ -0,0 +1,32 @@ +#!/bin/sh +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +# Remove all networking interfaces and stop fstab from mounting '/' + + +set -e +if [ "$NFSBOOT_CONFIGURE" ]; then + # Remove all networking interfaces but loopback + cat > "$1/etc/network/interfaces" < "$1/etc/fstab" +fi -- cgit v1.2.1 From c995c45cf0ba039296153b4e6f463fa6e538ff14 Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Tue, 26 Mar 2013 14:11:34 +0000 Subject: Add nfsboot write extension The 'location' command-line argument refers to the hostname of the 'nfsboot server', a baserock system that has an nfs server, the directory '/srv/nfsboot/nfs', and a tftp server that hosts files from the directory '/srv/nfsboot/tftp'. The write extension will read the hostname of the target system and copy its root filesystem to /srv/nfsboot/nfs/ and its kernel to /srv/nfsboot/tftp/. It will then configure the nfs server to export that nfs root. --- nfsboot.write | 161 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 161 insertions(+) create mode 100755 nfsboot.write diff --git a/nfsboot.write b/nfsboot.write new file mode 100755 index 00000000..60b4d00d --- /dev/null +++ b/nfsboot.write @@ -0,0 +1,161 @@ +#!/usr/bin/python +# Copyright (C) 2012-2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +'''A Morph deployment write extension for deploying to an nfsboot server + +An nfsboot server is defined as a baserock system that has tftp and nfs +servers running, the tftp server is exporting the contents of +/srv/nfsboot/tftp/ and the user has sufficient permissions to create nfs roots +in /srv/nfsboot/nfs/ + +''' + + +import cliapp +import os +import glob + +import morphlib.writeexts + + +class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): + + '''Create an NFS root and kernel on TFTP during Morph's deployment. + + The location command line argument is the hostname of the nfsboot server. + The user is expected to provide the location argument + using the following syntax: + + HOST + + where: + + * HOST is the host of the nfsboot server + + The extension will connect to root@HOST via ssh to copy the kernel and + rootfs, and configure the nfs server. + + It requires root because it uses systemd, and reads/writes to /etc. + + ''' + + def process_args(self, args): + if len(args) != 2: + raise cliapp.AppException('Wrong number of command line args') + + temp_root, location = args + hostname = self.get_hostname(temp_root) + if hostname == 'baserock': + raise cliapp.AppException('It is forbidden to nfsboot a system ' + 'with hostname "baserock"') + + self.test_good_server(location) + self.copy_kernel(temp_root, location, hostname) + self.copy_rootfs(temp_root, location, hostname) + self.configure_nfs(location, hostname) + + def get_hostname(self, temp_root): + hostnamepath = os.path.join(temp_root, 'etc', 'hostname') + with open(hostnamepath) as f: + return f.readline().strip() + + def copy_kernel(self, temp_root, location, hostname): + bootdir = os.path.join(temp_root, 'boot') + image_names = ['vmlinuz', 'zImage', 'uImage'] + for name in image_names: + try_path = os.path.join(bootdir, name) + if os.path.exists(try_path): + kernel_src = try_path + break + else: + raise cliapp.AppException( + 'Could not find a kernel in the system: none of ' + '%s found' % ', '.join(image_names)) + kernel_dest = os.path.join('/srv/nfsboot/tftp', hostname) + rsync_dest = 'root@%s:%s' % (location, kernel_dest) + cliapp.runcmd( + ['rsync', kernel_src, rsync_dest]) + + def copy_rootfs(self, temp_root, location, hostname): + rootfs_src = temp_root + '/' + rootfs_dest = os.path.join('/srv/nfsboot/nfs', hostname) + rsync_dest = 'root@%s:%s' % (location, rootfs_dest) + cliapp.runcmd( + ['rsync', '-a', rootfs_src, rsync_dest]) + + def configure_nfs(self, location, hostname): + rootfs_dest = os.path.join('/srv/nfsboot/nfs', hostname) + exports_path = '/etc/exports' + # If that path is not already exported: + try: + cliapp.ssh_runcmd( + 'root@%s' % location, ['grep', '-q', rootfs_dest, + exports_path]) + except cliapp.AppException: + ip_mask = '*' + options = 'rw,no_subtree_check,no_root_squash,async' + exports_string = '%s %s(%s)\n' % (rootfs_dest, ip_mask, options) + exports_append_sh = '''\ +set -eu +target = "$1" +temp = $(mktemp) +cat "$target" > "$temp" +cat >> "$temp" +mv "$temp" "$target" +''' + cliapp.ssh_runcmd( + 'root@%s' % location, + ['sh', '-c', exports_append_sh, '--', exports_path], + feed_stdin=exports_string) + cliapp.ssh_runcmd( + 'root@%s' % location, ['systemctl', 'reload', + 'nfs-server.service']) + + def test_good_server(self, server): + # Can be ssh'ed into + try: + cliapp.ssh_runcmd('root@%s' % server, ['true']) + except cliapp.AppException: + raise cliapp.AppException('You are unable to ssh into server %s' + % server) + + # Is an NFS server + try: + cliapp.ssh_runcmd( + 'root@%s' % server, ['test', '-e', '/etc/exports']) + except cliapp.AppException: + raise cliapp.AppException('server %s is not an nfs server' + % server) + try: + cliapp.ssh_runcmd( + 'root@%s' % server, ['systemctl', 'is-enabled', + 'nfs-server.service']) + + except cliapp.AppException: + raise cliapp.AppException('server %s does not control its ' + 'nfs server by systemd' % server) + + # TFTP server exports /srv/nfsboot/tftp + try: + cliapp.ssh_runcmd( + 'root@%s' % server, ['test' , '-d', '/srv/nfsboot/tftp']) + except cliapp.AppException: + raise cliapp.AppException('server %s does not export ' + '/srv/nfsboot/tftp' % server) + +NFSBootWriteExtension().run() + -- cgit v1.2.1 From 6255080a8227cccbbf892332b3f83b4c52763897 Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Mon, 8 Apr 2013 16:49:57 +0000 Subject: Fix nfsboot write-extension Reviewed-by: Lars Wirzenius --- nfsboot.write | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nfsboot.write b/nfsboot.write index 60b4d00d..293b1acf 100755 --- a/nfsboot.write +++ b/nfsboot.write @@ -111,8 +111,8 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): exports_string = '%s %s(%s)\n' % (rootfs_dest, ip_mask, options) exports_append_sh = '''\ set -eu -target = "$1" -temp = $(mktemp) +target="$1" +temp=$(mktemp) cat "$target" > "$temp" cat >> "$temp" mv "$temp" "$target" @@ -122,7 +122,7 @@ mv "$temp" "$target" ['sh', '-c', exports_append_sh, '--', exports_path], feed_stdin=exports_string) cliapp.ssh_runcmd( - 'root@%s' % location, ['systemctl', 'reload', + 'root@%s' % location, ['systemctl', 'restart', 'nfs-server.service']) def test_good_server(self, server): -- cgit v1.2.1 From e8c0f666f4407ed24ca20ba90adf809f45a02e45 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 12 Apr 2013 13:04:54 +0000 Subject: Fix copyright year We only have written this this year, so it should be copyright this year only. --- nfsboot.write | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nfsboot.write b/nfsboot.write index 293b1acf..34200793 100755 --- a/nfsboot.write +++ b/nfsboot.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -- cgit v1.2.1 From e1ceaaf5547d4ea030fbe0dee5f46aab67c974de Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 8 May 2013 13:45:31 +0100 Subject: Fix unfortunate typo in ssh configuration extension This tiny typo unfortunately makes root unable to use ssh, since ssh refuses to authenticate if your private key is globally readable, and the typo causes the private key to be given the public key's permissions. --- ssh.configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ssh.configure b/ssh.configure index 75b46b11..6012f298 100755 --- a/ssh.configure +++ b/ssh.configure @@ -114,7 +114,7 @@ class SshConfigurationExtension(cliapp.Application): s = len(base) nw_dst = os.path.join(dest, new + base[st:s-fi-4]) shutil.copy(key, nw_dst + '.pub') - os.chmod(nw_dst, 0644) + os.chmod(nw_dst + '.pub', 0644) def comb_auth_key(self, keys, dest): '''Combines authorized_keys file in new VM''' -- cgit v1.2.1 From 937544ac7a370f433eda4240148a33c64ff43c56 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 8 May 2013 17:04:45 +0100 Subject: Add AUTOSTART to kvm and libvirt write extensions If AUTOSTART is 'yes' then the VM will be started once it is created. If it is 'no' or undefined, then it will need to be manually started. If it is any other value, then an exception is raised. --- kvm.write | 19 ++++++++++--------- virtualbox-ssh.write | 9 +++++++-- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/kvm.write b/kvm.write index 630f6ae7..e2f7435c 100755 --- a/kvm.write +++ b/kvm.write @@ -56,6 +56,7 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): temp_root, location = args ssh_host, vm_name, vm_path = self.parse_location(location) + autostart = self.parse_autostart() fd, raw_disk = tempfile.mkstemp() os.close(fd) @@ -63,7 +64,7 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): try: self.transfer(raw_disk, ssh_host, vm_path) - self.create_libvirt_guest(ssh_host, vm_name, vm_path) + self.create_libvirt_guest(ssh_host, vm_name, vm_path, autostart) except BaseException: sys.stderr.write('Error deploying to libvirt') os.remove(raw_disk) @@ -95,7 +96,7 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): with open(raw_disk, 'rb') as f: cliapp.runcmd(['rsync', '-zS', raw_disk, target]) - def create_libvirt_guest(self, ssh_host, vm_name, vm_path): + def create_libvirt_guest(self, ssh_host, vm_name, vm_path, autostart): '''Create the libvirt virtual machine.''' self.status(msg='Creating libvirt/kvm virtual machine') @@ -107,13 +108,13 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): ram_mebibytes = str(self.get_ram_size() / (1024**2)) - cliapp.runcmd( - ['ssh', ssh_host, - 'virt-install', '--connect qemu:///system', '--import', - '--name', vm_name, '--vnc', '--noreboot', - '--ram=%s' % ram_mebibytes, - '--disk path=%s,bus=ide' % vm_path] + - attach_opts) + cmdline = ['ssh', ssh_host, + 'virt-install', '--connect qemu:///system', '--import', + '--name', vm_name, '--vnc', '--ram=%s' % ram_mebibytes, + '--disk path=%s,bus=ide' % vm_path] + attach_opts + if not autostart: + cmdline += '--noreboot' + cliapp.runcmd(cmdline) KvmPlusSshWriteExtension().run() diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 37f56524..cb17b69b 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -62,6 +62,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): temp_root, location = args ssh_host, vm_name, vdi_path = self.parse_location(location) + autostart = self.parse_autostart() fd, raw_disk = tempfile.mkstemp() os.close(fd) @@ -70,7 +71,8 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): try: self.transfer_and_convert_to_vdi( raw_disk, ssh_host, vdi_path) - self.create_virtualbox_guest(ssh_host, vm_name, vdi_path) + self.create_virtualbox_guest(ssh_host, vm_name, vdi_path, + autostart) except BaseException: sys.stderr.write('Error deploying to VirtualBox') os.remove(raw_disk) @@ -105,7 +107,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): str(os.path.getsize(raw_disk))], stdin=f) - def create_virtualbox_guest(self, ssh_host, vm_name, vdi_path): + def create_virtualbox_guest(self, ssh_host, vm_name, vdi_path, autostart): '''Create the VirtualBox virtual machine.''' self.status(msg='Create VirtualBox virtual machine') @@ -134,6 +136,9 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): '--medium', disk] commands.append(cmd) + if autostart: + commands.append(['startvm', vm_name]) + for command in commands: argv = ['ssh', ssh_host, 'VBoxManage'] + command cliapp.runcmd(argv) -- cgit v1.2.1 From 45258312f5e0f1a72d9440bb688ed2863200d980 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Thu, 9 May 2013 14:55:35 +0000 Subject: Allow to add public ssh keys to known hosts file --- ssh.configure | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/ssh.configure b/ssh.configure index 6012f298..7a7e1667 100755 --- a/ssh.configure +++ b/ssh.configure @@ -24,6 +24,7 @@ import os import sys import shutil import glob +import re import logging import morphlib @@ -63,6 +64,20 @@ class SshConfigurationExtension(cliapp.Application): if authkey: self.check_dir(dest, mode) self.comb_auth_key(authkey, dest) + + # Fills the known_hosts file + key = 'root_known_host_*_key.pub' + src = os.path.join(os.environ['SSH_KEY_DIR'], key) + known_hosts_keys = glob.glob(src) + known_hosts_path = os.path.join(args[0], 'root/.ssh/known_hosts') + known_hosts_file = open(known_hosts_path, "a") + for filename in known_hosts_keys: + hostname = re.search('root_known_host_(.+?)_key.pub', filename).group(1) + known_hosts_file.write(hostname + " ") + f = open(filename, "r") + known_hosts_file.write(f.read()) + f.close() + known_hosts_file.close() else: self.status(msg="No SSH key directory found.") pass -- cgit v1.2.1 From bd0e81a9b5e232124bf4c83e7125cb6fa4c00147 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Thu, 9 May 2013 15:25:24 +0000 Subject: Merge branch 'support-known-hosts' of git://git.baserock.org/baserock/baserock/morph A column width error was fixed up in the merge. --- ssh.configure | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ssh.configure b/ssh.configure index 7a7e1667..29a3589a 100755 --- a/ssh.configure +++ b/ssh.configure @@ -72,7 +72,8 @@ class SshConfigurationExtension(cliapp.Application): known_hosts_path = os.path.join(args[0], 'root/.ssh/known_hosts') known_hosts_file = open(known_hosts_path, "a") for filename in known_hosts_keys: - hostname = re.search('root_known_host_(.+?)_key.pub', filename).group(1) + hostname = re.search('root_known_host_(.+?)_key.pub', + filename).group(1) known_hosts_file.write(hostname + " ") f = open(filename, "r") known_hosts_file.write(f.read()) -- cgit v1.2.1 From 38a4ce37b1785a200f2bf141fb41b7bb0fde4512 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Thu, 9 May 2013 18:02:06 +0000 Subject: Improve the ssh configuration file. - Don't fail if the root/.ssh directory does not exist when adding entries to the known_hosts file - Use shutil.copyfilobj to copy the file contents - Use the python with statatement when opening files --- ssh.configure | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/ssh.configure b/ssh.configure index 29a3589a..55fbbb30 100755 --- a/ssh.configure +++ b/ssh.configure @@ -69,16 +69,17 @@ class SshConfigurationExtension(cliapp.Application): key = 'root_known_host_*_key.pub' src = os.path.join(os.environ['SSH_KEY_DIR'], key) known_hosts_keys = glob.glob(src) - known_hosts_path = os.path.join(args[0], 'root/.ssh/known_hosts') - known_hosts_file = open(known_hosts_path, "a") - for filename in known_hosts_keys: - hostname = re.search('root_known_host_(.+?)_key.pub', - filename).group(1) - known_hosts_file.write(hostname + " ") - f = open(filename, "r") - known_hosts_file.write(f.read()) - f.close() - known_hosts_file.close() + if known_hosts_keys: + self.check_dir(dest, mode) + known_hosts_path = os.path.join(dest, 'known_hosts') + with open(known_hosts_path, "a") as known_hosts_file: + for filename in known_hosts_keys: + hostname = re.search('root_known_host_(.+?)_key.pub', + filename).group(1) + known_hosts_file.write(hostname + " ") + with open(filename, "r") as f: + shutil.copyfileobj(f, known_hosts_file) + else: self.status(msg="No SSH key directory found.") pass -- cgit v1.2.1 From fb9d20ec4aa279928a692bff5961d413334e3be7 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Thu, 9 May 2013 23:04:48 +0000 Subject: Fix permissions on the ssh configure extension --- ssh.configure | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ssh.configure b/ssh.configure index 55fbbb30..2f3167e7 100755 --- a/ssh.configure +++ b/ssh.configure @@ -111,10 +111,12 @@ class SshConfigurationExtension(cliapp.Application): for key in keys: shutil.copy(key, dest) - os.chmod(dest, 0600) + path = os.path.join(dest, os.path.basename(key)) + os.chmod(path, 0600) for key in pubkeys: shutil.copy(key, dest) - os.chmod(dest, 0644) + path = os.path.join(dest, os.path.basename(key)) + os.chmod(path, 0644) def copy_rename_keys(self, keys, pubkeys, dest, new, snip): '''Copies SSH keys to new VM and renames them''' -- cgit v1.2.1 From c0d4db83c412f8d688b4495a0d2b4bc2342e527b Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Fri, 10 May 2013 04:50:21 +0000 Subject: Add a new configuration extension to drop configuration files to the image /etc --- drop-config-files.configure | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100755 drop-config-files.configure diff --git a/drop-config-files.configure b/drop-config-files.configure new file mode 100755 index 00000000..0094cf6b --- /dev/null +++ b/drop-config-files.configure @@ -0,0 +1,27 @@ +#!/bin/sh +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +# Copy all files located in $SRC_CONFIG_DIR to the image /etc. + + +set -e + +if [ "x${SRC_CONFIG_DIR}" != x ] +then + cp -r "$SRC_CONFIG_DIR"/* "$1/etc/" +fi + -- cgit v1.2.1 From aacfad8da27cf275b9c11d7220f05d21a28d55ad Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Fri, 10 May 2013 09:08:59 +0000 Subject: Rename the drop-config-files extension to add-config-files --- add-config-files.configure | 27 +++++++++++++++++++++++++++ drop-config-files.configure | 27 --------------------------- 2 files changed, 27 insertions(+), 27 deletions(-) create mode 100755 add-config-files.configure delete mode 100755 drop-config-files.configure diff --git a/add-config-files.configure b/add-config-files.configure new file mode 100755 index 00000000..0094cf6b --- /dev/null +++ b/add-config-files.configure @@ -0,0 +1,27 @@ +#!/bin/sh +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +# Copy all files located in $SRC_CONFIG_DIR to the image /etc. + + +set -e + +if [ "x${SRC_CONFIG_DIR}" != x ] +then + cp -r "$SRC_CONFIG_DIR"/* "$1/etc/" +fi + diff --git a/drop-config-files.configure b/drop-config-files.configure deleted file mode 100755 index 0094cf6b..00000000 --- a/drop-config-files.configure +++ /dev/null @@ -1,27 +0,0 @@ -#!/bin/sh -# Copyright (C) 2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -# Copy all files located in $SRC_CONFIG_DIR to the image /etc. - - -set -e - -if [ "x${SRC_CONFIG_DIR}" != x ] -then - cp -r "$SRC_CONFIG_DIR"/* "$1/etc/" -fi - -- cgit v1.2.1 From bbeec556514f7ccf58d38688f39da2b747010e45 Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Mon, 13 May 2013 10:32:45 +0000 Subject: Make rsync delete files that shouldn't exist --- nfsboot.write | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nfsboot.write b/nfsboot.write index 34200793..e18ff798 100755 --- a/nfsboot.write +++ b/nfsboot.write @@ -95,7 +95,7 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): rootfs_dest = os.path.join('/srv/nfsboot/nfs', hostname) rsync_dest = 'root@%s:%s' % (location, rootfs_dest) cliapp.runcmd( - ['rsync', '-a', rootfs_src, rsync_dest]) + ['rsync', '-aXSPH', '--delete', rootfs_src, rsync_dest]) def configure_nfs(self, location, hostname): rootfs_dest = os.path.join('/srv/nfsboot/nfs', hostname) -- cgit v1.2.1 From 2ad0b7db5010c89836be5fae17b138d82c4a033a Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Mon, 13 May 2013 14:31:43 +0000 Subject: Make nfsboot use the new hierarchy --- nfsboot.write | 85 +++++++++++++++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 74 insertions(+), 11 deletions(-) diff --git a/nfsboot.write b/nfsboot.write index e18ff798..e2ce7db2 100755 --- a/nfsboot.write +++ b/nfsboot.write @@ -53,6 +53,8 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): ''' + _nfsboot_root = '/srv/nfsboot' + def process_args(self, args): if len(args) != 2: raise cliapp.AppException('Wrong number of command line args') @@ -64,8 +66,12 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): 'with hostname "baserock"') self.test_good_server(location) - self.copy_kernel(temp_root, location, hostname) - self.copy_rootfs(temp_root, location, hostname) + version = 'version1' + versioned_root = os.path.join(self._nfsboot_root, hostname, 'systems', + version) + self.copy_rootfs(temp_root, location, versioned_root, hostname) + self.copy_kernel(temp_root, location, versioned_root, version, + hostname) self.configure_nfs(location, hostname) def get_hostname(self, temp_root): @@ -73,7 +79,8 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): with open(hostnamepath) as f: return f.readline().strip() - def copy_kernel(self, temp_root, location, hostname): + def copy_kernel(self, temp_root, location, versioned_root, version, + hostname): bootdir = os.path.join(temp_root, 'boot') image_names = ['vmlinuz', 'zImage', 'uImage'] for name in image_names: @@ -85,30 +92,86 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): raise cliapp.AppException( 'Could not find a kernel in the system: none of ' '%s found' % ', '.join(image_names)) - kernel_dest = os.path.join('/srv/nfsboot/tftp', hostname) + + kernel_dest = os.path.join(versioned_root, 'orig', 'kernel') rsync_dest = 'root@%s:%s' % (location, kernel_dest) + self.status(msg='Copying kernel') cliapp.runcmd( ['rsync', kernel_src, rsync_dest]) - def copy_rootfs(self, temp_root, location, hostname): + # Link the kernel to the right place + self.status(msg='Creating links to kernel in tftp directory') + tftp_dir = os.path.join(self._nfsboot_root , 'tftp') + versioned_kernel_name = "%s-%s" % (hostname, version) + kernel_name = hostname + try: + cliapp.ssh_runcmd('root@%s' % location, + ['ln', '-f', kernel_dest, + os.path.join(tftp_dir, versioned_kernel_name)]) + + cliapp.ssh_runcmd('root@%s' % location, + ['ln', '-sf', versioned_kernel_name, + os.path.join(tftp_dir, kernel_name)]) + except cliapp.AppException: + raise cliapp.AppException('Could not create symlinks to the ' + 'kernel at %s in %s on %s' + % (kernel_dest, tftp_dir, location)) + + def copy_rootfs(self, temp_root, location, versioned_root, hostname): rootfs_src = temp_root + '/' - rootfs_dest = os.path.join('/srv/nfsboot/nfs', hostname) - rsync_dest = 'root@%s:%s' % (location, rootfs_dest) + orig_path = os.path.join(versioned_root, 'orig') + run_path = os.path.join(versioned_root, 'run') + + self.status(msg='Creating destination directories') + try: + cliapp.ssh_runcmd('root@%s' % location, + ['mkdir', '-p', orig_path, run_path]) + except cliapp.AppException: + raise cliapp.AppException('Could not create dirs %s and %s on %s' + % (orig_path, run_path, location)) + + self.status(msg='Creating \'orig\' rootfs') cliapp.runcmd( - ['rsync', '-aXSPH', '--delete', rootfs_src, rsync_dest]) + ['rsync', '-aXSPH', '--delete', rootfs_src, + 'root@%s:%s' % (location, orig_path)]) + + self.status(msg='Creating \'run\' rootfs') + try: + cliapp.ssh_runcmd('root@%s' % location, + ['rm', '-rf', run_path]) + cliapp.ssh_runcmd('root@%s' % location, + ['cp', '-al', orig_path, run_path]) + cliapp.ssh_runcmd('root@%s' % location, + ['rm', '-rf', os.path.join(run_path, 'etc')]) + cliapp.ssh_runcmd('root@%s' % location, + ['cp', '-a', os.path.join(orig_path, 'etc'), + os.path.join(run_path, 'etc')]) + except cliapp.AppException: + raise cliapp.AppException('Could not create \'run\' rootfs' + ' from \'orig\'') + + self.status(msg='Linking \'default-run\' to latest system') + try: + cliapp.ssh_runcmd('root@%s' % location, + ['ln', '-sfn', run_path, + os.path.join(self._nfsboot_root, hostname, 'systems', + 'default-run')]) + except cliapp.AppException: + raise cliapp.AppException('Could not link \'default-run\' to %s' + % run_path) def configure_nfs(self, location, hostname): - rootfs_dest = os.path.join('/srv/nfsboot/nfs', hostname) + exported_path = os.path.join(self._nfsboot_root, hostname) exports_path = '/etc/exports' # If that path is not already exported: try: cliapp.ssh_runcmd( - 'root@%s' % location, ['grep', '-q', rootfs_dest, + 'root@%s' % location, ['grep', '-q', exported_path, exports_path]) except cliapp.AppException: ip_mask = '*' options = 'rw,no_subtree_check,no_root_squash,async' - exports_string = '%s %s(%s)\n' % (rootfs_dest, ip_mask, options) + exports_string = '%s %s(%s)\n' % (exported_path, ip_mask, options) exports_append_sh = '''\ set -eu target="$1" -- cgit v1.2.1 From bfcf3a24c78de00293c3ca59dabf8d3c31711fd8 Mon Sep 17 00:00:00 2001 From: "Daniel Silverstone (br7vm)" Date: Fri, 17 May 2013 12:55:04 +0000 Subject: EXTS: simple-network.configure Simple networking configuration extension, taking NETWORK_CONFIG from the environment and writing /etc/network/interfaces out during deployment configuration. --- simple-network.configure | 143 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 143 insertions(+) create mode 100755 simple-network.configure diff --git a/simple-network.configure b/simple-network.configure new file mode 100755 index 00000000..b98b202c --- /dev/null +++ b/simple-network.configure @@ -0,0 +1,143 @@ +#!/usr/bin/python +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''A Morph deployment configuration extension to handle /etc/network/interfaces + +This extension prepares /etc/network/interfaces with the interfaces specified +during deployment. + +If no network configuration is provided, eth0 will be configured for DHCP +with the hostname of the system. +''' + + +import os +import sys +import cliapp + +import morphlib + + +class SimpleNetworkError(morphlib.Error): + '''Errors associated with simple network setup''' + pass + + +class SimpleNetworkConfigurationExtension(cliapp.Application): + '''Configure /etc/network/interfaces + + Reading NETWORK_CONFIG, this extension sets up /etc/network/interfaces. + ''' + + def process_args(self, args): + network_config = os.environ.get( + "NETWORK_CONFIG", "lo:loopback;eth0:dhcp,hostname=$(hostname)") + + self.status(msg="Processing NETWORK_CONFIG=%(nc)s", nc=network_config) + + stanzas = self.parse_network_stanzas(network_config) + iface_file = self.generate_iface_file(stanzas) + + with open(os.path.join(args[0], "etc/network/interfaces"), "w") as f: + f.write(iface_file) + + def generate_iface_file(self, stanzas): + """Generate an interfaces file from the provided stanzas. + + The interfaces will be sorted by name, with loopback sorted first. + """ + + def cmp_iface_names(a, b): + a = a['name'] + b = b['name'] + if a == "lo": + return -1 + elif b == "lo": + return 1 + else: + return cmp(a,b) + + return "\n".join(self.generate_iface_stanza(stanza) + for stanza in sorted(stanzas, cmp=cmp_iface_names)) + + def generate_iface_stanza(self, stanza): + """Generate an interfaces stanza from the provided data.""" + + name = stanza['name'] + itype = stanza['type'] + lines = ["auto %s" % name, "iface %s inet %s" % (name, itype)] + lines += [" %s %s" % elem for elem in stanza['args'].items()] + lines += [""] + return "\n".join(lines) + + + def parse_network_stanzas(self, config): + """Parse a network config environment variable into stanzas. + + Network config stanzas are semi-colon separated. + """ + + return [self.parse_network_stanza(s) for s in config.split(";")] + + def parse_network_stanza(self, stanza): + """Parse a network config stanza into name, type and arguments. + + Each stanza is of the form name:type[,arg=value]... + + For example: + lo:loopback + eth0:dhcp + eth1:static,address=10.0.0.1,netmask=255.255.0.0 + """ + elements = stanza.split(",") + lead = elements.pop(0).split(":") + if len(lead) != 2: + raise SimpleNetworkError("Stanza '%s' is missing its type" % + stanza) + iface = lead[0] + iface_type = lead[1] + + if iface_type not in ['loopback', 'static', 'dhcp']: + raise SimpleNetworkError("Stanza '%s' has unknown interface type" + " '%s'" % (stanza, iface_type)) + + argpairs = [element.split("=", 1) for element in elements] + output_stanza = { "name": iface, + "type": iface_type, + "args": {} } + for argpair in argpairs: + if len(argpair) != 2: + raise SimpleNetworkError("Stanza '%s' has bad argument '%r'" + % (stanza, argpair.pop(0))) + if argpair[0] in output_stanza["args"]: + raise SimpleNetworkError("Stanza '%s' has repeated argument" + " %s" % (stanza, argpair[0])) + output_stanza["args"][argpair[0]] = argpair[1] + + return output_stanza + + def status(self, **kwargs): + '''Provide status output. + + The ``msg`` keyword argument is the actual message, + the rest are values for fields in the message as interpolated + by %. + + ''' + + self.output.write('%s\n' % (kwargs['msg'] % kwargs)) + +SimpleNetworkConfigurationExtension().run() -- cgit v1.2.1 From 7e0decbb40fcb779f8e4c4b0fd3718022f3cdd88 Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Thu, 16 May 2013 15:14:14 +0000 Subject: Can define version by the environment variable 'VERSION' If the version already exists on the device, deployment is aborted --- nfsboot.write | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/nfsboot.write b/nfsboot.write index e2ce7db2..cfc3fc32 100755 --- a/nfsboot.write +++ b/nfsboot.write @@ -66,14 +66,27 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): 'with hostname "baserock"') self.test_good_server(location) - version = 'version1' + version = os.environ['VERSION'] or 'version1' versioned_root = os.path.join(self._nfsboot_root, hostname, 'systems', version) + if self.version_exists(versioned_root, location): + raise cliapp.AppException('Version %s already exists on' + ' this device. Deployment aborted' + % version) self.copy_rootfs(temp_root, location, versioned_root, hostname) self.copy_kernel(temp_root, location, versioned_root, version, hostname) self.configure_nfs(location, hostname) + def version_exists(self, versioned_root, location): + try: + cliapp.ssh_runcmd('root@%s' % location, + ['test', '-d', versioned_root]) + except cliapp.AppException: + return False + + return True + def get_hostname(self, temp_root): hostnamepath = os.path.join(temp_root, 'etc', 'hostname') with open(hostnamepath) as f: -- cgit v1.2.1 From f52890784d6bbcd5f636badc9bc6b721a5417afc Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Mon, 20 May 2013 16:26:51 +0000 Subject: nfsboot deployment creates local state dirs --- nfsboot.write | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/nfsboot.write b/nfsboot.write index cfc3fc32..61c5306a 100755 --- a/nfsboot.write +++ b/nfsboot.write @@ -92,6 +92,14 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): with open(hostnamepath) as f: return f.readline().strip() + def create_local_state(self, location, hostname): + statedir = os.path.join(self._nfsboot_root, hostname, 'state') + subdirs = [os.path.join(statedir, 'home'), + os.path.join(statedir, 'opt'), + os.path.join(statedir, 'srv')] + cliapp.ssh_runcmd('root@%s' % location, + ['mkdir', '-p'] + subdirs) + def copy_kernel(self, temp_root, location, versioned_root, version, hostname): bootdir = os.path.join(temp_root, 'boot') -- cgit v1.2.1 From 1706b1074fbf01d1cd2bbd13bbb58eb5c28f4718 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Mon, 20 May 2013 11:20:28 +0000 Subject: Implement raw disk upgrades. We perform this by cloning an existing orig directory and then updating the contents using rsync --- rawdisk.write | 54 ++++++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 50 insertions(+), 4 deletions(-) diff --git a/rawdisk.write b/rawdisk.write index a55473f2..76438a5e 100755 --- a/rawdisk.write +++ b/rawdisk.write @@ -18,6 +18,7 @@ '''A Morph deployment write extension for raw disk images.''' +import cliapp import os import sys import time @@ -30,8 +31,10 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): '''Create a raw disk image during Morph's deployment. + If the image already exists, it is upgraded. + The location command line argument is the pathname of the disk image - to be created. + to be created/upgraded. ''' @@ -40,9 +43,52 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): raise cliapp.AppException('Wrong number of command line args') temp_root, location = args - - self.create_local_system(temp_root, location) - self.status(msg='Disk image has been created at %s' % location) + if os.path.isfile(location): + self.upgrade_local_system(location, temp_root) + else: + self.create_local_system(temp_root, location) + self.status(msg='Disk image has been created at %s' % location) + + def upgrade_local_system(self, raw_disk, temp_root): + mp = self.mount(raw_disk) + + version_label = self.get_version_label(mp) + self.status(msg='Updating image to a new version with label %s' % + version_label) + + version_root = os.path.join(mp, 'systems', version_label) + os.mkdir(version_root) + + old_orig = os.path.join(mp, 'systems', 'version1', 'orig') + new_orig = os.path.join(version_root, 'orig') + cliapp.runcmd( + ['btrfs', 'subvolume', 'snapshot', old_orig, new_orig]) + + cliapp.runcmd( + ['rsync', '-a', '--checksum', '--numeric-ids', '--delete', + temp_root + os.path.sep, new_orig]) + + self.create_run(version_root) + + if self.bootloader_is_wanted(): + self.install_kernel(version_root, temp_root) + self.install_extlinux(mp, version_label) + + self.unmount(mp) + + def get_version_label(self, mp): + version_label = os.environ.get('VERSION_LABEL') + + if version_label is None: + self.unmount(mp) + raise cliapp.AppException('VERSION_LABEL was not given') + + if os.path.exists(os.path.join(mp, 'systems', version_label)): + self.unmount(mp) + raise cliapp.AppException('VERSION_LABEL %s already exists' + % version_label) + + return version_label RawDiskWriteExtension().run() -- cgit v1.2.1 From 2672ffe119deff9faafc7382842574297d30e497 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Thu, 23 May 2013 15:20:52 +0000 Subject: Use the name factory for the first system version. --- nfsboot.write | 2 +- rawdisk.write | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nfsboot.write b/nfsboot.write index e2ce7db2..cf71301b 100755 --- a/nfsboot.write +++ b/nfsboot.write @@ -66,7 +66,7 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): 'with hostname "baserock"') self.test_good_server(location) - version = 'version1' + version = 'factory' versioned_root = os.path.join(self._nfsboot_root, hostname, 'systems', version) self.copy_rootfs(temp_root, location, versioned_root, hostname) diff --git a/rawdisk.write b/rawdisk.write index 76438a5e..a43a9cce 100755 --- a/rawdisk.write +++ b/rawdisk.write @@ -59,7 +59,7 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): version_root = os.path.join(mp, 'systems', version_label) os.mkdir(version_root) - old_orig = os.path.join(mp, 'systems', 'version1', 'orig') + old_orig = os.path.join(mp, 'systems', 'factory', 'orig') new_orig = os.path.join(version_root, 'orig') cliapp.runcmd( ['btrfs', 'subvolume', 'snapshot', old_orig, new_orig]) -- cgit v1.2.1 From ee54044ddaed11bd83161e98854e4411ebb91cde Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Fri, 24 May 2013 17:02:04 +0000 Subject: Fix kvm working without autostart --- kvm.write | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kvm.write b/kvm.write index e2f7435c..ae287fe5 100755 --- a/kvm.write +++ b/kvm.write @@ -113,7 +113,7 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): '--name', vm_name, '--vnc', '--ram=%s' % ram_mebibytes, '--disk path=%s,bus=ide' % vm_path] + attach_opts if not autostart: - cmdline += '--noreboot' + cmdline += ['--noreboot'] cliapp.runcmd(cmdline) -- cgit v1.2.1 From 9d911d9f79e98fa2059724352abd18f882f99387 Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Fri, 24 May 2013 17:02:04 +0000 Subject: Fix kvm working without autostart --- kvm.write | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kvm.write b/kvm.write index e2f7435c..ae287fe5 100755 --- a/kvm.write +++ b/kvm.write @@ -113,7 +113,7 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): '--name', vm_name, '--vnc', '--ram=%s' % ram_mebibytes, '--disk path=%s,bus=ide' % vm_path] + attach_opts if not autostart: - cmdline += '--noreboot' + cmdline += ['--noreboot'] cliapp.runcmd(cmdline) -- cgit v1.2.1 From 6bd0b52aa907a27c355b2ab00a151757b9bb24fc Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Fri, 24 May 2013 15:44:57 +0000 Subject: Add ssh-rsync write extension This is used to perform upgrades on running baserock systems. It requires rsync on the target system --- ssh-rsync.write | 181 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100755 ssh-rsync.write diff --git a/ssh-rsync.write b/ssh-rsync.write new file mode 100755 index 00000000..6fe1153d --- /dev/null +++ b/ssh-rsync.write @@ -0,0 +1,181 @@ +#!/usr/bin/python +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +'''A Morph deployment write extension for upgrading systems over ssh.''' + + +import cliapp +import os +import sys +import time +import tempfile + +import morphlib.writeexts + +class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): + + '''Upgrade a running baserock system with ssh and rsync. + + It assumes the system is baserock-based and has a btrfs partition. + + The location command line argument is the 'user@hostname' string + that will be passed to ssh and rsync + + ''' + + def process_args(self, args): + if len(args) != 2: + raise cliapp.AppException('Wrong number of command line args') + + temp_root, location = args + + self.check_valid_target(location) + self.upgrade_remote_system(location, temp_root) + + def upgrade_remote_system(self, location, temp_root): + root_disk = self.find_root_disk(location) + version_label = os.environ.get('VERSION_LABEL') + + try: + self.status(msg='Creating remote mount point') + remote_mnt = cliapp.ssh_runcmd(location, ['mktemp', '-d']).strip() + + self.status(msg='Mounting root disk') + cliapp.ssh_runcmd(location, ['mount', root_disk, remote_mnt]) + + version_root = os.path.join(remote_mnt, 'systems', version_label) + run_dir = os.path.join(version_root, 'run') + orig_dir = os.path.join(version_root, 'orig') + try: + self.status(msg='Creating %s' % version_root) + cliapp.ssh_runcmd(location, ['mkdir', version_root]) + + self.create_remote_orig(location, version_root, remote_mnt, + temp_root) + + self.status(msg='Creating "run" subvolume') + cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', + 'snapshot', orig_dir, run_dir]) + + self.install_remote_kernel(location, version_root, temp_root) + except Exception as e: + try: + cliapp.ssh_runcmd(location, + ['btrfs', 'subvolume', 'delete', run_dir]) + cliapp.ssh_runcmd(location, + ['btrfs', 'subvolume', 'delete', orig_dir]) + cliapp.ssh_runcmd(location, ['rm', '-rf', version_root]) + except: + pass + raise e + + if self.bootloader_is_wanted(): + self.update_remote_extlinux(location, remote_mnt, + version_label) + except: + raise + else: + self.status(msg='Removing temporary mounts') + cliapp.ssh_runcmd(location, ['umount', root_disk]) + cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) + + def update_remote_extlinux(self, location, remote_mnt, version_label): + '''Install/reconfigure extlinux on location''' + + self.status(msg='Creating extlinux.conf') + config = os.path.join(remote_mnt, 'extlinux.conf') + temp_file = tempfile.mkstemp()[1] + with open(temp_file, 'w') as f: + f.write('default linux\n') + f.write('timeout 1\n') + f.write('label linux\n') + f.write('kernel /systems/' + version_label + '/kernel\n') + f.write('append root=/dev/sda ' + 'rootflags=subvol=systems/' + version_label + '/run ' + 'init=/sbin/init rw\n') + + cliapp.ssh_runcmd(location, ['mv', config, config+'~']) + + try: + cliapp.runcmd(['rsync', '-a', temp_file, + '%s:%s' % (location, config)]) + except Exception as e: + try: + cliapp.ssh_runcmd(location, ['mv', config+'~', config]) + except: + pass + raise e + + def create_remote_orig(self, location, version_root, remote_mnt, + temp_root): + '''Create the subvolume version_root/orig on location''' + + self.status(msg='Creating "orig" subvolume') + old_orig = self.get_old_orig(location, remote_mnt) + new_orig = os.path.join(version_root, 'orig') + cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', 'snapshot', + old_orig, new_orig]) + + cliapp.runcmd(['rsync', '-a', '--checksum', '--numeric-ids', + '--delete', temp_root, '%s:%s' % (location, new_orig)]) + + def get_old_orig(self, location, remote_mnt): + '''Identify which subvolume to snapshot from''' + + # rawdisk upgrades use 'factory' + return os.path.join(remote_mnt, 'systems', 'factory', 'orig') + + def find_root_disk(self, location): + '''Read /proc/mounts on location to find which device contains "/"''' + + self.status(msg='Finding device that contains "/"') + contents = cliapp.ssh_runcmd(location, ['cat', '/proc/mounts']) + for line in contents.splitlines(): + line_words = line.split() + if (line_words[1] == '/' and line_words[0] != 'rootfs'): + return line_words[0] + + def install_remote_kernel(self, location, version_root, temp_root): + '''Install the kernel in temp_root inside version_root on location''' + + self.status(msg='Installing kernel') + image_names = ['vmlinuz', 'zImage', 'uImage'] + kernel_dest = os.path.join(version_root, 'kernel') + for name in image_names: + try_path = os.path.join(temp_root, 'boot', name) + if os.path.exists(try_path): + cliapp.runcmd(['rsync', '-a', try_path, + '%s:%s' % (location, kernel_dest)]) + + def check_valid_target(self, location): + try: + cliapp.ssh_runcmd(location, ['true']) + except Exception as e: + raise cliapp.AppException('%s does not respond to ssh:\n%s' + % (location, e)) + + try: + cliapp.ssh_runcmd(location, ['test', '-d', '/baserock']) + except: + raise cliapp.AppException('%s is not a baserock system' % location) + + try: + cliapp.ssh_runcmd(location, ['which', 'rsync']) + except: + raise cliapp.AppException('%s does not have rsync') + +SshRsyncWriteExtension().run() -- cgit v1.2.1 From 308b1b2a6545566c813e66da5b354175c3cef168 Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Thu, 30 May 2013 11:51:21 +0100 Subject: Add tar write extension --- tar.write | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100755 tar.write diff --git a/tar.write b/tar.write new file mode 100755 index 00000000..5b775e52 --- /dev/null +++ b/tar.write @@ -0,0 +1,19 @@ +#!/bin/sh +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# A Morph write extension to deploy to a .tar file + +tar -C "$1" -cf "$2" -- cgit v1.2.1 From 053333fdac45e58639726eef4726b47d6a2d6385 Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Thu, 30 May 2013 11:54:48 +0100 Subject: Tar write extension fails if arguments not set --- tar.write | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tar.write b/tar.write index 5b775e52..7a2f01e1 100755 --- a/tar.write +++ b/tar.write @@ -16,4 +16,6 @@ # A Morph write extension to deploy to a .tar file +set -eu + tar -C "$1" -cf "$2" -- cgit v1.2.1 From b8f4f02829303a5bbd06c0343f34c5354c2d8a0b Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Fri, 31 May 2013 15:57:53 +0100 Subject: Fix tar write extension --- tar.write | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tar.write b/tar.write index 7a2f01e1..333626b5 100755 --- a/tar.write +++ b/tar.write @@ -18,4 +18,4 @@ set -eu -tar -C "$1" -cf "$2" +tar -C "$1" -cf "$2" . -- cgit v1.2.1 From 7412ca5200695a99983c0bfb98a708808be6d639 Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Mon, 3 Jun 2013 15:47:49 +0000 Subject: kvm.write: use ssh_runcmd --- kvm.write | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/kvm.write b/kvm.write index ae287fe5..4bfff8c4 100755 --- a/kvm.write +++ b/kvm.write @@ -108,13 +108,12 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): ram_mebibytes = str(self.get_ram_size() / (1024**2)) - cmdline = ['ssh', ssh_host, - 'virt-install', '--connect qemu:///system', '--import', + cmdline = ['virt-install', '--connect', 'qemu:///system', '--import', '--name', vm_name, '--vnc', '--ram=%s' % ram_mebibytes, - '--disk path=%s,bus=ide' % vm_path] + attach_opts + '--disk', 'path=%s,bus=ide' % vm_path] + attach_opts if not autostart: cmdline += ['--noreboot'] - cliapp.runcmd(cmdline) + cliapp.ssh_runcmd(ssh_host, cmdline) KvmPlusSshWriteExtension().run() -- cgit v1.2.1 From 480aea39aa2b9298e0828bff1c2b1a2ec61a2124 Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Mon, 3 Jun 2013 15:49:53 +0000 Subject: kvm.write: Set VM to autostart if specified --- kvm.write | 3 +++ 1 file changed, 3 insertions(+) diff --git a/kvm.write b/kvm.write index 4bfff8c4..f2683d8e 100755 --- a/kvm.write +++ b/kvm.write @@ -115,6 +115,9 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): cmdline += ['--noreboot'] cliapp.ssh_runcmd(ssh_host, cmdline) + if autostart: + cliapp.ssh_runcmd(ssh_host, + ['virsh', '--connect', 'qemu:///system', 'autostart', vm_name]) KvmPlusSshWriteExtension().run() -- cgit v1.2.1 From c3c678ccd9fc2cc898b3d6d14cc15711fd992b9e Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Wed, 29 May 2013 16:35:36 +0000 Subject: Improve network setup of the virtualbox write extension. The VirtualBox deployment write extension will configure networking in the following manner: - One host-only network interface will be used to group the virtual machines together (and to give access between the host and the virtual machines). It will be bound to eth0 of the virtual machine. - One NAT (or Bridge) network interface will be used to allow the virtual machines access to the wider network. This will be bound to eth1 of the virtual machine. Now deployment to virtual box will require that both HOST_IPADDR and NETMASK environment variables be set, and also configuration for eth0 and eth1 is expected in the NETWORK_CONFIG environment variable. This commit also replaces some run_cmd with ssh_runcmd. --- virtualbox-ssh.write | 72 +++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 7 deletions(-) diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index cb17b69b..3ee2eae0 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -101,24 +101,26 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Transfer disk and convert to VDI') with open(raw_disk, 'rb') as f: - cliapp.runcmd( - ['ssh', ssh_host, - 'VBoxManage', 'convertfromraw', 'stdin', vdi_path, + cliapp.ssh_runcmd(ssh_host, + ['VBoxManage', 'convertfromraw', 'stdin', vdi_path, str(os.path.getsize(raw_disk))], stdin=f) def create_virtualbox_guest(self, ssh_host, vm_name, vdi_path, autostart): '''Create the VirtualBox virtual machine.''' - + self.status(msg='Create VirtualBox virtual machine') ram_mebibytes = str(self.get_ram_size() / (1024**2)) + hostonly_iface = self.get_host_interface(ssh_host) + commands = [ ['createvm', '--name', vm_name, '--ostype', 'Linux26_64', '--register'], ['modifyvm', vm_name, '--ioapic', 'on', '--memory', ram_mebibytes, - '--nic1', 'nat'], + '--nic1', 'hostonly', '--hostonlyadapter1', hostonly_iface, + '--nic2', 'nat', '--natnet2', 'default'], ['storagectl', vm_name, '--name', '"SATA Controller"', '--add', 'sata', '--bootable', 'on', '--sataportcount', '2'], ['storageattach', vm_name, '--storagectl', '"SATA Controller"', @@ -140,9 +142,65 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): commands.append(['startvm', vm_name]) for command in commands: - argv = ['ssh', ssh_host, 'VBoxManage'] + command - cliapp.runcmd(argv) + argv = ['VBoxManage'] + command + cliapp.ssh_runcmd(ssh_host, argv) + + def get_host_interface(self, ssh_host): + host_ipaddr = os.environ.get('HOST_IPADDR') + netmask = os.environ.get('NETMASK') + network_config = os.environ.get("NETWORK_CONFIG") + if network_config is None: + raise cliapp.AppException('NETWORK_CONFIG was not given') + + if "eth0:" not in network_config: + raise cliapp.AppException( + 'NETWORK_CONFIG does not contain ' + 'the eth0 configuration') + + if "eth1:" not in network_config: + raise cliapp.AppException( + 'NETWORK_CONFIG does not contain ' + 'the eth1 configuration') + + if host_ipaddr is None: + raise cliapp.AppException('HOST_IPADDR was not given') + + if netmask is None: + raise cliapp.AppException('NETMASK was not given') + + # 'VBoxManage list hostonlyifs' retrieves a list with the hostonly + # interfaces on the host. For each interface, the following lines + # are shown on top: + # + # Name: vboxnet0 + # GUID: 786f6276-656e-4074-8000-0a0027000000 + # Dhcp: Disabled + # IPAddress: 192.168.100.1 + # + # The following command tries to retrieve the hostonly interface + # name (e.g. vboxnet0) associated with the given ip address. + iface = None + lines = cliapp.ssh_runcmd(ssh_host, + ['VBoxManage', 'list', 'hostonlyifs']).splitlines() + for i, v in enumerate(lines): + if host_ipaddr in v: + iface = lines[i-3].split()[1] + break + + if iface is None: + iface = cliapp.ssh_runcmd(ssh_host, + ['VBoxManage', 'hostonlyif', 'create']) + # 'VBoxManage hostonlyif create' shows the name of the + # created hostonly interface inside single quotes + iface = iface[iface.find("'") + 1 : iface.rfind("'")] + cliapp.ssh_runcmd(ssh_host, + ['VBoxManage', 'hostonlyif', + 'ipconfig', iface, + '--ip', host_ipaddr, + '--netmask', netmask]) + + return iface VirtualBoxPlusSshWriteExtension().run() -- cgit v1.2.1 From 72e2d820516ef534c5b557703a973e0499ac398a Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Thu, 6 Jun 2013 11:14:21 +0100 Subject: Add the install-files extension to morph --- install-files.configure | 112 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 112 insertions(+) create mode 100755 install-files.configure diff --git a/install-files.configure b/install-files.configure new file mode 100755 index 00000000..669fc518 --- /dev/null +++ b/install-files.configure @@ -0,0 +1,112 @@ +#!/usr/bin/python +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +''' A Morph configuration extension for adding arbitrary files to a system + +It will read the manifest files specified in the environment variable +INSTALL_FILES, then use the contens of those files to determine which files +to install into the target system. + +''' + +import cliapp +import os +import re +import sys +import shlex +import shutil +import stat + +class InstallFilesConfigureExtension(cliapp.Application): + + '''Install the files specified in the manifests listed in INSTALL_FILES + + The manifest is formatted as: + + + + Where the filename is how the file is found inside whatever directory + the manifest is stored in, and also the path within the system to + install to. + + Directories on the target must be created if they do not exist. + + This extension supports files, symlinks and directories. + + ''' + + def process_args(self, args): + if not 'INSTALL_FILES' in os.environ: + return + target_root = args[0] + manifests = shlex.split(os.environ['INSTALL_FILES']) + for manifest in manifests: + self.install_manifest(manifest, target_root) + + def install_manifest(self, manifest, target_root): + manifest_dir = os.path.dirname(manifest) + with open(manifest) as f: + entries = f.readlines() + for entry in entries: + self.install_entry(entry, manifest_dir, target_root) + + def install_entry(self, entry, manifest_root, target_root): + entry_data = re.split('\W+', entry.strip(), maxsplit=3) + mode = int(entry_data[0], 8) + uid = int(entry_data[1]) + gid = int(entry_data[2]) + path = entry_data[3] + dest_path = os.path.join(target_root, './' + path) + if stat.S_ISDIR(mode): + if os.path.exists(dest_path): + dest_stat = os.stat(dest_path) + if (mode != dest_stat.st_mode + or uid != dest_stat.st_uid + or gid != dest_stat.st_gid): + raise cliapp.AppException('"%s" exists and is not ' + 'identical to directory ' + '"%s"' % (dest_path, entry)) + else: + os.mkdir(dest_path, mode) + os.chown(dest_path, uid, gid) + os.chmod(dest_path, mode) + + elif stat.S_ISLNK(mode): + if os.path.lexists(dest_path): + raise cliapp.AppException('Symlink already exists at %s' + % dest_path) + else: + linkdest = os.readlink(os.path.join(manifest_root, + './' + path)) + os.symlink(linkdest, dest_path) + os.lchown(dest_path, uid, gid) + + elif stat.S_ISREG(mode): + if os.path.lexists(dest_path): + raise cliapp.AppException('File already exists at %s' + % dest_path) + else: + shutil.copyfile(os.path.join(manifest_root, './' + path), + dest_path) + os.chown(dest_path, uid, gid) + os.chmod(dest_path, mode) + + else: + raise cliapp.AppException('Mode given in "%s" is not a file,' + ' symlink or directory' % entry) + +InstallFilesConfigureExtension().run() -- cgit v1.2.1 From 221ca74c31a7be787bfa354f53c292f532fa2e2d Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Thu, 13 Jun 2013 10:40:09 +0000 Subject: Set up a symlink to the default system version in rawdisk/kvm/vbox deployments Also Change them to use the "default" symlink in the extlinux.conf they create, instead of hardcoding the current system version name --- rawdisk.write | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/rawdisk.write b/rawdisk.write index a43a9cce..62d39b31 100755 --- a/rawdisk.write +++ b/rawdisk.write @@ -70,9 +70,17 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): self.create_run(version_root) + default_path = os.path.join(mp, 'systems', 'default') + if os.path.exists(default_path): + os.remove(default_path) + else: + # we are upgrading and old system that does + # not have an updated extlinux config file + self.install_extlinux(mp) + os.symlink(version_label, default_path) + if self.bootloader_is_wanted(): self.install_kernel(version_root, temp_root) - self.install_extlinux(mp, version_label) self.unmount(mp) -- cgit v1.2.1 From f041f27a744ad7fa9d95f59d29026a192cd4aa09 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Thu, 13 Jun 2013 15:01:40 +0000 Subject: Change the symbolic link target and directory the default system on NFS server. With this we'll have "default -> factory" instead of "default-run" -> "factory/run". Also change to use VERSION_LABEL instead of VERSION and "factory" instead of "version1", to be more consistent with other parts of the code. --- nfsboot.write | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/nfsboot.write b/nfsboot.write index 61c5306a..f43d3c98 100755 --- a/nfsboot.write +++ b/nfsboot.write @@ -66,15 +66,15 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): 'with hostname "baserock"') self.test_good_server(location) - version = os.environ['VERSION'] or 'version1' + version_label = os.getenv('VERSION_LABEL', 'factory') versioned_root = os.path.join(self._nfsboot_root, hostname, 'systems', - version) + version_label) if self.version_exists(versioned_root, location): raise cliapp.AppException('Version %s already exists on' ' this device. Deployment aborted' - % version) + % version_label) self.copy_rootfs(temp_root, location, versioned_root, hostname) - self.copy_kernel(temp_root, location, versioned_root, version, + self.copy_kernel(temp_root, location, versioned_root, version_label, hostname) self.configure_nfs(location, hostname) @@ -171,15 +171,15 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): raise cliapp.AppException('Could not create \'run\' rootfs' ' from \'orig\'') - self.status(msg='Linking \'default-run\' to latest system') + self.status(msg='Linking \'default\' to latest system') try: cliapp.ssh_runcmd('root@%s' % location, - ['ln', '-sfn', run_path, + ['ln', '-sfn', versioned_root, os.path.join(self._nfsboot_root, hostname, 'systems', - 'default-run')]) + 'default')]) except cliapp.AppException: - raise cliapp.AppException('Could not link \'default-run\' to %s' - % run_path) + raise cliapp.AppException('Could not link \'default\' to %s' + % versioned_root) def configure_nfs(self, location, hostname): exported_path = os.path.join(self._nfsboot_root, hostname) -- cgit v1.2.1 From bdca86375dd2d7fe53c5404db413656d53989bc9 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Fri, 14 Jun 2013 19:15:09 +0000 Subject: Only upgrade an older extlinux configuration if we want one. --- rawdisk.write | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rawdisk.write b/rawdisk.write index 62d39b31..a74d6905 100755 --- a/rawdisk.write +++ b/rawdisk.write @@ -76,7 +76,8 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): else: # we are upgrading and old system that does # not have an updated extlinux config file - self.install_extlinux(mp) + if self.bootloader_is_wanted(): + self.install_extlinux(mp) os.symlink(version_label, default_path) if self.bootloader_is_wanted(): -- cgit v1.2.1 From 64b72cdc4ae7a0d376239f31d1e607bae9d8d602 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Tue, 18 Jun 2013 16:09:51 +0100 Subject: Create a symbolic link to the default system version when upgrading running systems. --- ssh-rsync.write | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ssh-rsync.write b/ssh-rsync.write index 6fe1153d..4348714c 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -72,6 +72,10 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): 'snapshot', orig_dir, run_dir]) self.install_remote_kernel(location, version_root, temp_root) + default_path = os.path.join(remote_mnt, 'systems', 'default') + cliapp.ssh_runcmd(location, ['ln', '-s', '-f', + version_label, + default_path]) except Exception as e: try: cliapp.ssh_runcmd(location, @@ -103,9 +107,9 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): f.write('default linux\n') f.write('timeout 1\n') f.write('label linux\n') - f.write('kernel /systems/' + version_label + '/kernel\n') + f.write('kernel /systems/default/kernel\n') f.write('append root=/dev/sda ' - 'rootflags=subvol=systems/' + version_label + '/run ' + 'rootflags=subvol=systems/default/run ' 'init=/sbin/init rw\n') cliapp.ssh_runcmd(location, ['mv', config, config+'~']) -- cgit v1.2.1 From 251d6a684eda959057810e736184eac316e80c75 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Tue, 18 Jun 2013 16:10:42 +0100 Subject: Support upgrades in older running versions. Verify if are using and older extlinux configuration and upgrade it if the case, by checking if the "default" symbolic link exists on the target. Note that with the symbolic link we don't need to update extlinux configuration after an upgrade --- ssh-rsync.write | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/ssh-rsync.write b/ssh-rsync.write index 4348714c..1a921996 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -73,6 +73,17 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.install_remote_kernel(location, version_root, temp_root) default_path = os.path.join(remote_mnt, 'systems', 'default') + if self.bootloader_is_wanted(): + output = ssh_runcmd(location, ['sh', '-c', + 'test -e "$1" && stat -c %F "$1"' + ' || ' + 'echo missing file', + '-', default_path]) + if output != "symbolic link": + # we are upgrading and old system that does + # not have an updated extlinux config file + self.update_remote_extlinux(location, remote_mnt, + version_label) cliapp.ssh_runcmd(location, ['ln', '-s', '-f', version_label, default_path]) @@ -87,9 +98,6 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): pass raise e - if self.bootloader_is_wanted(): - self.update_remote_extlinux(location, remote_mnt, - version_label) except: raise else: -- cgit v1.2.1 From 5e664629324a2cab7b4b79c01d458cc00c38e9c4 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Wed, 19 Jun 2013 12:59:34 +0100 Subject: Fix a typo --- ssh-rsync.write | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ssh-rsync.write b/ssh-rsync.write index 1a921996..6bef51db 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -74,7 +74,7 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.install_remote_kernel(location, version_root, temp_root) default_path = os.path.join(remote_mnt, 'systems', 'default') if self.bootloader_is_wanted(): - output = ssh_runcmd(location, ['sh', '-c', + output = cliapp.ssh_runcmd(location, ['sh', '-c', 'test -e "$1" && stat -c %F "$1"' ' || ' 'echo missing file', -- cgit v1.2.1 From 0b1db252d61c3aa02ae7c865314ee07e96244be3 Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Wed, 19 Jun 2013 12:23:11 +0000 Subject: kvm.write uses NIC_CONFIG --- kvm.write | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/kvm.write b/kvm.write index f2683d8e..52768042 100755 --- a/kvm.write +++ b/kvm.write @@ -106,6 +106,11 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): for disk in attach_disks: attach_opts.extend(['--disk', 'path=%s' % disk]) + if 'NIC_CONFIG' in os.environ: + nics = os.environ['NIC_CONFIG'].split() + for nic in nics: + attach_opts.extend(['--network', nic]) + ram_mebibytes = str(self.get_ram_size() / (1024**2)) cmdline = ['virt-install', '--connect', 'qemu:///system', '--import', -- cgit v1.2.1 From 3e8721c40abdc474ad3431d62d102e10aee7488f Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Sun, 9 Jun 2013 22:56:33 +0000 Subject: Write extensions: pass -s to rsync -s, or --protect-args prevents the file path components of destination or source paths being interpreted by the remote shell. This is for wildcards or other shell features, but it breaks when paths have whitespace. We tend to always use absolute paths, so all uses of rsync now pass -s. kvm.write needs it, since the disk can be written to a path with spaces. Nfsboot and ssh-rsync need it because version labels are used, which may have spaces, and temporary directories are used, which could have spaces in weird TMPDIR configurations. --- kvm.write | 2 +- nfsboot.write | 4 ++-- ssh-rsync.write | 6 +++--- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/kvm.write b/kvm.write index 52768042..67ac40e7 100755 --- a/kvm.write +++ b/kvm.write @@ -94,7 +94,7 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Transferring disk image') target = '%s:%s' % (ssh_host, vm_path) with open(raw_disk, 'rb') as f: - cliapp.runcmd(['rsync', '-zS', raw_disk, target]) + cliapp.runcmd(['rsync', '-szS', raw_disk, target]) def create_libvirt_guest(self, ssh_host, vm_name, vm_path, autostart): '''Create the libvirt virtual machine.''' diff --git a/nfsboot.write b/nfsboot.write index f43d3c98..34a72972 100755 --- a/nfsboot.write +++ b/nfsboot.write @@ -118,7 +118,7 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): rsync_dest = 'root@%s:%s' % (location, kernel_dest) self.status(msg='Copying kernel') cliapp.runcmd( - ['rsync', kernel_src, rsync_dest]) + ['rsync', '-s', kernel_src, rsync_dest]) # Link the kernel to the right place self.status(msg='Creating links to kernel in tftp directory') @@ -153,7 +153,7 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Creating \'orig\' rootfs') cliapp.runcmd( - ['rsync', '-aXSPH', '--delete', rootfs_src, + ['rsync', '-asXSPH', '--delete', rootfs_src, 'root@%s:%s' % (location, orig_path)]) self.status(msg='Creating \'run\' rootfs') diff --git a/ssh-rsync.write b/ssh-rsync.write index 6bef51db..fba550cd 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -123,7 +123,7 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): cliapp.ssh_runcmd(location, ['mv', config, config+'~']) try: - cliapp.runcmd(['rsync', '-a', temp_file, + cliapp.runcmd(['rsync', '-as', temp_file, '%s:%s' % (location, config)]) except Exception as e: try: @@ -142,7 +142,7 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', 'snapshot', old_orig, new_orig]) - cliapp.runcmd(['rsync', '-a', '--checksum', '--numeric-ids', + cliapp.runcmd(['rsync', '-as', '--checksum', '--numeric-ids', '--delete', temp_root, '%s:%s' % (location, new_orig)]) def get_old_orig(self, location, remote_mnt): @@ -170,7 +170,7 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): for name in image_names: try_path = os.path.join(temp_root, 'boot', name) if os.path.exists(try_path): - cliapp.runcmd(['rsync', '-a', try_path, + cliapp.runcmd(['rsync', '-as', try_path, '%s:%s' % (location, kernel_dest)]) def check_valid_target(self, location): -- cgit v1.2.1 From 2a799319bd19ce9d303aa63d30ab7c556b17b6bb Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Sat, 15 Jun 2013 09:33:10 +0000 Subject: Don't dereference the default symbolic link when updating it Or else this ln -s -f update1 /mp/systems/default will do this '/pp/systems/default/update1' -> 'update1 When we want '/pp/systems/default' -> 'update1 --- ssh-rsync.write | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ssh-rsync.write b/ssh-rsync.write index fba550cd..83091c4b 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -84,8 +84,7 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): # not have an updated extlinux config file self.update_remote_extlinux(location, remote_mnt, version_label) - cliapp.ssh_runcmd(location, ['ln', '-s', '-f', - version_label, + cliapp.ssh_runcmd(location, ['ln', '-sfn', version_label, default_path]) except Exception as e: try: -- cgit v1.2.1 From 89ad5f816fff7bd7897b2d4cb02ae5cc6b6799d2 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Sat, 15 Jun 2013 10:20:31 +0000 Subject: Unmount the remote mouting point instead of the root disk Unmounting the root disk as the side effect of turn it to be read only --- ssh-rsync.write | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ssh-rsync.write b/ssh-rsync.write index 83091c4b..77266d33 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -101,7 +101,7 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): raise else: self.status(msg='Removing temporary mounts') - cliapp.ssh_runcmd(location, ['umount', root_disk]) + cliapp.ssh_runcmd(location, ['umount', remote_mnt]) cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) def update_remote_extlinux(self, location, remote_mnt, version_label): -- cgit v1.2.1 From d8a87880248ec754affc302fa8966bf5ebd83046 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Sun, 16 Jun 2013 11:41:32 +0000 Subject: Add a missing trailing slash to the source directory of rsync Accordingly the rsync manual: "A trailing slash on the source changes this behavior to avoid creating an additional directory level at the destination. You can think of a trailing / on a source as meaning "copy the contents of this directory" as opposed to "copy the directory by name". --- ssh-rsync.write | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ssh-rsync.write b/ssh-rsync.write index 77266d33..b8d30e22 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -142,7 +142,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): old_orig, new_orig]) cliapp.runcmd(['rsync', '-as', '--checksum', '--numeric-ids', - '--delete', temp_root, '%s:%s' % (location, new_orig)]) + '--delete', temp_root + os.path.sep, + '%s:%s' % (location, new_orig)]) def get_old_orig(self, location, remote_mnt): '''Identify which subvolume to snapshot from''' -- cgit v1.2.1 From 4f630811332b7ebb21fc47551bccb8e14a456410 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Sat, 15 Jun 2013 15:48:01 +0000 Subject: Run the merge mode of baserock-system-config-sync when upgrading running systems. --- ssh-rsync.write | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/ssh-rsync.write b/ssh-rsync.write index b8d30e22..9697e21b 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -71,6 +71,15 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', 'snapshot', orig_dir, run_dir]) + self.status(msg='Updating system configuration') + bscs_loc = os.path.join(run_dir, 'usr', 'bin', + 'baserock-system-config-sync') + try: + cliapp.ssh_runcmd(location, ['sh', bscs_loc, 'merge', + version_label]) + except: + self.status(msg='Updating system configuration failed') + self.install_remote_kernel(location, version_root, temp_root) default_path = os.path.join(remote_mnt, 'systems', 'default') if self.bootloader_is_wanted(): -- cgit v1.2.1 From ce80fe3e235ff747afbea9b20f992f5af41fe946 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Fri, 28 Jun 2013 15:00:36 +0000 Subject: Improvements to ssh-rsync extension --- ssh-rsync.write | 150 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 80 insertions(+), 70 deletions(-) diff --git a/ssh-rsync.write b/ssh-rsync.write index 9697e21b..211dbe5e 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -50,65 +50,74 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): root_disk = self.find_root_disk(location) version_label = os.environ.get('VERSION_LABEL') + self.status(msg='Creating remote mount point') + remote_mnt = cliapp.ssh_runcmd(location, ['mktemp', '-d']).strip() try: - self.status(msg='Creating remote mount point') - remote_mnt = cliapp.ssh_runcmd(location, ['mktemp', '-d']).strip() - self.status(msg='Mounting root disk') cliapp.ssh_runcmd(location, ['mount', root_disk, remote_mnt]) + except Exception as e: + try: + cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) + except: + pass + raise e + try: version_root = os.path.join(remote_mnt, 'systems', version_label) run_dir = os.path.join(version_root, 'run') orig_dir = os.path.join(version_root, 'orig') + + self.status(msg='Creating %s' % version_root) + cliapp.ssh_runcmd(location, ['mkdir', version_root]) + + self.create_remote_orig(location, version_root, remote_mnt, + temp_root) + + self.status(msg='Creating "run" subvolume') + cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', + 'snapshot', orig_dir, run_dir]) + + self.status(msg='Updating system configuration') + bscs_loc = os.path.join(run_dir, 'usr', 'bin', + 'baserock-system-config-sync') + + output = cliapp.ssh_runcmd(location, ['sh', '-c', + '"$1" merge "$2" &> /dev/null || echo -n cmdfailed', + '-', bscs_loc, version_label]) + if output == "cmdfailed": + self.status(msg='Updating system configuration failed') + + self.install_remote_kernel(location, version_root, temp_root) + default_path = os.path.join(remote_mnt, 'systems', 'default') + if self.bootloader_is_wanted(): + output = cliapp.ssh_runcmd(location, ['sh', '-c', + 'test -e "$1" && stat -c %F "$1" ' + '|| echo missing file', + '-', default_path]) + if output != "symbolic link": + # we are upgrading and old system that does + # not have an updated extlinux config file + self.update_remote_extlinux(location, remote_mnt, + version_label) + cliapp.ssh_runcmd(location, ['ln', '-sfn', version_label, + default_path]) + except Exception as e: + try: + cliapp.ssh_runcmd(location, + ['btrfs', 'subvolume', 'delete', run_dir]) + except: + pass + try: + cliapp.ssh_runcmd(location, + ['btrfs', 'subvolume', 'delete', orig_dir]) + except: + pass try: - self.status(msg='Creating %s' % version_root) - cliapp.ssh_runcmd(location, ['mkdir', version_root]) - - self.create_remote_orig(location, version_root, remote_mnt, - temp_root) - - self.status(msg='Creating "run" subvolume') - cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', - 'snapshot', orig_dir, run_dir]) - - self.status(msg='Updating system configuration') - bscs_loc = os.path.join(run_dir, 'usr', 'bin', - 'baserock-system-config-sync') - try: - cliapp.ssh_runcmd(location, ['sh', bscs_loc, 'merge', - version_label]) - except: - self.status(msg='Updating system configuration failed') - - self.install_remote_kernel(location, version_root, temp_root) - default_path = os.path.join(remote_mnt, 'systems', 'default') - if self.bootloader_is_wanted(): - output = cliapp.ssh_runcmd(location, ['sh', '-c', - 'test -e "$1" && stat -c %F "$1"' - ' || ' - 'echo missing file', - '-', default_path]) - if output != "symbolic link": - # we are upgrading and old system that does - # not have an updated extlinux config file - self.update_remote_extlinux(location, remote_mnt, - version_label) - cliapp.ssh_runcmd(location, ['ln', '-sfn', version_label, - default_path]) - except Exception as e: - try: - cliapp.ssh_runcmd(location, - ['btrfs', 'subvolume', 'delete', run_dir]) - cliapp.ssh_runcmd(location, - ['btrfs', 'subvolume', 'delete', orig_dir]) - cliapp.ssh_runcmd(location, ['rm', '-rf', version_root]) - except: - pass - raise e - - except: - raise - else: + cliapp.ssh_runcmd(location, ['rm', '-rf', version_root]) + except: + pass + raise e + finally: self.status(msg='Removing temporary mounts') cliapp.ssh_runcmd(location, ['umount', remote_mnt]) cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) @@ -118,8 +127,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Creating extlinux.conf') config = os.path.join(remote_mnt, 'extlinux.conf') - temp_file = tempfile.mkstemp()[1] - with open(temp_file, 'w') as f: + temp_fd, temp_path = tempfile.mkstemp() + with os.fdopen(temp_fd, 'w') as f: f.write('default linux\n') f.write('timeout 1\n') f.write('label linux\n') @@ -128,14 +137,13 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): 'rootflags=subvol=systems/default/run ' 'init=/sbin/init rw\n') - cliapp.ssh_runcmd(location, ['mv', config, config+'~']) - try: - cliapp.runcmd(['rsync', '-as', temp_file, - '%s:%s' % (location, config)]) + cliapp.runcmd(['rsync', '-as', temp_path, + '%s:%s~' % (location, config)]) + cliapp.ssh_runcmd(location, ['mv', config+'~', config]) except Exception as e: try: - cliapp.ssh_runcmd(location, ['mv', config+'~', config]) + cliapp.ssh_runcmd(location, ['rm', '-f', config+'~']) except: pass raise e @@ -168,19 +176,19 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): for line in contents.splitlines(): line_words = line.split() if (line_words[1] == '/' and line_words[0] != 'rootfs'): - return line_words[0] + return line_words[0] def install_remote_kernel(self, location, version_root, temp_root): '''Install the kernel in temp_root inside version_root on location''' self.status(msg='Installing kernel') - image_names = ['vmlinuz', 'zImage', 'uImage'] + image_names = ('vmlinuz', 'zImage', 'uImage') kernel_dest = os.path.join(version_root, 'kernel') for name in image_names: try_path = os.path.join(temp_root, 'boot', name) if os.path.exists(try_path): cliapp.runcmd(['rsync', '-as', try_path, - '%s:%s' % (location, kernel_dest)]) + '%s:%s' % (location, kernel_dest)]) def check_valid_target(self, location): try: @@ -189,14 +197,16 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): raise cliapp.AppException('%s does not respond to ssh:\n%s' % (location, e)) - try: - cliapp.ssh_runcmd(location, ['test', '-d', '/baserock']) - except: - raise cliapp.AppException('%s is not a baserock system' % location) - - try: - cliapp.ssh_runcmd(location, ['which', 'rsync']) - except: - raise cliapp.AppException('%s does not have rsync') + output = cliapp.ssh_runcmd(location, ['sh', '-c', + 'test -d /baserock || echo -n dirnotfound']) + if output == 'dirnotfound': + raise cliapp.AppException('%s is not a baserock system' + % location) + + output = cliapp.ssh_runcmd(location, ['sh', '-c', + 'type rsync &> /dev/null || echo -n cmdnotfound']) + if output == 'cmdnotfound': + raise cliapp.AppException('%s does not have rsync' + % location) SshRsyncWriteExtension().run() -- cgit v1.2.1 From 7f10cd9a320664609f83dc24f6d25e79b49cb7d2 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Tue, 2 Jul 2013 07:24:30 +0000 Subject: Allow to set the number of cpus for virtualbox and kvm deployments. --- kvm.write | 7 +++++-- virtualbox-ssh.write | 5 ++++- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/kvm.write b/kvm.write index 67ac40e7..9a6be135 100755 --- a/kvm.write +++ b/kvm.write @@ -113,8 +113,11 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): ram_mebibytes = str(self.get_ram_size() / (1024**2)) - cmdline = ['virt-install', '--connect', 'qemu:///system', '--import', - '--name', vm_name, '--vnc', '--ram=%s' % ram_mebibytes, + vcpu_count = str(self.get_vcpu_count()) + + cmdline = ['virt-install', '--connect', 'qemu:///system', + '--import', '--name', vm_name, '--vnc', + '--ram', ram_mebibytes, '--vcpus', vcpu_count, '--disk', 'path=%s,bus=ide' % vm_path] + attach_opts if not autostart: cmdline += ['--noreboot'] diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 3ee2eae0..1abe233e 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -113,12 +113,15 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): ram_mebibytes = str(self.get_ram_size() / (1024**2)) + vcpu_count = str(self.get_vcpu_count()) + hostonly_iface = self.get_host_interface(ssh_host) commands = [ ['createvm', '--name', vm_name, '--ostype', 'Linux26_64', '--register'], - ['modifyvm', vm_name, '--ioapic', 'on', '--memory', ram_mebibytes, + ['modifyvm', vm_name, '--ioapic', 'on', + '--memory', ram_mebibytes, '--cpus', vcpu_count, '--nic1', 'hostonly', '--hostonlyadapter1', hostonly_iface, '--nic2', 'nat', '--natnet2', 'default'], ['storagectl', vm_name, '--name', '"SATA Controller"', -- cgit v1.2.1 From f0800ade5348fd632d7c49307e55e32746d073a7 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Mon, 19 Aug 2013 10:23:31 +0000 Subject: Fix naming of SATA Controller in VirtualBox deployments Patch from Paul Sherwood. --- virtualbox-ssh.write | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 1abe233e..2374db31 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -124,9 +124,9 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): '--memory', ram_mebibytes, '--cpus', vcpu_count, '--nic1', 'hostonly', '--hostonlyadapter1', hostonly_iface, '--nic2', 'nat', '--natnet2', 'default'], - ['storagectl', vm_name, '--name', '"SATA Controller"', + ['storagectl', vm_name, '--name', 'SATA Controller', '--add', 'sata', '--bootable', 'on', '--sataportcount', '2'], - ['storageattach', vm_name, '--storagectl', '"SATA Controller"', + ['storageattach', vm_name, '--storagectl', 'SATA Controller', '--port', '0', '--device', '0', '--type', 'hdd', '--medium', vdi_path], ] @@ -134,7 +134,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): attach_disks = self.parse_attach_disks() for device_no, disk in enumerate(attach_disks, 1): cmd = ['storageattach', vm_name, - '--storagectl', '"SATA Controller"', + '--storagectl', 'SATA Controller', '--port', str(device_no), '--device', '0', '--type', 'hdd', -- cgit v1.2.1 From f6e74b24509e35938fbbb0e7239eb8e4644afb31 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Thu, 29 Aug 2013 16:35:26 +0000 Subject: exts: Add openstack configure/write exts openstackssh.write: Write extension which deploy a raw image of baserock directly to an OpenStack machine using python-glanceclient. The raw image deployed has modified its bootloader to use virtio disks. vdaboot.configure: Configuration extension to change the mount point of "/" to use virtio disks (/dev/vda). --- openstack.write | 140 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ vdaboot.configure | 34 +++++++++++++ 2 files changed, 174 insertions(+) create mode 100755 openstack.write create mode 100755 vdaboot.configure diff --git a/openstack.write b/openstack.write new file mode 100755 index 00000000..8ee8767e --- /dev/null +++ b/openstack.write @@ -0,0 +1,140 @@ +#!/usr/bin/python +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +'''A Morph deployment write extension for deploying to OpenStack.''' + + +import cliapp +import os +import tempfile +import urlparse + +import morphlib.writeexts + + +class OpenStackWriteExtension(morphlib.writeexts.WriteExtension): + + '''Configure a raw disk image into an OpenStack host. + + The raw disk image is created during Morph's deployment and the + image is deployed in OpenStack using python-glanceclient. + + The location command line argument is the authentification url + of the OpenStack server using the following syntax: + + http://HOST:PORT/VERSION + + where + + * HOST is the host running OpenStack + * PORT is the port which is using OpenStack for authentifications. + * VERSION is the authentification version of OpenStack (Only v2.0 + supported) + + This extension needs in the environment the following variables: + + * OPENSTACK_USER is the username to use in the deployment. + * OPENSTACK_TENANT is the project name to use in the deployment. + * OPENSTACK_IMAGENAME is the name of the image to create. + * OPENSTACK_PASSWORD is the password of the user. + + + The extension will connect to OpenStack using python-glanceclient + to configure a raw image. + + ''' + + def process_args(self, args): + if len(args) != 2: + raise cliapp.AppException('Wrong number of command line args') + + temp_root, location = args + self.check_location(location) + + os_params = self.get_openstack_parameters() + + fd, raw_disk = tempfile.mkstemp() + os.close(fd) + self.create_local_system(temp_root, raw_disk) + self.status(msg='Temporary disk image has been created at %s' + % raw_disk) + + self.set_extlinux_root_to_virtio(raw_disk) + + self.configure_openstack_image(raw_disk, location, os_params) + + def set_extlinux_root_to_virtio(self, raw_disk): + '''Re-configures extlinux to use virtio disks''' + self.status(msg='Updating extlinux.conf') + mp = self.mount(raw_disk) + try: + path = os.path.join(mp, 'extlinux.conf') + + with open(path) as f: + extlinux_conf = f.read() + + extlinux_conf = extlinux_conf.replace('root=/dev/sda', + 'root=/dev/vda') + with open(path, "w") as f: + f.write(extlinux_conf) + + finally: + self.unmount(mp) + + def get_openstack_parameters(self): + '''Check the environment variables needed and returns all. + + The environment variables are described in the class documentation. + ''' + + keys = ('OPENSTACK_USER', 'OPENSTACK_TENANT', + 'OPENSTACK_IMAGENAME', 'OPENSTACK_PASSWORD') + for key in keys: + if key not in os.environ: + raise cliapp.AppException(key + ' was not given') + return (os.environ[key] for key in keys) + + def check_location(self, location): + x = urlparse.urlparse(location) + if x.scheme != 'http': + raise cliapp.AppException('URL schema must be http in %s' \ + % location) + if (x.path != '/v2.0' and x.path != '/v2.0/'): + raise cliapp.AppException('API version must be v2.0 in %s'\ + % location) + + def configure_openstack_image(self, raw_disk, auth_url, os_params): + '''Configure the image in OpenStack using glance-client''' + self.status(msg='Configuring OpenStack image...') + + username, tenant_name, image_name, password = os_params + cmdline = ['glance', + '--os-username', username, + '--os-tenant-name', tenant_name, + '--os-password', password, + '--os-auth-url', auth_url, + 'image-create', + '--name=%s' % image_name, + '--disk-format=raw', + '--container-format', 'bare', + '--file', raw_disk] + cliapp.runcmd(cmdline) + + self.status(msg='Image configured.') + +OpenStackWriteExtension().run() + diff --git a/vdaboot.configure b/vdaboot.configure new file mode 100755 index 00000000..cb54bf0d --- /dev/null +++ b/vdaboot.configure @@ -0,0 +1,34 @@ +#!/bin/sh +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +# Change the "/" mount point to /dev/vda to use virtio disks. + +set -e + +if [ "$OPENSTACK_USER" ] +then + # Modifying fstab + if [ -f "$1/etc/fstab" ] + then + mv "$1/etc/fstab" "$1/etc/fstab.old" + awk 'BEGIN {print "/dev/vda / btrfs defaults,rw,noatime 0 1"}; + $2 != "/" {print $0 };' "$1/etc/fstab.old" > "$1/etc/fstab" + rm "$1/etc/fstab.old" + else + echo "/dev/vda / btrfs defaults,rw,noatime 0 1"> "$1/etc/fstab" + fi +fi -- cgit v1.2.1 From dc3d9cd1de7fcc4d0fed2ff4d958c73817415d9c Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Mon, 16 Sep 2013 14:51:24 +0000 Subject: exts: Remove tab from vdaboot.configure This snuck in since the test suite could not be run when TMPDIR was on a tmpfs. --- vdaboot.configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/vdaboot.configure b/vdaboot.configure index cb54bf0d..b88eb3a8 100755 --- a/vdaboot.configure +++ b/vdaboot.configure @@ -20,7 +20,7 @@ set -e if [ "$OPENSTACK_USER" ] -then +then # Modifying fstab if [ -f "$1/etc/fstab" ] then -- cgit v1.2.1 From 2492bd568b82c1f071a4c67800baaf6fae3e690c Mon Sep 17 00:00:00 2001 From: Dan Firth Date: Tue, 8 Oct 2013 16:00:31 +0000 Subject: Removed ssh.configure extension --- ssh.configure | 162 ---------------------------------------------------------- 1 file changed, 162 deletions(-) delete mode 100755 ssh.configure diff --git a/ssh.configure b/ssh.configure deleted file mode 100755 index 2f3167e7..00000000 --- a/ssh.configure +++ /dev/null @@ -1,162 +0,0 @@ -#!/usr/bin/python -# Copyright (C) 2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -'''A Morph deployment configuration to copy SSH keys. - -Keys are copied from the host to the new system. -''' - -import cliapp -import os -import sys -import shutil -import glob -import re -import logging - -import morphlib - -class SshConfigurationExtension(cliapp.Application): - - '''Copy over SSH keys to new system from host. - - The extension requires SSH_KEY_DIR to be set at the command line as it - will otherwise pass with only a status update. SSH_KEY_DIR should be - set to the location of the SSH keys to be passed to the new system. - - ''' - - def process_args(self, args): - if 'SSH_KEY_DIR' in os.environ: - # Copies ssh_host keys. - key = 'ssh_host_*_key' - mode = 0755 - dest = os.path.join(args[0], 'etc/ssh/') - sshhost, sshhostpub = self.find_keys(key) - if sshhost or sshhostpub: - self.check_dir(dest, mode) - self.copy_keys(sshhost, sshhostpub, dest) - - # Copies root keys. - key = 'root_*_key' - mode = 0700 - dest = os.path.join(args[0], 'root/.ssh/') - roothost, roothostpub = self.find_keys(key) - key = 'root_authorized_key_*.pub' - authkey, bleh = self.find_keys(key) - if roothost or roothostpub: - self.check_dir(dest, mode) - self.copy_rename_keys(roothost, - roothostpub, dest, 'id_', [5, 4]) - if authkey: - self.check_dir(dest, mode) - self.comb_auth_key(authkey, dest) - - # Fills the known_hosts file - key = 'root_known_host_*_key.pub' - src = os.path.join(os.environ['SSH_KEY_DIR'], key) - known_hosts_keys = glob.glob(src) - if known_hosts_keys: - self.check_dir(dest, mode) - known_hosts_path = os.path.join(dest, 'known_hosts') - with open(known_hosts_path, "a") as known_hosts_file: - for filename in known_hosts_keys: - hostname = re.search('root_known_host_(.+?)_key.pub', - filename).group(1) - known_hosts_file.write(hostname + " ") - with open(filename, "r") as f: - shutil.copyfileobj(f, known_hosts_file) - - else: - self.status(msg="No SSH key directory found.") - pass - - def find_keys(self, key_name): - '''Uses glob to find public and - private SSH keys and returns their path''' - - src = os.path.join(os.environ['SSH_KEY_DIR'], key_name) - keys = glob.glob(src) - pubkeys = glob.glob(src + '.pub') - if not (keys or pubkeys): - self.status(msg="No SSH keys of pattern %(src)s found.", src=src) - return keys, pubkeys - - def check_dir(self, dest, mode): - '''Checks if destination directory exists - and creates it if necessary''' - - if os.path.exists(dest) == False: - self.status(msg="Creating SSH key directory: %(dest)s", dest=dest) - os.mkdir(dest) - os.chmod(dest, mode) - else: - pass - - def copy_keys(self, keys, pubkeys, dest): - '''Copies SSH keys to new VM''' - - for key in keys: - shutil.copy(key, dest) - path = os.path.join(dest, os.path.basename(key)) - os.chmod(path, 0600) - for key in pubkeys: - shutil.copy(key, dest) - path = os.path.join(dest, os.path.basename(key)) - os.chmod(path, 0644) - - def copy_rename_keys(self, keys, pubkeys, dest, new, snip): - '''Copies SSH keys to new VM and renames them''' - - st, fi = snip - for key in keys: - base = os.path.basename(key) - s = len(base) - nw_dst = os.path.join(dest, new + base[st:s-fi]) - shutil.copy(key, nw_dst) - os.chmod(nw_dst, 0600) - for key in pubkeys: - base = os.path.basename(key) - s = len(base) - nw_dst = os.path.join(dest, new + base[st:s-fi-4]) - shutil.copy(key, nw_dst + '.pub') - os.chmod(nw_dst + '.pub', 0644) - - def comb_auth_key(self, keys, dest): - '''Combines authorized_keys file in new VM''' - - dest = os.path.join(dest, 'authorized_keys') - fout = open(dest, 'a') - for key in keys: - fin = open(key, 'r') - data = fin.read() - fout.write(data) - fin.close() - fout.close() - os.chmod(dest, 0600) - - def status(self, **kwargs): - '''Provide status output. - - The ``msg`` keyword argument is the actual message, - the rest are values for fields in the message as interpolated - by %. - - ''' - - self.output.write('%s\n' % (kwargs['msg'] % kwargs)) - -SshConfigurationExtension().run() -- cgit v1.2.1 From 1e57089d8aa73ffc49ef80622206d43e1bd15bf0 Mon Sep 17 00:00:00 2001 From: Dan Firth Date: Thu, 10 Oct 2013 16:07:37 +0000 Subject: Deployment failures will now remove the disk image --- kvm.write | 1 + rawdisk.write | 10 ++++++++-- virtualbox-ssh.write | 2 +- 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/kvm.write b/kvm.write index 9a6be135..4f877c22 100755 --- a/kvm.write +++ b/kvm.write @@ -68,6 +68,7 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): except BaseException: sys.stderr.write('Error deploying to libvirt') os.remove(raw_disk) + cliapp.ssh_runcmd(ssh_host, ['rm', '-f', vm_path]) raise else: os.remove(raw_disk) diff --git a/rawdisk.write b/rawdisk.write index a74d6905..8723ac0c 100755 --- a/rawdisk.write +++ b/rawdisk.write @@ -46,8 +46,14 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): if os.path.isfile(location): self.upgrade_local_system(location, temp_root) else: - self.create_local_system(temp_root, location) - self.status(msg='Disk image has been created at %s' % location) + try: + self.create_local_system(temp_root, location) + self.status(msg='Disk image has been created at %s' % location) + except Exception: + os.remove(location) + self.status(msg='Failure to create disk image at %s' % + location) + raise def upgrade_local_system(self, raw_disk, temp_root): mp = self.mount(raw_disk) diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 2374db31..f18ef804 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -76,10 +76,10 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): except BaseException: sys.stderr.write('Error deploying to VirtualBox') os.remove(raw_disk) + cliapp.ssh_runcmd(ssh_host, ['rm', '-f', vdi_path]) raise else: os.remove(raw_disk) - self.status( msg='Virtual machine %(vm_name)s has been created', vm_name=vm_name) -- cgit v1.2.1 From 14eb503d24a46dc9163d2445f40c14f92143a719 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 23 Oct 2013 17:17:41 +0000 Subject: Add fstab.configure This will allow the user to append text to /etc/fstab during a deployment, without having to write custom configuration extensions. --- fstab.configure | 40 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) create mode 100755 fstab.configure diff --git a/fstab.configure b/fstab.configure new file mode 100755 index 00000000..0100dacb --- /dev/null +++ b/fstab.configure @@ -0,0 +1,40 @@ +#!/usr/bin/python +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# =*= License: GPL-2 =*= + + +import os +import sys + + +def asciibetical(strings): + + def key(s): + return [ord(c) for c in s] + + return sorted(strings, key=key) + + +fstab_filename = os.path.join(sys.argv[1], 'etc', 'fstab') + +fstab_vars = sorted(x for x in os.environ if x.startswith('FSTAB_')) +with open(fstab_filename, 'a') as f: + for var in fstab_vars: + f.write('%s\n' % os.environ[var]) + +os.chown(fstab_filename, 0, 0) +os.chmod(fstab_filename, 0644) -- cgit v1.2.1 From 3f35c5ab40849f8f8d36a084cb3ab2ec8f17f7c6 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 24 Oct 2013 17:35:25 +0000 Subject: Merge remote-tracking branch 'origin/liw/fstab.configure' Reviewed-by: Richard Maw At his suggestion, fixed the call to sorted() to be a call to asciibetical(). --- fstab.configure | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fstab.configure b/fstab.configure index 0100dacb..a1287ea4 100755 --- a/fstab.configure +++ b/fstab.configure @@ -31,7 +31,7 @@ def asciibetical(strings): fstab_filename = os.path.join(sys.argv[1], 'etc', 'fstab') -fstab_vars = sorted(x for x in os.environ if x.startswith('FSTAB_')) +fstab_vars = asciibetical(x for x in os.environ if x.startswith('FSTAB_')) with open(fstab_filename, 'a') as f: for var in fstab_vars: f.write('%s\n' % os.environ[var]) -- cgit v1.2.1 From 45759fcbd49784401110ea399e04cd7e8fe85d44 Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Mon, 23 Dec 2013 14:18:59 +0000 Subject: Add optional overwrite option This option lets the install-files config extension overwrite existing files. A file will only be overwritten if the overwrite flag is specified for that file. Since the overwrite arg is optionally prepended to the manifest line, this patch should not break existing manifests With this patch default config files can be replaced with project specific config files --- install-files.configure | 42 +++++++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 11 deletions(-) diff --git a/install-files.configure b/install-files.configure index 669fc518..8ba61b4e 100755 --- a/install-files.configure +++ b/install-files.configure @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -35,9 +35,9 @@ class InstallFilesConfigureExtension(cliapp.Application): '''Install the files specified in the manifests listed in INSTALL_FILES - The manifest is formatted as: + Entries in the manifest are formatted as: - + [overwrite] Where the filename is how the file is found inside whatever directory the manifest is stored in, and also the path within the system to @@ -47,6 +47,18 @@ class InstallFilesConfigureExtension(cliapp.Application): This extension supports files, symlinks and directories. + For example, + + 0100644 0 0 /etc/issue + + creates a regular file at /etc/issue with 644 permissions, + uid 0 and gid 0, if the file doesn't already exist. + + overwrite 0100644 0 0 /etc/issue + + creates a regular file at /etc/issue with 644 permissions, + uid 0 and gid 0, if the file already exists it is overwritten. + ''' def process_args(self, args): @@ -65,14 +77,22 @@ class InstallFilesConfigureExtension(cliapp.Application): self.install_entry(entry, manifest_dir, target_root) def install_entry(self, entry, manifest_root, target_root): - entry_data = re.split('\W+', entry.strip(), maxsplit=3) - mode = int(entry_data[0], 8) - uid = int(entry_data[1]) - gid = int(entry_data[2]) - path = entry_data[3] + m = re.match('(overwrite )?([0-7]+) ([0-9]+) ([0-9]+) (\S+)', entry) + + if m: + overwrite = m.group(1) + mode = int(m.group(2), 8) # mode is octal + uid = int(m.group(3)) + gid = int(m.group(4)) + path = m.group(5) + else: + raise cliapp.AppException('Invalid manifest entry, ' + 'format: [overwrite] ' + '') + dest_path = os.path.join(target_root, './' + path) if stat.S_ISDIR(mode): - if os.path.exists(dest_path): + if os.path.exists(dest_path) and not overwrite: dest_stat = os.stat(dest_path) if (mode != dest_stat.st_mode or uid != dest_stat.st_uid @@ -86,7 +106,7 @@ class InstallFilesConfigureExtension(cliapp.Application): os.chmod(dest_path, mode) elif stat.S_ISLNK(mode): - if os.path.lexists(dest_path): + if os.path.lexists(dest_path) and not overwrite: raise cliapp.AppException('Symlink already exists at %s' % dest_path) else: @@ -96,7 +116,7 @@ class InstallFilesConfigureExtension(cliapp.Application): os.lchown(dest_path, uid, gid) elif stat.S_ISREG(mode): - if os.path.lexists(dest_path): + if os.path.lexists(dest_path) and not overwrite: raise cliapp.AppException('File already exists at %s' % dest_path) else: -- cgit v1.2.1 From 8fd0a807b70b19ac5a59da4e05a3abe5652ef34a Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 22 Jan 2014 17:42:18 +0000 Subject: virtualbox-ssh: Work around change in VBox options VirtualBox changed a command line option in 4.3 incompatibly, so we now have to check the version number and change an option from --sataportcount to --portcount if the version of VirtualBox running on the target is at least 4.3 This turns the version into a tuple and compares it against another, since it's more reliable than comparing strings, which will count '1.10' as earlier than '1.2', and more convenient than comparing the digits individually. --- virtualbox-ssh.write | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index f18ef804..369c0d61 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -106,6 +106,19 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): str(os.path.getsize(raw_disk))], stdin=f) + def virtualbox_version(self, ssh_host): + 'Get the version number of the VirtualBox running on the remote host.' + + # --version gives a build id, which looks something like + # 1.2.3r456789, so we need to strip the suffix off and get a tuple + # of the (major, minor, patch) version, since comparing with a + # tuple is more reliable than a string and more convenient than + # comparing against the major, minor and patch numbers directly + self.status(msg='Checking version of remote VirtualBox') + build_id = cliapp.ssh_runcmd(ssh_host, ['VBoxManage', '--version']) + version_string = re.match(r"^([0-9\.])+.*$", build_id.strip()).group(1) + return tuple(int(s or '0') for s in version_string.split('.')) + def create_virtualbox_guest(self, ssh_host, vm_name, vdi_path, autostart): '''Create the VirtualBox virtual machine.''' @@ -117,6 +130,11 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): hostonly_iface = self.get_host_interface(ssh_host) + if self.virtualbox_version(ssh_host) < (4, 3): + sataportcount_option = '--sataportcount' + else: + sataportcount_option = '--portcount' + commands = [ ['createvm', '--name', vm_name, '--ostype', 'Linux26_64', '--register'], @@ -125,7 +143,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): '--nic1', 'hostonly', '--hostonlyadapter1', hostonly_iface, '--nic2', 'nat', '--natnet2', 'default'], ['storagectl', vm_name, '--name', 'SATA Controller', - '--add', 'sata', '--bootable', 'on', '--sataportcount', '2'], + '--add', 'sata', '--bootable', 'on', sataportcount_option, '2'], ['storageattach', vm_name, '--storagectl', 'SATA Controller', '--port', '0', '--device', '0', '--type', 'hdd', '--medium', vdi_path], -- cgit v1.2.1 From fc7b833170c66acb4672184146d560c6fda20183 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Tue, 28 Jan 2014 18:17:09 +0000 Subject: Fix copyright year in previous commit --- virtualbox-ssh.write | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 369c0d61..204b2447 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -- cgit v1.2.1 From e419d582cfd0ea873b393b2773c1b0670d16afe0 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Wed, 12 Feb 2014 19:03:05 +0000 Subject: deploy: Finish off the Btrfs system layout implementation The shared state directories defined in writeexts.py (/var, /home etc.) are now separate Btrfs subvolumes that are mounted in place using fstab. There are some warnings on mounting /var and /srv about the mountpoint not being empty. Not yet investigated. If a configure extension has already added / to the fstab, use the device it chose rather than assuming /dev/sda. This is required for the vdaboot.configure extension that we use for OpenStack deployments. Similarly, if a configure extension has added an entry for a state directory in /etc/fstab already, we don't replace it with a /state/xxx directory. That's only done as a default behaviour. --- rawdisk.write | 2 ++ ssh-rsync.write | 2 ++ 2 files changed, 4 insertions(+) diff --git a/rawdisk.write b/rawdisk.write index 8723ac0c..1b4d58c0 100755 --- a/rawdisk.write +++ b/rawdisk.write @@ -56,6 +56,8 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): raise def upgrade_local_system(self, raw_disk, temp_root): + self.complete_fstab_for_btrfs_layout(temp_root) + mp = self.mount(raw_disk) version_label = self.get_version_label(mp) diff --git a/ssh-rsync.write b/ssh-rsync.write index 211dbe5e..fe72bc9a 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -47,6 +47,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.upgrade_remote_system(location, temp_root) def upgrade_remote_system(self, location, temp_root): + self.complete_fstab_for_btrfs_layout(temp_root) + root_disk = self.find_root_disk(location) version_label = os.environ.get('VERSION_LABEL') -- cgit v1.2.1 From 9886dd3e919f7dc66b9099e3c8ab1be79404ae31 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Fri, 14 Feb 2014 12:08:33 +0000 Subject: deploy: Depend on client OS version manager to deploy upgrades We now have a OS version manager tool in Baserock (in tbdiff.git). The code to deploy a new base OS version should live there, to minimise duplication between write extensions. --- ssh-rsync.write | 132 ++++++++++++++++++-------------------------------------- 1 file changed, 41 insertions(+), 91 deletions(-) diff --git a/ssh-rsync.write b/ssh-rsync.write index fe72bc9a..4961ee4d 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -26,6 +26,14 @@ import tempfile import morphlib.writeexts + +def ssh_runcmd_ignore_failure(location, command, **kwargs): + try: + return cliapp.ssh_runcmd(location, command, **kwargs) + except cliapp.AppException: + pass + + class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): '''Upgrade a running baserock system with ssh and rsync. @@ -58,15 +66,11 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Mounting root disk') cliapp.ssh_runcmd(location, ['mount', root_disk, remote_mnt]) except Exception as e: - try: - cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) - except: - pass + ssh_runcmd_ignore_failure(location, ['rmdir', remote_mnt]) raise e try: version_root = os.path.join(remote_mnt, 'systems', version_label) - run_dir = os.path.join(version_root, 'run') orig_dir = os.path.join(version_root, 'orig') self.status(msg='Creating %s' % version_root) @@ -75,81 +79,32 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.create_remote_orig(location, version_root, remote_mnt, temp_root) - self.status(msg='Creating "run" subvolume') - cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', - 'snapshot', orig_dir, run_dir]) - - self.status(msg='Updating system configuration') - bscs_loc = os.path.join(run_dir, 'usr', 'bin', - 'baserock-system-config-sync') - - output = cliapp.ssh_runcmd(location, ['sh', '-c', - '"$1" merge "$2" &> /dev/null || echo -n cmdfailed', - '-', bscs_loc, version_label]) - if output == "cmdfailed": - self.status(msg='Updating system configuration failed') - - self.install_remote_kernel(location, version_root, temp_root) - default_path = os.path.join(remote_mnt, 'systems', 'default') - if self.bootloader_is_wanted(): - output = cliapp.ssh_runcmd(location, ['sh', '-c', - 'test -e "$1" && stat -c %F "$1" ' - '|| echo missing file', - '-', default_path]) - if output != "symbolic link": - # we are upgrading and old system that does - # not have an updated extlinux config file - self.update_remote_extlinux(location, remote_mnt, - version_label) - cliapp.ssh_runcmd(location, ['ln', '-sfn', version_label, - default_path]) + # Use the system-version-manager from the new system we just + # installed, so that we can upgrade from systems that don't have + # it installed. + self.status(msg='Calling system-version-manager to deploy upgrade') + deployment = os.path.join('/systems', version_label, 'orig') + system_config_sync = os.path.join( + remote_mnt, 'systems', version_label, 'orig', 'usr', 'bin', + 'baserock-system-config-sync') + system_version_manager = os.path.join( + remote_mnt, 'systems', version_label, 'orig', 'usr', 'bin', + 'system-version-manager') + cliapp.ssh_runcmd(location, + ['env', 'BASEROCK_SYSTEM_CONFIG_SYNC='+system_config_sync, + system_version_manager, 'deploy', deployment]) except Exception as e: - try: - cliapp.ssh_runcmd(location, - ['btrfs', 'subvolume', 'delete', run_dir]) - except: - pass - try: - cliapp.ssh_runcmd(location, - ['btrfs', 'subvolume', 'delete', orig_dir]) - except: - pass - try: - cliapp.ssh_runcmd(location, ['rm', '-rf', version_root]) - except: - pass + self.status(msg='Deployment failed') + ssh_runcmd_ignore_failure( + location, ['btrfs', 'subvolume', 'delete', orig_dir]) + ssh_runcmd_ignore_failure( + location, ['rm', '-rf', version_root]) raise e finally: self.status(msg='Removing temporary mounts') cliapp.ssh_runcmd(location, ['umount', remote_mnt]) cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) - def update_remote_extlinux(self, location, remote_mnt, version_label): - '''Install/reconfigure extlinux on location''' - - self.status(msg='Creating extlinux.conf') - config = os.path.join(remote_mnt, 'extlinux.conf') - temp_fd, temp_path = tempfile.mkstemp() - with os.fdopen(temp_fd, 'w') as f: - f.write('default linux\n') - f.write('timeout 1\n') - f.write('label linux\n') - f.write('kernel /systems/default/kernel\n') - f.write('append root=/dev/sda ' - 'rootflags=subvol=systems/default/run ' - 'init=/sbin/init rw\n') - - try: - cliapp.runcmd(['rsync', '-as', temp_path, - '%s:%s~' % (location, config)]) - cliapp.ssh_runcmd(location, ['mv', config+'~', config]) - except Exception as e: - try: - cliapp.ssh_runcmd(location, ['rm', '-f', config+'~']) - except: - pass - raise e - def create_remote_orig(self, location, version_root, remote_mnt, temp_root): '''Create the subvolume version_root/orig on location''' @@ -180,18 +135,6 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): if (line_words[1] == '/' and line_words[0] != 'rootfs'): return line_words[0] - def install_remote_kernel(self, location, version_root, temp_root): - '''Install the kernel in temp_root inside version_root on location''' - - self.status(msg='Installing kernel') - image_names = ('vmlinuz', 'zImage', 'uImage') - kernel_dest = os.path.join(version_root, 'kernel') - for name in image_names: - try_path = os.path.join(temp_root, 'boot', name) - if os.path.exists(try_path): - cliapp.runcmd(['rsync', '-as', try_path, - '%s:%s' % (location, kernel_dest)]) - def check_valid_target(self, location): try: cliapp.ssh_runcmd(location, ['true']) @@ -205,10 +148,17 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): raise cliapp.AppException('%s is not a baserock system' % location) - output = cliapp.ssh_runcmd(location, ['sh', '-c', - 'type rsync &> /dev/null || echo -n cmdnotfound']) - if output == 'cmdnotfound': - raise cliapp.AppException('%s does not have rsync' - % location) + def check_command_exists(command): + test = 'type %s > /dev/null 2>&1 || echo -n cmdnotfound' % command + output = cliapp.ssh_runcmd(location, ['sh', '-c', test]) + if output == 'cmdnotfound': + raise cliapp.AppException( + "%s does not have %s" % (location, command)) + + # The deploy requires baserock-system-config-sync and + # system-version-manager in the new system only. The old system doesn't + # need to have them at all. + check_command_exists('rsync') + SshRsyncWriteExtension().run() -- cgit v1.2.1 From 539eed7e4045ef39377f4c7fe6cc9314799b9f4d Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Tue, 4 Mar 2014 11:49:02 +0000 Subject: deploy: Always set new system as default --- ssh-rsync.write | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ssh-rsync.write b/ssh-rsync.write index 4961ee4d..8dc0fe35 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -93,6 +93,11 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): cliapp.ssh_runcmd(location, ['env', 'BASEROCK_SYSTEM_CONFIG_SYNC='+system_config_sync, system_version_manager, 'deploy', deployment]) + + self.status(msg='Setting %s as the new default system' % + version_label) + cliapp.ssh_runcmd(location, + [system_version_manager, 'set-default', version_label]) except Exception as e: self.status(msg='Deployment failed') ssh_runcmd_ignore_failure( -- cgit v1.2.1 From e8b7abe65fcab62ecb84b7f0c2c6f9dde4e63ea7 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Mon, 17 Feb 2014 15:36:56 +0000 Subject: Make parse_autostart() into more general get_environment_boolean() Also, be more flexible when parsing environment booleans -- convert to lower case and match 0/1 and true/false as well as yes/no. --- kvm.write | 4 ++-- rawdisk.write | 2 +- virtualbox-ssh.write | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/kvm.write b/kvm.write index 4f877c22..94560972 100755 --- a/kvm.write +++ b/kvm.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -56,7 +56,7 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): temp_root, location = args ssh_host, vm_name, vm_path = self.parse_location(location) - autostart = self.parse_autostart() + autostart = self.get_environment_boolean('AUTOSTART') fd, raw_disk = tempfile.mkstemp() os.close(fd) diff --git a/rawdisk.write b/rawdisk.write index 1b4d58c0..bde9d67d 100755 --- a/rawdisk.write +++ b/rawdisk.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 204b2447..2a2f3f7b 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -62,7 +62,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): temp_root, location = args ssh_host, vm_name, vdi_path = self.parse_location(location) - autostart = self.parse_autostart() + autostart = self.get_environment_boolean('AUTOSTART') fd, raw_disk = tempfile.mkstemp() os.close(fd) -- cgit v1.2.1 From 9293be701e6b8ae2a1017bc5df9b80c85c735173 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Mon, 17 Feb 2014 13:31:47 +0000 Subject: deploy: Honour AUTOSTART in ssh-rsync extension Now you can deploy an upgrade, set it to be the default version and reboot into it all with one call to `morph deploy`. --- ssh-rsync.write | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ssh-rsync.write b/ssh-rsync.write index 8dc0fe35..509520ae 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -59,6 +59,7 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): root_disk = self.find_root_disk(location) version_label = os.environ.get('VERSION_LABEL') + autostart = self.get_environment_boolean('AUTOSTART') self.status(msg='Creating remote mount point') remote_mnt = cliapp.ssh_runcmd(location, ['mktemp', '-d']).strip() @@ -110,6 +111,10 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): cliapp.ssh_runcmd(location, ['umount', remote_mnt]) cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) + if autostart: + self.status(msg="Rebooting into new system ...") + ssh_runcmd_ignore_failure(location, ['reboot']) + def create_remote_orig(self, location, version_root, remote_mnt, temp_root): '''Create the subvolume version_root/orig on location''' -- cgit v1.2.1 From c808549169a7704cdd5928e0d75a30ecc8036487 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Mon, 17 Feb 2014 16:17:00 +0000 Subject: deploy: Check the --upgrade flag has been used correctly. Most write extensions don't handle both initial deployments and upgrades of a system. --- kvm.check | 35 +++++++++++++++++++++++++++++++++++ nfsboot.check | 34 ++++++++++++++++++++++++++++++++++ openstack.check | 35 +++++++++++++++++++++++++++++++++++ ssh-rsync.check | 36 ++++++++++++++++++++++++++++++++++++ tar.check | 24 ++++++++++++++++++++++++ virtualbox-ssh.check | 35 +++++++++++++++++++++++++++++++++++ 6 files changed, 199 insertions(+) create mode 100755 kvm.check create mode 100755 nfsboot.check create mode 100755 openstack.check create mode 100755 ssh-rsync.check create mode 100755 tar.check create mode 100755 virtualbox-ssh.check diff --git a/kvm.check b/kvm.check new file mode 100755 index 00000000..be7c51c2 --- /dev/null +++ b/kvm.check @@ -0,0 +1,35 @@ +#!/usr/bin/python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''Preparatory checks for Morph 'kvm' write extension''' + +import cliapp + +import morphlib.writeexts + + +class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + upgrade = self.get_environment_boolean('UPGRADE') + if upgrade: + raise cliapp.AppException( + 'Use the `ssh-rsync` write extension to deploy upgrades to an ' + 'existing remote system.') + +KvmPlusSshCheckExtension().run() diff --git a/nfsboot.check b/nfsboot.check new file mode 100755 index 00000000..092a1df7 --- /dev/null +++ b/nfsboot.check @@ -0,0 +1,34 @@ +#!/usr/bin/python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''Preparatory checks for Morph 'nfsboot' write extension''' + +import cliapp + +import morphlib.writeexts + + +class NFSBootCheckExtension(morphlib.writeexts.WriteExtension): + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + upgrade = self.get_environment_boolean('UPGRADE') + if upgrade: + raise cliapp.AppException( + 'Upgrading is not currently supported for NFS deployments.') + +NFSBootCheckExtension().run() diff --git a/openstack.check b/openstack.check new file mode 100755 index 00000000..a9a8fe1b --- /dev/null +++ b/openstack.check @@ -0,0 +1,35 @@ +#!/usr/bin/python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''Preparatory checks for Morph 'openstack' write extension''' + +import cliapp + +import morphlib.writeexts + + +class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + upgrade = self.get_environment_boolean('UPGRADE') + if upgrade: + raise cliapp.AppException( + 'Use the `ssh-rsync` write extension to deploy upgrades to an ' + 'existing remote system.') + +OpenStackCheckExtension().run() diff --git a/ssh-rsync.check b/ssh-rsync.check new file mode 100755 index 00000000..90029cb4 --- /dev/null +++ b/ssh-rsync.check @@ -0,0 +1,36 @@ +#!/usr/bin/python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''Preparatory checks for Morph 'ssh-rsync' write extension''' + +import cliapp + +import morphlib.writeexts + + +class SshRsyncCheckExtension(morphlib.writeexts.WriteExtension): + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + upgrade = self.get_environment_boolean('UPGRADE') + if not upgrade: + raise cliapp.AppException( + 'The ssh-rsync write is for upgrading existing remote ' + 'Baserock machines. It cannot be used for an initial ' + 'deployment.') + +SshRsyncCheckExtension().run() diff --git a/tar.check b/tar.check new file mode 100755 index 00000000..cbeaf163 --- /dev/null +++ b/tar.check @@ -0,0 +1,24 @@ +#!/bin/sh +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# Preparatory checks for Morph 'tar' write extension + +set -eu + +if [ "$UPGRADE" == "yes" ]; then + echo >&2 "ERROR: Cannot upgrade a tar file deployment." + exit 1 +fi diff --git a/virtualbox-ssh.check b/virtualbox-ssh.check new file mode 100755 index 00000000..1aeb8999 --- /dev/null +++ b/virtualbox-ssh.check @@ -0,0 +1,35 @@ +#!/usr/bin/python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''Preparatory checks for Morph 'virtualbox-ssh' write extension''' + +import cliapp + +import morphlib.writeexts + + +class VirtualBoxPlusSshCheckExtension(morphlib.writeexts.WriteExtension): + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + upgrade = self.get_environment_boolean('UPGRADE') + if upgrade: + raise cliapp.AppException( + 'Use the `ssh-rsync` write extension to deploy upgrades to an ' + 'existing remote system.') + +VirtualBoxPlusSshCheckExtension().run() -- cgit v1.2.1 From 0bccd86aa45da7ddde2486c21cee939dc871c53f Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Mon, 17 Feb 2014 17:03:54 +0000 Subject: deploy: Fix double exception in rawdisk.write If the disk image was not yet created then the os.remove() call fails and the original exception gets lost, causing confusion and sadness. Also print status earlier on failure --- rawdisk.write | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rawdisk.write b/rawdisk.write index bde9d67d..87edf7bf 100755 --- a/rawdisk.write +++ b/rawdisk.write @@ -50,9 +50,10 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): self.create_local_system(temp_root, location) self.status(msg='Disk image has been created at %s' % location) except Exception: - os.remove(location) self.status(msg='Failure to create disk image at %s' % location) + if os.path.exists(location): + os.remove(location) raise def upgrade_local_system(self, raw_disk, temp_root): -- cgit v1.2.1 From 8e48860748a4455420fdccfb00395f7f395e8e3c Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Mon, 10 Mar 2014 16:59:07 +0000 Subject: Add sysroot write extension This moves the deployed system to somewhere on the host. Any existing contents of the directory is deleted, so don't try to be clever and deploy a new system on top of / in place of a proper upgrade. It can be used to deploy a chroot, sysroot or container, but its current use is to allow for nested deployments to include another system in itself, since the parent deployment's "$1" is prepended to the sub-deployment's "$2". --- sysroot.write | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100755 sysroot.write diff --git a/sysroot.write b/sysroot.write new file mode 100755 index 00000000..1ae4864f --- /dev/null +++ b/sysroot.write @@ -0,0 +1,29 @@ +#!/bin/sh +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# A Morph write extension to deploy to another directory + +set -eu + +# Ensure the target is an empty directory +mkdir -p "$2" +find "$2" -mindepth 1 -delete + +# Move the contents of our source directory to our target +# Previously we would (cd "$1" && find -print0 | cpio -0pumd "$absolute_path") +# to do this, but the source directory is disposable anyway, so we can move +# its contents to save time +find "$1" -maxdepth 1 -mindepth 1 -exec mv {} "$2/." + -- cgit v1.2.1 From 5e686455f6cd4ef5870933d60544af78b167a545 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Tue, 18 Mar 2014 17:29:13 +0000 Subject: Remove fstab hack from nfsboot.configure There is no longer a default /etc/fstab in the Baserock fhs-dirs chunk, and the nfsboot.write extension does not use the default Btrfs system layout so no entry is added for / to /etc/fstab at deploy-time. We cannot have / in /etc/fstab for nfsboot deployments because it causes systemd to remount / during bootup, which breaks everything. --- nfsboot.configure | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/nfsboot.configure b/nfsboot.configure index 8dc6c67c..d6b254d4 100755 --- a/nfsboot.configure +++ b/nfsboot.configure @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -26,7 +26,4 @@ auto lo iface lo inet loopback EOF - # Stop fstab from mounting '/' - mv "$1/etc/fstab" "$1/etc/fstab.old" - awk '/^ *#/ || $2 != "/"' "$1/etc/fstab.old" > "$1/etc/fstab" fi -- cgit v1.2.1 From fbf84afbb9c0bc27816ffcf310a681335c4775ee Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Tue, 18 Mar 2014 17:40:58 +0000 Subject: Improve comment in nfsboot.configure --- nfsboot.configure | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/nfsboot.configure b/nfsboot.configure index d6b254d4..660d9c39 100755 --- a/nfsboot.configure +++ b/nfsboot.configure @@ -15,7 +15,9 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -# Remove all networking interfaces and stop fstab from mounting '/' +# Remove all networking interfaces. On nfsboot systems, eth0 is set up +# during kernel init, and the normal ifup@eth0.service systemd unit +# would break the NFS connection and cause the system to hang. set -e -- cgit v1.2.1 From 9faeac0a60b40256454e7049964898f3c749fd62 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Tue, 18 Mar 2014 17:32:59 +0000 Subject: Make sanity checks for nfsboot deployments run earlier Move some code to the '.check' extension to verify that the deployment can happen *before* spending 5 minutes unpacking and configuring the rootfs. This is not a perfect solution yet because when multiple systems are being deployed in a cluster, we do not check all systems and then deploy them all. Instead, we check one, then deploy it, then check the second, etc. --- nfsboot.check | 66 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ nfsboot.write | 59 ++++------------------------------------------------ 2 files changed, 70 insertions(+), 55 deletions(-) diff --git a/nfsboot.check b/nfsboot.check index 092a1df7..e240dd72 100755 --- a/nfsboot.check +++ b/nfsboot.check @@ -17,18 +17,84 @@ '''Preparatory checks for Morph 'nfsboot' write extension''' import cliapp +import os import morphlib.writeexts class NFSBootCheckExtension(morphlib.writeexts.WriteExtension): + + _nfsboot_root = '/srv/nfsboot' + def process_args(self, args): if len(args) != 1: raise cliapp.AppException('Wrong number of command line args') + location = args[0] + upgrade = self.get_environment_boolean('UPGRADE') if upgrade: raise cliapp.AppException( 'Upgrading is not currently supported for NFS deployments.') + hostname = os.environ.get('HOSTNAME', None) + if hostname is None: + raise cliapp.AppException('You must specify a HOSTNAME.') + if hostname == 'baserock': + raise cliapp.AppException('It is forbidden to nfsboot a system ' + 'with hostname "%s"' % hostname) + + self.test_good_server(location) + + version_label = os.getenv('VERSION_LABEL', 'factory') + versioned_root = os.path.join(self._nfsboot_root, hostname, 'systems', + version_label) + if self.version_exists(versioned_root, location): + raise cliapp.AppException( + 'Root file system for host %s (version %s) already exists on ' + 'the NFS server %s. Deployment aborted.' % (hostname, + version_label, location)) + + def test_good_server(self, server): + # Can be ssh'ed into + try: + cliapp.ssh_runcmd('root@%s' % server, ['true']) + except cliapp.AppException: + raise cliapp.AppException('You are unable to ssh into server %s' + % server) + + # Is an NFS server + try: + cliapp.ssh_runcmd( + 'root@%s' % server, ['test', '-e', '/etc/exports']) + except cliapp.AppException: + raise cliapp.AppException('server %s is not an nfs server' + % server) + try: + cliapp.ssh_runcmd( + 'root@%s' % server, ['systemctl', 'is-enabled', + 'nfs-server.service']) + + except cliapp.AppException: + raise cliapp.AppException('server %s does not control its ' + 'nfs server by systemd' % server) + + # TFTP server exports /srv/nfsboot/tftp + try: + cliapp.ssh_runcmd( + 'root@%s' % server, ['test' , '-d', '/srv/nfsboot/tftp']) + except cliapp.AppException: + raise cliapp.AppException('server %s does not export ' + '/srv/nfsboot/tftp' % server) + + def version_exists(self, versioned_root, location): + try: + cliapp.ssh_runcmd('root@%s' % location, + ['test', '-d', versioned_root]) + except cliapp.AppException: + return False + + return True + + NFSBootCheckExtension().run() diff --git a/nfsboot.write b/nfsboot.write index 34a72972..3bb66821 100755 --- a/nfsboot.write +++ b/nfsboot.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -60,38 +60,18 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): raise cliapp.AppException('Wrong number of command line args') temp_root, location = args - hostname = self.get_hostname(temp_root) - if hostname == 'baserock': - raise cliapp.AppException('It is forbidden to nfsboot a system ' - 'with hostname "baserock"') - self.test_good_server(location) version_label = os.getenv('VERSION_LABEL', 'factory') + hostname = os.environ.get('HOSTNAME') + versioned_root = os.path.join(self._nfsboot_root, hostname, 'systems', version_label) - if self.version_exists(versioned_root, location): - raise cliapp.AppException('Version %s already exists on' - ' this device. Deployment aborted' - % version_label) + self.copy_rootfs(temp_root, location, versioned_root, hostname) self.copy_kernel(temp_root, location, versioned_root, version_label, hostname) self.configure_nfs(location, hostname) - def version_exists(self, versioned_root, location): - try: - cliapp.ssh_runcmd('root@%s' % location, - ['test', '-d', versioned_root]) - except cliapp.AppException: - return False - - return True - - def get_hostname(self, temp_root): - hostnamepath = os.path.join(temp_root, 'etc', 'hostname') - with open(hostnamepath) as f: - return f.readline().strip() - def create_local_state(self, location, hostname): statedir = os.path.join(self._nfsboot_root, hostname, 'state') subdirs = [os.path.join(statedir, 'home'), @@ -209,37 +189,6 @@ mv "$temp" "$target" 'root@%s' % location, ['systemctl', 'restart', 'nfs-server.service']) - def test_good_server(self, server): - # Can be ssh'ed into - try: - cliapp.ssh_runcmd('root@%s' % server, ['true']) - except cliapp.AppException: - raise cliapp.AppException('You are unable to ssh into server %s' - % server) - - # Is an NFS server - try: - cliapp.ssh_runcmd( - 'root@%s' % server, ['test', '-e', '/etc/exports']) - except cliapp.AppException: - raise cliapp.AppException('server %s is not an nfs server' - % server) - try: - cliapp.ssh_runcmd( - 'root@%s' % server, ['systemctl', 'is-enabled', - 'nfs-server.service']) - - except cliapp.AppException: - raise cliapp.AppException('server %s does not control its ' - 'nfs server by systemd' % server) - - # TFTP server exports /srv/nfsboot/tftp - try: - cliapp.ssh_runcmd( - 'root@%s' % server, ['test' , '-d', '/srv/nfsboot/tftp']) - except cliapp.AppException: - raise cliapp.AppException('server %s does not export ' - '/srv/nfsboot/tftp' % server) NFSBootWriteExtension().run() -- cgit v1.2.1 From 5847e6d821748b386d68e8e982d8efcb3358e4e9 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Mon, 24 Mar 2014 16:24:49 +0000 Subject: Be consistent with nfsboot_root path We were building it from a variable in some places and hardcoding it in others; now we build it from a variable everywhere. --- nfsboot.check | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/nfsboot.check b/nfsboot.check index e240dd72..f84f187f 100755 --- a/nfsboot.check +++ b/nfsboot.check @@ -80,12 +80,13 @@ class NFSBootCheckExtension(morphlib.writeexts.WriteExtension): 'nfs server by systemd' % server) # TFTP server exports /srv/nfsboot/tftp + tftp_root = os.path.join(self._nfsboot_root, 'tftp') try: cliapp.ssh_runcmd( - 'root@%s' % server, ['test' , '-d', '/srv/nfsboot/tftp']) + 'root@%s' % server, ['test' , '-d', tftp_root]) except cliapp.AppException: - raise cliapp.AppException('server %s does not export ' - '/srv/nfsboot/tftp' % server) + raise cliapp.AppException('server %s does not export %s' % + (tftp_root, server)) def version_exists(self, versioned_root, location): try: -- cgit v1.2.1 From 703572829cb5e4d2a45109020a8f7a32c031f554 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Mon, 24 Mar 2014 16:26:30 +0000 Subject: Use os.environ[] instead of os.environ.get() to read a required variable --- nfsboot.write | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nfsboot.write b/nfsboot.write index 3bb66821..8d3d6df7 100755 --- a/nfsboot.write +++ b/nfsboot.write @@ -62,7 +62,7 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): temp_root, location = args version_label = os.getenv('VERSION_LABEL', 'factory') - hostname = os.environ.get('HOSTNAME') + hostname = os.environ['HOSTNAME'] versioned_root = os.path.join(self._nfsboot_root, hostname, 'systems', version_label) -- cgit v1.2.1 From faa30c752633303baf53cf61de5bae42f43f6ab0 Mon Sep 17 00:00:00 2001 From: Mark Doffman Date: Tue, 1 Apr 2014 03:27:53 +0000 Subject: Add help files for nfsboot, rawdisk and tar. Begin adding help documentation for configuration and write extensions starting with nfsboot.write, rawdisk.write and tar.write. --- nfsboot.write.help | 12 ++++++++++++ rawdisk.write.help | 7 +++++++ tar.write.help | 5 +++++ 3 files changed, 24 insertions(+) create mode 100644 nfsboot.write.help create mode 100644 rawdisk.write.help create mode 100644 tar.write.help diff --git a/nfsboot.write.help b/nfsboot.write.help new file mode 100644 index 00000000..598b1b23 --- /dev/null +++ b/nfsboot.write.help @@ -0,0 +1,12 @@ +help: | + Deploy a system image and kernel to an nfsboot server. + + An nfsboot server is defined as a baserock system that has + tftp and nfs servers running, the tftp server is exporting + the contents of /srv/nfsboot/tftp/ and the user has sufficient + permissions to create nfs roots in /srv/nfsboot/nfs/. + + The `location` argument is the hostname of the nfsboot server. + + The extension will connect to root@HOST via ssh to copy the + kernel and rootfs, and configure the nfs server. diff --git a/rawdisk.write.help b/rawdisk.write.help new file mode 100644 index 00000000..a514a4e8 --- /dev/null +++ b/rawdisk.write.help @@ -0,0 +1,7 @@ +help: | + Create a raw disk image during Morph's deployment. + + If the image already exists, it is upgraded. + + The `location` argument is a pathname to the image to be + created or upgraded. diff --git a/tar.write.help b/tar.write.help new file mode 100644 index 00000000..f052ac03 --- /dev/null +++ b/tar.write.help @@ -0,0 +1,5 @@ +help: | + Create a .tar file of the deployed system. + + The `location` argument is a pathname to the .tar file to be + created. -- cgit v1.2.1 From 0328bdbf7c3d2def974ca3279fe8732f6f8fa968 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Fri, 11 Apr 2014 12:29:32 +0000 Subject: deploy: Check SSH connection for KVM deployment before starting Slight duplication is necessary, but it's only a few lines. We could move the duplicated code into the base class in 'morphlib.writeexts' if there was more duplication. --- kvm.check | 29 +++++++++++++++++++++++++++++ kvm.write | 9 +++------ 2 files changed, 32 insertions(+), 6 deletions(-) diff --git a/kvm.check b/kvm.check index be7c51c2..04c25069 100755 --- a/kvm.check +++ b/kvm.check @@ -17,11 +17,16 @@ '''Preparatory checks for Morph 'kvm' write extension''' import cliapp +import re +import urlparse import morphlib.writeexts class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): + + location_pattern = '^/(?P[^/]+)(?P/.+)$' + def process_args(self, args): if len(args) != 1: raise cliapp.AppException('Wrong number of command line args') @@ -32,4 +37,28 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): 'Use the `ssh-rsync` write extension to deploy upgrades to an ' 'existing remote system.') + location = args[0] + ssh_host, vm_name, vm_path = self.check_and_parse_location(location) + + try: + cliapp.ssh_runcmd(ssh_host, ['true']) + except cliapp.AppException: + raise cliapp.AppException('Unable to SSH to %s' % ssh_host) + + def check_and_parse_location(self, location): + '''Check and parse the location argument to get relevant data.''' + + x = urlparse.urlparse(location) + + if x.scheme != 'kvm+ssh': + raise cliapp.AppException( + 'URL schema must be kvm+ssh in %s' % location) + + m = re.match(self.location_pattern, x.path) + if not m: + raise cliapp.AppException('Cannot parse location %s' % location) + + return x.netloc, m.group('guest'), m.group('path') + + KvmPlusSshCheckExtension().run() diff --git a/kvm.write b/kvm.write index 94560972..94a55daa 100755 --- a/kvm.write +++ b/kvm.write @@ -50,6 +50,8 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): ''' + location_pattern = '^/(?P[^/]+)(?P/.+)$' + def process_args(self, args): if len(args) != 2: raise cliapp.AppException('Wrong number of command line args') @@ -79,14 +81,9 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): def parse_location(self, location): '''Parse the location argument to get relevant data.''' - + x = urlparse.urlparse(location) - if x.scheme != 'kvm+ssh': - raise cliapp.AppException( - 'URL schema must be vbox+ssh in %s' % location) m = re.match('^/(?P[^/]+)(?P/.+)$', x.path) - if not m: - raise cliapp.AppException('Cannot parse location %s' % location) return x.netloc, m.group('guest'), m.group('path') def transfer(self, raw_disk, ssh_host, vm_path): -- cgit v1.2.1 From 7223d97177b570020941562c9c054584d41c5d7c Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Mon, 14 Apr 2014 12:35:26 +0000 Subject: deploy: Share SSH connectivity check in the common writeexts.py code Also, change it to log the real error message in morph.log before raising a more general exception to the user. --- kvm.check | 5 +---- nfsboot.check | 7 +------ 2 files changed, 2 insertions(+), 10 deletions(-) diff --git a/kvm.check b/kvm.check index 04c25069..6ce52e7e 100755 --- a/kvm.check +++ b/kvm.check @@ -40,10 +40,7 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): location = args[0] ssh_host, vm_name, vm_path = self.check_and_parse_location(location) - try: - cliapp.ssh_runcmd(ssh_host, ['true']) - except cliapp.AppException: - raise cliapp.AppException('Unable to SSH to %s' % ssh_host) + self.check_ssh_connectivity(ssh_host) def check_and_parse_location(self, location): '''Check and parse the location argument to get relevant data.''' diff --git a/nfsboot.check b/nfsboot.check index f84f187f..806e560a 100755 --- a/nfsboot.check +++ b/nfsboot.check @@ -56,12 +56,7 @@ class NFSBootCheckExtension(morphlib.writeexts.WriteExtension): version_label, location)) def test_good_server(self, server): - # Can be ssh'ed into - try: - cliapp.ssh_runcmd('root@%s' % server, ['true']) - except cliapp.AppException: - raise cliapp.AppException('You are unable to ssh into server %s' - % server) + self.check_ssh_connectivity(server) # Is an NFS server try: -- cgit v1.2.1 From 8bb0d71ba1ada3c26b6bfa553c2ceb831c161818 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Mon, 14 Apr 2014 13:19:20 +0000 Subject: deploy: Extra checks for KVM deployments Test that a VM with the given name does not already exist, and check that the files specified in ATTACH_DISKS do already exist. --- kvm.check | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/kvm.check b/kvm.check index 6ce52e7e..957d0893 100755 --- a/kvm.check +++ b/kvm.check @@ -41,6 +41,8 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): ssh_host, vm_name, vm_path = self.check_and_parse_location(location) self.check_ssh_connectivity(ssh_host) + self.check_no_existing_libvirt_vm(ssh_host, vm_name) + self.check_extra_disks_exist(ssh_host, self.parse_attach_disks()) def check_and_parse_location(self, location): '''Check and parse the location argument to get relevant data.''' @@ -57,5 +59,24 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): return x.netloc, m.group('guest'), m.group('path') + def check_no_existing_libvirt_vm(self, ssh_host, vm_name): + try: + cliapp.ssh_runcmd(ssh_host, + ['virsh', '--connect', 'qemu:///system', 'domstate', vm_name]) + except cliapp.AppException as e: + pass + else: + raise cliapp.AppException( + 'Host %s already has a VM named %s. You can use the ssh-rsync ' + 'write extension to deploy upgrades to existing machines.' % + (ssh_host, vm_name)) + + def check_extra_disks_exist(self, ssh_host, filename_list): + for filename in filename_list: + try: + cliapp.ssh_runcmd(ssh_host, ['ls', filename]) + except cliapp.AppException as e: + raise cliapp.AppException('Did not find file %s on host %s' % + (filename, ssh_host)) KvmPlusSshCheckExtension().run() -- cgit v1.2.1 From a0e42d6be4e73d58f3568264a399cf6a39fc3e42 Mon Sep 17 00:00:00 2001 From: Paul Sherwood Date: Sun, 20 Apr 2014 15:40:50 +0000 Subject: fix the Virtualbox version checking --- virtualbox-ssh.write | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 2a2f3f7b..b9d53579 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -116,7 +116,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): # comparing against the major, minor and patch numbers directly self.status(msg='Checking version of remote VirtualBox') build_id = cliapp.ssh_runcmd(ssh_host, ['VBoxManage', '--version']) - version_string = re.match(r"^([0-9\.])+.*$", build_id.strip()).group(1) + version_string = re.match(r"^([0-9\.]+).*$", build_id.strip()).group(1) return tuple(int(s or '0') for s in version_string.split('.')) def create_virtualbox_guest(self, ssh_host, vm_name, vdi_path, autostart): @@ -130,7 +130,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): hostonly_iface = self.get_host_interface(ssh_host) - if self.virtualbox_version(ssh_host) < (4, 3): + if self.virtualbox_version(ssh_host) < (4, 3, 0): sataportcount_option = '--sataportcount' else: sataportcount_option = '--portcount' -- cgit v1.2.1 From 59697ddb89674493dd3621ffaf8fbf34f04668db Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Thu, 8 May 2014 18:43:15 +0100 Subject: Add help for install-files.configure Move the help out of the comment and into a help file, and add a clearer example. --- install-files.configure | 28 --------------------- install-files.configure.help | 60 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 60 insertions(+), 28 deletions(-) create mode 100644 install-files.configure.help diff --git a/install-files.configure b/install-files.configure index 8ba61b4e..04dc5f18 100755 --- a/install-files.configure +++ b/install-files.configure @@ -33,34 +33,6 @@ import stat class InstallFilesConfigureExtension(cliapp.Application): - '''Install the files specified in the manifests listed in INSTALL_FILES - - Entries in the manifest are formatted as: - - [overwrite] - - Where the filename is how the file is found inside whatever directory - the manifest is stored in, and also the path within the system to - install to. - - Directories on the target must be created if they do not exist. - - This extension supports files, symlinks and directories. - - For example, - - 0100644 0 0 /etc/issue - - creates a regular file at /etc/issue with 644 permissions, - uid 0 and gid 0, if the file doesn't already exist. - - overwrite 0100644 0 0 /etc/issue - - creates a regular file at /etc/issue with 644 permissions, - uid 0 and gid 0, if the file already exists it is overwritten. - - ''' - def process_args(self, args): if not 'INSTALL_FILES' in os.environ: return diff --git a/install-files.configure.help b/install-files.configure.help new file mode 100644 index 00000000..eb3aab0c --- /dev/null +++ b/install-files.configure.help @@ -0,0 +1,60 @@ +help: | + Install a set of files onto a system + + To use this extension you create a directory of files you want to install + onto the target system. + + In this example we want to copy some ssh keys onto a system + + % mkdir sshkeyfiles + % mkdir -p sshkeyfiles/root/.ssh + % cp id_rsa sshkeyfiles/root/.ssh + % cp id_rsa.pub sshkeyfiles/root/.ssh + + Now we need to create a manifest file to set the file modes + and persmissions. The manifest file should be created inside the + directory that contains the files we're trying to install. + + cat << EOF > sshkeyfiles/manifest + 0040755 0 0 /root/.ssh + 0100600 0 0 /root/.ssh/id_rsa + 0100644 0 0 /root/.ssh/id_rsa.pub + EOF + + Then we add the path to our manifest to our cluster morph, + this path should be relative to the system definitions repository. + + INSTALL_FILES: sshkeysfiles/manifest + + More generally entries in the manifest are formatted as: + [overwrite] + + NOTE: Directories on the target must be created if they do not exist. + + The extension supports files, symlinks and directories. + + For example, + + 0100644 0 0 /etc/issue + + creates a regular file at /etc/issue with 644 permissions, + uid 0 and gid 0, if the file doesn't already exist. + + overwrite 0100644 0 0 /etc/issue + + creates a regular file at /etc/issue with 644 permissions, + uid 0 and gid 0, if the file already exists it is overwritten. + + 0100755 0 0 /usr/bin/foo + + creates an executable file at /usr/bin/foo + + 0040755 0 0 /etc/foodir + + creates a directory with 755 permissions + + 0120000 0 0 /usr/bin/bar + + creates a symlink at /usr/bin/bar + + NOTE: You will still need to make a symlink in the manifest directory. -- cgit v1.2.1 From 38c8a61410e4ba73d87d43a6101ecefb28433f13 Mon Sep 17 00:00:00 2001 From: Daniel Silverstone Date: Fri, 16 May 2014 09:39:37 +0000 Subject: VirtualBox Write Extension: Vagrant support Add support to the VirtualBox write extension to notice if we are doing a Vagrant Basebox installation and not do the clever network setup we normally do to allow machines to talk to one another since this confuses Vagrant quite a bit if it is left in. --- virtualbox-ssh.write | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index b9d53579..47584b83 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -63,7 +63,9 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): temp_root, location = args ssh_host, vm_name, vdi_path = self.parse_location(location) autostart = self.get_environment_boolean('AUTOSTART') - + + vagrant = self.get_environment_boolean('VAGRANT') + fd, raw_disk = tempfile.mkstemp() os.close(fd) self.create_local_system(temp_root, raw_disk) @@ -72,7 +74,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): self.transfer_and_convert_to_vdi( raw_disk, ssh_host, vdi_path) self.create_virtualbox_guest(ssh_host, vm_name, vdi_path, - autostart) + autostart, vagrant) except BaseException: sys.stderr.write('Error deploying to VirtualBox') os.remove(raw_disk) @@ -119,7 +121,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): version_string = re.match(r"^([0-9\.]+).*$", build_id.strip()).group(1) return tuple(int(s or '0') for s in version_string.split('.')) - def create_virtualbox_guest(self, ssh_host, vm_name, vdi_path, autostart): + def create_virtualbox_guest(self, ssh_host, vm_name, vdi_path, autostart, vagrant): '''Create the VirtualBox virtual machine.''' self.status(msg='Create VirtualBox virtual machine') @@ -128,7 +130,8 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): vcpu_count = str(self.get_vcpu_count()) - hostonly_iface = self.get_host_interface(ssh_host) + if not vagrant: + hostonly_iface = self.get_host_interface(ssh_host) if self.virtualbox_version(ssh_host) < (4, 3, 0): sataportcount_option = '--sataportcount' @@ -139,15 +142,20 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): ['createvm', '--name', vm_name, '--ostype', 'Linux26_64', '--register'], ['modifyvm', vm_name, '--ioapic', 'on', - '--memory', ram_mebibytes, '--cpus', vcpu_count, - '--nic1', 'hostonly', '--hostonlyadapter1', hostonly_iface, - '--nic2', 'nat', '--natnet2', 'default'], + '--memory', ram_mebibytes, '--cpus', vcpu_count], ['storagectl', vm_name, '--name', 'SATA Controller', '--add', 'sata', '--bootable', 'on', sataportcount_option, '2'], ['storageattach', vm_name, '--storagectl', 'SATA Controller', '--port', '0', '--device', '0', '--type', 'hdd', '--medium', vdi_path], ] + if vagrant: + commands[1].extend(['--nic1', 'nat', + '--natnet1', 'default']) + else: + commands[1].extend(['--nic1', 'hostonly', + '--hostonlyadapter1', hostonly_iface, + '--nic2', 'nat', '--natnet2', 'default']) attach_disks = self.parse_attach_disks() for device_no, disk in enumerate(attach_disks, 1): -- cgit v1.2.1 From ddaf9840bbe3ad3a45fff341b1cfdf2985fd760b Mon Sep 17 00:00:00 2001 From: Daniel Silverstone Date: Fri, 16 May 2014 16:18:54 +0000 Subject: Fix long line --- virtualbox-ssh.write | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 47584b83..42585f5e 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -121,7 +121,8 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): version_string = re.match(r"^([0-9\.]+).*$", build_id.strip()).group(1) return tuple(int(s or '0') for s in version_string.split('.')) - def create_virtualbox_guest(self, ssh_host, vm_name, vdi_path, autostart, vagrant): + def create_virtualbox_guest(self, ssh_host, vm_name, vdi_path, autostart, + vagrant): '''Create the VirtualBox virtual machine.''' self.status(msg='Create VirtualBox virtual machine') -- cgit v1.2.1 From 6efbcd6ef631a79f73d2429622296ddfbde09003 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Tue, 20 May 2014 09:37:49 +0000 Subject: deploy: Do sanity checks earlier in ssh-rsync (upgrade) extension --- ssh-rsync.check | 24 ++++++++++++++++++++++++ ssh-rsync.write | 26 -------------------------- 2 files changed, 24 insertions(+), 26 deletions(-) diff --git a/ssh-rsync.check b/ssh-rsync.check index 90029cb4..6a776ce9 100755 --- a/ssh-rsync.check +++ b/ssh-rsync.check @@ -33,4 +33,28 @@ class SshRsyncCheckExtension(morphlib.writeexts.WriteExtension): 'Baserock machines. It cannot be used for an initial ' 'deployment.') + location = args[0] + self.check_ssh_connectivity(location) + self.check_is_baserock_system(location) + + # The new system that being deployed as an upgrade must contain + # baserock-system-config-sync and system-version-manager. However, the + # old system simply needs to have SSH and rsync. + self.check_command_exists(location, 'rsync') + + def check_is_baserock_system(self, location): + output = cliapp.ssh_runcmd(location, ['sh', '-c', + 'test -d /baserock || echo -n dirnotfound']) + if output == 'dirnotfound': + raise cliapp.AppException('%s is not a baserock system' + % location) + + def check_command_exists(self, location, command): + test = 'type %s > /dev/null 2>&1 || echo -n cmdnotfound' % command + output = cliapp.ssh_runcmd(location, ['sh', '-c', test]) + if output == 'cmdnotfound': + raise cliapp.AppException( + "%s does not have %s" % (location, command)) + + SshRsyncCheckExtension().run() diff --git a/ssh-rsync.write b/ssh-rsync.write index 509520ae..c139b6c0 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -51,7 +51,6 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): temp_root, location = args - self.check_valid_target(location) self.upgrade_remote_system(location, temp_root) def upgrade_remote_system(self, location, temp_root): @@ -145,30 +144,5 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): if (line_words[1] == '/' and line_words[0] != 'rootfs'): return line_words[0] - def check_valid_target(self, location): - try: - cliapp.ssh_runcmd(location, ['true']) - except Exception as e: - raise cliapp.AppException('%s does not respond to ssh:\n%s' - % (location, e)) - - output = cliapp.ssh_runcmd(location, ['sh', '-c', - 'test -d /baserock || echo -n dirnotfound']) - if output == 'dirnotfound': - raise cliapp.AppException('%s is not a baserock system' - % location) - - def check_command_exists(command): - test = 'type %s > /dev/null 2>&1 || echo -n cmdnotfound' % command - output = cliapp.ssh_runcmd(location, ['sh', '-c', test]) - if output == 'cmdnotfound': - raise cliapp.AppException( - "%s does not have %s" % (location, command)) - - # The deploy requires baserock-system-config-sync and - # system-version-manager in the new system only. The old system doesn't - # need to have them at all. - check_command_exists('rsync') - SshRsyncWriteExtension().run() -- cgit v1.2.1 From dd380544e7c3a9f6c47e23639bb0a45958c3e446 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Mon, 2 Jun 2014 10:53:29 +0000 Subject: Add initramfs write extension This creates a gzipped cpio archive that may be used as an initramfs. It is hard-coded to use gzip to compress the initramfs, since it's the most common way to do it. This is unfortunate, since the busybox gzip utility only allows maximum compression, which is rather slow and doesn't give progress reporting, so you can easily think it's gotten stuck. It's possible to use other compression formats, but they need the kernel to be built with them supported, and in the case of lz4, unusual userland tools to create it, since the version of lz4 supported in the kernel is not what the standard lz4 tools produce. --- initramfs.write | 27 +++++++++++++++++++++++++++ initramfs.write.help | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100755 initramfs.write create mode 100644 initramfs.write.help diff --git a/initramfs.write b/initramfs.write new file mode 100755 index 00000000..815772f2 --- /dev/null +++ b/initramfs.write @@ -0,0 +1,27 @@ +#!/bin/sh +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# =*= License: GPL-2 =*= + +set -e + +ROOTDIR="$1" +INITRAMFS_PATH="$2" + +(cd "$ROOTDIR" && + find . -print0 | + cpio -0 -H newc -o | + gzip -c) >"$INITRAMFS_PATH" diff --git a/initramfs.write.help b/initramfs.write.help new file mode 100644 index 00000000..29a9d266 --- /dev/null +++ b/initramfs.write.help @@ -0,0 +1,35 @@ +help: | + Create an initramfs for a system by taking an existing system and + converting it to the appropriate format. + + The system must have a `/init` executable as the userland entry-point. + This can have a different path, if `rdinit=$path` is added to + the kernel command line. This can be added to the `rawdisk`, + `virtualbox-ssh` and `kvm` write extensions with the `KERNEL_CMDLINE` + option. + + It is possible to use a ramfs as the final rootfs without a `/init` + executable, by setting `root=/dev/mem`, or `rdinit=/sbin/init`, + but this is beyond the scope for the `initramfs.write` extension. + + The intended use of initramfs.write is to be part of a nested + deployment, so the parent system has an initramfs stored as + `/boot/initramfs.gz`. See the following example: + + name: initramfs-test + kind: cluster + systems: + - morph: minimal-system-x86_64-generic + deploy: + system: + type: rawdisk + location: initramfs-system-x86_64.img + DISK_SIZE: 1G + HOSTNAME: initramfs-system + INITRAMFS_PATH: boot/initramfs.gz + subsystems: + - morph: initramfs-x86_64 + deploy: + initramfs: + type: initramfs + location: boot/initramfs.gz -- cgit v1.2.1 From 3a8c02eb8d9beac6ec6c0570c9a50de72b23fc52 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Mon, 2 Jun 2014 11:09:29 +0000 Subject: Add initramfs support to write extensions that produce disks If INITRAMFS_PATH is specified and the file exists, then the produced kernel command line will use root=UUID=$uuid_of_created_disk rather than root=/dev/sda, which may be incorrect. Help files have been updated to mention the new option. This leads to an unfortunate duplication of the path to the initramfs, in both the location field of the nested deployment and the INITRAMFS_PATH of the disk image creation. However, an initramfs could be produced by a chunk and put in the same place, so it doesn't make sense to couple the rawdisk and initramfs write extensions to remove this duplication. Similarly, there may be multiple valid initramfs in the rootfs e.g. extlinux loads a hypervisor, which is Linux + initramfs, and the initramfs then boots a guest Linux system, which uses a different initramfs. This makes it important to explicitly let the rootfs write extensions know which to use, or not as the case may be. util-linux's blkid is required, since the busybox version ignores the options to filter its output, and parsing the output is undesirable. Because syslinux's btrfs subvolume support is limited to being able to use a non-0 default subvolume, the initramfs has to be copied out of the run-time rootfs subvolume and into the boot subvolume. This pushed the required disk space of a minimal system over the 512M threshold because we do not have the userland tooling support to be able to do a btrfs file contents clone. --- kvm.write.help | 4 ++++ rawdisk.write.help | 4 ++++ virtualbox-ssh.write.help | 4 ++++ 3 files changed, 12 insertions(+) create mode 100644 kvm.write.help create mode 100644 virtualbox-ssh.write.help diff --git a/kvm.write.help b/kvm.write.help new file mode 100644 index 00000000..8b5053a5 --- /dev/null +++ b/kvm.write.help @@ -0,0 +1,4 @@ +help: | + The INITRAMFS_PATH option can be used to specify the location of an + initramfs for syslinux to tell Linux to use, rather than booting + the rootfs directly. diff --git a/rawdisk.write.help b/rawdisk.write.help index a514a4e8..298d441c 100644 --- a/rawdisk.write.help +++ b/rawdisk.write.help @@ -5,3 +5,7 @@ help: | The `location` argument is a pathname to the image to be created or upgraded. + + The INITRAMFS_PATH option can be used to specify the location of an + initramfs for syslinux to tell Linux to use, rather than booting + the rootfs directly. diff --git a/virtualbox-ssh.write.help b/virtualbox-ssh.write.help new file mode 100644 index 00000000..8b5053a5 --- /dev/null +++ b/virtualbox-ssh.write.help @@ -0,0 +1,4 @@ +help: | + The INITRAMFS_PATH option can be used to specify the location of an + initramfs for syslinux to tell Linux to use, rather than booting + the rootfs directly. -- cgit v1.2.1 From 6063df929fb29d152b0b1b7fadf2f2c3dc7327c4 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 4 Jun 2014 16:06:29 +0000 Subject: initramfs.write: create parent directories of location --- initramfs.write | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/initramfs.write b/initramfs.write index 815772f2..f8af6d84 100755 --- a/initramfs.write +++ b/initramfs.write @@ -23,5 +23,5 @@ INITRAMFS_PATH="$2" (cd "$ROOTDIR" && find . -print0 | - cpio -0 -H newc -o | - gzip -c) >"$INITRAMFS_PATH" + cpio -0 -H newc -o) | + gzip -c | install -D -m644 /dev/stdin "$INITRAMFS_PATH" -- cgit v1.2.1 From cc059a443b1dbcd214075dcaf95d601e47ca655f Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Tue, 10 Jun 2014 17:18:37 +0000 Subject: Check for presence of btrfs before trying to use it If btrfs is not present in the kernel we end up with strange output like this: Error creating disk image2014-06-10 16:00:40 [devel-system-x86_64-generic][my-raw-disk-image][rawdisk.write]Failure to create disk image at /src/tmp/testdev.img ERROR: Command failed: mount -o loop /src/tmp/testdev.img /src/tmp/deployments/tmpQ7wXO1/tmp4lVDcu/tmpvHSzDE mount: mounting /dev/loop0 on /src/tmp/deployments/tmpQ7wXO1/tmp4lVDcu/tmpvHSzDE failed: Device or resource busy To avoid this confusing error, Morph should explicitly check first. --- kvm.check | 2 ++ openstack.check | 2 ++ rawdisk.check | 31 +++++++++++++++++++++++++++++++ virtualbox-ssh.check | 2 ++ 4 files changed, 37 insertions(+) create mode 100755 rawdisk.check diff --git a/kvm.check b/kvm.check index 957d0893..1bb4007a 100755 --- a/kvm.check +++ b/kvm.check @@ -31,6 +31,8 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): if len(args) != 1: raise cliapp.AppException('Wrong number of command line args') + self.require_btrfs_in_deployment_host_kernel() + upgrade = self.get_environment_boolean('UPGRADE') if upgrade: raise cliapp.AppException( diff --git a/openstack.check b/openstack.check index a9a8fe1b..b5173011 100755 --- a/openstack.check +++ b/openstack.check @@ -26,6 +26,8 @@ class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): if len(args) != 1: raise cliapp.AppException('Wrong number of command line args') + self.require_btrfs_in_deployment_host_kernel() + upgrade = self.get_environment_boolean('UPGRADE') if upgrade: raise cliapp.AppException( diff --git a/rawdisk.check b/rawdisk.check new file mode 100755 index 00000000..6a656ee7 --- /dev/null +++ b/rawdisk.check @@ -0,0 +1,31 @@ +#!/usr/bin/python +# Copyright (C) 2014 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +'''Preparatory checks for Morph 'rawdisk' write extension''' + +import cliapp + +import morphlib.writeexts + + +class RawdiskCheckExtension(morphlib.writeexts.WriteExtension): + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + self.require_btrfs_in_deployment_host_kernel() + +RawdiskCheckExtension().run() diff --git a/virtualbox-ssh.check b/virtualbox-ssh.check index 1aeb8999..57d54db1 100755 --- a/virtualbox-ssh.check +++ b/virtualbox-ssh.check @@ -26,6 +26,8 @@ class VirtualBoxPlusSshCheckExtension(morphlib.writeexts.WriteExtension): if len(args) != 1: raise cliapp.AppException('Wrong number of command line args') + self.require_btrfs_in_deployment_host_kernel() + upgrade = self.get_environment_boolean('UPGRADE') if upgrade: raise cliapp.AppException( -- cgit v1.2.1 From 0da35ea65a40526cd395b3f47b51743366efccbc Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Tue, 12 Aug 2014 13:59:09 +0000 Subject: Merge remote-tracking branch 'origin/baserock/james/writeexts_support_jetson' --- rawdisk.write | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/rawdisk.write b/rawdisk.write index 87edf7bf..1c2c5a84 100755 --- a/rawdisk.write +++ b/rawdisk.write @@ -85,11 +85,12 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): else: # we are upgrading and old system that does # not have an updated extlinux config file - if self.bootloader_is_wanted(): - self.install_extlinux(mp) + if self.bootloader_config_is_wanted(): + self.generate_bootloader_config(mp) + self.install_bootloader(mp) os.symlink(version_label, default_path) - if self.bootloader_is_wanted(): + if self.bootloader_config_is_wanted(): self.install_kernel(version_root, temp_root) self.unmount(mp) -- cgit v1.2.1 From 59d03b0ae4d1643bba0f0b2b83e85b7068092819 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Tue, 19 Aug 2014 15:05:05 +0000 Subject: deploy: Check correct usage of --upgrade for rawdisk deployments This avoids confusion when the user expected to be doing an initial deployment and wasn't aware that a file with the same name as the target already existed. Previously rawdisk.write would try to mount the file and upgrade it. Now we require the user to pass '--upgrade' when they intend to upgrade, as with other deployment extensions. --- rawdisk.check | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/rawdisk.check b/rawdisk.check index 6a656ee7..5e75abe2 100755 --- a/rawdisk.check +++ b/rawdisk.check @@ -20,6 +20,8 @@ import cliapp import morphlib.writeexts +import os + class RawdiskCheckExtension(morphlib.writeexts.WriteExtension): def process_args(self, args): @@ -28,4 +30,23 @@ class RawdiskCheckExtension(morphlib.writeexts.WriteExtension): self.require_btrfs_in_deployment_host_kernel() + location = args[0] + upgrade = self.get_environment_boolean('UPGRADE') + if upgrade: + if not os.path.isfile(location): + raise cliapp.AppException( + 'Cannot upgrade %s: it is not an existing disk image' % + location) + + version_label = os.environ.get('VERSION_LABEL') + if version_label is None: + raise cliapp.AppException( + 'VERSION_LABEL was not given. It is required when ' + 'upgrading an existing system.') + else: + if os.path.exists(location): + raise cliapp.AppException( + 'Target %s already exists. Pass --upgrade if you want to ' + 'update an existing image.' % location) + RawdiskCheckExtension().run() -- cgit v1.2.1 From 8c4f5dd9adce099693c53d14c1a549d5b4fa88d1 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Fri, 29 Aug 2014 23:18:45 +0100 Subject: Add `morph upgrade` command, deprecate `morph deploy --upgrade` The arguments to `morph deploy` can get quite long, any way we can make it shorter and clearer is useful. We can also avoid having the strange --no-upgrade flag in future. --- rawdisk.check | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/rawdisk.check b/rawdisk.check index 5e75abe2..acdc4de1 100755 --- a/rawdisk.check +++ b/rawdisk.check @@ -46,7 +46,7 @@ class RawdiskCheckExtension(morphlib.writeexts.WriteExtension): else: if os.path.exists(location): raise cliapp.AppException( - 'Target %s already exists. Pass --upgrade if you want to ' - 'update an existing image.' % location) + 'Target %s already exists. Use `morph upgrade` if you ' + 'want to update an existing image.' % location) RawdiskCheckExtension().run() -- cgit v1.2.1 From 7f566024ace800ca358e1bc73ce7e5c237ed6e21 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Fri, 5 Sep 2014 14:50:31 +0100 Subject: Check OpenStack credentials in openstack.check If the credentials are wrong, then morph will fail before attempting the OpenStack deployment. To achieve that openstack.check will attempt to run `glance image-list`. --- openstack.check | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ openstack.write | 14 +------------- 2 files changed, 49 insertions(+), 13 deletions(-) diff --git a/openstack.check b/openstack.check index b5173011..d9d3ef24 100755 --- a/openstack.check +++ b/openstack.check @@ -17,6 +17,8 @@ '''Preparatory checks for Morph 'openstack' write extension''' import cliapp +import os +import urlparse import morphlib.writeexts @@ -34,4 +36,50 @@ class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): 'Use the `ssh-rsync` write extension to deploy upgrades to an ' 'existing remote system.') + location = args[0] + self.check_location(location) + + os_params = self.get_openstack_parameters() + + self.check_openstack_parameters(location, os_params) + + def get_openstack_parameters(self): + '''Check the environment variables needed and returns all. + + The environment variables are described in the class documentation. + ''' + + keys = ('OPENSTACK_USER', 'OPENSTACK_TENANT', + 'OPENSTACK_IMAGENAME', 'OPENSTACK_PASSWORD') + for key in keys: + if key not in os.environ: + raise cliapp.AppException(key + ' was not given') + return (os.environ[key] for key in keys) + + + def check_location(self, location): + x = urlparse.urlparse(location) + if x.scheme not in ['http', 'https']: + raise cliapp.AppException('URL schema must be http or https in %s' \ + % location) + if (x.path != '/v2.0' and x.path != '/v2.0/'): + raise cliapp.AppException('API version must be v2.0 in %s'\ + % location) + + def check_openstack_parameters(self, auth_url, os_params): + '''Check OpenStack credentials using glance image-list''' + self.status(msg='Checking OpenStack credentials...') + + username, tenant_name, image_name, password = os_params + cmdline = ['glance', + '--os-username', username, + '--os-tenant-name', tenant_name, + '--os-password', password, + '--os-auth-url', auth_url, + 'image-list'] + try: + cliapp.runcmd(cmdline) + except cliapp.AppException: + raise cliapp.AppException('Wrong OpenStack credentals.') + OpenStackCheckExtension().run() diff --git a/openstack.write b/openstack.write index 8ee8767e..ac2e2c8a 100755 --- a/openstack.write +++ b/openstack.write @@ -96,27 +96,15 @@ class OpenStackWriteExtension(morphlib.writeexts.WriteExtension): self.unmount(mp) def get_openstack_parameters(self): - '''Check the environment variables needed and returns all. + '''Get the environment variables needed. The environment variables are described in the class documentation. ''' keys = ('OPENSTACK_USER', 'OPENSTACK_TENANT', 'OPENSTACK_IMAGENAME', 'OPENSTACK_PASSWORD') - for key in keys: - if key not in os.environ: - raise cliapp.AppException(key + ' was not given') return (os.environ[key] for key in keys) - def check_location(self, location): - x = urlparse.urlparse(location) - if x.scheme != 'http': - raise cliapp.AppException('URL schema must be http in %s' \ - % location) - if (x.path != '/v2.0' and x.path != '/v2.0/'): - raise cliapp.AppException('API version must be v2.0 in %s'\ - % location) - def configure_openstack_image(self, raw_disk, auth_url, os_params): '''Configure the image in OpenStack using glance-client''' self.status(msg='Configuring OpenStack image...') -- cgit v1.2.1 From 65ed235de623fd152dd2967b9ff2e1f60626c658 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 8 Aug 2014 13:17:01 +0000 Subject: Transfer sparse files faster for kvm, vbox deployment The KVM and VirtualBox deployments use sparse files for raw disk images. This means they can store a large disk (say, tens or hundreds of gigabytes) without using more disk space than is required for the actual content (e.g., a gigabyte or so for the files in the root filesystem). The kernel and filesystem make the unwritten parts of the disk image look as if they are filled with zero bytes. This is good. However, during deployment those sparse files get transferred as if there really are a lot of zeroes. Those zeroes take a lot of time to transfer. rsync, for example, does not handle large holes efficiently. This change introduces a couple of helper tools (morphlib/xfer-hole and morphlib/recv-hole), which transfer the holes more efficiently. The xfer-hole program reads a file and outputs records like these: DATA 123 binary data (exaclyt 123 bytes and no newline at the end) HOLE 3245 xfer-hole can do this efficiently, without having to read through all the zeroes in the holes, using the SEEK_DATA and SEEK_HOLE arguments to lseek. Using this, the holes take only take a few bytes each, making it possible to transfer a disk image faster. In my benchmarks, transferring a 100G byte disk image took about 100 seconds for KVM, and 220 seconds for VirtualBox (which needs to more work at the receiver to convert the raw disk to a VDI). Both benchmarks were from a VM on my laptop to the laptop itself. The interesting bit here is that the receiver (recv-hole) is simple enough that it can be implemented in a bit of shell script, and the text of the shell script can be run on the remote end by giving it to ssh as a command line argument. This means there is no need to install any special tools on the receiver, which makes using this improvement much simpler. --- kvm.write | 13 ++++++++++--- virtualbox-ssh.write | 16 +++++++++++----- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/kvm.write b/kvm.write index 94a55daa..3e3b3eb1 100755 --- a/kvm.write +++ b/kvm.write @@ -90,9 +90,16 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): '''Transfer raw disk image to libvirt host.''' self.status(msg='Transferring disk image') - target = '%s:%s' % (ssh_host, vm_path) - with open(raw_disk, 'rb') as f: - cliapp.runcmd(['rsync', '-szS', raw_disk, target]) + + xfer_hole_path = morphlib.util.get_data_path('xfer-hole') + recv_hole = morphlib.util.get_data('recv-hole') + + cliapp.runcmd( + ['python', xfer_hole_path, raw_disk], + ['ssh', ssh_host, + 'sh', '-c', cliapp.shell_quote(recv_hole), + 'dummy-argv0', 'file', vm_path], + stdout=None, stderr=None) def create_libvirt_guest(self, ssh_host, vm_name, vm_path, autostart): '''Create the libvirt virtual machine.''' diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 42585f5e..1aebe490 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -102,11 +102,17 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): '''Transfer raw disk image to VirtualBox host, and convert to VDI.''' self.status(msg='Transfer disk and convert to VDI') - with open(raw_disk, 'rb') as f: - cliapp.ssh_runcmd(ssh_host, - ['VBoxManage', 'convertfromraw', 'stdin', vdi_path, - str(os.path.getsize(raw_disk))], - stdin=f) + + st = os.lstat(raw_disk) + xfer_hole_path = morphlib.util.get_data_path('xfer-hole') + recv_hole = morphlib.util.get_data('recv-hole') + + cliapp.runcmd( + ['python', xfer_hole_path, raw_disk], + ['ssh', ssh_host, + 'sh', '-c', cliapp.shell_quote(recv_hole), + 'dummy-argv0', 'vbox', vdi_path, str(st.st_size)], + stdout=None, stderr=None) def virtualbox_version(self, ssh_host): 'Get the version number of the VirtualBox running on the remote host.' -- cgit v1.2.1 From 93d932f6a62e60357542e15f028551b7221c1720 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 5 Sep 2014 14:17:53 +0000 Subject: Fix shell quoting for ssh remote command lines Found by Richard Maw. --- kvm.write | 8 +++++--- virtualbox-ssh.write | 9 ++++++--- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/kvm.write b/kvm.write index 3e3b3eb1..16f188b5 100755 --- a/kvm.write +++ b/kvm.write @@ -94,11 +94,13 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): xfer_hole_path = morphlib.util.get_data_path('xfer-hole') recv_hole = morphlib.util.get_data('recv-hole') + ssh_remote_cmd = [ + 'sh', '-c', recv_hole, 'dummy-argv0', 'file', vm_path + ] + cliapp.runcmd( ['python', xfer_hole_path, raw_disk], - ['ssh', ssh_host, - 'sh', '-c', cliapp.shell_quote(recv_hole), - 'dummy-argv0', 'file', vm_path], + ['ssh', ssh_host] + map(cliapp.shell_quote, ssh_remote_cmd), stdout=None, stderr=None) def create_libvirt_guest(self, ssh_host, vm_name, vm_path, autostart): diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 1aebe490..39ea8f86 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -107,11 +107,14 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): xfer_hole_path = morphlib.util.get_data_path('xfer-hole') recv_hole = morphlib.util.get_data('recv-hole') + ssh_remote_cmd = [ + 'sh', '-c', recv_hole, + 'dummy-argv0', 'vbox', vdi_path, str(st.st_size), + ] + cliapp.runcmd( ['python', xfer_hole_path, raw_disk], - ['ssh', ssh_host, - 'sh', '-c', cliapp.shell_quote(recv_hole), - 'dummy-argv0', 'vbox', vdi_path, str(st.st_size)], + ['ssh', ssh_host] + map(cliapp.shell_quote, ssh_remote_cmd), stdout=None, stderr=None) def virtualbox_version(self, ssh_host): -- cgit v1.2.1 From 03e794ca09e1583872fcc1c560a5ec7016983cb3 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Tue, 9 Sep 2014 17:49:22 +0100 Subject: Fix openstack write/check exts to pass the tests --- openstack.check | 2 +- openstack.write | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/openstack.check b/openstack.check index d9d3ef24..edc37cc1 100755 --- a/openstack.check +++ b/openstack.check @@ -60,7 +60,7 @@ class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): def check_location(self, location): x = urlparse.urlparse(location) if x.scheme not in ['http', 'https']: - raise cliapp.AppException('URL schema must be http or https in %s' \ + raise cliapp.AppException('URL schema must be http or https in %s'\ % location) if (x.path != '/v2.0' and x.path != '/v2.0/'): raise cliapp.AppException('API version must be v2.0 in %s'\ diff --git a/openstack.write b/openstack.write index ac2e2c8a..dc18f9aa 100755 --- a/openstack.write +++ b/openstack.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013 - 2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -- cgit v1.2.1 From abffc90e8a8150a1279f1d9c9722239e832e7172 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Tue, 9 Sep 2014 17:51:28 +0100 Subject: Fix up openstack.write extension The openstack.write extension was calling a nonexistent method 'check_location'. This metod was moved to openstack.check in the commit ba7d1d1ed3bad002ce36e5d4adf4e3794625091a. --- openstack.write | 1 - 1 file changed, 1 deletion(-) diff --git a/openstack.write b/openstack.write index dc18f9aa..516fe367 100755 --- a/openstack.write +++ b/openstack.write @@ -63,7 +63,6 @@ class OpenStackWriteExtension(morphlib.writeexts.WriteExtension): raise cliapp.AppException('Wrong number of command line args') temp_root, location = args - self.check_location(location) os_params = self.get_openstack_parameters() -- cgit v1.2.1 From 4fff47462e598d475e930893383f9c27e6f2c381 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Wed, 1 Oct 2014 08:21:29 +0000 Subject: ssh-rsync: gett UUID of the disk before writing fstab With this patch, the fstab of the system to be deployed as an upgrade will be conifgured using the UUID of the disk. Now when doing an upgrade is not needed to specify the ROOT_DEVICE. --- ssh-rsync.write | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/ssh-rsync.write b/ssh-rsync.write index c139b6c0..468e5a1f 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -54,9 +54,12 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.upgrade_remote_system(location, temp_root) def upgrade_remote_system(self, location, temp_root): - self.complete_fstab_for_btrfs_layout(temp_root) - root_disk = self.find_root_disk(location) + uuid = cliapp.ssh_runcmd(location, ['blkid', '-s', 'UUID', '-o', + 'value', root_disk]).strip() + + self.complete_fstab_for_btrfs_layout(temp_root, uuid) + version_label = os.environ.get('VERSION_LABEL') autostart = self.get_environment_boolean('AUTOSTART') -- cgit v1.2.1 From bd6bae145a4f7064caa4ee49f7e815452d4469e8 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Wed, 8 Oct 2014 10:28:23 +0000 Subject: deploy: Make ssh-rsync upgrade extension handle unset VERSION_LABEL It now gives an error message. Previously it would fail with a backtrace like this: 2014-10-08 09:51:37 [systems/genivi-baseline-system-armv7lhf-jetson.morph][self]Removing temporary mounts Traceback (most recent call last): File "/usr/lib/python2.7/site-packages/cliapp/app.py", line 190, in _run self.process_args(args) File "/src/morph/morphlib/exts/ssh-rsync.write", line 54, in process_args self.upgrade_remote_system(location, temp_root) File "/src/morph/morphlib/exts/ssh-rsync.write", line 107, in upgrade_remote_system location, ['btrfs', 'subvolume', 'delete', orig_dir]) UnboundLocalError: local variable 'orig_dir' referenced before assignment --- ssh-rsync.check | 7 ++++++- ssh-rsync.write | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/ssh-rsync.check b/ssh-rsync.check index 6a776ce9..11446c28 100755 --- a/ssh-rsync.check +++ b/ssh-rsync.check @@ -18,8 +18,9 @@ import cliapp -import morphlib.writeexts +import os +import morphlib.writeexts class SshRsyncCheckExtension(morphlib.writeexts.WriteExtension): def process_args(self, args): @@ -33,6 +34,10 @@ class SshRsyncCheckExtension(morphlib.writeexts.WriteExtension): 'Baserock machines. It cannot be used for an initial ' 'deployment.') + if os.environ.get('VERSION_LABEL', '') == '': + raise cliapp.AppException( + 'A VERSION_LABEL must be set when deploying an upgrade.') + location = args[0] self.check_ssh_connectivity(location) self.check_is_baserock_system(location) diff --git a/ssh-rsync.write b/ssh-rsync.write index 468e5a1f..775619ec 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -60,7 +60,7 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.complete_fstab_for_btrfs_layout(temp_root, uuid) - version_label = os.environ.get('VERSION_LABEL') + version_label = os.environ['VERSION_LABEL'] autostart = self.get_environment_boolean('AUTOSTART') self.status(msg='Creating remote mount point') -- cgit v1.2.1 From 7ec3d106f49fd4af2d193afc976c2230a9369a7e Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Thu, 9 Oct 2014 15:09:03 +0000 Subject: ssh-rsync: Don't delete version if it exists A quirk in the resource cleanup code meant that if you gave the same version label when deploying a new version, then it would fail, then remove the old version, as it had assumed that it was the one to create those directories. This patch fixes this issue by making short context managers for all the resource allocation, so cleanup is done by walking up the context managers, so only the mount and the temporary directory need to be cleaned up if the `mkdir "$VERSION_ROOT"` fails. I've tested this with a deploy of a version that doesn't already exist, and the version I'm currently running, so I can conclusively say it's fixed that problem. --- ssh-rsync.write | 182 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 105 insertions(+), 77 deletions(-) diff --git a/ssh-rsync.write b/ssh-rsync.write index 775619ec..2391d48c 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -18,6 +18,7 @@ '''A Morph deployment write extension for upgrading systems over ssh.''' +import contextlib import cliapp import os import sys @@ -45,107 +46,134 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): ''' - def process_args(self, args): - if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') - - temp_root, location = args - - self.upgrade_remote_system(location, temp_root) - - def upgrade_remote_system(self, location, temp_root): - root_disk = self.find_root_disk(location) - uuid = cliapp.ssh_runcmd(location, ['blkid', '-s', 'UUID', '-o', - 'value', root_disk]).strip() - - self.complete_fstab_for_btrfs_layout(temp_root, uuid) + def find_root_disk(self, location): + '''Read /proc/mounts on location to find which device contains "/"''' - version_label = os.environ['VERSION_LABEL'] - autostart = self.get_environment_boolean('AUTOSTART') + self.status(msg='Finding device that contains "/"') + contents = cliapp.ssh_runcmd(location, ['cat', '/proc/mounts']) + for line in contents.splitlines(): + line_words = line.split() + if (line_words[1] == '/' and line_words[0] != 'rootfs'): + return line_words[0] + @contextlib.contextmanager + def _remote_mount_point(self, location): self.status(msg='Creating remote mount point') remote_mnt = cliapp.ssh_runcmd(location, ['mktemp', '-d']).strip() try: - self.status(msg='Mounting root disk') - cliapp.ssh_runcmd(location, ['mount', root_disk, remote_mnt]) - except Exception as e: - ssh_runcmd_ignore_failure(location, ['rmdir', remote_mnt]) - raise e + yield remote_mnt + finally: + self.status(msg='Removing remote mount point') + cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) + @contextlib.contextmanager + def _remote_mount(self, location, root_disk, mountpoint): + self.status(msg='Mounting root disk') + cliapp.ssh_runcmd(location, ['mount', root_disk, mountpoint]) try: - version_root = os.path.join(remote_mnt, 'systems', version_label) - orig_dir = os.path.join(version_root, 'orig') - - self.status(msg='Creating %s' % version_root) - cliapp.ssh_runcmd(location, ['mkdir', version_root]) - - self.create_remote_orig(location, version_root, remote_mnt, - temp_root) - - # Use the system-version-manager from the new system we just - # installed, so that we can upgrade from systems that don't have - # it installed. - self.status(msg='Calling system-version-manager to deploy upgrade') - deployment = os.path.join('/systems', version_label, 'orig') - system_config_sync = os.path.join( - remote_mnt, 'systems', version_label, 'orig', 'usr', 'bin', - 'baserock-system-config-sync') - system_version_manager = os.path.join( - remote_mnt, 'systems', version_label, 'orig', 'usr', 'bin', - 'system-version-manager') - cliapp.ssh_runcmd(location, - ['env', 'BASEROCK_SYSTEM_CONFIG_SYNC='+system_config_sync, - system_version_manager, 'deploy', deployment]) - - self.status(msg='Setting %s as the new default system' % - version_label) - cliapp.ssh_runcmd(location, - [system_version_manager, 'set-default', version_label]) - except Exception as e: - self.status(msg='Deployment failed') - ssh_runcmd_ignore_failure( - location, ['btrfs', 'subvolume', 'delete', orig_dir]) - ssh_runcmd_ignore_failure( - location, ['rm', '-rf', version_root]) - raise e + yield finally: - self.status(msg='Removing temporary mounts') - cliapp.ssh_runcmd(location, ['umount', remote_mnt]) - cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) + self.status(msg='Unmounting root disk') + cliapp.ssh_runcmd(location, ['umount', mountpoint]) + + @contextlib.contextmanager + def _created_version_root(self, location, remote_mnt, version_label): + version_root = os.path.join(remote_mnt, 'systems', version_label) + self.status(msg='Creating %(root)s', root=version_root) + cliapp.ssh_runcmd(location, ['mkdir', version_root]) + try: + yield version_root + except BaseException as e: + # catch all, we always want to clean up + self.status(msg='Cleaning up %(root)s', root=version_root) + ssh_runcmd_ignore_failure(location, ['rmdir', version_root]) + raise - if autostart: - self.status(msg="Rebooting into new system ...") - ssh_runcmd_ignore_failure(location, ['reboot']) + def get_old_orig(self, location, remote_mnt): + '''Identify which subvolume to snapshot from''' - def create_remote_orig(self, location, version_root, remote_mnt, - temp_root): - '''Create the subvolume version_root/orig on location''' + # rawdisk upgrades use 'factory' + return os.path.join(remote_mnt, 'systems', 'factory', 'orig') + @contextlib.contextmanager + def _created_orig_subvolume(self, location, remote_mnt, version_root): self.status(msg='Creating "orig" subvolume') old_orig = self.get_old_orig(location, remote_mnt) new_orig = os.path.join(version_root, 'orig') cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', 'snapshot', old_orig, new_orig]) + try: + yield new_orig + except BaseException as e: + ssh_runcmd_ignore_failure( + location, ['btrfs', 'subvolume', 'delete', new_orig]) + raise + def populate_remote_orig(self, location, new_orig, temp_root): + '''Populate the subvolume version_root/orig on location''' + + self.status(msg='Populating "orig" subvolume') cliapp.runcmd(['rsync', '-as', '--checksum', '--numeric-ids', '--delete', temp_root + os.path.sep, '%s:%s' % (location, new_orig)]) - def get_old_orig(self, location, remote_mnt): - '''Identify which subvolume to snapshot from''' + @contextlib.contextmanager + def _deployed_version(self, location, version_label, + system_config_sync, system_version_manager): + self.status(msg='Calling system-version-manager to deploy upgrade') + deployment = os.path.join('/systems', version_label, 'orig') + cliapp.ssh_runcmd(location, + ['env', 'BASEROCK_SYSTEM_CONFIG_SYNC='+system_config_sync, + system_version_manager, 'deploy', deployment]) + try: + yield deployment + except BaseException as e: + self.status(msg='Cleaning up failed version installation') + cliapp.ssh_runcmd(location, + [system_version_manager, 'remove', version_label]) + raise - # rawdisk upgrades use 'factory' - return os.path.join(remote_mnt, 'systems', 'factory', 'orig') + def upgrade_remote_system(self, location, temp_root): + root_disk = self.find_root_disk(location) + uuid = cliapp.ssh_runcmd(location, ['blkid', '-s', 'UUID', '-o', + 'value', root_disk]).strip() - def find_root_disk(self, location): - '''Read /proc/mounts on location to find which device contains "/"''' + self.complete_fstab_for_btrfs_layout(temp_root, uuid) - self.status(msg='Finding device that contains "/"') - contents = cliapp.ssh_runcmd(location, ['cat', '/proc/mounts']) - for line in contents.splitlines(): - line_words = line.split() - if (line_words[1] == '/' and line_words[0] != 'rootfs'): - return line_words[0] + version_label = os.environ['VERSION_LABEL'] + autostart = self.get_environment_boolean('AUTOSTART') + + with self._remote_mount_point(location) as remote_mnt, \ + self._remote_mount(location, root_disk, remote_mnt), \ + self._created_version_root(location, remote_mnt, + version_label) as version_root, \ + self._created_orig_subvolume(location, remote_mnt, + version_root) as orig: + self.populate_remote_orig(location, orig, temp_root) + system_config_sync = os.path.join( + remote_mnt, 'systems', version_label, 'orig', + 'usr', 'bin', 'baserock-system-config-sync') + system_version_manager = os.path.join( + remote_mnt, 'systems', version_label, 'orig', + 'usr', 'bin', 'system-version-manager') + with self._deployed_version(location, version_label, + system_config_sync, system_version_manager): + self.status(msg='Setting %(v)s as the new default system', + v=version_label) + cliapp.ssh_runcmd(location, [system_version_manager, + 'set-default', version_label]) + + if autostart: + self.status(msg="Rebooting into new system ...") + ssh_runcmd_ignore_failure(location, ['reboot']) + + def process_args(self, args): + if len(args) != 2: + raise cliapp.AppException('Wrong number of command line args') + + temp_root, location = args + + self.upgrade_remote_system(location, temp_root) SshRsyncWriteExtension().run() -- cgit v1.2.1 From aa7005c9b4207ac32621433a95a0c9b44007f6b4 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Fri, 10 Oct 2014 13:09:25 +0000 Subject: Merge branch 'baserock/richardmaw/fix-ssh-rsync-destroying-versions' Reviewed-by: Sam Thursfield Reviewed-by: Jim MacArthur Reviewed-by: Richard Ipsum --- ssh-rsync.write | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/ssh-rsync.write b/ssh-rsync.write index 2391d48c..0ce89c7f 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -150,17 +150,17 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self._created_orig_subvolume(location, remote_mnt, version_root) as orig: self.populate_remote_orig(location, orig, temp_root) - system_config_sync = os.path.join( - remote_mnt, 'systems', version_label, 'orig', - 'usr', 'bin', 'baserock-system-config-sync') - system_version_manager = os.path.join( - remote_mnt, 'systems', version_label, 'orig', - 'usr', 'bin', 'system-version-manager') + system_root = os.path.join(remote_mnt, 'systems', + version_label, 'orig') + config_sync = os.path.join(system_root, 'usr', 'bin', + 'baserock-system-config-sync') + version_manager = os.path.join(system_root, 'usr', 'bin', + 'system-version-manager') with self._deployed_version(location, version_label, - system_config_sync, system_version_manager): + config_sync, version_manager): self.status(msg='Setting %(v)s as the new default system', v=version_label) - cliapp.ssh_runcmd(location, [system_version_manager, + cliapp.ssh_runcmd(location, [version_manager, 'set-default', version_label]) if autostart: -- cgit v1.2.1 From ec477897b9d6afd3ab176ab512d12f36121618c1 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Wed, 5 Nov 2014 10:24:07 +0000 Subject: Use the default symlink when creating the orig subvolume. This patch solves the issue caused by upgrading a system without a factory version. Currently we are only using the factory version to snapshot its orig subvolume to make faster the transfer of the new content (rsync won't have to send everything). The default symlink may not be present, but it can't be deleted easily using system-version-manager. This is a quick fix, but in the future we may want to not harcode the path from where we snapshot the orig subvolume. Or improve system-version-manager to make sure that the default symlink is always present. --- rawdisk.write | 2 +- ssh-rsync.write | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/rawdisk.write b/rawdisk.write index 1c2c5a84..12db4398 100755 --- a/rawdisk.write +++ b/rawdisk.write @@ -68,7 +68,7 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): version_root = os.path.join(mp, 'systems', version_label) os.mkdir(version_root) - old_orig = os.path.join(mp, 'systems', 'factory', 'orig') + old_orig = os.path.join(mp, 'systems', 'default', 'orig') new_orig = os.path.join(version_root, 'orig') cliapp.runcmd( ['btrfs', 'subvolume', 'snapshot', old_orig, new_orig]) diff --git a/ssh-rsync.write b/ssh-rsync.write index 0ce89c7f..2d7258ba 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -92,8 +92,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): def get_old_orig(self, location, remote_mnt): '''Identify which subvolume to snapshot from''' - # rawdisk upgrades use 'factory' - return os.path.join(remote_mnt, 'systems', 'factory', 'orig') + # rawdisk upgrades use 'default' + return os.path.join(remote_mnt, 'systems', 'default', 'orig') @contextlib.contextmanager def _created_orig_subvolume(self, location, remote_mnt, version_root): -- cgit v1.2.1 From 00530b7da0ccd540619a4c5b35fcc9b284d4e1cf Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Tue, 25 Nov 2014 14:02:25 +0000 Subject: Whitespace and line endings? --- kvm.write | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/kvm.write b/kvm.write index 16f188b5..eeb7c6b5 100755 --- a/kvm.write +++ b/kvm.write @@ -31,23 +31,23 @@ import morphlib.writeexts class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): '''Create a KVM/LibVirt virtual machine during Morph's deployment. - + The location command line argument is the pathname of the disk image to be created. The user is expected to provide the location argument using the following syntax: - + kvm+ssh://HOST/GUEST/PATH - + where: - + * HOST is the host on which KVM/LibVirt is running * GUEST is the name of the guest virtual machine on that host * PATH is the path to the disk image that should be created, on that host - + The extension will connect to HOST via ssh to run libvirt's command line management tools. - + ''' location_pattern = '^/(?P[^/]+)(?P/.+)$' @@ -55,15 +55,15 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): def process_args(self, args): if len(args) != 2: raise cliapp.AppException('Wrong number of command line args') - + temp_root, location = args ssh_host, vm_name, vm_path = self.parse_location(location) autostart = self.get_environment_boolean('AUTOSTART') - + fd, raw_disk = tempfile.mkstemp() os.close(fd) self.create_local_system(temp_root, raw_disk) - + try: self.transfer(raw_disk, ssh_host, vm_path) self.create_libvirt_guest(ssh_host, vm_name, vm_path, autostart) @@ -105,7 +105,7 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): def create_libvirt_guest(self, ssh_host, vm_name, vm_path, autostart): '''Create the libvirt virtual machine.''' - + self.status(msg='Creating libvirt/kvm virtual machine') attach_disks = self.parse_attach_disks() @@ -135,4 +135,3 @@ class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): ['virsh', '--connect', 'qemu:///system', 'autostart', vm_name]) KvmPlusSshWriteExtension().run() - -- cgit v1.2.1 From f03570edacad3fe9cf5468c11324a027e028acc1 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Tue, 25 Nov 2014 14:52:59 +0000 Subject: Document the write extension in the write.help file - Move docstring from .write to .write.help - Rework the content and formatting of the help information --- kvm.write | 20 -------------------- kvm.write.help | 29 ++++++++++++++++++++++++++--- 2 files changed, 26 insertions(+), 23 deletions(-) diff --git a/kvm.write b/kvm.write index eeb7c6b5..61c96676 100755 --- a/kvm.write +++ b/kvm.write @@ -30,26 +30,6 @@ import morphlib.writeexts class KvmPlusSshWriteExtension(morphlib.writeexts.WriteExtension): - '''Create a KVM/LibVirt virtual machine during Morph's deployment. - - The location command line argument is the pathname of the disk image - to be created. The user is expected to provide the location argument - using the following syntax: - - kvm+ssh://HOST/GUEST/PATH - - where: - - * HOST is the host on which KVM/LibVirt is running - * GUEST is the name of the guest virtual machine on that host - * PATH is the path to the disk image that should be created, - on that host - - The extension will connect to HOST via ssh to run libvirt's - command line management tools. - - ''' - location_pattern = '^/(?P[^/]+)(?P/.+)$' def process_args(self, args): diff --git a/kvm.write.help b/kvm.write.help index 8b5053a5..ad0e25f0 100644 --- a/kvm.write.help +++ b/kvm.write.help @@ -1,4 +1,27 @@ help: | - The INITRAMFS_PATH option can be used to specify the location of an - initramfs for syslinux to tell Linux to use, rather than booting - the rootfs directly. + + Create a KVM/LibVirt virtual machine during Morph's deployment. + + Parameters: + + * location: a custom URL scheme of the form `kvm+ssh://HOST/GUEST/PATH`, + where: + * HOST is the name of the host on which KVM/LibVirt is running + * GUEST is the name of the guest VM on that host + * PATH is the path to the disk image that should be created, + on that host. For example, + `kvm+ssh://alice@192.168.122.1/testsys/home/alice/testys.img` where + * `alice@192.168.122.1` is the target host as given to ssh, + **from within the development host** (which may be + different from the target host's normal address); + * `testsys` is the name of the new guest VM'; + * `/home/alice/testys.img` is the pathname of the disk image files + on the target host. + * HOSTNAME: the hostname of the **guest** VM within the network into which + it is being deployed + * DISK_SIZE: the size of the VM's primary virtual hard disk + * RAM_SIZE: The amount of RAM that the virtual machine should allocate for + itself from the host. + * VCPUS: the number of virtual CPUs for the VM + * INITRAMFS_PATH: the location of an initramfs for the bootloader to tell + Linux to use, rather than booting the rootfs directly. -- cgit v1.2.1 From 78f4fc473c71781722caff5d6ba819f21c381ab6 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Wed, 26 Nov 2014 18:02:09 +0000 Subject: Add more information on existing parameters. Add AUTOSTART parameter --- kvm.write.help | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/kvm.write.help b/kvm.write.help index ad0e25f0..0b428e48 100644 --- a/kvm.write.help +++ b/kvm.write.help @@ -17,11 +17,19 @@ help: | * `testsys` is the name of the new guest VM'; * `/home/alice/testys.img` is the pathname of the disk image files on the target host. - * HOSTNAME: the hostname of the **guest** VM within the network into which - it is being deployed - * DISK_SIZE: the size of the VM's primary virtual hard disk - * RAM_SIZE: The amount of RAM that the virtual machine should allocate for - itself from the host. - * VCPUS: the number of virtual CPUs for the VM - * INITRAMFS_PATH: the location of an initramfs for the bootloader to tell - Linux to use, rather than booting the rootfs directly. + * HOSTNAME=name: the hostname of the **guest** VM within the network into + which it is being deployed + * DISK_SIZE=X: the size of the VM's primary virtual hard disk. `X` should + use a suffix of `K`, `M`, or `G` (in upper or lower case) to indicate + kilo-, mega-, or gigabytes. For example, `DISK_SIZE=100G` would create a + 100 gigabyte disk image. **This parameter is mandatory**. + * RAM_SIZE=X: The amount of RAM that the virtual machine should allocate for + itself from the host. `X` is interpreted in the same was as for + DISK_SIZE`, and defaults to `1G` + * VCPUS=n: the number of virtual CPUs for the VM. Allowed values 1-32. Do + not use more CPU cores than you have available physically (real cores, no + hyperthreads) + * INITRAMFS_PATH=path: the location of an initramfs for the bootloader to + tell Linux to use, rather than booting the rootfs directly. + * AUTOSTART=` - allowed values are `yes` and `no` (default). If the + value is 'yes', the VM will be started when it has been deployed -- cgit v1.2.1 From d9069e2434fd152b0e3039839b6596c84376c4c3 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Fri, 28 Nov 2014 10:10:29 +0000 Subject: Improve readability by adding blank lines --- kvm.write.help | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/kvm.write.help b/kvm.write.help index 0b428e48..a9b63303 100644 --- a/kvm.write.help +++ b/kvm.write.help @@ -17,19 +17,25 @@ help: | * `testsys` is the name of the new guest VM'; * `/home/alice/testys.img` is the pathname of the disk image files on the target host. + * HOSTNAME=name: the hostname of the **guest** VM within the network into which it is being deployed + * DISK_SIZE=X: the size of the VM's primary virtual hard disk. `X` should use a suffix of `K`, `M`, or `G` (in upper or lower case) to indicate kilo-, mega-, or gigabytes. For example, `DISK_SIZE=100G` would create a 100 gigabyte disk image. **This parameter is mandatory**. + * RAM_SIZE=X: The amount of RAM that the virtual machine should allocate for itself from the host. `X` is interpreted in the same was as for DISK_SIZE`, and defaults to `1G` + * VCPUS=n: the number of virtual CPUs for the VM. Allowed values 1-32. Do not use more CPU cores than you have available physically (real cores, no hyperthreads) + * INITRAMFS_PATH=path: the location of an initramfs for the bootloader to tell Linux to use, rather than booting the rootfs directly. + * AUTOSTART=` - allowed values are `yes` and `no` (default). If the value is 'yes', the VM will be started when it has been deployed -- cgit v1.2.1 From a2ac64a4baa72bb805d0aee2b9862c25a0fff89f Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Fri, 28 Nov 2014 10:14:53 +0000 Subject: Refere to .write.help file for documentation --- kvm.write | 6 +++++- virtualbox-ssh.write | 31 ++++++------------------------- 2 files changed, 11 insertions(+), 26 deletions(-) diff --git a/kvm.write b/kvm.write index 61c96676..30b43d6c 100755 --- a/kvm.write +++ b/kvm.write @@ -15,7 +15,11 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -'''A Morph deployment write extension for deploying to KVM+libvirt.''' +'''A Morph deployment write extension for deploying to KVM+libvirt. + +See file kvm.write.help for documentation + +''' import cliapp diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 39ea8f86..1b4de89c 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -20,6 +20,8 @@ VirtualBox is assumed to be running on a remote machine, which is accessed over ssh. The machine gets created, but not started. +See file virtualbox-ssh.write.help for documentation + ''' @@ -36,30 +38,10 @@ import morphlib.writeexts class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): - '''Create a VirtualBox virtual machine during Morph's deployment. - - The location command line argument is the pathname of the disk image - to be created. The user is expected to provide the location argument - using the following syntax: - - vbox+ssh://HOST/GUEST/PATH - - where: - - * HOST is the host on which VirtualBox is running - * GUEST is the name of the guest virtual machine on that host - * PATH is the path to the disk image that should be created, - on that host - - The extension will connect to HOST via ssh to run VirtualBox's - command line management tools. - - ''' - def process_args(self, args): if len(args) != 2: raise cliapp.AppException('Wrong number of command line args') - + temp_root, location = args ssh_host, vm_name, vdi_path = self.parse_location(location) autostart = self.get_environment_boolean('AUTOSTART') @@ -88,7 +70,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): def parse_location(self, location): '''Parse the location argument to get relevant data.''' - + x = urlparse.urlparse(location) if x.scheme != 'vbox+ssh': raise cliapp.AppException( @@ -169,11 +151,11 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): attach_disks = self.parse_attach_disks() for device_no, disk in enumerate(attach_disks, 1): - cmd = ['storageattach', vm_name, + cmd = ['storageattach', vm_name, '--storagectl', 'SATA Controller', '--port', str(device_no), '--device', '0', - '--type', 'hdd', + '--type', 'hdd', '--medium', disk] commands.append(cmd) @@ -242,4 +224,3 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): return iface VirtualBoxPlusSshWriteExtension().run() - -- cgit v1.2.1 From 5f4399accaf2f501bbf307c70c6d9b2d90ded27c Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Fri, 28 Nov 2014 10:15:26 +0000 Subject: More complete help documentation --- virtualbox-ssh.write.help | 87 +++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 84 insertions(+), 3 deletions(-) diff --git a/virtualbox-ssh.write.help b/virtualbox-ssh.write.help index 8b5053a5..95b2e247 100644 --- a/virtualbox-ssh.write.help +++ b/virtualbox-ssh.write.help @@ -1,4 +1,85 @@ help: | - The INITRAMFS_PATH option can be used to specify the location of an - initramfs for syslinux to tell Linux to use, rather than booting - the rootfs directly. + + Deploy a Baserock system as a *new* VirtualBox virtual machine. + (Use the `ssh-rsync` write extension to deploy upgrades to an *existing* + VM) + + Connects to HOST via ssh to run VirtualBox's command line management tools. + + Parameters: + + * location: a custom URL scheme of the form `vbox+ssh://HOST/GUEST/PATH`, + where: + * HOST is the name of the host on which VirtualBox is running + * GUEST is the name of the guest VM on that host + * PATH is the path to the disk image that should be created, + on that host. For example, + `vbox+ssh://alice@192.168.122.1/testsys/home/alice/testys.img` where + * `alice@192.168.122.1` is the target host as given to ssh, + **from within the development host** (which may be + different from the target host's normal address); + * `testsys` is the name of the new guest VM'; + * `/home/alice/testys.img` is the pathname of the disk image files + on the target host. + + * HOSTNAME=name: the hostname of the **guest** VM within the network into + which it is being deployed. + + * DISK_SIZE=X: **(MANDATORY)** the size of the VM's primary virtual hard + disk. `X` should use a suffix of `K`, `M`, or `G` (in upper or lower case) + to indicate kilo-, mega-, or gigabytes. For example, `DISK_SIZE=100G` + would create a 100 gigabyte virtual hard disk. + + * RAM_SIZE=X: The amount of RAM that the virtual machine should allocate for + itself from the host. `X` is interpreted in the same as for DISK_SIZE`, + and defaults to `1G`. + + * VCPUS=n: the number of virtual CPUs for the VM. Allowed values 1-32. Do + not use more CPU cores than you have available physically (real cores, no + hyperthreads). + + * INITRAMFS_PATH=path: the location of an initramfs for the bootloader to + tell Linux to use, rather than booting the rootfs directly. + + * AUTOSTART= - allowed values are `yes` and `no` (default). If the + value is 'yes', the VM will be started when it has been deployed. + + * VAGRANT= - allowed values are `yes` and `no` (default). If the + value is 'yes', then networking is configured so that the VM will work + with Vagrant. Otherwise networking is configured to run directly in + VirtualBox. + + * HOST_IPADDR= - the IP address of the VM host. + + * NETMASK= - the netmask of the VM host. + + * NETWORK_CONFIG= - `net_config` is used to set up the VM's + network interfaces. It is a string containing semi-colon separated + 'stanzas' where each stanza provides information about a network + interface. Each stanza is of the form name:type[,arg=value] e.g. + + lo:loopback + eth0:dhcp + eth1:static,address=10.0.0.1,netmask=255.255.0.0 + + An example of the NETWORK_CONFIG parameter + + `"lo:loopback;eth0:static,address=192.168.100.2,netmask=255.255.255.0;eth1:dhcp,hostname=$(hostname)"` + + It is useful to configure one interface to use NAT to give the VM access + to the outside world and another interface to use the Virtual Box host + adapter to allow you to access the Trove from the host machine. + + The NAT interface eth1 is set up to use dhcp, the host-only adapter + interface is configured statically. + + Note: you must give the host-only adapter interface an address that lies + **on the same network** as the host adapter. So if the host adapter has an + IP of 192.168.100.1 eth0 should have an address such as 192.168.100.42. + + The settings of the host adapter, including its IP can be changed either + in the VirtualBox manager UI (https://www.virtualbox.org/manual/ch03.html#settings-network) + or via the VBoxManage command line (https://www.virtualbox.org/manual/ch08.html#idp57572192) + + See Chapter 6 of the VirtualBox User Manual for more information about + virtual networking (https://www.virtualbox.org/manual/ch06.html) -- cgit v1.2.1 From e1d759e58c9feb4069cf18c430f5e410733d1d9f Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 1 Dec 2014 13:43:44 +0000 Subject: Action review comments - Document different ways of calling parameters - Allowed values for boolean parameters --- virtualbox-ssh.write.help | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/virtualbox-ssh.write.help b/virtualbox-ssh.write.help index 95b2e247..a19d8fb2 100644 --- a/virtualbox-ssh.write.help +++ b/virtualbox-ssh.write.help @@ -41,13 +41,12 @@ help: | * INITRAMFS_PATH=path: the location of an initramfs for the bootloader to tell Linux to use, rather than booting the rootfs directly. - * AUTOSTART= - allowed values are `yes` and `no` (default). If the - value is 'yes', the VM will be started when it has been deployed. + * AUTOSTART= - boolean. If it is set, the VM will be started when + it has been deployed. - * VAGRANT= - allowed values are `yes` and `no` (default). If the - value is 'yes', then networking is configured so that the VM will work - with Vagrant. Otherwise networking is configured to run directly in - VirtualBox. + * VAGRANT= - boolean. If it is set, then networking is configured + so that the VM will work with Vagrant. Otherwise networking is configured + to run directly in VirtualBox. * HOST_IPADDR= - the IP address of the VM host. -- cgit v1.2.1 From 5d19be62bfd39f834ce6b2af020b41c6365e73af Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 1 Dec 2014 13:52:10 +0000 Subject: Merge branch 'pf-document-extensions' Reviewed-by: Richard Maw --- kvm.write.help | 11 ++++++++--- virtualbox-ssh.write.help | 2 ++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/kvm.write.help b/kvm.write.help index a9b63303..db2c0d97 100644 --- a/kvm.write.help +++ b/kvm.write.help @@ -1,6 +1,8 @@ help: | - Create a KVM/LibVirt virtual machine during Morph's deployment. + Deploy a Baserock system as a *new* KVM/LibVirt virtual machine. + + Use the `ssh-rsync` write extension to deploy upgrades to an *existing* VM Parameters: @@ -37,5 +39,8 @@ help: | * INITRAMFS_PATH=path: the location of an initramfs for the bootloader to tell Linux to use, rather than booting the rootfs directly. - * AUTOSTART=` - allowed values are `yes` and `no` (default). If the - value is 'yes', the VM will be started when it has been deployed + * AUTOSTART=` - boolean. If it is set, the VM will be started when + it has been deployed. + + (See `morph help deploy` for details of how to pass parameters to write extensions) + diff --git a/virtualbox-ssh.write.help b/virtualbox-ssh.write.help index a19d8fb2..ad2a83eb 100644 --- a/virtualbox-ssh.write.help +++ b/virtualbox-ssh.write.help @@ -82,3 +82,5 @@ help: | See Chapter 6 of the VirtualBox User Manual for more information about virtual networking (https://www.virtualbox.org/manual/ch06.html) + + (See `morph help deploy` for details of how to pass parameters to write extensions) -- cgit v1.2.1 From 79817575561496fdc91787d93cfeef2934290ec5 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Wed, 19 Nov 2014 16:21:47 +0000 Subject: Modify rawdisk.write to allow the deployment to devices This patch also modifies rawdisk.write to use the UPGRADE environment variable to figure out when is doing an upgrade or a fresh deployment. This change is important because os.path.isfile doesn't work with devices. --- rawdisk.write | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/rawdisk.write b/rawdisk.write index 12db4398..e1a75fe0 100755 --- a/rawdisk.write +++ b/rawdisk.write @@ -43,17 +43,25 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): raise cliapp.AppException('Wrong number of command line args') temp_root, location = args - if os.path.isfile(location): + upgrade = self.get_environment_boolean('UPGRADE') + + if upgrade: self.upgrade_local_system(location, temp_root) else: try: - self.create_local_system(temp_root, location) - self.status(msg='Disk image has been created at %s' % location) + if not self.is_device(location): + with self.created_disk_image(location): + self.format_btrfs(location) + self.create_system(temp_root, location) + self.status(msg='Disk image has been created at %s' % + location) + else: + self.format_btrfs(location) + self.create_system(temp_root, location) + self.status(msg='System deployed to %s' % location) except Exception: - self.status(msg='Failure to create disk image at %s' % + self.status(msg='Failure to deploy system to %s' % location) - if os.path.exists(location): - os.remove(location) raise def upgrade_local_system(self, raw_disk, temp_root): -- cgit v1.2.1 From b80fb87d383b74cf2827443ae24a3a304bdb0fc8 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Wed, 19 Nov 2014 17:19:40 +0000 Subject: Update rawdisk.check to support device deployments --- rawdisk.check | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) diff --git a/rawdisk.check b/rawdisk.check index acdc4de1..094adb72 100755 --- a/rawdisk.check +++ b/rawdisk.check @@ -33,10 +33,11 @@ class RawdiskCheckExtension(morphlib.writeexts.WriteExtension): location = args[0] upgrade = self.get_environment_boolean('UPGRADE') if upgrade: - if not os.path.isfile(location): - raise cliapp.AppException( - 'Cannot upgrade %s: it is not an existing disk image' % - location) + if not self.is_device(location): + if not os.path.isfile(location): + raise cliapp.AppException( + 'Cannot upgrade %s: it is not an existing disk image' % + location) version_label = os.environ.get('VERSION_LABEL') if version_label is None: @@ -44,9 +45,10 @@ class RawdiskCheckExtension(morphlib.writeexts.WriteExtension): 'VERSION_LABEL was not given. It is required when ' 'upgrading an existing system.') else: - if os.path.exists(location): - raise cliapp.AppException( - 'Target %s already exists. Use `morph upgrade` if you ' - 'want to update an existing image.' % location) + if not self.is_device(location): + if os.path.exists(location): + raise cliapp.AppException( + 'Target %s already exists. Use `morph upgrade` if you ' + 'want to update an existing image.' % location) RawdiskCheckExtension().run() -- cgit v1.2.1 From beb86b3ad69f0712d432bdef1dad609171bfeb7a Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Mon, 1 Dec 2014 15:25:21 +0000 Subject: writeexts.py: convert 'mount' to context manager --- openstack.write | 6 +----- rawdisk.write | 65 ++++++++++++++++++++++++++------------------------------- 2 files changed, 31 insertions(+), 40 deletions(-) diff --git a/openstack.write b/openstack.write index 516fe367..b1941d3c 100755 --- a/openstack.write +++ b/openstack.write @@ -79,8 +79,7 @@ class OpenStackWriteExtension(morphlib.writeexts.WriteExtension): def set_extlinux_root_to_virtio(self, raw_disk): '''Re-configures extlinux to use virtio disks''' self.status(msg='Updating extlinux.conf') - mp = self.mount(raw_disk) - try: + with self.mount(raw_disk) as mp: path = os.path.join(mp, 'extlinux.conf') with open(path) as f: @@ -91,9 +90,6 @@ class OpenStackWriteExtension(morphlib.writeexts.WriteExtension): with open(path, "w") as f: f.write(extlinux_conf) - finally: - self.unmount(mp) - def get_openstack_parameters(self): '''Get the environment variables needed. diff --git a/rawdisk.write b/rawdisk.write index e1a75fe0..b17f8aa7 100755 --- a/rawdisk.write +++ b/rawdisk.write @@ -67,51 +67,46 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): def upgrade_local_system(self, raw_disk, temp_root): self.complete_fstab_for_btrfs_layout(temp_root) - mp = self.mount(raw_disk) + with self.mount(raw_disk) as mp: + version_label = self.get_version_label(mp) + self.status(msg='Updating image to a new version with label %s' % + version_label) + + version_root = os.path.join(mp, 'systems', version_label) + os.mkdir(version_root) + + old_orig = os.path.join(mp, 'systems', 'default', 'orig') + new_orig = os.path.join(version_root, 'orig') + cliapp.runcmd( + ['btrfs', 'subvolume', 'snapshot', old_orig, new_orig]) + + cliapp.runcmd( + ['rsync', '-a', '--checksum', '--numeric-ids', '--delete', + temp_root + os.path.sep, new_orig]) + + self.create_run(version_root) + + default_path = os.path.join(mp, 'systems', 'default') + if os.path.exists(default_path): + os.remove(default_path) + else: + # we are upgrading and old system that does + # not have an updated extlinux config file + if self.bootloader_config_is_wanted(): + self.generate_bootloader_config(mp) + self.install_bootloader(mp) + os.symlink(version_label, default_path) - version_label = self.get_version_label(mp) - self.status(msg='Updating image to a new version with label %s' % - version_label) - - version_root = os.path.join(mp, 'systems', version_label) - os.mkdir(version_root) - - old_orig = os.path.join(mp, 'systems', 'default', 'orig') - new_orig = os.path.join(version_root, 'orig') - cliapp.runcmd( - ['btrfs', 'subvolume', 'snapshot', old_orig, new_orig]) - - cliapp.runcmd( - ['rsync', '-a', '--checksum', '--numeric-ids', '--delete', - temp_root + os.path.sep, new_orig]) - - self.create_run(version_root) - - default_path = os.path.join(mp, 'systems', 'default') - if os.path.exists(default_path): - os.remove(default_path) - else: - # we are upgrading and old system that does - # not have an updated extlinux config file if self.bootloader_config_is_wanted(): - self.generate_bootloader_config(mp) - self.install_bootloader(mp) - os.symlink(version_label, default_path) - - if self.bootloader_config_is_wanted(): - self.install_kernel(version_root, temp_root) - - self.unmount(mp) + self.install_kernel(version_root, temp_root) def get_version_label(self, mp): version_label = os.environ.get('VERSION_LABEL') if version_label is None: - self.unmount(mp) raise cliapp.AppException('VERSION_LABEL was not given') if os.path.exists(os.path.join(mp, 'systems', version_label)): - self.unmount(mp) raise cliapp.AppException('VERSION_LABEL %s already exists' % version_label) -- cgit v1.2.1 From 5a070828829e4c28d6e0a44c047a116ee1bbd1ae Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Mon, 1 Dec 2014 17:43:20 +0000 Subject: Fix line lengths to be shorter than 80 columns --- kvm.write.help | 7 ++++--- virtualbox-ssh.write.help | 39 ++++++++++++++++++++++----------------- 2 files changed, 26 insertions(+), 20 deletions(-) diff --git a/kvm.write.help b/kvm.write.help index db2c0d97..26a54d9c 100644 --- a/kvm.write.help +++ b/kvm.write.help @@ -28,8 +28,8 @@ help: | kilo-, mega-, or gigabytes. For example, `DISK_SIZE=100G` would create a 100 gigabyte disk image. **This parameter is mandatory**. - * RAM_SIZE=X: The amount of RAM that the virtual machine should allocate for - itself from the host. `X` is interpreted in the same was as for + * RAM_SIZE=X: The amount of RAM that the virtual machine should allocate + for itself from the host. `X` is interpreted in the same was as for DISK_SIZE`, and defaults to `1G` * VCPUS=n: the number of virtual CPUs for the VM. Allowed values 1-32. Do @@ -42,5 +42,6 @@ help: | * AUTOSTART=` - boolean. If it is set, the VM will be started when it has been deployed. - (See `morph help deploy` for details of how to pass parameters to write extensions) + (See `morph help deploy` for details of how to pass parameters to write + extensions) diff --git a/virtualbox-ssh.write.help b/virtualbox-ssh.write.help index ad2a83eb..b4c59553 100644 --- a/virtualbox-ssh.write.help +++ b/virtualbox-ssh.write.help @@ -26,17 +26,17 @@ help: | which it is being deployed. * DISK_SIZE=X: **(MANDATORY)** the size of the VM's primary virtual hard - disk. `X` should use a suffix of `K`, `M`, or `G` (in upper or lower case) - to indicate kilo-, mega-, or gigabytes. For example, `DISK_SIZE=100G` - would create a 100 gigabyte virtual hard disk. + disk. `X` should use a suffix of `K`, `M`, or `G` (in upper or lower + case) to indicate kilo-, mega-, or gigabytes. For example, + `DISK_SIZE=100G` would create a 100 gigabyte virtual hard disk. - * RAM_SIZE=X: The amount of RAM that the virtual machine should allocate for - itself from the host. `X` is interpreted in the same as for DISK_SIZE`, - and defaults to `1G`. + * RAM_SIZE=X: The amount of RAM that the virtual machine should allocate + for itself from the host. `X` is interpreted in the same as for + DISK_SIZE, and defaults to `1G`. * VCPUS=n: the number of virtual CPUs for the VM. Allowed values 1-32. Do - not use more CPU cores than you have available physically (real cores, no - hyperthreads). + not use more CPU cores than you have available physically (real cores, + no hyperthreads). * INITRAMFS_PATH=path: the location of an initramfs for the bootloader to tell Linux to use, rather than booting the rootfs directly. @@ -45,8 +45,8 @@ help: | it has been deployed. * VAGRANT= - boolean. If it is set, then networking is configured - so that the VM will work with Vagrant. Otherwise networking is configured - to run directly in VirtualBox. + so that the VM will work with Vagrant. Otherwise networking is + configured to run directly in VirtualBox. * HOST_IPADDR= - the IP address of the VM host. @@ -61,9 +61,10 @@ help: | eth0:dhcp eth1:static,address=10.0.0.1,netmask=255.255.0.0 - An example of the NETWORK_CONFIG parameter + An example of the NETWORK_CONFIG parameter (It should be in one line) - `"lo:loopback;eth0:static,address=192.168.100.2,netmask=255.255.255.0;eth1:dhcp,hostname=$(hostname)"` + `"lo:loopback;eth0:static,address=192.168.100.2,netmask=255.255.255.0; + eth1:dhcp,hostname=$(hostname)"` It is useful to configure one interface to use NAT to give the VM access to the outside world and another interface to use the Virtual Box host @@ -73,14 +74,18 @@ help: | interface is configured statically. Note: you must give the host-only adapter interface an address that lies - **on the same network** as the host adapter. So if the host adapter has an - IP of 192.168.100.1 eth0 should have an address such as 192.168.100.42. + **on the same network** as the host adapter. So if the host adapter has + an IP of 192.168.100.1 eth0 should have an address such as + 192.168.100.42. The settings of the host adapter, including its IP can be changed either - in the VirtualBox manager UI (https://www.virtualbox.org/manual/ch03.html#settings-network) - or via the VBoxManage command line (https://www.virtualbox.org/manual/ch08.html#idp57572192) + in the VirtualBox manager UI + (https://www.virtualbox.org/manual/ch03.html#settings-network) + or via the VBoxManage command line + (https://www.virtualbox.org/manual/ch08.html#idp57572192) See Chapter 6 of the VirtualBox User Manual for more information about virtual networking (https://www.virtualbox.org/manual/ch06.html) - (See `morph help deploy` for details of how to pass parameters to write extensions) + (See `morph help deploy` for details of how to pass parameters to write + extensions) -- cgit v1.2.1 From bf19f21a4820f2cd0a08a8a5fb397932fc851cb6 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Tue, 2 Dec 2014 13:58:13 +0000 Subject: OpenStack write extension documentation --- openstack.write | 34 ++-------------------------------- openstack.write.help | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 32 deletions(-) create mode 100644 openstack.write.help diff --git a/openstack.write b/openstack.write index b1941d3c..faf47f54 100755 --- a/openstack.write +++ b/openstack.write @@ -28,40 +28,11 @@ import morphlib.writeexts class OpenStackWriteExtension(morphlib.writeexts.WriteExtension): - '''Configure a raw disk image into an OpenStack host. - - The raw disk image is created during Morph's deployment and the - image is deployed in OpenStack using python-glanceclient. - - The location command line argument is the authentification url - of the OpenStack server using the following syntax: - - http://HOST:PORT/VERSION - - where - - * HOST is the host running OpenStack - * PORT is the port which is using OpenStack for authentifications. - * VERSION is the authentification version of OpenStack (Only v2.0 - supported) - - This extension needs in the environment the following variables: - - * OPENSTACK_USER is the username to use in the deployment. - * OPENSTACK_TENANT is the project name to use in the deployment. - * OPENSTACK_IMAGENAME is the name of the image to create. - * OPENSTACK_PASSWORD is the password of the user. - - - The extension will connect to OpenStack using python-glanceclient - to configure a raw image. - - ''' def process_args(self, args): if len(args) != 2: raise cliapp.AppException('Wrong number of command line args') - + temp_root, location = args os_params = self.get_openstack_parameters() @@ -69,7 +40,7 @@ class OpenStackWriteExtension(morphlib.writeexts.WriteExtension): fd, raw_disk = tempfile.mkstemp() os.close(fd) self.create_local_system(temp_root, raw_disk) - self.status(msg='Temporary disk image has been created at %s' + self.status(msg='Temporary disk image has been created at %s' % raw_disk) self.set_extlinux_root_to_virtio(raw_disk) @@ -120,4 +91,3 @@ class OpenStackWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Image configured.') OpenStackWriteExtension().run() - diff --git a/openstack.write.help b/openstack.write.help new file mode 100644 index 00000000..579512f5 --- /dev/null +++ b/openstack.write.help @@ -0,0 +1,37 @@ +help: | + + Deploy a Baserock system as a *new* OpenStack virtual machine. + (Use the `ssh-rsync` write extension to deploy upgrades to an *existing* + VM) + + Deploys the system to the OpenStack host using python-glanceclient. + + Parameters: + + * location: the authentication url of the OpenStack server using the + following syntax: + + http://HOST:PORT/VERSION + + where + + * HOST is the host running OpenStack + * PORT is the port which is using OpenStack for authentifications. + * VERSION is the authentification version of OpenStack (Only v2.0 + supported) + + * OPENSTACK_USER=username: the username to use in the `--os-username` + argument to `glance`. + + * OPENSTACK_TENANT=tenant: the project name to use in the `--os-tenant-name` + argument to `glance`. + + * OPENSTACK_IMAGENAME=imagename: the name of the image to use in the + `--name` argument to `glance`. + + * OPENSTACK_PASSWORD=password: the password of the OpenStack user. (We + recommend passing this on the command-line, rather than setting an + environment variable or storing it in a cluster cluster definition file.) + + (See `morph help deploy` for details of how to pass parameters to write + extensions) -- cgit v1.2.1 From f33d468ea48fba8b75aa3aa1492983bbccae46aa Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Thu, 4 Dec 2014 13:49:59 +0000 Subject: ssh-rsync write extension documentation --- ssh-rsync.write.help | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 ssh-rsync.write.help diff --git a/ssh-rsync.write.help b/ssh-rsync.write.help new file mode 100644 index 00000000..4ef666e8 --- /dev/null +++ b/ssh-rsync.write.help @@ -0,0 +1,36 @@ +help: | + + Upgrade a Baserock system which is already deployed: + - as a KVM/LibVirt, OpenStack or vbox-ssh virtual machine; + - on a Jetson board. + + Copies a binary delta over to the target system and arranges for it + to be bootable. + + The recommended way to use this extension is by calling `morph upgrade`. + Using `morph deploy --upgrade` is deprecated. + + The upgrade will fail if: + - no VM is deployed and running at `location`; + - the target system is not a Baserock system; + - the target's filesystem and its layout are not compatible with that + created by `morph deploy`." + + See also the 'Upgrading a Baserock installation' section of the 'Using + Baserock` page at wiki.baserock.org + http://wiki.baserock.org/devel-with/#index8h2 + + Parameters: + + * location: the 'user@hostname' string that will be used by ssh and rsync. + 'user' will always be `root` and `hostname` the hostname or address of the + system being upgraded. + + * VERSION_LABEL=label - **(MANDATORY)** should contain only alpha-numeric + characters and the '-' (hyphen) character. + + * AUTOSTART=` - boolean. If it is set, the VM will be started when + it has been deployed. + + (See `morph help deploy` for details of how to pass parameters to write + extensions) -- cgit v1.2.1 From 3599f68c40dc74f5634801b9c3ced856cf91155d Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Fri, 5 Dec 2014 09:02:34 +0000 Subject: rawdisk write extension documentation --- rawdisk.write.help | 42 ++++++++++++++++++++++++++++++++++-------- 1 file changed, 34 insertions(+), 8 deletions(-) diff --git a/rawdisk.write.help b/rawdisk.write.help index 298d441c..f225276d 100644 --- a/rawdisk.write.help +++ b/rawdisk.write.help @@ -1,11 +1,37 @@ help: | - Create a raw disk image during Morph's deployment. - - If the image already exists, it is upgraded. - The `location` argument is a pathname to the image to be - created or upgraded. + Write a system produced by Morph to a physical disk, or to a file that can + be used as a virtual disk. The target will be formatted as a single Btrfs + partition, with the system image written to a subvolume in /systems, and + other subvolumes created for /home, /opt, /root, /srv and /var. - The INITRAMFS_PATH option can be used to specify the location of an - initramfs for syslinux to tell Linux to use, rather than booting - the rootfs directly. + When written to a physical drive, the drive can be used as the boot device + for a 'real' machine. + + When written to a file, the file can be used independently of `morph` to + create virtual machines with KVM / libvirt, OpenStack or, after converting + it to VDI format, VirtualBox. + + `morph deploy` will fail if the file specified by `location` already exists. + + If used in `morph upgrade`, the rootfs produced by 'morph build' is added to + the existing raw disk image or device as an additional btrfs sub-volume. + `morph upgrade` will fail if the file specified by `location` does not + exist, or is not a Baserock raw disk image. (Most users are unlikely to need + or use this functionality: it is useful mainly for developers working on the + Baserock tools.) + + Parameters: + + * location: the pathname of the disk image to be created/upgraded, or the + path to the physical device. + + * VERSION_LABEL=label - should contain only alpha-numeric + characters and the '-' (hyphen) character. Mandatory if being used with + `morph update` + + * INITRAMFS_PATH=path: the location of an initramfs for the bootloader to + tell Linux to use, rather than booting the rootfs directly. + + (See `morph help deploy` for details of how to pass parameters to write + extensions) -- cgit v1.2.1 From a4534ef98b71b6d7bc57ae863fcdce353e174afb Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Mon, 8 Dec 2014 08:32:22 +0000 Subject: initramfs write extension documentation --- initramfs.write.help | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/initramfs.write.help b/initramfs.write.help index 29a9d266..a4a89f9d 100644 --- a/initramfs.write.help +++ b/initramfs.write.help @@ -1,4 +1,5 @@ help: | + Create an initramfs for a system by taking an existing system and converting it to the appropriate format. @@ -33,3 +34,8 @@ help: | initramfs: type: initramfs location: boot/initramfs.gz + + Parameters: + + * location: the path where the initramfs will be installed (e.g. + `boot/initramfs.gz`) in the above example -- cgit v1.2.1 From bfc74c3a2a1c5058e18653f4fb28e0390b66d520 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Fri, 5 Dec 2014 15:34:07 +0000 Subject: Add a reference to write.help file --- openstack.write | 1 + rawdisk.write | 12 ++---------- ssh-rsync.write | 10 ++-------- 3 files changed, 5 insertions(+), 18 deletions(-) diff --git a/openstack.write b/openstack.write index faf47f54..d29d2661 100755 --- a/openstack.write +++ b/openstack.write @@ -28,6 +28,7 @@ import morphlib.writeexts class OpenStackWriteExtension(morphlib.writeexts.WriteExtension): + '''See openstack.write.help for documentation''' def process_args(self, args): if len(args) != 2: diff --git a/rawdisk.write b/rawdisk.write index b17f8aa7..d91a4d5f 100755 --- a/rawdisk.write +++ b/rawdisk.write @@ -29,19 +29,12 @@ import morphlib.writeexts class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): - '''Create a raw disk image during Morph's deployment. - - If the image already exists, it is upgraded. - - The location command line argument is the pathname of the disk image - to be created/upgraded. - - ''' + '''See rawdisk.write.help for documentation''' def process_args(self, args): if len(args) != 2: raise cliapp.AppException('Wrong number of command line args') - + temp_root, location = args upgrade = self.get_environment_boolean('UPGRADE') @@ -114,4 +107,3 @@ class RawDiskWriteExtension(morphlib.writeexts.WriteExtension): RawDiskWriteExtension().run() - diff --git a/ssh-rsync.write b/ssh-rsync.write index 2d7258ba..c4577026 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -37,14 +37,8 @@ def ssh_runcmd_ignore_failure(location, command, **kwargs): class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): - '''Upgrade a running baserock system with ssh and rsync. - - It assumes the system is baserock-based and has a btrfs partition. - - The location command line argument is the 'user@hostname' string - that will be passed to ssh and rsync - - ''' + '''See ssh-rsync.write.help for documentation''' + def find_root_disk(self, location): '''Read /proc/mounts on location to find which device contains "/"''' -- cgit v1.2.1 From de88869eff96b3f9c09f61f4c9f67c0a710e1974 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Tue, 9 Dec 2014 16:19:11 +0000 Subject: openstack.check: Be more careful when claiming credentials are invalid In order to check the user's credentials at the start of deployment, we try to run `glance image-list`. I found a situation where this command failed despite my credentials being correct. Morph outputted a misleading error message that said 'Wrong OpenStack credentials' The code now checks that the error returned by 'glance' does indeed look like a credentials error. If it doesn't, the full error output is displayed. The error I encountered now gets a message like this: ERROR: openstack.check failed with code 1: ERROR: Failed to connect to OpenStack instance at https://example.com:5000/v2.0: [('SSL routines', 'SSL3_GET_SERVER_CERTIFICATE', 'certificate verify failed')] (If you are curious, I fixed this by running `update-ca-certificates`.) --- openstack.check | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/openstack.check b/openstack.check index edc37cc1..a6856c31 100755 --- a/openstack.check +++ b/openstack.check @@ -77,9 +77,16 @@ class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): '--os-password', password, '--os-auth-url', auth_url, 'image-list'] - try: - cliapp.runcmd(cmdline) - except cliapp.AppException: - raise cliapp.AppException('Wrong OpenStack credentals.') + + exit, out, err = cliapp.runcmd_unchecked(cmdline) + + if exit != 0: + if err.startswith('The request you have made requires ' \ + 'authentication. (HTTP 401)'): + raise cliapp.AppException('Invalid OpenStack credentials.') + else: + raise cliapp.AppException( + 'Failed to connect to OpenStack instance at %s: %s' % + (auth_url, err)) OpenStackCheckExtension().run() -- cgit v1.2.1 From 6c1cc0f3706d1019158b6a31735f416b974a1226 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Tue, 9 Dec 2014 17:49:36 +0000 Subject: deploy: Note that some of the extensions listed live in definitions.git The user should be aware of this because if they aren't building baserock:baserock/definitions or a repo forked from it, those extensions won't be available. Also fix some long lines that I seem to have failed to commit already. --- openstack.write.help | 8 ++++---- rawdisk.write.help | 13 +++++++------ ssh-rsync.write.help | 4 ++-- 3 files changed, 13 insertions(+), 12 deletions(-) diff --git a/openstack.write.help b/openstack.write.help index 579512f5..75ad9f0c 100644 --- a/openstack.write.help +++ b/openstack.write.help @@ -16,15 +16,15 @@ help: | where * HOST is the host running OpenStack - * PORT is the port which is using OpenStack for authentifications. - * VERSION is the authentification version of OpenStack (Only v2.0 + * PORT is the port which is using OpenStack for authentications. + * VERSION is the authentication version of OpenStack (Only v2.0 supported) * OPENSTACK_USER=username: the username to use in the `--os-username` argument to `glance`. - * OPENSTACK_TENANT=tenant: the project name to use in the `--os-tenant-name` - argument to `glance`. + * OPENSTACK_TENANT=tenant: the project name to use in the + `--os-tenant-name` argument to `glance`. * OPENSTACK_IMAGENAME=imagename: the name of the image to use in the `--name` argument to `glance`. diff --git a/rawdisk.write.help b/rawdisk.write.help index f225276d..81f35024 100644 --- a/rawdisk.write.help +++ b/rawdisk.write.help @@ -12,14 +12,15 @@ help: | create virtual machines with KVM / libvirt, OpenStack or, after converting it to VDI format, VirtualBox. - `morph deploy` will fail if the file specified by `location` already exists. + `morph deploy` will fail if the file specified by `location` already + exists. - If used in `morph upgrade`, the rootfs produced by 'morph build' is added to - the existing raw disk image or device as an additional btrfs sub-volume. + If used in `morph upgrade`, the rootfs produced by 'morph build' is added + to the existing raw disk image or device as an additional btrfs sub-volume. `morph upgrade` will fail if the file specified by `location` does not - exist, or is not a Baserock raw disk image. (Most users are unlikely to need - or use this functionality: it is useful mainly for developers working on the - Baserock tools.) + exist, or is not a Baserock raw disk image. (Most users are unlikely to + need or use this functionality: it is useful mainly for developers working + on the Baserock tools.) Parameters: diff --git a/ssh-rsync.write.help b/ssh-rsync.write.help index 4ef666e8..d03508c0 100644 --- a/ssh-rsync.write.help +++ b/ssh-rsync.write.help @@ -23,8 +23,8 @@ help: | Parameters: * location: the 'user@hostname' string that will be used by ssh and rsync. - 'user' will always be `root` and `hostname` the hostname or address of the - system being upgraded. + 'user' will always be `root` and `hostname` the hostname or address of + the system being upgraded. * VERSION_LABEL=label - **(MANDATORY)** should contain only alpha-numeric characters and the '-' (hyphen) character. -- cgit v1.2.1 From 37ca9dae1b891d73f891448d32fbbd8d7120850c Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Fri, 12 Dec 2014 11:26:58 +0000 Subject: Document DTB_PATH write extension parameter --- kvm.write.help | 6 +++++- rawdisk.write.help | 5 +++++ virtualbox-ssh.write.help | 5 +++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/kvm.write.help b/kvm.write.help index 26a54d9c..6df6c53c 100644 --- a/kvm.write.help +++ b/kvm.write.help @@ -42,6 +42,10 @@ help: | * AUTOSTART=` - boolean. If it is set, the VM will be started when it has been deployed. + * DTB_PATH=path: **(MANDATORY)** for systems that require a device tree + binary - Give the full path (without a leading /) to the location of the + DTB in the built system image . The deployment will fail if `path` does + not exist. + (See `morph help deploy` for details of how to pass parameters to write extensions) - diff --git a/rawdisk.write.help b/rawdisk.write.help index 81f35024..fe47c890 100644 --- a/rawdisk.write.help +++ b/rawdisk.write.help @@ -34,5 +34,10 @@ help: | * INITRAMFS_PATH=path: the location of an initramfs for the bootloader to tell Linux to use, rather than booting the rootfs directly. + * DTB_PATH=path: **(MANDATORY)** for systems that require a device tree + binary - Give the full path (without a leading /) to the location of the + DTB in the built system image . The deployment will fail if `path` does + not exist. + (See `morph help deploy` for details of how to pass parameters to write extensions) diff --git a/virtualbox-ssh.write.help b/virtualbox-ssh.write.help index b4c59553..7131f8b8 100644 --- a/virtualbox-ssh.write.help +++ b/virtualbox-ssh.write.help @@ -41,6 +41,11 @@ help: | * INITRAMFS_PATH=path: the location of an initramfs for the bootloader to tell Linux to use, rather than booting the rootfs directly. + * DTB_PATH=path: **(MANDATORY)** for systems that require a device tree + binary - Give the full path (without a leading /) to the location of the + DTB in the built system image . The deployment will fail if `path` does + not exist. + * AUTOSTART= - boolean. If it is set, the VM will be started when it has been deployed. -- cgit v1.2.1 From 859568b79d22299c082a4108dd057b9145a13eda Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Fri, 12 Dec 2014 11:30:13 +0000 Subject: Document BOOTLOADER_INSTALL and BOOTLOADER_CONFIG_FORMAT write extension parameters --- kvm.write.help | 16 ++++++++++++++++ rawdisk.write.help | 16 ++++++++++++++++ virtualbox-ssh.write.help | 16 ++++++++++++++++ 3 files changed, 48 insertions(+) diff --git a/kvm.write.help b/kvm.write.help index 6df6c53c..8ddcb89c 100644 --- a/kvm.write.help +++ b/kvm.write.help @@ -47,5 +47,21 @@ help: | DTB in the built system image . The deployment will fail if `path` does not exist. + * BOOTLOADER_INSTALL=value: the bootloader to be installed + **(MANDATORY)** for non-x86 systems + + allowed values = + - 'extlinux' (default) - the extlinux bootloader will + be installed + - 'none' - no bootloader will be installed by `morph deploy`. A + bootloader must be installed manually. This value must be used when + deploying non-x86 systems such as ARM. + + * BOOTLOADER_CONFIG_FORMAT=value: the bootloader format to be used. + If not specified for x86-32 and x86-64 systems, 'extlinux' will be used + + allowed values = + - 'extlinux' + (See `morph help deploy` for details of how to pass parameters to write extensions) diff --git a/rawdisk.write.help b/rawdisk.write.help index fe47c890..d6c78573 100644 --- a/rawdisk.write.help +++ b/rawdisk.write.help @@ -39,5 +39,21 @@ help: | DTB in the built system image . The deployment will fail if `path` does not exist. + * BOOTLOADER_INSTALL=value: the bootloader to be installed + **(MANDATORY)** for non-x86 systems + + allowed values = + - 'extlinux' (default) - the extlinux bootloader will + be installed + - 'none' - no bootloader will be installed by `morph deploy`. A + bootloader must be installed manually. This value must be used when + deploying non-x86 systems such as ARM. + + * BOOTLOADER_CONFIG_FORMAT=value: the bootloader format to be used. + If not specified for x86-32 and x86-64 systems, 'extlinux' will be used + + allowed values = + - 'extlinux' + (See `morph help deploy` for details of how to pass parameters to write extensions) diff --git a/virtualbox-ssh.write.help b/virtualbox-ssh.write.help index 7131f8b8..4dddd987 100644 --- a/virtualbox-ssh.write.help +++ b/virtualbox-ssh.write.help @@ -46,6 +46,22 @@ help: | DTB in the built system image . The deployment will fail if `path` does not exist. + * BOOTLOADER_INSTALL=value: the bootloader to be installed + **(MANDATORY)** for non-x86 systems + + allowed values = + - 'extlinux' (default) - the extlinux bootloader will + be installed + - 'none' - no bootloader will be installed by `morph deploy`. A + bootloader must be installed manually. This value must be used when + deploying non-x86 systems such as ARM. + + * BOOTLOADER_CONFIG_FORMAT=value: the bootloader format to be used. + If not specified for x86-32 and x86-64 systems, 'extlinux' will be used + + allowed values = + - 'extlinux' + * AUTOSTART= - boolean. If it is set, the VM will be started when it has been deployed. -- cgit v1.2.1 From cb8811daf402a8485f0ca9fd678843f11aaaf5ea Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Fri, 12 Dec 2014 11:36:04 +0000 Subject: Document KERNEL_ARGS write extension parameter --- kvm.write.help | 9 +++++++++ rawdisk.write.help | 9 +++++++++ virtualbox-ssh.write.help | 9 +++++++++ 3 files changed, 27 insertions(+) diff --git a/kvm.write.help b/kvm.write.help index 8ddcb89c..04393b8a 100644 --- a/kvm.write.help +++ b/kvm.write.help @@ -63,5 +63,14 @@ help: | allowed values = - 'extlinux' + * KERNEL_ARGS=args: optional additional kernel command-line parameters to + be appended to the default set. The default set is: + + 'rw init=/sbin/init rootfstype=btrfs \ + rootflags=subvol=systems/default/run \ + root=[name or UUID of root filesystem]' + + (See https://www.kernel.org/doc/Documentation/kernel-parameters.txt) + (See `morph help deploy` for details of how to pass parameters to write extensions) diff --git a/rawdisk.write.help b/rawdisk.write.help index d6c78573..54af81c4 100644 --- a/rawdisk.write.help +++ b/rawdisk.write.help @@ -55,5 +55,14 @@ help: | allowed values = - 'extlinux' + * KERNEL_ARGS=args: optional additional kernel command-line parameters to + be appended to the default set. The default set is: + + 'rw init=/sbin/init rootfstype=btrfs \ + rootflags=subvol=systems/default/run \ + root=[name or UUID of root filesystem]' + + (See https://www.kernel.org/doc/Documentation/kernel-parameters.txt) + (See `morph help deploy` for details of how to pass parameters to write extensions) diff --git a/virtualbox-ssh.write.help b/virtualbox-ssh.write.help index 4dddd987..cb50acc0 100644 --- a/virtualbox-ssh.write.help +++ b/virtualbox-ssh.write.help @@ -62,6 +62,15 @@ help: | allowed values = - 'extlinux' + * KERNEL_ARGS=args: optional additional kernel command-line parameters to + be appended to the default set. The default set is: + + 'rw init=/sbin/init rootfstype=btrfs \ + rootflags=subvol=systems/default/run \ + root=[name or UUID of root filesystem]' + + (See https://www.kernel.org/doc/Documentation/kernel-parameters.txt) + * AUTOSTART= - boolean. If it is set, the VM will be started when it has been deployed. -- cgit v1.2.1 From 701ebe1464a97dfd40336007c8da89d9c9685ad8 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Fri, 12 Dec 2014 11:56:39 +0000 Subject: Add 'do not use' warnings to nfsboot write extension --- nfsboot.write | 13 +++++++++++-- nfsboot.write.help | 9 ++++++++- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/nfsboot.write b/nfsboot.write index 8d3d6df7..9fa6fc84 100755 --- a/nfsboot.write +++ b/nfsboot.write @@ -17,6 +17,16 @@ '''A Morph deployment write extension for deploying to an nfsboot server +*** DO NOT USE *** +- This was written before 'proper' deployment mechanisms were in place +It is unlikely to work at all and will not work correctly + +Use the pxeboot werite extension instead + +*** + + + An nfsboot server is defined as a baserock system that has tftp and nfs servers running, the tftp server is exporting the contents of /srv/nfsboot/tftp/ and the user has sufficient permissions to create nfs roots @@ -125,7 +135,7 @@ class NFSBootWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Creating destination directories') try: - cliapp.ssh_runcmd('root@%s' % location, + cliapp.ssh_runcmd('root@%s' % location, ['mkdir', '-p', orig_path, run_path]) except cliapp.AppException: raise cliapp.AppException('Could not create dirs %s and %s on %s' @@ -191,4 +201,3 @@ mv "$temp" "$target" NFSBootWriteExtension().run() - diff --git a/nfsboot.write.help b/nfsboot.write.help index 598b1b23..310fd7a4 100644 --- a/nfsboot.write.help +++ b/nfsboot.write.help @@ -1,6 +1,13 @@ help: | + *** DO NOT USE *** + - This was written before 'proper' deployment mechanisms were in place. + It is unlikely to work at all, and will not work correctly. + + Use the pxeboot write extension instead + + *** Deploy a system image and kernel to an nfsboot server. - + An nfsboot server is defined as a baserock system that has tftp and nfs servers running, the tftp server is exporting the contents of /srv/nfsboot/tftp/ and the user has sufficient -- cgit v1.2.1 From 28d69d11444fcd5a6531f145ce521cc4d4346dde Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Thu, 18 Dec 2014 10:12:41 +0000 Subject: Remove trailing \ I meant to do this as part of the previous merge. --- openstack.check | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openstack.check b/openstack.check index a6856c31..3850d481 100755 --- a/openstack.check +++ b/openstack.check @@ -81,7 +81,7 @@ class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): exit, out, err = cliapp.runcmd_unchecked(cmdline) if exit != 0: - if err.startswith('The request you have made requires ' \ + if err.startswith('The request you have made requires ' 'authentication. (HTTP 401)'): raise cliapp.AppException('Invalid OpenStack credentials.') else: -- cgit v1.2.1 From 6a49bc67b3fc5073612195510d556129bfdd28bb Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Thu, 1 Jan 2015 00:25:54 +0000 Subject: Add some checks to the sysroot deployment extension Ensure that a) the deployment directory must not exist b) the extension can not be used to upgrade a system --- sysroot.check | 30 ++++++++++++++++++++++++++++++ sysroot.write | 4 +--- 2 files changed, 31 insertions(+), 3 deletions(-) create mode 100755 sysroot.check diff --git a/sysroot.check b/sysroot.check new file mode 100755 index 00000000..bfacd3fc --- /dev/null +++ b/sysroot.check @@ -0,0 +1,30 @@ +#!/bin/sh +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# Preparatory checks for Morph 'sysroot' write extension + +set -eu + +location="$1" +if [ -d "$location" ]; then + echo >&2 "ERROR: Deployment directory already exists: $location" + exit 1 +fi + +if [ "$UPGRADE" == "yes" ]; then + echo >&2 "ERROR: Cannot upgrade a sysroot deployment" + exit 1 +fi diff --git a/sysroot.write b/sysroot.write index 1ae4864f..be315365 100755 --- a/sysroot.write +++ b/sysroot.write @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -18,9 +18,7 @@ set -eu -# Ensure the target is an empty directory mkdir -p "$2" -find "$2" -mindepth 1 -delete # Move the contents of our source directory to our target # Previously we would (cd "$1" && find -print0 | cpio -0pumd "$absolute_path") -- cgit v1.2.1 From 544ac8182471a0552b8a22fd574029e74a30d8ed Mon Sep 17 00:00:00 2001 From: Jim MacArthur Date: Wed, 21 Jan 2015 18:08:01 +0000 Subject: Remove checks for NETWORK_CONFIG and eth0 and eth1 in it network_config isn't used anywhere in this function. The purpose of this function (getting the name of an appropriate host-only network interface) doesn't seem to depend on it either. eth0 and eth1 won't always be present (several Baserock systems will have enp0s3, etc). So I think these checks should be removed. --- virtualbox-ssh.write | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 1b4de89c..fa54c296 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -169,20 +169,6 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): def get_host_interface(self, ssh_host): host_ipaddr = os.environ.get('HOST_IPADDR') netmask = os.environ.get('NETMASK') - network_config = os.environ.get("NETWORK_CONFIG") - - if network_config is None: - raise cliapp.AppException('NETWORK_CONFIG was not given') - - if "eth0:" not in network_config: - raise cliapp.AppException( - 'NETWORK_CONFIG does not contain ' - 'the eth0 configuration') - - if "eth1:" not in network_config: - raise cliapp.AppException( - 'NETWORK_CONFIG does not contain ' - 'the eth1 configuration') if host_ipaddr is None: raise cliapp.AppException('HOST_IPADDR was not given') -- cgit v1.2.1 From 0e6d9749b509382f65177337ec6465cdbea44b7b Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Wed, 28 Jan 2015 11:32:07 +0000 Subject: Check file can be created at location --- kvm.check | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/kvm.check b/kvm.check index 1bb4007a..3c6accbf 100755 --- a/kvm.check +++ b/kvm.check @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -43,6 +43,7 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): ssh_host, vm_name, vm_path = self.check_and_parse_location(location) self.check_ssh_connectivity(ssh_host) + self.check_can_create_file_at_given_path(ssh_host, vm_path) self.check_no_existing_libvirt_vm(ssh_host, vm_name) self.check_extra_disks_exist(ssh_host, self.parse_attach_disks()) @@ -73,6 +74,26 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): 'write extension to deploy upgrades to existing machines.' % (ssh_host, vm_name)) + def check_can_create_file_at_given_path(self, ssh_host, vm_path): + + def check_can_write_to_given_path(): + try: + cliapp.ssh_runcmd(ssh_host, ['touch', vm_path]) + except cliapp.AppException as e: + raise cliapp.AppException("Can't write to location %s on %s" + % (vm_path, ssh_host)) + else: + cliapp.ssh_runcmd(ssh_host, ['rm', vm_path]) + + try: + cliapp.ssh_runcmd(ssh_host, ['test', '-e', vm_path]) + except cliapp.AppException as e: + # vm_path doesn't already exist, so let's test we can write + check_can_write_to_given_path() + else: + raise cliapp.AppException('%s already exists on %s' + % (vm_path, ssh_host)) + def check_extra_disks_exist(self, ssh_host, filename_list): for filename in filename_list: try: -- cgit v1.2.1 From ca9e43bf901d10bd1c0ec580255f6096bb53c65f Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Wed, 28 Jan 2015 17:28:00 +0000 Subject: Add check for virtual networks An exception will be raised if any needed networks are not started --- kvm.check | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/kvm.check b/kvm.check index 3c6accbf..b8877a89 100755 --- a/kvm.check +++ b/kvm.check @@ -17,6 +17,7 @@ '''Preparatory checks for Morph 'kvm' write extension''' import cliapp +import os import re import urlparse @@ -46,6 +47,7 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): self.check_can_create_file_at_given_path(ssh_host, vm_path) self.check_no_existing_libvirt_vm(ssh_host, vm_name) self.check_extra_disks_exist(ssh_host, self.parse_attach_disks()) + self.check_virtual_networks_are_started(ssh_host) def check_and_parse_location(self, location): '''Check and parse the location argument to get relevant data.''' @@ -102,4 +104,50 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): raise cliapp.AppException('Did not find file %s on host %s' % (filename, ssh_host)) + def check_virtual_networks_are_started(self, ssh_host): + + def check_virtual_network_is_started(network_name): + cmd = ['virsh', '-c', 'qemu:///system', 'net-info', network_name] + net_info = cliapp.ssh_runcmd(ssh_host, cmd).split('\n') + + def pretty_concat(lines): + return '\n'.join(['\t%s' % line for line in lines]) + + for line in net_info: + m = re.match('^Active:\W*(\w+)\W*', line) + if m: + break + else: + raise cliapp.AppException( + "Got unexpected output parsing output of `%s':\n%s" + % (' '.join(cmd), pretty_concat(net_info))) + + network_active = m.group(1) == 'yes' + + if not network_active: + raise cliapp.AppException("Network '%s' is not started" + % network_name) + + def name(nic_entry): + if ',' in nic_entry: + # NETWORK_NAME,mac=12:34,model=e1000... + return nic_entry[:nic_entry.find(',')] + else: + return nic_entry # NETWORK_NAME + + if 'NIC_CONFIG' in os.environ: + nics = os.environ['NIC_CONFIG'].split() + + # --network bridge= is used to specify a bridge + # --network user is used to specify a form of NAT + # (see the virt-install(1) man page) + networks = [name(n) for n in nics if not n.startswith('bridge=') + and not n.startswith('user')] + else: + networks = ['default'] + + for network in networks: + check_virtual_network_is_started(network) + + KvmPlusSshCheckExtension().run() -- cgit v1.2.1 From ee1ccdbeda5cc559194db64c6ecf38f28cc391f0 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Tue, 3 Feb 2015 17:35:58 +0000 Subject: Update copyright years so ./check passes --- virtualbox-ssh.write | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index fa54c296..7eafcff3 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -- cgit v1.2.1 From 3e8dcd44d14929170c693f6b1ab25be6593bfc9e Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Wed, 28 Jan 2015 11:32:07 +0000 Subject: Check file can be created at location --- kvm.check | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/kvm.check b/kvm.check index 1bb4007a..3c6accbf 100755 --- a/kvm.check +++ b/kvm.check @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -43,6 +43,7 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): ssh_host, vm_name, vm_path = self.check_and_parse_location(location) self.check_ssh_connectivity(ssh_host) + self.check_can_create_file_at_given_path(ssh_host, vm_path) self.check_no_existing_libvirt_vm(ssh_host, vm_name) self.check_extra_disks_exist(ssh_host, self.parse_attach_disks()) @@ -73,6 +74,26 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): 'write extension to deploy upgrades to existing machines.' % (ssh_host, vm_name)) + def check_can_create_file_at_given_path(self, ssh_host, vm_path): + + def check_can_write_to_given_path(): + try: + cliapp.ssh_runcmd(ssh_host, ['touch', vm_path]) + except cliapp.AppException as e: + raise cliapp.AppException("Can't write to location %s on %s" + % (vm_path, ssh_host)) + else: + cliapp.ssh_runcmd(ssh_host, ['rm', vm_path]) + + try: + cliapp.ssh_runcmd(ssh_host, ['test', '-e', vm_path]) + except cliapp.AppException as e: + # vm_path doesn't already exist, so let's test we can write + check_can_write_to_given_path() + else: + raise cliapp.AppException('%s already exists on %s' + % (vm_path, ssh_host)) + def check_extra_disks_exist(self, ssh_host, filename_list): for filename in filename_list: try: -- cgit v1.2.1 From b3736e8ff76418364dc1f9e0ce41cc0a89f309b7 Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Wed, 28 Jan 2015 17:28:00 +0000 Subject: Add check for virtual networks An exception will be raised if any needed networks are not started --- kvm.check | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/kvm.check b/kvm.check index 3c6accbf..b8877a89 100755 --- a/kvm.check +++ b/kvm.check @@ -17,6 +17,7 @@ '''Preparatory checks for Morph 'kvm' write extension''' import cliapp +import os import re import urlparse @@ -46,6 +47,7 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): self.check_can_create_file_at_given_path(ssh_host, vm_path) self.check_no_existing_libvirt_vm(ssh_host, vm_name) self.check_extra_disks_exist(ssh_host, self.parse_attach_disks()) + self.check_virtual_networks_are_started(ssh_host) def check_and_parse_location(self, location): '''Check and parse the location argument to get relevant data.''' @@ -102,4 +104,50 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): raise cliapp.AppException('Did not find file %s on host %s' % (filename, ssh_host)) + def check_virtual_networks_are_started(self, ssh_host): + + def check_virtual_network_is_started(network_name): + cmd = ['virsh', '-c', 'qemu:///system', 'net-info', network_name] + net_info = cliapp.ssh_runcmd(ssh_host, cmd).split('\n') + + def pretty_concat(lines): + return '\n'.join(['\t%s' % line for line in lines]) + + for line in net_info: + m = re.match('^Active:\W*(\w+)\W*', line) + if m: + break + else: + raise cliapp.AppException( + "Got unexpected output parsing output of `%s':\n%s" + % (' '.join(cmd), pretty_concat(net_info))) + + network_active = m.group(1) == 'yes' + + if not network_active: + raise cliapp.AppException("Network '%s' is not started" + % network_name) + + def name(nic_entry): + if ',' in nic_entry: + # NETWORK_NAME,mac=12:34,model=e1000... + return nic_entry[:nic_entry.find(',')] + else: + return nic_entry # NETWORK_NAME + + if 'NIC_CONFIG' in os.environ: + nics = os.environ['NIC_CONFIG'].split() + + # --network bridge= is used to specify a bridge + # --network user is used to specify a form of NAT + # (see the virt-install(1) man page) + networks = [name(n) for n in nics if not n.startswith('bridge=') + and not n.startswith('user')] + else: + networks = ['default'] + + for network in networks: + check_virtual_network_is_started(network) + + KvmPlusSshCheckExtension().run() -- cgit v1.2.1 From 8ca2082cdb2cf65b8ec9e1c392349208c0a00373 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Wed, 4 Mar 2015 13:28:57 +0000 Subject: Add copyright headers to write.help and configure.help files --- initramfs.write.help | 14 ++++++++++++++ install-files.configure.help | 14 ++++++++++++++ kvm.write.help | 14 ++++++++++++++ nfsboot.write.help | 14 ++++++++++++++ openstack.write.help | 14 ++++++++++++++ rawdisk.write.help | 14 ++++++++++++++ ssh-rsync.write.help | 14 ++++++++++++++ tar.write.help | 16 +++++++++++++++- virtualbox-ssh.write.help | 14 ++++++++++++++ 9 files changed, 127 insertions(+), 1 deletion(-) diff --git a/initramfs.write.help b/initramfs.write.help index a4a89f9d..54d3ae8c 100644 --- a/initramfs.write.help +++ b/initramfs.write.help @@ -1,3 +1,17 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see . + help: | Create an initramfs for a system by taking an existing system and diff --git a/install-files.configure.help b/install-files.configure.help index eb3aab0c..991c26c8 100644 --- a/install-files.configure.help +++ b/install-files.configure.help @@ -1,3 +1,17 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see . + help: | Install a set of files onto a system diff --git a/kvm.write.help b/kvm.write.help index 04393b8a..812a5309 100644 --- a/kvm.write.help +++ b/kvm.write.help @@ -1,3 +1,17 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see . + help: | Deploy a Baserock system as a *new* KVM/LibVirt virtual machine. diff --git a/nfsboot.write.help b/nfsboot.write.help index 310fd7a4..186c479a 100644 --- a/nfsboot.write.help +++ b/nfsboot.write.help @@ -1,3 +1,17 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see . + help: | *** DO NOT USE *** - This was written before 'proper' deployment mechanisms were in place. diff --git a/openstack.write.help b/openstack.write.help index 75ad9f0c..26983060 100644 --- a/openstack.write.help +++ b/openstack.write.help @@ -1,3 +1,17 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see . + help: | Deploy a Baserock system as a *new* OpenStack virtual machine. diff --git a/rawdisk.write.help b/rawdisk.write.help index 54af81c4..52ed73fb 100644 --- a/rawdisk.write.help +++ b/rawdisk.write.help @@ -1,3 +1,17 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see . + help: | Write a system produced by Morph to a physical disk, or to a file that can diff --git a/ssh-rsync.write.help b/ssh-rsync.write.help index d03508c0..f3f79ed5 100644 --- a/ssh-rsync.write.help +++ b/ssh-rsync.write.help @@ -1,3 +1,17 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see . + help: | Upgrade a Baserock system which is already deployed: diff --git a/tar.write.help b/tar.write.help index f052ac03..b45c61fa 100644 --- a/tar.write.help +++ b/tar.write.help @@ -1,5 +1,19 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see . + help: | Create a .tar file of the deployed system. - + The `location` argument is a pathname to the .tar file to be created. diff --git a/virtualbox-ssh.write.help b/virtualbox-ssh.write.help index cb50acc0..2dbf988c 100644 --- a/virtualbox-ssh.write.help +++ b/virtualbox-ssh.write.help @@ -1,3 +1,17 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see . + help: | Deploy a Baserock system as a *new* VirtualBox virtual machine. -- cgit v1.2.1 From ed741d8d090086e2380f7b9d68ddc3bd122acb9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 13 Mar 2015 18:18:55 +0000 Subject: Use the modern way of the GPL copyright header: URL instead real address Change-Id: I992dc0c1d40f563ade56a833162d409b02be90a0 --- add-config-files.configure | 5 ++--- fstab.configure | 5 ++--- initramfs.write | 5 ++--- install-files.configure | 5 ++--- kvm.check | 3 +-- kvm.write | 5 ++--- nfsboot.check | 5 ++--- nfsboot.configure | 5 ++--- nfsboot.write | 5 ++--- openstack.check | 5 ++--- openstack.write | 5 ++--- rawdisk.check | 5 ++--- rawdisk.write | 5 ++--- set-hostname.configure | 5 ++--- simple-network.configure | 5 ++--- ssh-rsync.check | 5 ++--- ssh-rsync.write | 5 ++--- sysroot.check | 3 +-- sysroot.write | 3 +-- tar.check | 5 ++--- tar.write | 5 ++--- vdaboot.configure | 5 ++--- virtualbox-ssh.check | 5 ++--- virtualbox-ssh.write | 3 +-- 24 files changed, 44 insertions(+), 68 deletions(-) diff --git a/add-config-files.configure b/add-config-files.configure index 0094cf6b..2cf96fd1 100755 --- a/add-config-files.configure +++ b/add-config-files.configure @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . # Copy all files located in $SRC_CONFIG_DIR to the image /etc. diff --git a/fstab.configure b/fstab.configure index a1287ea4..3bbc9102 100755 --- a/fstab.configure +++ b/fstab.configure @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . # # =*= License: GPL-2 =*= diff --git a/initramfs.write b/initramfs.write index f8af6d84..1059defa 100755 --- a/initramfs.write +++ b/initramfs.write @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . # # =*= License: GPL-2 =*= diff --git a/install-files.configure b/install-files.configure index 04dc5f18..58cf373a 100755 --- a/install-files.configure +++ b/install-files.configure @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . ''' A Morph configuration extension for adding arbitrary files to a system diff --git a/kvm.check b/kvm.check index b8877a89..62d76453 100755 --- a/kvm.check +++ b/kvm.check @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''Preparatory checks for Morph 'kvm' write extension''' diff --git a/kvm.write b/kvm.write index 30b43d6c..0d0c095b 100755 --- a/kvm.write +++ b/kvm.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''A Morph deployment write extension for deploying to KVM+libvirt. diff --git a/nfsboot.check b/nfsboot.check index 806e560a..e273f61c 100755 --- a/nfsboot.check +++ b/nfsboot.check @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''Preparatory checks for Morph 'nfsboot' write extension''' diff --git a/nfsboot.configure b/nfsboot.configure index 660d9c39..6a68dc48 100755 --- a/nfsboot.configure +++ b/nfsboot.configure @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . # Remove all networking interfaces. On nfsboot systems, eth0 is set up diff --git a/nfsboot.write b/nfsboot.write index 49d71174..d928775e 100755 --- a/nfsboot.write +++ b/nfsboot.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''A Morph deployment write extension for deploying to an nfsboot server diff --git a/openstack.check b/openstack.check index 3850d481..4c21b604 100755 --- a/openstack.check +++ b/openstack.check @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''Preparatory checks for Morph 'openstack' write extension''' diff --git a/openstack.write b/openstack.write index d29d2661..67e07c18 100755 --- a/openstack.write +++ b/openstack.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013 - 2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''A Morph deployment write extension for deploying to OpenStack.''' diff --git a/rawdisk.check b/rawdisk.check index 094adb72..9be0ce91 100755 --- a/rawdisk.check +++ b/rawdisk.check @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''Preparatory checks for Morph 'rawdisk' write extension''' diff --git a/rawdisk.write b/rawdisk.write index d91a4d5f..6f2d45ba 100755 --- a/rawdisk.write +++ b/rawdisk.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''A Morph deployment write extension for raw disk images.''' diff --git a/set-hostname.configure b/set-hostname.configure index e44c5d56..4b2424d8 100755 --- a/set-hostname.configure +++ b/set-hostname.configure @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . # Set hostname on system from HOSTNAME. diff --git a/simple-network.configure b/simple-network.configure index b98b202c..13884e9d 100755 --- a/simple-network.configure +++ b/simple-network.configure @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''A Morph deployment configuration extension to handle /etc/network/interfaces diff --git a/ssh-rsync.check b/ssh-rsync.check index 11446c28..c3bdfd29 100755 --- a/ssh-rsync.check +++ b/ssh-rsync.check @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''Preparatory checks for Morph 'ssh-rsync' write extension''' diff --git a/ssh-rsync.write b/ssh-rsync.write index c4577026..6d596500 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''A Morph deployment write extension for upgrading systems over ssh.''' diff --git a/sysroot.check b/sysroot.check index bfacd3fc..8ed965bd 100755 --- a/sysroot.check +++ b/sysroot.check @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . # Preparatory checks for Morph 'sysroot' write extension diff --git a/sysroot.write b/sysroot.write index be315365..0ad8d630 100755 --- a/sysroot.write +++ b/sysroot.write @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . # A Morph write extension to deploy to another directory diff --git a/tar.check b/tar.check index cbeaf163..f2304d46 100755 --- a/tar.check +++ b/tar.check @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . # Preparatory checks for Morph 'tar' write extension diff --git a/tar.write b/tar.write index 333626b5..01b545b4 100755 --- a/tar.write +++ b/tar.write @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . # A Morph write extension to deploy to a .tar file diff --git a/vdaboot.configure b/vdaboot.configure index b88eb3a8..60de925b 100755 --- a/vdaboot.configure +++ b/vdaboot.configure @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . # Change the "/" mount point to /dev/vda to use virtio disks. diff --git a/virtualbox-ssh.check b/virtualbox-ssh.check index 57d54db1..a97f3294 100755 --- a/virtualbox-ssh.check +++ b/virtualbox-ssh.check @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''Preparatory checks for Morph 'virtualbox-ssh' write extension''' diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 7eafcff3..774f2b4f 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''A Morph deployment write extension for deploying to VirtualBox via ssh. -- cgit v1.2.1 From bbc0af46f600aff7c98f8790319fea53dafeca36 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 13 Mar 2015 18:18:55 +0000 Subject: Use the modern way of the GPL copyright header: URL instead real address Change-Id: I992dc0c1d40f563ade56a833162d409b02be90a0 --- add-config-files.configure | 5 ++--- fstab.configure | 5 ++--- initramfs.write | 5 ++--- install-files.configure | 5 ++--- kvm.check | 3 +-- kvm.write | 5 ++--- nfsboot.check | 5 ++--- nfsboot.configure | 5 ++--- nfsboot.write | 5 ++--- openstack.check | 5 ++--- openstack.write | 5 ++--- rawdisk.check | 5 ++--- rawdisk.write | 5 ++--- set-hostname.configure | 5 ++--- simple-network.configure | 5 ++--- ssh-rsync.check | 5 ++--- ssh-rsync.write | 5 ++--- sysroot.check | 3 +-- sysroot.write | 3 +-- tar.check | 5 ++--- tar.write | 5 ++--- vdaboot.configure | 5 ++--- virtualbox-ssh.check | 5 ++--- virtualbox-ssh.write | 3 +-- 24 files changed, 44 insertions(+), 68 deletions(-) diff --git a/add-config-files.configure b/add-config-files.configure index 0094cf6b..2cf96fd1 100755 --- a/add-config-files.configure +++ b/add-config-files.configure @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . # Copy all files located in $SRC_CONFIG_DIR to the image /etc. diff --git a/fstab.configure b/fstab.configure index a1287ea4..3bbc9102 100755 --- a/fstab.configure +++ b/fstab.configure @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . # # =*= License: GPL-2 =*= diff --git a/initramfs.write b/initramfs.write index f8af6d84..1059defa 100755 --- a/initramfs.write +++ b/initramfs.write @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . # # =*= License: GPL-2 =*= diff --git a/install-files.configure b/install-files.configure index 04dc5f18..58cf373a 100755 --- a/install-files.configure +++ b/install-files.configure @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . ''' A Morph configuration extension for adding arbitrary files to a system diff --git a/kvm.check b/kvm.check index b8877a89..62d76453 100755 --- a/kvm.check +++ b/kvm.check @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''Preparatory checks for Morph 'kvm' write extension''' diff --git a/kvm.write b/kvm.write index 30b43d6c..0d0c095b 100755 --- a/kvm.write +++ b/kvm.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''A Morph deployment write extension for deploying to KVM+libvirt. diff --git a/nfsboot.check b/nfsboot.check index 806e560a..e273f61c 100755 --- a/nfsboot.check +++ b/nfsboot.check @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''Preparatory checks for Morph 'nfsboot' write extension''' diff --git a/nfsboot.configure b/nfsboot.configure index 660d9c39..6a68dc48 100755 --- a/nfsboot.configure +++ b/nfsboot.configure @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . # Remove all networking interfaces. On nfsboot systems, eth0 is set up diff --git a/nfsboot.write b/nfsboot.write index 49d71174..d928775e 100755 --- a/nfsboot.write +++ b/nfsboot.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''A Morph deployment write extension for deploying to an nfsboot server diff --git a/openstack.check b/openstack.check index 3850d481..4c21b604 100755 --- a/openstack.check +++ b/openstack.check @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''Preparatory checks for Morph 'openstack' write extension''' diff --git a/openstack.write b/openstack.write index d29d2661..67e07c18 100755 --- a/openstack.write +++ b/openstack.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013 - 2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''A Morph deployment write extension for deploying to OpenStack.''' diff --git a/rawdisk.check b/rawdisk.check index 094adb72..9be0ce91 100755 --- a/rawdisk.check +++ b/rawdisk.check @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''Preparatory checks for Morph 'rawdisk' write extension''' diff --git a/rawdisk.write b/rawdisk.write index d91a4d5f..6f2d45ba 100755 --- a/rawdisk.write +++ b/rawdisk.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''A Morph deployment write extension for raw disk images.''' diff --git a/set-hostname.configure b/set-hostname.configure index e44c5d56..4b2424d8 100755 --- a/set-hostname.configure +++ b/set-hostname.configure @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . # Set hostname on system from HOSTNAME. diff --git a/simple-network.configure b/simple-network.configure index b98b202c..13884e9d 100755 --- a/simple-network.configure +++ b/simple-network.configure @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''A Morph deployment configuration extension to handle /etc/network/interfaces diff --git a/ssh-rsync.check b/ssh-rsync.check index 11446c28..c3bdfd29 100755 --- a/ssh-rsync.check +++ b/ssh-rsync.check @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''Preparatory checks for Morph 'ssh-rsync' write extension''' diff --git a/ssh-rsync.write b/ssh-rsync.write index c4577026..6d596500 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''A Morph deployment write extension for upgrading systems over ssh.''' diff --git a/sysroot.check b/sysroot.check index bfacd3fc..8ed965bd 100755 --- a/sysroot.check +++ b/sysroot.check @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . # Preparatory checks for Morph 'sysroot' write extension diff --git a/sysroot.write b/sysroot.write index be315365..0ad8d630 100755 --- a/sysroot.write +++ b/sysroot.write @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . # A Morph write extension to deploy to another directory diff --git a/tar.check b/tar.check index cbeaf163..f2304d46 100755 --- a/tar.check +++ b/tar.check @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . # Preparatory checks for Morph 'tar' write extension diff --git a/tar.write b/tar.write index 333626b5..01b545b4 100755 --- a/tar.write +++ b/tar.write @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . # A Morph write extension to deploy to a .tar file diff --git a/vdaboot.configure b/vdaboot.configure index b88eb3a8..60de925b 100755 --- a/vdaboot.configure +++ b/vdaboot.configure @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013,2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . # Change the "/" mount point to /dev/vda to use virtio disks. diff --git a/virtualbox-ssh.check b/virtualbox-ssh.check index 57d54db1..a97f3294 100755 --- a/virtualbox-ssh.check +++ b/virtualbox-ssh.check @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''Preparatory checks for Morph 'virtualbox-ssh' write extension''' diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 7eafcff3..774f2b4f 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''A Morph deployment write extension for deploying to VirtualBox via ssh. -- cgit v1.2.1 From f1dce77123a814527f96d415b69ee10a06b8d36c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 6 Mar 2015 12:21:05 +0000 Subject: simple-network.configure: Move the generation of /etc/network/interfaces to a function --- simple-network.configure | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/simple-network.configure b/simple-network.configure index 13884e9d..c533ff22 100755 --- a/simple-network.configure +++ b/simple-network.configure @@ -48,8 +48,13 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): self.status(msg="Processing NETWORK_CONFIG=%(nc)s", nc=network_config) stanzas = self.parse_network_stanzas(network_config) - iface_file = self.generate_iface_file(stanzas) + self.generate_interfaces_file(args, stanzas) + + def generate_interfaces_file(self, args, stanzas): + """Generate /etc/network/interfaces file""" + + iface_file = self.generate_iface_file(stanzas) with open(os.path.join(args[0], "etc/network/interfaces"), "w") as f: f.write(iface_file) -- cgit v1.2.1 From e36de9eb1129db24b47dfa84cd96c5e2e2e6a327 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 6 Mar 2015 12:39:58 +0000 Subject: simple-network.configure: Generate networkd .network files as well --- simple-network.configure | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/simple-network.configure b/simple-network.configure index c533ff22..5b02142c 100755 --- a/simple-network.configure +++ b/simple-network.configure @@ -50,6 +50,7 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): stanzas = self.parse_network_stanzas(network_config) self.generate_interfaces_file(args, stanzas) + self.generate_networkd_files(args, stanzas) def generate_interfaces_file(self, args, stanzas): """Generate /etc/network/interfaces file""" @@ -87,6 +88,38 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): lines += [""] return "\n".join(lines) + def generate_networkd_files(self, args, stanzas): + """Generate .network files""" + + for i, stanza in enumerate(stanzas, 50): + iface_file = self.generate_networkd_file(stanza) + + if iface_file is None: + continue + + path = os.path.join(args[0], "etc", "systemd", "network", + "%s-%s.network" % (i, stanza['name'])) + + with open(path, "w") as f: + f.write(iface_file) + + def generate_networkd_file(self, stanza): + """Generate an .network file from the provided data.""" + + name = stanza['name'] + itype = stanza['type'] + pairs = stanza['args'].items() + + if itype == "loopback": + return + + lines = ["[Match]"] + lines += ["Name=%s\n" % name] + lines += ["[Network]"] + if itype == "dhcp": + lines += ["DHCP=yes"] + + return "\n".join(lines) def parse_network_stanzas(self, config): """Parse a network config environment variable into stanzas. -- cgit v1.2.1 From 8d36b6f71a2858843d9a7c6a878ebb2c85fda2a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 6 Mar 2015 12:56:19 +0000 Subject: simple-network.configure: Add function to convert mask to cidr suffix 255.255.255.0 -> 24 --- simple-network.configure | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/simple-network.configure b/simple-network.configure index 5b02142c..0e514e0b 100755 --- a/simple-network.configure +++ b/simple-network.configure @@ -121,6 +121,13 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): return "\n".join(lines) + def convert_net_mask_to_cidr_suffix(self, mask): + """Convert dotted decimal form of a subnet mask to CIDR suffix notation + + For example: 255.255.255.0 -> 24 + """ + return sum(bin(int(x)).count('1') for x in mask.split('.')) + def parse_network_stanzas(self, config): """Parse a network config environment variable into stanzas. -- cgit v1.2.1 From c33a852ba7b72258165d33f4e6810455483abc3d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 6 Mar 2015 12:55:01 +0000 Subject: simple-network.configure: process pairs of parameters in a function --- simple-network.configure | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/simple-network.configure b/simple-network.configure index 0e514e0b..a058cba7 100755 --- a/simple-network.configure +++ b/simple-network.configure @@ -118,9 +118,38 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): lines += ["[Network]"] if itype == "dhcp": lines += ["DHCP=yes"] + else: + lines += self.generate_networkd_entries(pairs) return "\n".join(lines) + def generate_networkd_entries(self, pairs): + """Generate networkd configuration entries with the other parameters""" + + address = None + netmask = None + gateway = None + lines = [] + for pair in pairs: + if pair[0] == 'address': + address = pair[1] + elif pair[0] == 'netmask': + netmask = pair[1] + elif pair[0] == 'gateway': + gateway = pair[1] + + if address and netmask: + network_suffix = self.convert_net_mask_to_cidr_suffix (netmask); + address_line = address + '/' + str(network_suffix) + lines += ["Address=%s" % address_line] + elif address or netmask: + raise Exception('address and netmask must be specified together') + + if gateway is not None: + lines += ["Gateway=%s" % gateway] + + return lines + def convert_net_mask_to_cidr_suffix(self, mask): """Convert dotted decimal form of a subnet mask to CIDR suffix notation -- cgit v1.2.1 From fcc6c405d5efd0043419bf7696a875fb18b7930a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 6 Mar 2015 13:18:08 +0000 Subject: simple-network.configure: Generate default network config files in a function Use DHCP by defaul in the default interfaces: - for /etc/networ/interfaces: "lo:loopback;eth0:dhcp,hostname=$(hostname)" - for networkd: "e*:dhcp" --- simple-network.configure | 26 ++++++++++++++++++++------ 1 file changed, 20 insertions(+), 6 deletions(-) diff --git a/simple-network.configure b/simple-network.configure index a058cba7..fbbe6c4e 100755 --- a/simple-network.configure +++ b/simple-network.configure @@ -42,15 +42,29 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): ''' def process_args(self, args): - network_config = os.environ.get( - "NETWORK_CONFIG", "lo:loopback;eth0:dhcp,hostname=$(hostname)") + network_config = os.environ.get("NETWORK_CONFIG") - self.status(msg="Processing NETWORK_CONFIG=%(nc)s", nc=network_config) + if network_config is None: + self.generate_default_network_config(args) + else: + self.status(msg="Processing NETWORK_CONFIG=%(nc)s", nc=network_config) + + stanzas = self.parse_network_stanzas(network_config) + + self.generate_interfaces_file(args, stanzas) + self.generate_networkd_files(args, stanzas) + + def generate_default_network_config(self, args): + """Generate default network configuration: DHCP in all the interfaces""" + + default_network_config_interfaces = "lo:loopback;eth0:dhcp,hostname=$(hostname)" + default_network_config_networkd = "e*:dhcp" - stanzas = self.parse_network_stanzas(network_config) + stanzas_interfaces = self.parse_network_stanzas(default_network_config_interfaces) + stanzas_networkd = self.parse_network_stanzas(default_network_config_networkd) - self.generate_interfaces_file(args, stanzas) - self.generate_networkd_files(args, stanzas) + self.generate_interfaces_file(args, stanzas_interfaces) + self.generate_networkd_files(args, stanzas_networkd) def generate_interfaces_file(self, args, stanzas): """Generate /etc/network/interfaces file""" -- cgit v1.2.1 From fe364c83daf5f66c685a00e3b368322d53139401 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Wed, 11 Mar 2015 19:48:02 +0000 Subject: simple-network.configure: Update documentation --- simple-network.configure | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/simple-network.configure b/simple-network.configure index fbbe6c4e..7c078c93 100755 --- a/simple-network.configure +++ b/simple-network.configure @@ -13,13 +13,15 @@ # You should have received a copy of the GNU General Public License along # with this program. If not, see . -'''A Morph deployment configuration extension to handle /etc/network/interfaces +'''A Morph deployment configuration extension to handle network configutation -This extension prepares /etc/network/interfaces with the interfaces specified -during deployment. +This extension prepares /etc/network/interfaces and networkd .network files +in /etc/systemd/network/ with the interfaces specified during deployment. If no network configuration is provided, eth0 will be configured for DHCP -with the hostname of the system. +with the hostname of the system in the case of /etc/network/interfaces. +In the case of networkd, any interface starting by e* will be configured +for DHCP ''' @@ -36,9 +38,10 @@ class SimpleNetworkError(morphlib.Error): class SimpleNetworkConfigurationExtension(cliapp.Application): - '''Configure /etc/network/interfaces + '''Configure /etc/network/interfaces and generate networkd .network files - Reading NETWORK_CONFIG, this extension sets up /etc/network/interfaces. + Reading NETWORK_CONFIG, this extension sets up /etc/network/interfaces + and .network files in /etc/systemd/network/. ''' def process_args(self, args): -- cgit v1.2.1 From 2021f504c66ab367e4039ff91b49f97fd926779a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Wed, 11 Mar 2015 19:48:17 +0000 Subject: simple-network.configure: Rename networkd file generated by systemd chunk Rename instead removal in case the user is already using a 10-dhcp.network file --- simple-network.configure | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/simple-network.configure b/simple-network.configure index 7c078c93..a347ebf9 100755 --- a/simple-network.configure +++ b/simple-network.configure @@ -47,6 +47,8 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): def process_args(self, args): network_config = os.environ.get("NETWORK_CONFIG") + self.rename_networkd_chunk_file(args) + if network_config is None: self.generate_default_network_config(args) else: @@ -57,6 +59,33 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): self.generate_interfaces_file(args, stanzas) self.generate_networkd_files(args, stanzas) + def rename_networkd_chunk_file(self, args): + """Rename the 10-dchp.network file generated in the systemd chunk + + The systemd chunk will place something in 10-dhcp.network, which will + have higher precedence than anything added in this extension (we + start at 50-*). + + We should check for that file and rename it instead remove it in + case the file is being used by the user. + + Until both the following happen, we should continue to rename that + default config file: + + 1. simple-network.configure is always run when systemd is included + 2. We've been building systems without systemd including that default + networkd config for long enough that nobody should be including + that config file. + """ + file_path = os.path.join(args[0], "etc", "systemd", "network", + "10-dhcp.network") + try: + os.rename(file_path, file_path + ".morph") + self.status(msg="Renaming networkd file from systemd chunk: %(f)s \ + to %(f)s.morph", f=file_path) + except OSError: + pass + def generate_default_network_config(self, args): """Generate default network configuration: DHCP in all the interfaces""" -- cgit v1.2.1 From 76bc6121a0cee508cc3feb1cac85b6a3d093b5cc Mon Sep 17 00:00:00 2001 From: Adam Coldrick Date: Tue, 17 Mar 2015 14:03:29 +0000 Subject: Fix line lengths in morphlib/exts/simple-network.configure Some lines were more than 79 characters long. This was causing ./check --full to fail on master of morph. This commit fixes the lines in question. Change-Id: I80969d2d89d3922e021b716c250796188d7a7f4c --- simple-network.configure | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/simple-network.configure b/simple-network.configure index a347ebf9..61113325 100755 --- a/simple-network.configure +++ b/simple-network.configure @@ -52,7 +52,8 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): if network_config is None: self.generate_default_network_config(args) else: - self.status(msg="Processing NETWORK_CONFIG=%(nc)s", nc=network_config) + self.status(msg="Processing NETWORK_CONFIG=%(nc)s", + nc=network_config) stanzas = self.parse_network_stanzas(network_config) @@ -87,13 +88,16 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): pass def generate_default_network_config(self, args): - """Generate default network configuration: DHCP in all the interfaces""" + """Generate default network config: DHCP in all the interfaces""" - default_network_config_interfaces = "lo:loopback;eth0:dhcp,hostname=$(hostname)" + default_network_config_interfaces = "lo:loopback;" \ + "eth0:dhcp,hostname=$(hostname)" default_network_config_networkd = "e*:dhcp" - stanzas_interfaces = self.parse_network_stanzas(default_network_config_interfaces) - stanzas_networkd = self.parse_network_stanzas(default_network_config_networkd) + stanzas_interfaces = self.parse_network_stanzas( + default_network_config_interfaces) + stanzas_networkd = self.parse_network_stanzas( + default_network_config_networkd) self.generate_interfaces_file(args, stanzas_interfaces) self.generate_networkd_files(args, stanzas_networkd) -- cgit v1.2.1 From ed6de447693fcf4a0b9240cc9943f065e9585fee Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Sat, 14 Mar 2015 17:30:43 +0000 Subject: Make fstab.configure use write_from_dict --- fstab.configure | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/fstab.configure b/fstab.configure index 3bbc9102..b9154eee 100755 --- a/fstab.configure +++ b/fstab.configure @@ -1,5 +1,6 @@ -#!/usr/bin/python -# Copyright (C) 2013,2015 Codethink Limited +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright © 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,21 +20,9 @@ import os import sys +import morphlib -def asciibetical(strings): +envvars = {k: v for (k, v) in os.environ.iteritems() if k.startswith('FSTAB_')} - def key(s): - return [ord(c) for c in s] - - return sorted(strings, key=key) - - -fstab_filename = os.path.join(sys.argv[1], 'etc', 'fstab') - -fstab_vars = asciibetical(x for x in os.environ if x.startswith('FSTAB_')) -with open(fstab_filename, 'a') as f: - for var in fstab_vars: - f.write('%s\n' % os.environ[var]) - -os.chown(fstab_filename, 0, 0) -os.chmod(fstab_filename, 0644) +conf_file = os.path.join(sys.argv[1], 'etc/fstab') +morphlib.util.write_from_dict(conf_file, envvars) -- cgit v1.2.1 From 813573c351e9f790992e66ee225965340b55aceb Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Sat, 14 Mar 2015 17:31:12 +0000 Subject: Add hosts.configure This adds a new config extension to allow deployments to write to /etc/hosts by adding HOSTS_x: to a cluster morph in a similar manner to the fstab.configure extension. --- hosts.configure | 48 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100755 hosts.configure diff --git a/hosts.configure b/hosts.configure new file mode 100755 index 00000000..6b068d04 --- /dev/null +++ b/hosts.configure @@ -0,0 +1,48 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright © 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# +# =*= License: GPL-2 =*= + + +import os +import sys +import socket + +import morphlib + +def validate(var, line): + xs = line.split() + if len(xs) == 0: + raise morphlib.Error("`%s: %s': line is empty" % (var, line)) + + ip = xs[0] + hostnames = xs[1:] + + if len(hostnames) == 0: + raise morphlib.Error("`%s: %s': missing hostname" % (var, line)) + + family = socket.AF_INET6 if ':' in ip else socket.AF_INET + + try: + socket.inet_pton(family, ip) + except socket.error: + raise morphlib.Error("`%s: %s' invalid ip" % (var, ip)) + +envvars = {k: v for (k, v) in os.environ.iteritems() if k.startswith('HOSTS_')} + +conf_file = os.path.join(sys.argv[1], 'etc/hosts') +morphlib.util.write_from_dict(conf_file, envvars, validate) -- cgit v1.2.1 From 8e691df0c964bbb8b20e8e66e37402a5fc854b37 Mon Sep 17 00:00:00 2001 From: Javier Jardon Date: Mon, 23 Mar 2015 23:14:05 +0000 Subject: simple-network: separate creation of directory and file path Change-Id: Ic715815bbad3ef1ee9ab457b62a194eaef45744c --- simple-network.configure | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/simple-network.configure b/simple-network.configure index 61113325..42bf1b82 100755 --- a/simple-network.configure +++ b/simple-network.configure @@ -106,7 +106,10 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): """Generate /etc/network/interfaces file""" iface_file = self.generate_iface_file(stanzas) - with open(os.path.join(args[0], "etc/network/interfaces"), "w") as f: + + directory_path = os.path.join(args[0], "etc", "network") + file_path = os.path.join(directory_path, "interfaces") + with open(file_path, "w") as f: f.write(iface_file) def generate_iface_file(self, stanzas): @@ -147,10 +150,11 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): if iface_file is None: continue - path = os.path.join(args[0], "etc", "systemd", "network", - "%s-%s.network" % (i, stanza['name'])) + directory_path = os.path.join(args[0], "etc", "systemd", "network") + file_path = os.path.join(directory_path, + "%s-%s.network" % (i, stanza['name'])) - with open(path, "w") as f: + with open(file_path, "w") as f: f.write(iface_file) def generate_networkd_file(self, stanza): -- cgit v1.2.1 From da48a28de7461bd46ef777a2ab2f5ca0e5c1185c Mon Sep 17 00:00:00 2001 From: Javier Jardon Date: Mon, 23 Mar 2015 23:02:57 +0000 Subject: simple-network: Add functionn to create a path if it doesnt exist Change-Id: If011a5518fd30914c89b00099b9d05cff5cd3959 --- simple-network.configure | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/simple-network.configure b/simple-network.configure index 42bf1b82..b4988125 100755 --- a/simple-network.configure +++ b/simple-network.configure @@ -27,6 +27,7 @@ for DHCP import os import sys +import errno import cliapp import morphlib @@ -256,6 +257,16 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): return output_stanza + def make_sure_path_exists(self, path): + try: + os.makedirs(path) + except OSError as e: + if e.errno == errno.EEXIST and os.path.isdir(path): + pass + else: + raise SimpleNetworkError("Unable to create directory '%s'" + % path) + def status(self, **kwargs): '''Provide status output. -- cgit v1.2.1 From 500943a125a1b21c08adcc400a86569693aebfa4 Mon Sep 17 00:00:00 2001 From: Javier Jardon Date: Mon, 23 Mar 2015 23:19:33 +0000 Subject: simple-network: Use function to ensure directory path will exist Change-Id: I44693d15aa5e92d5f09720065788adff34f8685c --- simple-network.configure | 2 ++ 1 file changed, 2 insertions(+) diff --git a/simple-network.configure b/simple-network.configure index b4988125..130b96c9 100755 --- a/simple-network.configure +++ b/simple-network.configure @@ -109,6 +109,7 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): iface_file = self.generate_iface_file(stanzas) directory_path = os.path.join(args[0], "etc", "network") + self.make_sure_path_exists(directory_path) file_path = os.path.join(directory_path, "interfaces") with open(file_path, "w") as f: f.write(iface_file) @@ -152,6 +153,7 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): continue directory_path = os.path.join(args[0], "etc", "systemd", "network") + self.make_sure_path_exists(directory_path) file_path = os.path.join(directory_path, "%s-%s.network" % (i, stanza['name'])) -- cgit v1.2.1 From bb3fa0e026b1bab96b4bbbcb1920efcf5f995f37 Mon Sep 17 00:00:00 2001 From: Javier Jardon Date: Mon, 23 Mar 2015 23:27:11 +0000 Subject: simple-network: only try to rename "10-dhcp.network" if it actually exist Change-Id: I1521c0bdec4d7a6812f8988a2349e66b08161de8 --- simple-network.configure | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/simple-network.configure b/simple-network.configure index 130b96c9..1ba94e86 100755 --- a/simple-network.configure +++ b/simple-network.configure @@ -81,12 +81,14 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): """ file_path = os.path.join(args[0], "etc", "systemd", "network", "10-dhcp.network") - try: - os.rename(file_path, file_path + ".morph") - self.status(msg="Renaming networkd file from systemd chunk: %(f)s \ - to %(f)s.morph", f=file_path) - except OSError: - pass + + if os.path.isfile(file_path): + try: + os.rename(file_path, file_path + ".morph") + self.status(msg="Renaming networkd file from systemd chunk: \ + %(f)s to %(f)s.morph", f=file_path) + except OSError: + pass def generate_default_network_config(self, args): """Generate default network config: DHCP in all the interfaces""" -- cgit v1.2.1 From a69b342f542f4895ef2bfd9f18a0e56d38b4bc7d Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Mon, 30 Mar 2015 13:34:40 +0000 Subject: Fix: strip 'network=' from NIC_CONFIG Also ensure NIC_CONFIG begins with 'network=', 'bridge=' or is 'user' Change-Id: I3bcbd25eb2c9a05b7fa276697f97a1080cb0316e --- kvm.check | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/kvm.check b/kvm.check index 62d76453..83562e44 100755 --- a/kvm.check +++ b/kvm.check @@ -129,14 +129,22 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): def name(nic_entry): if ',' in nic_entry: - # NETWORK_NAME,mac=12:34,model=e1000... - return nic_entry[:nic_entry.find(',')] + # network=NETWORK_NAME,mac=12:34,model=e1000... + return nic_entry[:nic_entry.find(',')].lstrip('network=') else: - return nic_entry # NETWORK_NAME + return nic_entry.lstrip('network=') # NETWORK_NAME if 'NIC_CONFIG' in os.environ: nics = os.environ['NIC_CONFIG'].split() + for n in nics: + if not (n.startswith('network=') + or n.startswith('bridge=') + or n == 'user'): + raise cliapp.AppException('malformed NIC_CONFIG: %s\n' + " (expected 'bridge=BRIDGE' 'network=NAME'" + " or 'user')" % n) + # --network bridge= is used to specify a bridge # --network user is used to specify a form of NAT # (see the virt-install(1) man page) -- cgit v1.2.1 From 2b35eb5ebef0de0ad78466f41ccfd7792fbf2e40 Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Fri, 3 Apr 2015 20:05:09 +0100 Subject: Make kvm deploy check that host has virt-install This allows us to catch a case where virt-install hasn't been installed on the host we're deploying to much earlier in the deployment process. Change-Id: I413ad804a7e8bef4fc2d1231411e01d30d0cb9e8 --- kvm.check | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/kvm.check b/kvm.check index 83562e44..67cb3d38 100755 --- a/kvm.check +++ b/kvm.check @@ -47,6 +47,7 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): self.check_no_existing_libvirt_vm(ssh_host, vm_name) self.check_extra_disks_exist(ssh_host, self.parse_attach_disks()) self.check_virtual_networks_are_started(ssh_host) + self.check_host_has_virtinstall(ssh_host) def check_and_parse_location(self, location): '''Check and parse the location argument to get relevant data.''' @@ -156,5 +157,13 @@ class KvmPlusSshCheckExtension(morphlib.writeexts.WriteExtension): for network in networks: check_virtual_network_is_started(network) + def check_host_has_virtinstall(self, ssh_host): + try: + cliapp.ssh_runcmd(ssh_host, ['which', 'virt-install']) + except cliapp.AppException: + raise cliapp.AppException( + 'virt-install does not seem to be installed on host %s' + % ssh_host) + KvmPlusSshCheckExtension().run() -- cgit v1.2.1 From ef08b64d0c81515a360ceea1d0068a7a9d4233d9 Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Sat, 11 Apr 2015 14:23:48 +0100 Subject: Add template option to install-files conf ext This adds an optional 'template' option to the install-files manifest format. A file declared as a template will be rendered using jinja2 with variables substituted in from the environment. Change-Id: I2ed6fe58f5fff315b42b7e4ec478ada851e0a70d --- install-files.configure | 42 ++++++++++++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 10 deletions(-) diff --git a/install-files.configure b/install-files.configure index 58cf373a..c2970243 100755 --- a/install-files.configure +++ b/install-files.configure @@ -30,6 +30,12 @@ import shlex import shutil import stat +try: + import jinja2 + jinja_available = True +except ImportError: + jinja_available = False + class InstallFilesConfigureExtension(cliapp.Application): def process_args(self, args): @@ -48,18 +54,20 @@ class InstallFilesConfigureExtension(cliapp.Application): self.install_entry(entry, manifest_dir, target_root) def install_entry(self, entry, manifest_root, target_root): - m = re.match('(overwrite )?([0-7]+) ([0-9]+) ([0-9]+) (\S+)', entry) + m = re.match('(template )?(overwrite )?' + '([0-7]+) ([0-9]+) ([0-9]+) (\S+)', entry) if m: - overwrite = m.group(1) - mode = int(m.group(2), 8) # mode is octal - uid = int(m.group(3)) - gid = int(m.group(4)) - path = m.group(5) + template = m.group(1) + overwrite = m.group(2) + mode = int(m.group(3), 8) # mode is octal + uid = int(m.group(4)) + gid = int(m.group(5)) + path = m.group(6) else: raise cliapp.AppException('Invalid manifest entry, ' - 'format: [overwrite] ' - '') + 'format: [template] [overwrite] ' + ' ') dest_path = os.path.join(target_root, './' + path) if stat.S_ISDIR(mode): @@ -91,8 +99,22 @@ class InstallFilesConfigureExtension(cliapp.Application): raise cliapp.AppException('File already exists at %s' % dest_path) else: - shutil.copyfile(os.path.join(manifest_root, './' + path), - dest_path) + if template: + if not jinja_available: + raise cliapp.AppException( + "Failed to install template file `%s': " + 'install-files templates require jinja2' + % path) + + loader = jinja2.FileSystemLoader(manifest_root) + env = jinja2.Environment(loader=loader, + keep_trailing_newline=True) + + env.get_template(path).stream(os.environ).dump(dest_path) + else: + shutil.copyfile(os.path.join(manifest_root, './' + path), + dest_path) + os.chown(dest_path, uid, gid) os.chmod(dest_path, mode) -- cgit v1.2.1 From 64465445f2a95d74cb4a5bae3ab0d1783d6de68e Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Wed, 15 Apr 2015 15:58:34 +0100 Subject: Add dns option to simple-network conf ext Change-Id: I0f4490d76caca802536b21085ea0d770fb8c0798 --- simple-network.configure | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/simple-network.configure b/simple-network.configure index 1ba94e86..4a70f311 100755 --- a/simple-network.configure +++ b/simple-network.configure @@ -188,7 +188,9 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): address = None netmask = None gateway = None + dns = None lines = [] + for pair in pairs: if pair[0] == 'address': address = pair[1] @@ -196,6 +198,8 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): netmask = pair[1] elif pair[0] == 'gateway': gateway = pair[1] + elif pair[0] == 'dns': + dns = pair[1] if address and netmask: network_suffix = self.convert_net_mask_to_cidr_suffix (netmask); @@ -204,9 +208,12 @@ class SimpleNetworkConfigurationExtension(cliapp.Application): elif address or netmask: raise Exception('address and netmask must be specified together') - if gateway is not None: + if gateway: lines += ["Gateway=%s" % gateway] + if dns: + lines += ["DNS=%s" % dns] + return lines def convert_net_mask_to_cidr_suffix(self, mask): -- cgit v1.2.1 From 985d512ad9969b9216720a7dc9274b41bb2802eb Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Mon, 13 Apr 2015 12:31:46 +0000 Subject: Add distbuild-trove-nfsboot.write The nfsboot.write deployment extension has been deprecated for a while because it's not generally useful. It's only used for deploying distbuild nodes to a Trove, as far as I know. We still need to support setting up a bunch of machines that boot over NFS from a Trove. But we can do this in a special-purpose .write extension. The new distbuild-trove-nfsboot.write is much more efficient than the more generic nfsboot.write: instead of treating each system individually (thus copying an almost identical ~2GB rootfs to the Trove once per node) it copies the system image to the Trove once, and /then/ sets up a rootfs per node. Upgrades are now supported, although the code assumes distbuild nodes are stateless (as they should be) so nothing special is done for upgrades, other than checking that there is already a version of the given system in existance. The new extension does not create an orig/ and run/ version of each system, because there is no need when the deployed system is stateless. There could be further gains in efficiency, but I don't have time to do them right now. This write extension is full of compromises, its goal is to better support the existing users who have a Trove and a distbuild network deployed via NFS. It is specifically not intended to be useful for other purposes. Change-Id: I9a50c58b714ed272212d1d6c55b289aaa96051b1 --- distbuild-trove-nfsboot.check | 150 ++++++++++++++++++++++ distbuild-trove-nfsboot.help | 49 ++++++++ distbuild-trove-nfsboot.write | 283 ++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 482 insertions(+) create mode 100755 distbuild-trove-nfsboot.check create mode 100644 distbuild-trove-nfsboot.help create mode 100755 distbuild-trove-nfsboot.write diff --git a/distbuild-trove-nfsboot.check b/distbuild-trove-nfsboot.check new file mode 100755 index 00000000..38c491e5 --- /dev/null +++ b/distbuild-trove-nfsboot.check @@ -0,0 +1,150 @@ +#!/usr/bin/python +# Copyright (C) 2014-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + +'''Preparatory checks for Morph 'distbuild-trove-nfsboot' write extension''' + +import cliapp +import logging +import os + +import morphlib.writeexts + + +class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension): + + nfsboot_root = '/srv/nfsboot' + remote_user = 'root' + + required_vars = [ + 'DISTBUILD_CONTROLLER', + 'DISTBUILD_GIT_SERVER', + 'DISTBUILD_SHARED_ARTIFACT_CACHE', + 'DISTBUILD_TROVE_ID', + 'DISTBUILD_WORKERS', + 'DISTBUILD_WORKER_SSH_KEY', + ] + + def system_path(self, system_name, version_label=None): + if version_label: + return os.path.join(self.nfsboot_root, system_name, 'systems', + version_label, 'run') + else: + return os.path.join(self.nfsboot_root, system_name) + + def process_args(self, args): + if len(args) != 1: + raise cliapp.AppException('Wrong number of command line args') + + nfs_host = args[0] + nfs_netloc = '%s@%s' % (self.remote_user, nfs_host) + + version_label = os.getenv('VERSION_LABEL', 'factory') + + missing_vars = [var for var in self.required_vars + if not var in os.environ] + if missing_vars: + raise cliapp.AppException( + 'Please set: %s' % ', '.join(missing_vars)) + + controllers = os.getenv('DISTBUILD_CONTROLLER').split() + workers = os.getenv('DISTBUILD_WORKERS').split() + + if len(controllers) != 1: + raise cliapp.AppException('Please specify exactly one controller.') + + if len(workers) == 0: + raise cliapp.AppException('Please specify at least one worker.') + + upgrade = self.get_environment_boolean('UPGRADE') + + self.check_good_server(nfs_netloc) + + system_names = set(controllers + workers) + for system_name in system_names: + if upgrade: + self.check_upgradeable(nfs_netloc, system_name, version_label) + else: + system_path = self.system_path(system_name) + + if self.remote_directory_exists(nfs_netloc, system_path): + if self.get_environment_boolean('OVERWRITE') == False: + raise cliapp.AppException( + 'System %s already exists at %s:%s. Try `morph ' + 'upgrade` instead of `morph deploy`.' % ( + system_name, nfs_netloc, system_path)) + + def check_good_server(self, netloc): + # FIXME: assumes root + self.check_ssh_connectivity(netloc.split('@')[-1]) + + # Is an NFS server + try: + cliapp.ssh_runcmd( + netloc, ['test', '-e', '/etc/exports']) + except cliapp.AppException: + raise cliapp.AppException('server %s is not an nfs server' + % netloc) + try: + cliapp.ssh_runcmd( + netloc, ['systemctl', 'is-enabled', 'nfs-server.service']) + + except cliapp.AppException: + raise cliapp.AppException('server %s does not control its ' + 'nfs server by systemd' % netloc) + + # TFTP server exports /srv/nfsboot/tftp + tftp_root = os.path.join(self.nfsboot_root, 'tftp') + try: + cliapp.ssh_runcmd( + netloc, ['test' , '-d', tftp_root]) + except cliapp.AppException: + raise cliapp.AppException('server %s does not export %s' % + (netloc, tftp_root)) + + def check_upgradeable(self, nfs_netloc, system_name, version_label): + '''Check that there is already a version of the system present. + + Distbuild nodes are stateless, so an upgrade is actually pretty much + the same as an initial deployment. This test is just a sanity check. + + ''' + system_path = self.system_path(system_name) + system_version_path = self.system_path(system_name, version_label) + + if not self.remote_directory_exists(nfs_netloc, system_path): + raise cliapp.AppException( + 'System %s not found at %s:%s, cannot deploy an upgrade.' % ( + system_name, nfs_netloc, system_path)) + + if self.remote_directory_exists(nfs_netloc, system_version_path): + if self.get_environment_boolean('OVERWRITE'): + pass + else: + raise cliapp.AppException( + 'System %s version %s already exists at %s:%s.' % ( + system_name, version_label, nfs_netloc, + system_version_path)) + + def remote_directory_exists(self, nfs_netloc, path): + try: + cliapp.ssh_runcmd(nfs_netloc, ['test', '-d', path]) + except cliapp.AppException as e: + logging.debug('SSH exception: %s', e) + return False + + return True + + +DistbuildTroveNFSBootCheckExtension().run() diff --git a/distbuild-trove-nfsboot.help b/distbuild-trove-nfsboot.help new file mode 100644 index 00000000..62f1455c --- /dev/null +++ b/distbuild-trove-nfsboot.help @@ -0,0 +1,49 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see . + +help: | + Deploy a distbuild network, using a Trove to serve the kernel and rootfs. + + The `location` argument is the hostname of the Trove system. + + The following configuration values must be specified: + + - DISTBUILD_CONTROLLER: hostname of controller system + - DISTBUILD_WORKERS: hostnames of each worker system + - DISTBUILD_GIT_SERVER: Trove hostname + - DISTBUILD_SHARED_ARTIFACT_CACHE: Trove hostname + - DISTBUILD_TROVE_ID: Trove ID + - DISTBUILD_WORKER_SSH_KEY: SSH key to be used for ssh:// repos + + A note on TROVE_ID: the current distbuild-setup service requires that + a single 'Trove ID' is specified. This is used in Morph for expanding + keyed URLs. If you set TROVE_ID=foo for example, foo:bar will be expanded + to git://$GIT_SERVER/foo, in addition to the standard baserock: and + upstream: prefixes that you can use. + + The WORKER_SSH_KEY must be provided, even if you don't need it. The + distbuild-setup service could be changed to make it optional. + + The following configuration values are optional: + + - HOST_MAP: a list of key=value pairs mapping hostnames to IP addresses, + or fully-qualified domain names. Useful if you + cannot rely on hostname resolution working for your deploment. + + The extension will connect to root@location via ssh to copy the kernel and + rootfs, and configure the nfs server. It will duplicate the kernel and + rootfs once for each node in the distbuild network. + + The deployment mechanism makes assumptions about the bootloader + configuration of the target machines. diff --git a/distbuild-trove-nfsboot.write b/distbuild-trove-nfsboot.write new file mode 100755 index 00000000..a5a5b094 --- /dev/null +++ b/distbuild-trove-nfsboot.write @@ -0,0 +1,283 @@ +#!/usr/bin/python +# Copyright (C) 2013-2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + + +'''Morph .write extension for a distbuild network booting off a Trove with NFS. + +''' + + +import os +import sys +import tempfile + +import cliapp +import morphlib.writeexts + + +class DistbuildTroveNFSBootWriteExtension(morphlib.writeexts.WriteExtension): + + '''Create an NFS root and kernel on TFTP during Morph's deployment. + + See distbuild-trove-nfsboot.help for documentation. + + ''' + + nfsboot_root = '/srv/nfsboot' + remote_user = 'root' + + def system_path(self, system_name, version_label=None): + if version_label: + # The 'run' directory is kind of a historical artifact. Baserock + # systems that have Btrfs root disks maintain an orig/ and a run/ + # subvolume, so that one can find changes that have been made at + # runtime. For distbuild systems, this isn't necessary because the + # root filesystems of the nodes are effectively stateless. However, + # existing systems have bootloaders configured to look for the + # 'run' directory, so we need to keep creating it. + return os.path.join(self.nfsboot_root, system_name, 'systems', + version_label, 'run') + else: + return os.path.join(self.nfsboot_root, system_name) + + def process_args(self, args): + if len(args) != 2: + raise cliapp.AppException('Wrong number of command line args') + + local_system_path, nfs_host = args + + nfs_netloc = '%s@%s' % (self.remote_user, nfs_host) + + version_label = os.getenv('VERSION_LABEL', 'factory') + + controller_name = os.getenv('DISTBUILD_CONTROLLER') + worker_names = os.getenv('DISTBUILD_WORKERS').split() + system_names = set([controller_name] + worker_names) + + git_server = os.getenv('DISTBUILD_GIT_SERVER') + shared_artifact_cache = os.getenv('DISTBUILD_SHARED_ARTIFACT_CACHE') + trove_id = os.getenv('DISTBUILD_TROVE_ID') + worker_ssh_key_path = os.getenv('DISTBUILD_WORKER_SSH_KEY') + + host_map = self.parse_host_map_string(os.getenv('HOST_MAP', '')) + + kernel_relpath = self.find_kernel(local_system_path) + + copied_rootfs = None + for system_name in system_names: + remote_system_path = self.system_path(system_name, version_label) + if copied_rootfs is None: + self.transfer_system( + nfs_netloc, local_system_path, remote_system_path) + copied_rootfs = remote_system_path + else: + self.duplicate_remote_system( + nfs_netloc, copied_rootfs, remote_system_path) + + for system_name in system_names: + remote_system_path = self.system_path(system_name, version_label) + self.link_kernel_to_tftpboot_path( + nfs_netloc, system_name, version_label, kernel_relpath) + self.set_hostname( + nfs_netloc, system_name, remote_system_path) + self.write_distbuild_config( + nfs_netloc, system_name, remote_system_path, git_server, + shared_artifact_cache, trove_id, worker_ssh_key_path, + controller_name, worker_names, host_map=host_map) + + self.configure_nfs_exports(nfs_netloc, system_names) + + for system_name in system_names: + self.update_default_version(nfs_netloc, system_name, version_label) + + def parse_host_map_string(self, host_map_string): + '''Parse the HOST_MAP variable + + Returns a dict mapping hostname to value (where value is an IP + address, a fully-qualified domain name, an alternate hostname, or + whatever). + + ''' + pairs = host_map_string.split(' ') + return morphlib.util.parse_environment_pairs({}, pairs) + + def transfer_system(self, nfs_netloc, local_system_path, + remote_system_path): + self.status(msg='Copying rootfs to %(nfs_netloc)s', + nfs_netloc=nfs_netloc) + cliapp.ssh_runcmd( + nfs_netloc, ['mkdir', '-p', remote_system_path]) + # The deployed rootfs may have been created by OSTree, so definitely + # don't pass --hard-links to `rsync`. + cliapp.runcmd( + ['rsync', '--archive', '--delete', '--info=progress2', + '--protect-args', '--partial', '--sparse', '--xattrs', + local_system_path + '/', + '%s:%s' % (nfs_netloc, remote_system_path)], stdout=sys.stdout) + + def duplicate_remote_system(self, nfs_netloc, source_system_path, + target_system_path): + self.status(msg='Duplicating rootfs to %(target_system_path)s', + target_system_path=target_system_path) + cliapp.ssh_runcmd(nfs_netloc, + ['mkdir', '-p', target_system_path]) + # We can't pass --info=progress2 here, because it may not be available + # in the remote 'rsync'. The --info setting was added in RSync 3.1.0, + # old versions of Baserock have RSync 3.0.9. So the user doesn't get + # any progress info on stdout for the 'duplicate' stage. + cliapp.ssh_runcmd(nfs_netloc, + ['rsync', '--archive', '--delete', '--protect-args', '--partial', + '--sparse', '--xattrs', source_system_path + '/', + target_system_path], stdout=sys.stdout) + + def find_kernel(self, local_system_path): + bootdir = os.path.join(local_system_path, 'boot') + image_names = ['vmlinuz', 'zImage', 'uImage'] + + for name in image_names: + try_path = os.path.join(bootdir, name) + if os.path.exists(try_path): + kernel_path = os.path.relpath(try_path, local_system_path) + break + else: + raise cliapp.AppException( + 'Could not find a kernel in the system: none of ' + '%s found' % ', '.join(image_names)) + return kernel_path + + def link_kernel_to_tftpboot_path(self, nfs_netloc, system_name, + version_label, kernel_relpath): + '''Create links for TFTP server for a system's kernel.''' + + remote_system_path = self.system_path(system_name, version_label) + kernel_dest = os.path.join(remote_system_path, kernel_relpath) + + self.status(msg='Creating links to %(name)s kernel in tftp directory', + name=system_name) + tftp_dir = os.path.join(self.nfsboot_root , 'tftp') + + versioned_kernel_name = "%s-%s" % (system_name, version_label) + kernel_name = system_name + + cliapp.ssh_runcmd(nfs_netloc, + ['ln', '-f', kernel_dest, + os.path.join(tftp_dir, versioned_kernel_name)]) + + cliapp.ssh_runcmd(nfs_netloc, + ['ln', '-sf', versioned_kernel_name, + os.path.join(tftp_dir, kernel_name)]) + + def set_remote_file_contents(self, nfs_netloc, path, text): + with tempfile.NamedTemporaryFile() as f: + f.write(text) + f.flush() + cliapp.runcmd( + ['scp', f.name, '%s:%s' % (nfs_netloc, path)]) + + def set_hostname(self, nfs_netloc, system_name, system_path): + hostname_path = os.path.join(system_path, 'etc', 'hostname') + self.set_remote_file_contents( + nfs_netloc, hostname_path, system_name + '\n') + + def write_distbuild_config(self, nfs_netloc, system_name, system_path, + git_server, shared_artifact_cache, trove_id, + worker_ssh_key_path, controller_name, + worker_names, host_map = {}): + '''Write /etc/distbuild/distbuild.conf on the node. + + This .write extension takes advantage of the 'generic' mode of + distbuild.configure. Each node is not configured until first-boot, + when distbuild-setup.service runs and configures the node based on the + contents of /etc/distbuild/distbuild.conf. + + ''' + def host(hostname): + return host_map.get(hostname, hostname) + + config = { + 'ARTIFACT_CACHE_SERVER': host(shared_artifact_cache), + 'CONTROLLERHOST': host(controller_name), + 'TROVE_HOST': host(git_server), + 'TROVE_ID': trove_id, + 'DISTBUILD_CONTROLLER': system_name == controller_name, + 'DISTBUILD_WORKER': system_name in worker_names, + 'WORKERS': ', '.join(map(host, worker_names)), + 'WORKER_SSH_KEY': '/etc/distbuild/worker.key', + } + + config_text = '\n'.join( + '%s: %s' % (key, value) for key, value in config.iteritems()) + config_text = \ + '# Generated by distbuild-trove-nfsboot.write\n' + \ + config_text + '\n' + path = os.path.join(system_path, 'etc', 'distbuild') + cliapp.ssh_runcmd( + nfs_netloc, ['mkdir', '-p', path]) + cliapp.runcmd( + ['scp', worker_ssh_key_path, '%s:%s' % (nfs_netloc, path)]) + self.set_remote_file_contents( + nfs_netloc, os.path.join(path, 'distbuild.conf'), config_text) + + def configure_nfs_exports(self, nfs_netloc, system_names): + '''Ensure the Trove is set up to export the NFS roots we need. + + This doesn't handle setting up the TFTP daemon. We assume that is + already running. + + ''' + for system_name in system_names: + exported_path = self.system_path(system_name) + exports_path = '/etc/exports' + + # Rather ugly SSH hackery follows to ensure each system path is + # listed in /etc/exports. + try: + cliapp.ssh_runcmd( + nfs_netloc, ['grep', '-q', exported_path, exports_path]) + except cliapp.AppException: + ip_mask = '*' + options = 'rw,no_subtree_check,no_root_squash,async' + exports_string = '%s %s(%s)\n' % (exported_path, ip_mask, + options) + exports_append_sh = '''\ + set -eu + target="$1" + temp=$(mktemp) + cat "$target" > "$temp" + cat >> "$temp" + mv "$temp" "$target" + ''' + cliapp.ssh_runcmd( + nfs_netloc, + ['sh', '-c', exports_append_sh, '--', exports_path], + feed_stdin=exports_string) + + cliapp.ssh_runcmd(nfs_netloc, + ['systemctl', 'restart', 'nfs-server.service']) + + def update_default_version(self, remote_netloc, system_name, + version_label): + self.status(msg='Linking \'default\' to %(version)s for %(system)s', + version=version_label, system=system_name) + system_path = self.system_path(system_name) + system_version_path = os.path.join(system_path, 'systems', + version_label) + default_path = os.path.join(system_path, 'systems', 'default') + + cliapp.ssh_runcmd(remote_netloc, + ['ln', '-sfn', system_version_path, default_path]) + + +DistbuildTroveNFSBootWriteExtension().run() -- cgit v1.2.1 From cea5244d54568d4ac0e0182c754c623a0fdae79e Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Thu, 23 Apr 2015 13:52:16 +0000 Subject: Rename help file to work with 'help-extensions' subcommand Change-Id: Ibf7cf1f81998678f9354d77f52e54344294e89f7 --- distbuild-trove-nfsboot.help | 49 -------------------------------------- distbuild-trove-nfsboot.write.help | 49 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 49 insertions(+), 49 deletions(-) delete mode 100644 distbuild-trove-nfsboot.help create mode 100644 distbuild-trove-nfsboot.write.help diff --git a/distbuild-trove-nfsboot.help b/distbuild-trove-nfsboot.help deleted file mode 100644 index 62f1455c..00000000 --- a/distbuild-trove-nfsboot.help +++ /dev/null @@ -1,49 +0,0 @@ -# Copyright (C) 2014, 2015 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, see . - -help: | - Deploy a distbuild network, using a Trove to serve the kernel and rootfs. - - The `location` argument is the hostname of the Trove system. - - The following configuration values must be specified: - - - DISTBUILD_CONTROLLER: hostname of controller system - - DISTBUILD_WORKERS: hostnames of each worker system - - DISTBUILD_GIT_SERVER: Trove hostname - - DISTBUILD_SHARED_ARTIFACT_CACHE: Trove hostname - - DISTBUILD_TROVE_ID: Trove ID - - DISTBUILD_WORKER_SSH_KEY: SSH key to be used for ssh:// repos - - A note on TROVE_ID: the current distbuild-setup service requires that - a single 'Trove ID' is specified. This is used in Morph for expanding - keyed URLs. If you set TROVE_ID=foo for example, foo:bar will be expanded - to git://$GIT_SERVER/foo, in addition to the standard baserock: and - upstream: prefixes that you can use. - - The WORKER_SSH_KEY must be provided, even if you don't need it. The - distbuild-setup service could be changed to make it optional. - - The following configuration values are optional: - - - HOST_MAP: a list of key=value pairs mapping hostnames to IP addresses, - or fully-qualified domain names. Useful if you - cannot rely on hostname resolution working for your deploment. - - The extension will connect to root@location via ssh to copy the kernel and - rootfs, and configure the nfs server. It will duplicate the kernel and - rootfs once for each node in the distbuild network. - - The deployment mechanism makes assumptions about the bootloader - configuration of the target machines. diff --git a/distbuild-trove-nfsboot.write.help b/distbuild-trove-nfsboot.write.help new file mode 100644 index 00000000..62f1455c --- /dev/null +++ b/distbuild-trove-nfsboot.write.help @@ -0,0 +1,49 @@ +# Copyright (C) 2014, 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see . + +help: | + Deploy a distbuild network, using a Trove to serve the kernel and rootfs. + + The `location` argument is the hostname of the Trove system. + + The following configuration values must be specified: + + - DISTBUILD_CONTROLLER: hostname of controller system + - DISTBUILD_WORKERS: hostnames of each worker system + - DISTBUILD_GIT_SERVER: Trove hostname + - DISTBUILD_SHARED_ARTIFACT_CACHE: Trove hostname + - DISTBUILD_TROVE_ID: Trove ID + - DISTBUILD_WORKER_SSH_KEY: SSH key to be used for ssh:// repos + + A note on TROVE_ID: the current distbuild-setup service requires that + a single 'Trove ID' is specified. This is used in Morph for expanding + keyed URLs. If you set TROVE_ID=foo for example, foo:bar will be expanded + to git://$GIT_SERVER/foo, in addition to the standard baserock: and + upstream: prefixes that you can use. + + The WORKER_SSH_KEY must be provided, even if you don't need it. The + distbuild-setup service could be changed to make it optional. + + The following configuration values are optional: + + - HOST_MAP: a list of key=value pairs mapping hostnames to IP addresses, + or fully-qualified domain names. Useful if you + cannot rely on hostname resolution working for your deploment. + + The extension will connect to root@location via ssh to copy the kernel and + rootfs, and configure the nfs server. It will duplicate the kernel and + rootfs once for each node in the distbuild network. + + The deployment mechanism makes assumptions about the bootloader + configuration of the target machines. -- cgit v1.2.1 From 8a56b40116e96905c1cb8d84c231f858a3ff8388 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Mon, 13 Apr 2015 18:29:36 +0100 Subject: Add install-essential-files configuration extension This is meant to add essential system files like /etc/profile, /etc/os-release ... Change-Id: I3d67b3a452b32205c5d3c7303d128bda80ce75de --- install-essential-files.configure | 42 ++++++++++++++++++++++++++++++++++ install-essential-files.configure.help | 20 ++++++++++++++++ 2 files changed, 62 insertions(+) create mode 100755 install-essential-files.configure create mode 100644 install-essential-files.configure.help diff --git a/install-essential-files.configure b/install-essential-files.configure new file mode 100755 index 00000000..2779b0d4 --- /dev/null +++ b/install-essential-files.configure @@ -0,0 +1,42 @@ +#!/usr/bin/env python2 +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . + + +''' A Morph configuration extension for adding essential files to a system + +It will read the manifest files located in essential-files/manifest, +then use the contens of those files to determine which files +to install into the target system. + +''' + +import subprocess +import os + +import cliapp + +class InstallEssentialFilesConfigureExtension(cliapp.Application): + + def process_args(self, args): + target_root = args[0] + os.environ["INSTALL_FILES"] = "essential-files/manifest" + self.install_essential_files(target_root) + + def install_essential_files(self, target_root): + command = os.path.join(os.path.dirname(__file__), + "install-files.configure") + subprocess.check_call([command, target_root]) + +InstallEssentialFilesConfigureExtension().run() diff --git a/install-essential-files.configure.help b/install-essential-files.configure.help new file mode 100644 index 00000000..1d123839 --- /dev/null +++ b/install-essential-files.configure.help @@ -0,0 +1,20 @@ +# Copyright (C) 2015 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, see . + +help: | + This installs files from the essential-files/ folder in your definitions.git + repo, according to essential-files/manifest. + + It wraps the install-files.configure extension. Take a look to that + extension help to know more about the format of the manifest file. -- cgit v1.2.1 From 7c6ab30c70e141533de6af3257515002259757e1 Mon Sep 17 00:00:00 2001 From: Francisco Redondo Marchena Date: Tue, 28 Apr 2015 18:37:53 +0000 Subject: Fix lines bigger than 79 characters in install-essential-files.configure.help This fixes morph check. Change-Id: I5f9c8f5e73c1976b623f9a75f97b5e9aa1887c44 --- install-essential-files.configure.help | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/install-essential-files.configure.help b/install-essential-files.configure.help index 1d123839..9148aeff 100644 --- a/install-essential-files.configure.help +++ b/install-essential-files.configure.help @@ -13,8 +13,8 @@ # with this program; if not, see . help: | - This installs files from the essential-files/ folder in your definitions.git - repo, according to essential-files/manifest. + This installs files from the essential-files/ folder in your + definitions.git repo, according to essential-files/manifest. It wraps the install-files.configure extension. Take a look to that extension help to know more about the format of the manifest file. -- cgit v1.2.1 From 7e475f6dc71ae5fb453d91db8cefbf731c9123d2 Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Wed, 28 May 2014 14:37:28 +0100 Subject: Add ssh keys conf ext Change-Id: I4e7888cbff2e4708154538f8f0a48aeaa1a8a811 --- sshkeys.configure | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100755 sshkeys.configure diff --git a/sshkeys.configure b/sshkeys.configure new file mode 100755 index 00000000..7a5a8379 --- /dev/null +++ b/sshkeys.configure @@ -0,0 +1,25 @@ +#!/bin/sh +# +# Copyright 2014 Codethink Ltd +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +set -e + +if [ "$SSHKEYS" ] +then + install -d -m 700 "$1/root/.ssh" + echo Adding Key in "$SSHKEYS" to authorized_keys file + cat $SSHKEYS >> "$1/root/.ssh/authorized_keys" +fi -- cgit v1.2.1 From 703edae4dc4a284c5710ffa41ed79bd095af8b43 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Wed, 29 Apr 2015 22:52:43 +0100 Subject: install-files.configure: make possible to overwrite symlinks os.symlink will fail if the origin file/link already exist Change-Id: I8175c8dce699e55c3e39e35dfd45c0c19b8bd96d --- install-files.configure | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/install-files.configure b/install-files.configure index c2970243..341cce61 100755 --- a/install-files.configure +++ b/install-files.configure @@ -24,6 +24,7 @@ to install into the target system. import cliapp import os +import errno import re import sys import shlex @@ -53,6 +54,14 @@ class InstallFilesConfigureExtension(cliapp.Application): for entry in entries: self.install_entry(entry, manifest_dir, target_root) + def force_symlink(self, source, link_name): + try: + os.symlink(source, link_name) + except OSError as e: + if e.errno == errno.EEXIST: + os.remove(link_name) + os.symlink(source, link_name) + def install_entry(self, entry, manifest_root, target_root): m = re.match('(template )?(overwrite )?' '([0-7]+) ([0-9]+) ([0-9]+) (\S+)', entry) @@ -91,7 +100,7 @@ class InstallFilesConfigureExtension(cliapp.Application): else: linkdest = os.readlink(os.path.join(manifest_root, './' + path)) - os.symlink(linkdest, dest_path) + self.force_symlink(linkdest, dest_path) os.lchown(dest_path, uid, gid) elif stat.S_ISREG(mode): -- cgit v1.2.1 From 7c360ce448114dd626661e688e0aad3b3754f302 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Wed, 29 Apr 2015 15:47:28 +0000 Subject: Fix sysroot.write trying to overwrite existing files Commit 807e6a90876c5469d242 changed the behaviour of sysroot.write to avoid deleting the contents of the sysroot. This was done so if you accidentally set 'sysroot=/' it wouldn't delete your whole system. It turns out that SDK deployments like clusters/sdk-example-cluster.morph depended on the contents of the directory being deleted. The system armv7lhf-cross-toolchain-system-x86_64.morph has a bunch of files installed by the cross-toolchain in /usr/armv7lhf-baserock-linux-gnueabi/sys-root. Previously sysroot.write would delete these, but since commit 807e6a90876c5469d242 it would fail with several errors like: mv: can't rename '/src/tmp/deployments/usr/armv7l.../sys-root/sbin' If we use 'cp -a' instead of 'mv' then it is slower to deploy, but there are no errors. I am still unsure why files from the cross-toolchain system are installed and then deleted. Although this patch fixes the immediate issue, I don't know if it's the right thing to do. It seems better to not install those files in the first place, if we do not need them. This commit also removes the check for the sysroot target location being empty. This doesn't work, because it runs /before/ the system being deployed is unpacked. Change-Id: I10671c2f3b2060cfb36f880675b83351c6cdd807 --- sysroot.check | 6 ------ sysroot.write | 6 +----- 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/sysroot.check b/sysroot.check index 8ed965bd..71b35175 100755 --- a/sysroot.check +++ b/sysroot.check @@ -17,12 +17,6 @@ set -eu -location="$1" -if [ -d "$location" ]; then - echo >&2 "ERROR: Deployment directory already exists: $location" - exit 1 -fi - if [ "$UPGRADE" == "yes" ]; then echo >&2 "ERROR: Cannot upgrade a sysroot deployment" exit 1 diff --git a/sysroot.write b/sysroot.write index 0ad8d630..019edbe9 100755 --- a/sysroot.write +++ b/sysroot.write @@ -19,8 +19,4 @@ set -eu mkdir -p "$2" -# Move the contents of our source directory to our target -# Previously we would (cd "$1" && find -print0 | cpio -0pumd "$absolute_path") -# to do this, but the source directory is disposable anyway, so we can move -# its contents to save time -find "$1" -maxdepth 1 -mindepth 1 -exec mv {} "$2/." + +cp -a "$1/*" "$2" -- cgit v1.2.1 From 069bab3f4673b1aba33d6573576f72234a8209cd Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Fri, 8 May 2015 11:32:37 +0000 Subject: Fix mistake in sysroot.write The * should not be in quotes. Change-Id: Ieebdc7532ba1bff5ba9742f72440ed00b0c0de2a --- sysroot.write | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sysroot.write b/sysroot.write index 019edbe9..46f1a780 100755 --- a/sysroot.write +++ b/sysroot.write @@ -19,4 +19,4 @@ set -eu mkdir -p "$2" -cp -a "$1/*" "$2" +cp -a "$1"/* "$2" -- cgit v1.2.1 From 30cba5d9a8757f6bafc8079377aa3d6705e8364c Mon Sep 17 00:00:00 2001 From: Richard Ipsum Date: Thu, 21 May 2015 14:56:25 +0100 Subject: Use keystoneclient python api in openstack.check Switching to the keystoneclient python api gives us a more reliable means of detecting auth failure. Change-Id: I5f734bbfe5568c855f524a3448357f7cf46ab254 --- openstack.check | 57 ++++++++++++++++++++++++++++----------------------------- 1 file changed, 28 insertions(+), 29 deletions(-) diff --git a/openstack.check b/openstack.check index 4c21b604..a3379763 100755 --- a/openstack.check +++ b/openstack.check @@ -18,11 +18,13 @@ import cliapp import os import urlparse +import keystoneclient import morphlib.writeexts class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): + def process_args(self, args): if len(args) != 1: raise cliapp.AppException('Wrong number of command line args') @@ -38,23 +40,30 @@ class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): location = args[0] self.check_location(location) - os_params = self.get_openstack_parameters() - - self.check_openstack_parameters(location, os_params) + self.check_imagename() + self.check_openstack_parameters(self._get_auth_parameters(location)) - def get_openstack_parameters(self): + def _get_auth_parameters(self, location): '''Check the environment variables needed and returns all. The environment variables are described in the class documentation. ''' - keys = ('OPENSTACK_USER', 'OPENSTACK_TENANT', - 'OPENSTACK_IMAGENAME', 'OPENSTACK_PASSWORD') - for key in keys: - if key not in os.environ: + auth_keys = {'OPENSTACK_USER': 'username', + 'OPENSTACK_TENANT': 'tenant_name', + 'OPENSTACK_PASSWORD': 'password'} + + for key in auth_keys: + if os.environ.get(key, '') == '': raise cliapp.AppException(key + ' was not given') - return (os.environ[key] for key in keys) + auth_params = {auth_keys[key]: os.environ[key] for key in auth_keys} + auth_params['auth_url'] = location + return auth_params + + def check_imagename(self): + if os.environ.get('OPENSTACK_IMAGENAME', '') == '': + raise cliapp.AppException('OPENSTACK_IMAGENAME was not given') def check_location(self, location): x = urlparse.urlparse(location) @@ -65,27 +74,17 @@ class OpenStackCheckExtension(morphlib.writeexts.WriteExtension): raise cliapp.AppException('API version must be v2.0 in %s'\ % location) - def check_openstack_parameters(self, auth_url, os_params): - '''Check OpenStack credentials using glance image-list''' + def check_openstack_parameters(self, auth_params): + ''' Check that we can connect to and authenticate with openstack ''' + self.status(msg='Checking OpenStack credentials...') - username, tenant_name, image_name, password = os_params - cmdline = ['glance', - '--os-username', username, - '--os-tenant-name', tenant_name, - '--os-password', password, - '--os-auth-url', auth_url, - 'image-list'] - - exit, out, err = cliapp.runcmd_unchecked(cmdline) - - if exit != 0: - if err.startswith('The request you have made requires ' - 'authentication. (HTTP 401)'): - raise cliapp.AppException('Invalid OpenStack credentials.') - else: - raise cliapp.AppException( - 'Failed to connect to OpenStack instance at %s: %s' % - (auth_url, err)) + try: + keystoneclient.v2_0.Client(**auth_params) + except keystoneclient.exceptions.Unauthorized: + errmsg = ('Failed to authenticate with OpenStack ' + '(are your credentials correct?)') + raise cliapp.AppException(errmsg) + OpenStackCheckExtension().run() -- cgit v1.2.1