From 2d40cac81a029f29c8e16593112a8c18e945f83b Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 24 Jan 2013 15:08:18 +0000 Subject: Add a write extension for VirtualBox --- virtualbox-ssh.write | 158 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 158 insertions(+) create mode 100755 virtualbox-ssh.write (limited to 'virtualbox-ssh.write') diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write new file mode 100755 index 00000000..e2f592e7 --- /dev/null +++ b/virtualbox-ssh.write @@ -0,0 +1,158 @@ +#!/usr/bin/python +# Copyright (C) 2012-2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +'''A Morph deployment write extension for deploying to VirtualBox via ssh. + +VirtualBox is assumed to be running on a remote machine, which is +accessed over ssh. The machine gets created, but not started. + +''' + + +import os +import re +import time +import tempfile +import urlparse + +import morphlib.writeexts + + +class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): + + '''Create a VirtualBox virtual machine during Morph's deployment. + + The location command line argument is the pathname of the disk image + to be created. The user is expected to provide the location argument + using the following syntax: + + vbox+ssh://HOST/GUEST/PATH + + where: + + * HOST is the host on which VirtualBox is running + * GUEST is the name of the guest virtual machine on that host + * PATH is the path to the disk image that should be created, + on that host + + The extension will connect to HOST via ssh to run VirtualBox's + command line management tools. + + ''' + + def process_args(self, args): + if len(args) != 2: + raise cliapp.AppException('Wrong number of command line args') + + temp_root, location = args + ssh_host, vm_name, vdi_path = self.parse_location(location) + + self.status( + msg='Temporary system root: %(temp_root)s', + temp_root=temp_root) + self.status( + msg='VirtualBox server: %(ssh_host)s', + ssh_host=ssh_host) + self.status( + msg='VirtualBox guest: %(vm_name)s', + vm_name=vm_name) + + size = self.get_disk_size() + self.status(msg='Disk size is %(size)d bytes', size=size) + + raw_disk = tempfile.mkstemp() + self.create_raw_disk_image(raw_disk, size) + try: + self.mkfs_btrfs(raw_disk) + mp = self.mount(raw_disk) + except BaseException: + self.status('EEEK') + os.remove(raw_disk) + raise + try: + self.create_factory(mp, temp_root) + self.create_fstab(mp) + self.install_extlinux(mp) + except BaseException, e: + self.status(msg='EEK') + self.unmount(mp) + os.remove(raw_disk) + raise + else: + self.unmount(mp) + + try: + self.transfer_and_convert_to_vdi( + raw_disk, size, ssh_host, vdi_path) + self.create_virtualbox_guest(ssh_host, vm_name, vdi_path) + except BaseException: + self.status('EEEK') + os.remove(raw_disk) + raise + else: + os.remove(raw_disk) + + self.status( + msg='Virtual machine %(vm_name)s has been created', + vm_name=vm_name) + + def parse_location(self, location): + '''Parse the location argument to get relevant data.''' + + x = urlparse.urlparse(location) + if x.scheme != 'vbox+ssh': + raise cliapp.AppException( + 'URL schema must be vbox+ssh in %s' % location) + m = re.match('^/(?P[^/]+)(?P/.+)$', x.path) + if not m: + raise cliapp.AppException('Cannot parse location %s' % location) + return x.netloc, m.group('guest'), m.group('path') + + def transfer_and_convert_to_vdi(self, raw_disk, size, ssh_host, vdi_path): + '''Transfer raw disk image to VirtualBox host, and convert to VDI.''' + + self.status(msg='Transfer disk and convert to VDI') + with open(raw_disk, 'rb') as f: + cliapp.runcmd( + ['ssh', ssh_host, + 'VBoxManage', 'convertfromraw', 'stdin', vdi_path, str(size)], + stdin=f) + + def create_virtualbox_guest(self, ssh_host, vm_name, vdi_path): + '''Create the VirtualBox virtual machine.''' + + self.status(msg='Create VirtualBox virtual machine') + + commands = [ + ['createvm', '--name', vm_name, '--ostype', 'Linux26_64', + '--register'], + ['modifyvm', vm_name, '--ioapic', 'on', '--memory', '1024', + '--nic1', 'nat'], + ['storagectl', vm_name, '--name', '"SATA Controller"', + '--add', 'sata', '--bootable', 'on', '--sataportcount', '2'], + ['storageattach', vm_name, '--storagectl', '"SATA Controller"', + '--port', '0', '--device', '0', '--type', 'hdd', '--medium', + vdi_path], + ] + + for command in commands: + argv = ['ssh', ssh_host, 'VBoxManage'] + command + cliapp.runcmd(argv) + + +VirtualBoxPlusSshWriteExtension().run() + -- cgit v1.2.1 From 769c0ae2808a3c3e383c470795510b72c45c5a76 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 7 Feb 2013 11:51:06 +0000 Subject: Remove debugging output Suggested-By: Richard Maw --- virtualbox-ssh.write | 10 ---------- 1 file changed, 10 deletions(-) (limited to 'virtualbox-ssh.write') diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index e2f592e7..dbfb90a7 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -61,16 +61,6 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): temp_root, location = args ssh_host, vm_name, vdi_path = self.parse_location(location) - self.status( - msg='Temporary system root: %(temp_root)s', - temp_root=temp_root) - self.status( - msg='VirtualBox server: %(ssh_host)s', - ssh_host=ssh_host) - self.status( - msg='VirtualBox guest: %(vm_name)s', - vm_name=vm_name) - size = self.get_disk_size() self.status(msg='Disk size is %(size)d bytes', size=size) -- cgit v1.2.1 From 099f2ef05cd52dd623b47901ecc361754aa729f7 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 7 Feb 2013 11:41:04 +0000 Subject: Refactor: Add WriteExtension.create_local_system method This allows code sharing amongst all the places that create a system in a raw disk image. This also adds the creation of a factory-run subvolume, and fixes error messages for errors that happen during a disk image creation. Suggested-By: Richard Maw Suggested-By: Sam Thursfield --- virtualbox-ssh.write | 29 +++++------------------------ 1 file changed, 5 insertions(+), 24 deletions(-) (limited to 'virtualbox-ssh.write') diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index dbfb90a7..5d118ec4 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -23,6 +23,7 @@ accessed over ssh. The machine gets created, but not started. ''' +import cliapp import os import re import time @@ -61,36 +62,16 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): temp_root, location = args ssh_host, vm_name, vdi_path = self.parse_location(location) - size = self.get_disk_size() - self.status(msg='Disk size is %(size)d bytes', size=size) - - raw_disk = tempfile.mkstemp() - self.create_raw_disk_image(raw_disk, size) - try: - self.mkfs_btrfs(raw_disk) - mp = self.mount(raw_disk) - except BaseException: - self.status('EEEK') - os.remove(raw_disk) - raise - try: - self.create_factory(mp, temp_root) - self.create_fstab(mp) - self.install_extlinux(mp) - except BaseException, e: - self.status(msg='EEK') - self.unmount(mp) - os.remove(raw_disk) - raise - else: - self.unmount(mp) + fd, raw_disk = tempfile.mkstemp() + os.close(fd) + self.create_local_system(temp_root) try: self.transfer_and_convert_to_vdi( raw_disk, size, ssh_host, vdi_path) self.create_virtualbox_guest(ssh_host, vm_name, vdi_path) except BaseException: - self.status('EEEK') + sys.stderr.write('Error deploying to VirtualBox') os.remove(raw_disk) raise else: -- cgit v1.2.1 From 311a41778532dc5d102f5c8e2cbe0d3dd58cadf1 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 7 Feb 2013 15:17:56 +0000 Subject: Add missing argument to create_local_system method Reported-By: Richard Maw --- virtualbox-ssh.write | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'virtualbox-ssh.write') diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 5d118ec4..c21dcc57 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -64,7 +64,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): fd, raw_disk = tempfile.mkstemp() os.close(fd) - self.create_local_system(temp_root) + self.create_local_system(temp_root, raw_disk) try: self.transfer_and_convert_to_vdi( -- cgit v1.2.1 From c57547b87cdad636a28a3f378104c4279b03b45f Mon Sep 17 00:00:00 2001 From: Ric Holland Date: Tue, 5 Mar 2013 10:32:03 +0000 Subject: Fix virtualbox deployment It was missing an import for sys. It did not have the size of the disk either, this has also been fixed --- virtualbox-ssh.write | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'virtualbox-ssh.write') diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index c21dcc57..9b99c7a1 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -26,6 +26,7 @@ accessed over ssh. The machine gets created, but not started. import cliapp import os import re +import sys import time import tempfile import urlparse @@ -68,7 +69,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): try: self.transfer_and_convert_to_vdi( - raw_disk, size, ssh_host, vdi_path) + raw_disk, ssh_host, vdi_path) self.create_virtualbox_guest(ssh_host, vm_name, vdi_path) except BaseException: sys.stderr.write('Error deploying to VirtualBox') @@ -93,14 +94,15 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): raise cliapp.AppException('Cannot parse location %s' % location) return x.netloc, m.group('guest'), m.group('path') - def transfer_and_convert_to_vdi(self, raw_disk, size, ssh_host, vdi_path): + def transfer_and_convert_to_vdi(self, raw_disk, ssh_host, vdi_path): '''Transfer raw disk image to VirtualBox host, and convert to VDI.''' self.status(msg='Transfer disk and convert to VDI') with open(raw_disk, 'rb') as f: cliapp.runcmd( ['ssh', ssh_host, - 'VBoxManage', 'convertfromraw', 'stdin', vdi_path, str(size)], + 'VBoxManage', 'convertfromraw', 'stdin', vdi_path, + str(os.path.getsize(raw_disk))], stdin=f) def create_virtualbox_guest(self, ssh_host, vm_name, vdi_path): -- cgit v1.2.1 From 717e3e91b3a606cca14f1a4c1d44a28cbbbe3c9e Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 13 Mar 2013 15:11:36 +0000 Subject: Implement ATTACH_DISKS in virtualbox-ssh.write --- virtualbox-ssh.write | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) (limited to 'virtualbox-ssh.write') diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 9b99c7a1..862d4f02 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -121,7 +121,17 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): '--port', '0', '--device', '0', '--type', 'hdd', '--medium', vdi_path], ] - + + attach_disks = self.parse_attach_disks() + for device_no, disk in enumerate(attach_disks, 1): + cmd = ['storageattach', vm_name, + '--storagectl', '"SATA Controller"', + '--port', str(device_no), + '--device', '0', + '--type', 'hdd', + '--medium', disk] + commands.append(cmd) + for command in commands: argv = ['ssh', ssh_host, 'VBoxManage'] + command cliapp.runcmd(argv) -- cgit v1.2.1 From 02a5188ba2a89af403919f8635e146de003dd05f Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 28 Mar 2013 13:44:15 +0000 Subject: Set RAM size in VirtualBox deployments --- virtualbox-ssh.write | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) (limited to 'virtualbox-ssh.write') diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 862d4f02..37f56524 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -110,10 +110,12 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Create VirtualBox virtual machine') + ram_mebibytes = str(self.get_ram_size() / (1024**2)) + commands = [ ['createvm', '--name', vm_name, '--ostype', 'Linux26_64', '--register'], - ['modifyvm', vm_name, '--ioapic', 'on', '--memory', '1024', + ['modifyvm', vm_name, '--ioapic', 'on', '--memory', ram_mebibytes, '--nic1', 'nat'], ['storagectl', vm_name, '--name', '"SATA Controller"', '--add', 'sata', '--bootable', 'on', '--sataportcount', '2'], -- cgit v1.2.1 From 937544ac7a370f433eda4240148a33c64ff43c56 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 8 May 2013 17:04:45 +0100 Subject: Add AUTOSTART to kvm and libvirt write extensions If AUTOSTART is 'yes' then the VM will be started once it is created. If it is 'no' or undefined, then it will need to be manually started. If it is any other value, then an exception is raised. --- virtualbox-ssh.write | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'virtualbox-ssh.write') diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 37f56524..cb17b69b 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -62,6 +62,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): temp_root, location = args ssh_host, vm_name, vdi_path = self.parse_location(location) + autostart = self.parse_autostart() fd, raw_disk = tempfile.mkstemp() os.close(fd) @@ -70,7 +71,8 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): try: self.transfer_and_convert_to_vdi( raw_disk, ssh_host, vdi_path) - self.create_virtualbox_guest(ssh_host, vm_name, vdi_path) + self.create_virtualbox_guest(ssh_host, vm_name, vdi_path, + autostart) except BaseException: sys.stderr.write('Error deploying to VirtualBox') os.remove(raw_disk) @@ -105,7 +107,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): str(os.path.getsize(raw_disk))], stdin=f) - def create_virtualbox_guest(self, ssh_host, vm_name, vdi_path): + def create_virtualbox_guest(self, ssh_host, vm_name, vdi_path, autostart): '''Create the VirtualBox virtual machine.''' self.status(msg='Create VirtualBox virtual machine') @@ -134,6 +136,9 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): '--medium', disk] commands.append(cmd) + if autostart: + commands.append(['startvm', vm_name]) + for command in commands: argv = ['ssh', ssh_host, 'VBoxManage'] + command cliapp.runcmd(argv) -- cgit v1.2.1 From c3c678ccd9fc2cc898b3d6d14cc15711fd992b9e Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Wed, 29 May 2013 16:35:36 +0000 Subject: Improve network setup of the virtualbox write extension. The VirtualBox deployment write extension will configure networking in the following manner: - One host-only network interface will be used to group the virtual machines together (and to give access between the host and the virtual machines). It will be bound to eth0 of the virtual machine. - One NAT (or Bridge) network interface will be used to allow the virtual machines access to the wider network. This will be bound to eth1 of the virtual machine. Now deployment to virtual box will require that both HOST_IPADDR and NETMASK environment variables be set, and also configuration for eth0 and eth1 is expected in the NETWORK_CONFIG environment variable. This commit also replaces some run_cmd with ssh_runcmd. --- virtualbox-ssh.write | 72 +++++++++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 65 insertions(+), 7 deletions(-) (limited to 'virtualbox-ssh.write') diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index cb17b69b..3ee2eae0 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -101,24 +101,26 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Transfer disk and convert to VDI') with open(raw_disk, 'rb') as f: - cliapp.runcmd( - ['ssh', ssh_host, - 'VBoxManage', 'convertfromraw', 'stdin', vdi_path, + cliapp.ssh_runcmd(ssh_host, + ['VBoxManage', 'convertfromraw', 'stdin', vdi_path, str(os.path.getsize(raw_disk))], stdin=f) def create_virtualbox_guest(self, ssh_host, vm_name, vdi_path, autostart): '''Create the VirtualBox virtual machine.''' - + self.status(msg='Create VirtualBox virtual machine') ram_mebibytes = str(self.get_ram_size() / (1024**2)) + hostonly_iface = self.get_host_interface(ssh_host) + commands = [ ['createvm', '--name', vm_name, '--ostype', 'Linux26_64', '--register'], ['modifyvm', vm_name, '--ioapic', 'on', '--memory', ram_mebibytes, - '--nic1', 'nat'], + '--nic1', 'hostonly', '--hostonlyadapter1', hostonly_iface, + '--nic2', 'nat', '--natnet2', 'default'], ['storagectl', vm_name, '--name', '"SATA Controller"', '--add', 'sata', '--bootable', 'on', '--sataportcount', '2'], ['storageattach', vm_name, '--storagectl', '"SATA Controller"', @@ -140,9 +142,65 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): commands.append(['startvm', vm_name]) for command in commands: - argv = ['ssh', ssh_host, 'VBoxManage'] + command - cliapp.runcmd(argv) + argv = ['VBoxManage'] + command + cliapp.ssh_runcmd(ssh_host, argv) + + def get_host_interface(self, ssh_host): + host_ipaddr = os.environ.get('HOST_IPADDR') + netmask = os.environ.get('NETMASK') + network_config = os.environ.get("NETWORK_CONFIG") + if network_config is None: + raise cliapp.AppException('NETWORK_CONFIG was not given') + + if "eth0:" not in network_config: + raise cliapp.AppException( + 'NETWORK_CONFIG does not contain ' + 'the eth0 configuration') + + if "eth1:" not in network_config: + raise cliapp.AppException( + 'NETWORK_CONFIG does not contain ' + 'the eth1 configuration') + + if host_ipaddr is None: + raise cliapp.AppException('HOST_IPADDR was not given') + + if netmask is None: + raise cliapp.AppException('NETMASK was not given') + + # 'VBoxManage list hostonlyifs' retrieves a list with the hostonly + # interfaces on the host. For each interface, the following lines + # are shown on top: + # + # Name: vboxnet0 + # GUID: 786f6276-656e-4074-8000-0a0027000000 + # Dhcp: Disabled + # IPAddress: 192.168.100.1 + # + # The following command tries to retrieve the hostonly interface + # name (e.g. vboxnet0) associated with the given ip address. + iface = None + lines = cliapp.ssh_runcmd(ssh_host, + ['VBoxManage', 'list', 'hostonlyifs']).splitlines() + for i, v in enumerate(lines): + if host_ipaddr in v: + iface = lines[i-3].split()[1] + break + + if iface is None: + iface = cliapp.ssh_runcmd(ssh_host, + ['VBoxManage', 'hostonlyif', 'create']) + # 'VBoxManage hostonlyif create' shows the name of the + # created hostonly interface inside single quotes + iface = iface[iface.find("'") + 1 : iface.rfind("'")] + cliapp.ssh_runcmd(ssh_host, + ['VBoxManage', 'hostonlyif', + 'ipconfig', iface, + '--ip', host_ipaddr, + '--netmask', netmask]) + + return iface VirtualBoxPlusSshWriteExtension().run() -- cgit v1.2.1 From 7f10cd9a320664609f83dc24f6d25e79b49cb7d2 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Tue, 2 Jul 2013 07:24:30 +0000 Subject: Allow to set the number of cpus for virtualbox and kvm deployments. --- virtualbox-ssh.write | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'virtualbox-ssh.write') diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 3ee2eae0..1abe233e 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -113,12 +113,15 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): ram_mebibytes = str(self.get_ram_size() / (1024**2)) + vcpu_count = str(self.get_vcpu_count()) + hostonly_iface = self.get_host_interface(ssh_host) commands = [ ['createvm', '--name', vm_name, '--ostype', 'Linux26_64', '--register'], - ['modifyvm', vm_name, '--ioapic', 'on', '--memory', ram_mebibytes, + ['modifyvm', vm_name, '--ioapic', 'on', + '--memory', ram_mebibytes, '--cpus', vcpu_count, '--nic1', 'hostonly', '--hostonlyadapter1', hostonly_iface, '--nic2', 'nat', '--natnet2', 'default'], ['storagectl', vm_name, '--name', '"SATA Controller"', -- cgit v1.2.1 From f0800ade5348fd632d7c49307e55e32746d073a7 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Mon, 19 Aug 2013 10:23:31 +0000 Subject: Fix naming of SATA Controller in VirtualBox deployments Patch from Paul Sherwood. --- virtualbox-ssh.write | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'virtualbox-ssh.write') diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 1abe233e..2374db31 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -124,9 +124,9 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): '--memory', ram_mebibytes, '--cpus', vcpu_count, '--nic1', 'hostonly', '--hostonlyadapter1', hostonly_iface, '--nic2', 'nat', '--natnet2', 'default'], - ['storagectl', vm_name, '--name', '"SATA Controller"', + ['storagectl', vm_name, '--name', 'SATA Controller', '--add', 'sata', '--bootable', 'on', '--sataportcount', '2'], - ['storageattach', vm_name, '--storagectl', '"SATA Controller"', + ['storageattach', vm_name, '--storagectl', 'SATA Controller', '--port', '0', '--device', '0', '--type', 'hdd', '--medium', vdi_path], ] @@ -134,7 +134,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): attach_disks = self.parse_attach_disks() for device_no, disk in enumerate(attach_disks, 1): cmd = ['storageattach', vm_name, - '--storagectl', '"SATA Controller"', + '--storagectl', 'SATA Controller', '--port', str(device_no), '--device', '0', '--type', 'hdd', -- cgit v1.2.1 From 1e57089d8aa73ffc49ef80622206d43e1bd15bf0 Mon Sep 17 00:00:00 2001 From: Dan Firth Date: Thu, 10 Oct 2013 16:07:37 +0000 Subject: Deployment failures will now remove the disk image --- virtualbox-ssh.write | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'virtualbox-ssh.write') diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 2374db31..f18ef804 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -76,10 +76,10 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): except BaseException: sys.stderr.write('Error deploying to VirtualBox') os.remove(raw_disk) + cliapp.ssh_runcmd(ssh_host, ['rm', '-f', vdi_path]) raise else: os.remove(raw_disk) - self.status( msg='Virtual machine %(vm_name)s has been created', vm_name=vm_name) -- cgit v1.2.1 From 8fd0a807b70b19ac5a59da4e05a3abe5652ef34a Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 22 Jan 2014 17:42:18 +0000 Subject: virtualbox-ssh: Work around change in VBox options VirtualBox changed a command line option in 4.3 incompatibly, so we now have to check the version number and change an option from --sataportcount to --portcount if the version of VirtualBox running on the target is at least 4.3 This turns the version into a tuple and compares it against another, since it's more reliable than comparing strings, which will count '1.10' as earlier than '1.2', and more convenient than comparing the digits individually. --- virtualbox-ssh.write | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) (limited to 'virtualbox-ssh.write') diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index f18ef804..369c0d61 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -106,6 +106,19 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): str(os.path.getsize(raw_disk))], stdin=f) + def virtualbox_version(self, ssh_host): + 'Get the version number of the VirtualBox running on the remote host.' + + # --version gives a build id, which looks something like + # 1.2.3r456789, so we need to strip the suffix off and get a tuple + # of the (major, minor, patch) version, since comparing with a + # tuple is more reliable than a string and more convenient than + # comparing against the major, minor and patch numbers directly + self.status(msg='Checking version of remote VirtualBox') + build_id = cliapp.ssh_runcmd(ssh_host, ['VBoxManage', '--version']) + version_string = re.match(r"^([0-9\.])+.*$", build_id.strip()).group(1) + return tuple(int(s or '0') for s in version_string.split('.')) + def create_virtualbox_guest(self, ssh_host, vm_name, vdi_path, autostart): '''Create the VirtualBox virtual machine.''' @@ -117,6 +130,11 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): hostonly_iface = self.get_host_interface(ssh_host) + if self.virtualbox_version(ssh_host) < (4, 3): + sataportcount_option = '--sataportcount' + else: + sataportcount_option = '--portcount' + commands = [ ['createvm', '--name', vm_name, '--ostype', 'Linux26_64', '--register'], @@ -125,7 +143,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): '--nic1', 'hostonly', '--hostonlyadapter1', hostonly_iface, '--nic2', 'nat', '--natnet2', 'default'], ['storagectl', vm_name, '--name', 'SATA Controller', - '--add', 'sata', '--bootable', 'on', '--sataportcount', '2'], + '--add', 'sata', '--bootable', 'on', sataportcount_option, '2'], ['storageattach', vm_name, '--storagectl', 'SATA Controller', '--port', '0', '--device', '0', '--type', 'hdd', '--medium', vdi_path], -- cgit v1.2.1 From fc7b833170c66acb4672184146d560c6fda20183 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Tue, 28 Jan 2014 18:17:09 +0000 Subject: Fix copyright year in previous commit --- virtualbox-ssh.write | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'virtualbox-ssh.write') diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 369c0d61..204b2447 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -- cgit v1.2.1 From e8b7abe65fcab62ecb84b7f0c2c6f9dde4e63ea7 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Mon, 17 Feb 2014 15:36:56 +0000 Subject: Make parse_autostart() into more general get_environment_boolean() Also, be more flexible when parsing environment booleans -- convert to lower case and match 0/1 and true/false as well as yes/no. --- virtualbox-ssh.write | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'virtualbox-ssh.write') diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 204b2447..2a2f3f7b 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -62,7 +62,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): temp_root, location = args ssh_host, vm_name, vdi_path = self.parse_location(location) - autostart = self.parse_autostart() + autostart = self.get_environment_boolean('AUTOSTART') fd, raw_disk = tempfile.mkstemp() os.close(fd) -- cgit v1.2.1 From a0e42d6be4e73d58f3568264a399cf6a39fc3e42 Mon Sep 17 00:00:00 2001 From: Paul Sherwood Date: Sun, 20 Apr 2014 15:40:50 +0000 Subject: fix the Virtualbox version checking --- virtualbox-ssh.write | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'virtualbox-ssh.write') diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 2a2f3f7b..b9d53579 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -116,7 +116,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): # comparing against the major, minor and patch numbers directly self.status(msg='Checking version of remote VirtualBox') build_id = cliapp.ssh_runcmd(ssh_host, ['VBoxManage', '--version']) - version_string = re.match(r"^([0-9\.])+.*$", build_id.strip()).group(1) + version_string = re.match(r"^([0-9\.]+).*$", build_id.strip()).group(1) return tuple(int(s or '0') for s in version_string.split('.')) def create_virtualbox_guest(self, ssh_host, vm_name, vdi_path, autostart): @@ -130,7 +130,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): hostonly_iface = self.get_host_interface(ssh_host) - if self.virtualbox_version(ssh_host) < (4, 3): + if self.virtualbox_version(ssh_host) < (4, 3, 0): sataportcount_option = '--sataportcount' else: sataportcount_option = '--portcount' -- cgit v1.2.1 From 38c8a61410e4ba73d87d43a6101ecefb28433f13 Mon Sep 17 00:00:00 2001 From: Daniel Silverstone Date: Fri, 16 May 2014 09:39:37 +0000 Subject: VirtualBox Write Extension: Vagrant support Add support to the VirtualBox write extension to notice if we are doing a Vagrant Basebox installation and not do the clever network setup we normally do to allow machines to talk to one another since this confuses Vagrant quite a bit if it is left in. --- virtualbox-ssh.write | 22 +++++++++++++++------- 1 file changed, 15 insertions(+), 7 deletions(-) (limited to 'virtualbox-ssh.write') diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index b9d53579..47584b83 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -63,7 +63,9 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): temp_root, location = args ssh_host, vm_name, vdi_path = self.parse_location(location) autostart = self.get_environment_boolean('AUTOSTART') - + + vagrant = self.get_environment_boolean('VAGRANT') + fd, raw_disk = tempfile.mkstemp() os.close(fd) self.create_local_system(temp_root, raw_disk) @@ -72,7 +74,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): self.transfer_and_convert_to_vdi( raw_disk, ssh_host, vdi_path) self.create_virtualbox_guest(ssh_host, vm_name, vdi_path, - autostart) + autostart, vagrant) except BaseException: sys.stderr.write('Error deploying to VirtualBox') os.remove(raw_disk) @@ -119,7 +121,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): version_string = re.match(r"^([0-9\.]+).*$", build_id.strip()).group(1) return tuple(int(s or '0') for s in version_string.split('.')) - def create_virtualbox_guest(self, ssh_host, vm_name, vdi_path, autostart): + def create_virtualbox_guest(self, ssh_host, vm_name, vdi_path, autostart, vagrant): '''Create the VirtualBox virtual machine.''' self.status(msg='Create VirtualBox virtual machine') @@ -128,7 +130,8 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): vcpu_count = str(self.get_vcpu_count()) - hostonly_iface = self.get_host_interface(ssh_host) + if not vagrant: + hostonly_iface = self.get_host_interface(ssh_host) if self.virtualbox_version(ssh_host) < (4, 3, 0): sataportcount_option = '--sataportcount' @@ -139,15 +142,20 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): ['createvm', '--name', vm_name, '--ostype', 'Linux26_64', '--register'], ['modifyvm', vm_name, '--ioapic', 'on', - '--memory', ram_mebibytes, '--cpus', vcpu_count, - '--nic1', 'hostonly', '--hostonlyadapter1', hostonly_iface, - '--nic2', 'nat', '--natnet2', 'default'], + '--memory', ram_mebibytes, '--cpus', vcpu_count], ['storagectl', vm_name, '--name', 'SATA Controller', '--add', 'sata', '--bootable', 'on', sataportcount_option, '2'], ['storageattach', vm_name, '--storagectl', 'SATA Controller', '--port', '0', '--device', '0', '--type', 'hdd', '--medium', vdi_path], ] + if vagrant: + commands[1].extend(['--nic1', 'nat', + '--natnet1', 'default']) + else: + commands[1].extend(['--nic1', 'hostonly', + '--hostonlyadapter1', hostonly_iface, + '--nic2', 'nat', '--natnet2', 'default']) attach_disks = self.parse_attach_disks() for device_no, disk in enumerate(attach_disks, 1): -- cgit v1.2.1 From ddaf9840bbe3ad3a45fff341b1cfdf2985fd760b Mon Sep 17 00:00:00 2001 From: Daniel Silverstone Date: Fri, 16 May 2014 16:18:54 +0000 Subject: Fix long line --- virtualbox-ssh.write | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'virtualbox-ssh.write') diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 47584b83..42585f5e 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -121,7 +121,8 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): version_string = re.match(r"^([0-9\.]+).*$", build_id.strip()).group(1) return tuple(int(s or '0') for s in version_string.split('.')) - def create_virtualbox_guest(self, ssh_host, vm_name, vdi_path, autostart, vagrant): + def create_virtualbox_guest(self, ssh_host, vm_name, vdi_path, autostart, + vagrant): '''Create the VirtualBox virtual machine.''' self.status(msg='Create VirtualBox virtual machine') -- cgit v1.2.1 From 65ed235de623fd152dd2967b9ff2e1f60626c658 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 8 Aug 2014 13:17:01 +0000 Subject: Transfer sparse files faster for kvm, vbox deployment The KVM and VirtualBox deployments use sparse files for raw disk images. This means they can store a large disk (say, tens or hundreds of gigabytes) without using more disk space than is required for the actual content (e.g., a gigabyte or so for the files in the root filesystem). The kernel and filesystem make the unwritten parts of the disk image look as if they are filled with zero bytes. This is good. However, during deployment those sparse files get transferred as if there really are a lot of zeroes. Those zeroes take a lot of time to transfer. rsync, for example, does not handle large holes efficiently. This change introduces a couple of helper tools (morphlib/xfer-hole and morphlib/recv-hole), which transfer the holes more efficiently. The xfer-hole program reads a file and outputs records like these: DATA 123 binary data (exaclyt 123 bytes and no newline at the end) HOLE 3245 xfer-hole can do this efficiently, without having to read through all the zeroes in the holes, using the SEEK_DATA and SEEK_HOLE arguments to lseek. Using this, the holes take only take a few bytes each, making it possible to transfer a disk image faster. In my benchmarks, transferring a 100G byte disk image took about 100 seconds for KVM, and 220 seconds for VirtualBox (which needs to more work at the receiver to convert the raw disk to a VDI). Both benchmarks were from a VM on my laptop to the laptop itself. The interesting bit here is that the receiver (recv-hole) is simple enough that it can be implemented in a bit of shell script, and the text of the shell script can be run on the remote end by giving it to ssh as a command line argument. This means there is no need to install any special tools on the receiver, which makes using this improvement much simpler. --- virtualbox-ssh.write | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) (limited to 'virtualbox-ssh.write') diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 42585f5e..1aebe490 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -102,11 +102,17 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): '''Transfer raw disk image to VirtualBox host, and convert to VDI.''' self.status(msg='Transfer disk and convert to VDI') - with open(raw_disk, 'rb') as f: - cliapp.ssh_runcmd(ssh_host, - ['VBoxManage', 'convertfromraw', 'stdin', vdi_path, - str(os.path.getsize(raw_disk))], - stdin=f) + + st = os.lstat(raw_disk) + xfer_hole_path = morphlib.util.get_data_path('xfer-hole') + recv_hole = morphlib.util.get_data('recv-hole') + + cliapp.runcmd( + ['python', xfer_hole_path, raw_disk], + ['ssh', ssh_host, + 'sh', '-c', cliapp.shell_quote(recv_hole), + 'dummy-argv0', 'vbox', vdi_path, str(st.st_size)], + stdout=None, stderr=None) def virtualbox_version(self, ssh_host): 'Get the version number of the VirtualBox running on the remote host.' -- cgit v1.2.1 From 93d932f6a62e60357542e15f028551b7221c1720 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 5 Sep 2014 14:17:53 +0000 Subject: Fix shell quoting for ssh remote command lines Found by Richard Maw. --- virtualbox-ssh.write | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) (limited to 'virtualbox-ssh.write') diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 1aebe490..39ea8f86 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -107,11 +107,14 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): xfer_hole_path = morphlib.util.get_data_path('xfer-hole') recv_hole = morphlib.util.get_data('recv-hole') + ssh_remote_cmd = [ + 'sh', '-c', recv_hole, + 'dummy-argv0', 'vbox', vdi_path, str(st.st_size), + ] + cliapp.runcmd( ['python', xfer_hole_path, raw_disk], - ['ssh', ssh_host, - 'sh', '-c', cliapp.shell_quote(recv_hole), - 'dummy-argv0', 'vbox', vdi_path, str(st.st_size)], + ['ssh', ssh_host] + map(cliapp.shell_quote, ssh_remote_cmd), stdout=None, stderr=None) def virtualbox_version(self, ssh_host): -- cgit v1.2.1 From a2ac64a4baa72bb805d0aee2b9862c25a0fff89f Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Fri, 28 Nov 2014 10:14:53 +0000 Subject: Refere to .write.help file for documentation --- virtualbox-ssh.write | 31 ++++++------------------------- 1 file changed, 6 insertions(+), 25 deletions(-) (limited to 'virtualbox-ssh.write') diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 39ea8f86..1b4de89c 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -20,6 +20,8 @@ VirtualBox is assumed to be running on a remote machine, which is accessed over ssh. The machine gets created, but not started. +See file virtualbox-ssh.write.help for documentation + ''' @@ -36,30 +38,10 @@ import morphlib.writeexts class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): - '''Create a VirtualBox virtual machine during Morph's deployment. - - The location command line argument is the pathname of the disk image - to be created. The user is expected to provide the location argument - using the following syntax: - - vbox+ssh://HOST/GUEST/PATH - - where: - - * HOST is the host on which VirtualBox is running - * GUEST is the name of the guest virtual machine on that host - * PATH is the path to the disk image that should be created, - on that host - - The extension will connect to HOST via ssh to run VirtualBox's - command line management tools. - - ''' - def process_args(self, args): if len(args) != 2: raise cliapp.AppException('Wrong number of command line args') - + temp_root, location = args ssh_host, vm_name, vdi_path = self.parse_location(location) autostart = self.get_environment_boolean('AUTOSTART') @@ -88,7 +70,7 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): def parse_location(self, location): '''Parse the location argument to get relevant data.''' - + x = urlparse.urlparse(location) if x.scheme != 'vbox+ssh': raise cliapp.AppException( @@ -169,11 +151,11 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): attach_disks = self.parse_attach_disks() for device_no, disk in enumerate(attach_disks, 1): - cmd = ['storageattach', vm_name, + cmd = ['storageattach', vm_name, '--storagectl', 'SATA Controller', '--port', str(device_no), '--device', '0', - '--type', 'hdd', + '--type', 'hdd', '--medium', disk] commands.append(cmd) @@ -242,4 +224,3 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): return iface VirtualBoxPlusSshWriteExtension().run() - -- cgit v1.2.1 From 544ac8182471a0552b8a22fd574029e74a30d8ed Mon Sep 17 00:00:00 2001 From: Jim MacArthur Date: Wed, 21 Jan 2015 18:08:01 +0000 Subject: Remove checks for NETWORK_CONFIG and eth0 and eth1 in it network_config isn't used anywhere in this function. The purpose of this function (getting the name of an appropriate host-only network interface) doesn't seem to depend on it either. eth0 and eth1 won't always be present (several Baserock systems will have enp0s3, etc). So I think these checks should be removed. --- virtualbox-ssh.write | 14 -------------- 1 file changed, 14 deletions(-) (limited to 'virtualbox-ssh.write') diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 1b4de89c..fa54c296 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -169,20 +169,6 @@ class VirtualBoxPlusSshWriteExtension(morphlib.writeexts.WriteExtension): def get_host_interface(self, ssh_host): host_ipaddr = os.environ.get('HOST_IPADDR') netmask = os.environ.get('NETMASK') - network_config = os.environ.get("NETWORK_CONFIG") - - if network_config is None: - raise cliapp.AppException('NETWORK_CONFIG was not given') - - if "eth0:" not in network_config: - raise cliapp.AppException( - 'NETWORK_CONFIG does not contain ' - 'the eth0 configuration') - - if "eth1:" not in network_config: - raise cliapp.AppException( - 'NETWORK_CONFIG does not contain ' - 'the eth1 configuration') if host_ipaddr is None: raise cliapp.AppException('HOST_IPADDR was not given') -- cgit v1.2.1 From ee1ccdbeda5cc559194db64c6ecf38f28cc391f0 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Tue, 3 Feb 2015 17:35:58 +0000 Subject: Update copyright years so ./check passes --- virtualbox-ssh.write | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'virtualbox-ssh.write') diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index fa54c296..7eafcff3 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2012-2014 Codethink Limited +# Copyright (C) 2012-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by -- cgit v1.2.1 From ed741d8d090086e2380f7b9d68ddc3bd122acb9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 13 Mar 2015 18:18:55 +0000 Subject: Use the modern way of the GPL copyright header: URL instead real address Change-Id: I992dc0c1d40f563ade56a833162d409b02be90a0 --- virtualbox-ssh.write | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'virtualbox-ssh.write') diff --git a/virtualbox-ssh.write b/virtualbox-ssh.write index 7eafcff3..774f2b4f 100755 --- a/virtualbox-ssh.write +++ b/virtualbox-ssh.write @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''A Morph deployment write extension for deploying to VirtualBox via ssh. -- cgit v1.2.1