From 6bd0b52aa907a27c355b2ab00a151757b9bb24fc Mon Sep 17 00:00:00 2001 From: Jonathan Maw Date: Fri, 24 May 2013 15:44:57 +0000 Subject: Add ssh-rsync write extension This is used to perform upgrades on running baserock systems. It requires rsync on the target system --- ssh-rsync.write | 181 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 181 insertions(+) create mode 100755 ssh-rsync.write (limited to 'ssh-rsync.write') diff --git a/ssh-rsync.write b/ssh-rsync.write new file mode 100755 index 00000000..6fe1153d --- /dev/null +++ b/ssh-rsync.write @@ -0,0 +1,181 @@ +#!/usr/bin/python +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +'''A Morph deployment write extension for upgrading systems over ssh.''' + + +import cliapp +import os +import sys +import time +import tempfile + +import morphlib.writeexts + +class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): + + '''Upgrade a running baserock system with ssh and rsync. + + It assumes the system is baserock-based and has a btrfs partition. + + The location command line argument is the 'user@hostname' string + that will be passed to ssh and rsync + + ''' + + def process_args(self, args): + if len(args) != 2: + raise cliapp.AppException('Wrong number of command line args') + + temp_root, location = args + + self.check_valid_target(location) + self.upgrade_remote_system(location, temp_root) + + def upgrade_remote_system(self, location, temp_root): + root_disk = self.find_root_disk(location) + version_label = os.environ.get('VERSION_LABEL') + + try: + self.status(msg='Creating remote mount point') + remote_mnt = cliapp.ssh_runcmd(location, ['mktemp', '-d']).strip() + + self.status(msg='Mounting root disk') + cliapp.ssh_runcmd(location, ['mount', root_disk, remote_mnt]) + + version_root = os.path.join(remote_mnt, 'systems', version_label) + run_dir = os.path.join(version_root, 'run') + orig_dir = os.path.join(version_root, 'orig') + try: + self.status(msg='Creating %s' % version_root) + cliapp.ssh_runcmd(location, ['mkdir', version_root]) + + self.create_remote_orig(location, version_root, remote_mnt, + temp_root) + + self.status(msg='Creating "run" subvolume') + cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', + 'snapshot', orig_dir, run_dir]) + + self.install_remote_kernel(location, version_root, temp_root) + except Exception as e: + try: + cliapp.ssh_runcmd(location, + ['btrfs', 'subvolume', 'delete', run_dir]) + cliapp.ssh_runcmd(location, + ['btrfs', 'subvolume', 'delete', orig_dir]) + cliapp.ssh_runcmd(location, ['rm', '-rf', version_root]) + except: + pass + raise e + + if self.bootloader_is_wanted(): + self.update_remote_extlinux(location, remote_mnt, + version_label) + except: + raise + else: + self.status(msg='Removing temporary mounts') + cliapp.ssh_runcmd(location, ['umount', root_disk]) + cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) + + def update_remote_extlinux(self, location, remote_mnt, version_label): + '''Install/reconfigure extlinux on location''' + + self.status(msg='Creating extlinux.conf') + config = os.path.join(remote_mnt, 'extlinux.conf') + temp_file = tempfile.mkstemp()[1] + with open(temp_file, 'w') as f: + f.write('default linux\n') + f.write('timeout 1\n') + f.write('label linux\n') + f.write('kernel /systems/' + version_label + '/kernel\n') + f.write('append root=/dev/sda ' + 'rootflags=subvol=systems/' + version_label + '/run ' + 'init=/sbin/init rw\n') + + cliapp.ssh_runcmd(location, ['mv', config, config+'~']) + + try: + cliapp.runcmd(['rsync', '-a', temp_file, + '%s:%s' % (location, config)]) + except Exception as e: + try: + cliapp.ssh_runcmd(location, ['mv', config+'~', config]) + except: + pass + raise e + + def create_remote_orig(self, location, version_root, remote_mnt, + temp_root): + '''Create the subvolume version_root/orig on location''' + + self.status(msg='Creating "orig" subvolume') + old_orig = self.get_old_orig(location, remote_mnt) + new_orig = os.path.join(version_root, 'orig') + cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', 'snapshot', + old_orig, new_orig]) + + cliapp.runcmd(['rsync', '-a', '--checksum', '--numeric-ids', + '--delete', temp_root, '%s:%s' % (location, new_orig)]) + + def get_old_orig(self, location, remote_mnt): + '''Identify which subvolume to snapshot from''' + + # rawdisk upgrades use 'factory' + return os.path.join(remote_mnt, 'systems', 'factory', 'orig') + + def find_root_disk(self, location): + '''Read /proc/mounts on location to find which device contains "/"''' + + self.status(msg='Finding device that contains "/"') + contents = cliapp.ssh_runcmd(location, ['cat', '/proc/mounts']) + for line in contents.splitlines(): + line_words = line.split() + if (line_words[1] == '/' and line_words[0] != 'rootfs'): + return line_words[0] + + def install_remote_kernel(self, location, version_root, temp_root): + '''Install the kernel in temp_root inside version_root on location''' + + self.status(msg='Installing kernel') + image_names = ['vmlinuz', 'zImage', 'uImage'] + kernel_dest = os.path.join(version_root, 'kernel') + for name in image_names: + try_path = os.path.join(temp_root, 'boot', name) + if os.path.exists(try_path): + cliapp.runcmd(['rsync', '-a', try_path, + '%s:%s' % (location, kernel_dest)]) + + def check_valid_target(self, location): + try: + cliapp.ssh_runcmd(location, ['true']) + except Exception as e: + raise cliapp.AppException('%s does not respond to ssh:\n%s' + % (location, e)) + + try: + cliapp.ssh_runcmd(location, ['test', '-d', '/baserock']) + except: + raise cliapp.AppException('%s is not a baserock system' % location) + + try: + cliapp.ssh_runcmd(location, ['which', 'rsync']) + except: + raise cliapp.AppException('%s does not have rsync') + +SshRsyncWriteExtension().run() -- cgit v1.2.1 From 64b72cdc4ae7a0d376239f31d1e607bae9d8d602 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Tue, 18 Jun 2013 16:09:51 +0100 Subject: Create a symbolic link to the default system version when upgrading running systems. --- ssh-rsync.write | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) (limited to 'ssh-rsync.write') diff --git a/ssh-rsync.write b/ssh-rsync.write index 6fe1153d..4348714c 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -72,6 +72,10 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): 'snapshot', orig_dir, run_dir]) self.install_remote_kernel(location, version_root, temp_root) + default_path = os.path.join(remote_mnt, 'systems', 'default') + cliapp.ssh_runcmd(location, ['ln', '-s', '-f', + version_label, + default_path]) except Exception as e: try: cliapp.ssh_runcmd(location, @@ -103,9 +107,9 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): f.write('default linux\n') f.write('timeout 1\n') f.write('label linux\n') - f.write('kernel /systems/' + version_label + '/kernel\n') + f.write('kernel /systems/default/kernel\n') f.write('append root=/dev/sda ' - 'rootflags=subvol=systems/' + version_label + '/run ' + 'rootflags=subvol=systems/default/run ' 'init=/sbin/init rw\n') cliapp.ssh_runcmd(location, ['mv', config, config+'~']) -- cgit v1.2.1 From 251d6a684eda959057810e736184eac316e80c75 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Tue, 18 Jun 2013 16:10:42 +0100 Subject: Support upgrades in older running versions. Verify if are using and older extlinux configuration and upgrade it if the case, by checking if the "default" symbolic link exists on the target. Note that with the symbolic link we don't need to update extlinux configuration after an upgrade --- ssh-rsync.write | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) (limited to 'ssh-rsync.write') diff --git a/ssh-rsync.write b/ssh-rsync.write index 4348714c..1a921996 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -73,6 +73,17 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.install_remote_kernel(location, version_root, temp_root) default_path = os.path.join(remote_mnt, 'systems', 'default') + if self.bootloader_is_wanted(): + output = ssh_runcmd(location, ['sh', '-c', + 'test -e "$1" && stat -c %F "$1"' + ' || ' + 'echo missing file', + '-', default_path]) + if output != "symbolic link": + # we are upgrading and old system that does + # not have an updated extlinux config file + self.update_remote_extlinux(location, remote_mnt, + version_label) cliapp.ssh_runcmd(location, ['ln', '-s', '-f', version_label, default_path]) @@ -87,9 +98,6 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): pass raise e - if self.bootloader_is_wanted(): - self.update_remote_extlinux(location, remote_mnt, - version_label) except: raise else: -- cgit v1.2.1 From 5e664629324a2cab7b4b79c01d458cc00c38e9c4 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Wed, 19 Jun 2013 12:59:34 +0100 Subject: Fix a typo --- ssh-rsync.write | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'ssh-rsync.write') diff --git a/ssh-rsync.write b/ssh-rsync.write index 1a921996..6bef51db 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -74,7 +74,7 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.install_remote_kernel(location, version_root, temp_root) default_path = os.path.join(remote_mnt, 'systems', 'default') if self.bootloader_is_wanted(): - output = ssh_runcmd(location, ['sh', '-c', + output = cliapp.ssh_runcmd(location, ['sh', '-c', 'test -e "$1" && stat -c %F "$1"' ' || ' 'echo missing file', -- cgit v1.2.1 From 3e8721c40abdc474ad3431d62d102e10aee7488f Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Sun, 9 Jun 2013 22:56:33 +0000 Subject: Write extensions: pass -s to rsync -s, or --protect-args prevents the file path components of destination or source paths being interpreted by the remote shell. This is for wildcards or other shell features, but it breaks when paths have whitespace. We tend to always use absolute paths, so all uses of rsync now pass -s. kvm.write needs it, since the disk can be written to a path with spaces. Nfsboot and ssh-rsync need it because version labels are used, which may have spaces, and temporary directories are used, which could have spaces in weird TMPDIR configurations. --- ssh-rsync.write | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'ssh-rsync.write') diff --git a/ssh-rsync.write b/ssh-rsync.write index 6bef51db..fba550cd 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -123,7 +123,7 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): cliapp.ssh_runcmd(location, ['mv', config, config+'~']) try: - cliapp.runcmd(['rsync', '-a', temp_file, + cliapp.runcmd(['rsync', '-as', temp_file, '%s:%s' % (location, config)]) except Exception as e: try: @@ -142,7 +142,7 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', 'snapshot', old_orig, new_orig]) - cliapp.runcmd(['rsync', '-a', '--checksum', '--numeric-ids', + cliapp.runcmd(['rsync', '-as', '--checksum', '--numeric-ids', '--delete', temp_root, '%s:%s' % (location, new_orig)]) def get_old_orig(self, location, remote_mnt): @@ -170,7 +170,7 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): for name in image_names: try_path = os.path.join(temp_root, 'boot', name) if os.path.exists(try_path): - cliapp.runcmd(['rsync', '-a', try_path, + cliapp.runcmd(['rsync', '-as', try_path, '%s:%s' % (location, kernel_dest)]) def check_valid_target(self, location): -- cgit v1.2.1 From 2a799319bd19ce9d303aa63d30ab7c556b17b6bb Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Sat, 15 Jun 2013 09:33:10 +0000 Subject: Don't dereference the default symbolic link when updating it Or else this ln -s -f update1 /mp/systems/default will do this '/pp/systems/default/update1' -> 'update1 When we want '/pp/systems/default' -> 'update1 --- ssh-rsync.write | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) (limited to 'ssh-rsync.write') diff --git a/ssh-rsync.write b/ssh-rsync.write index fba550cd..83091c4b 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -84,8 +84,7 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): # not have an updated extlinux config file self.update_remote_extlinux(location, remote_mnt, version_label) - cliapp.ssh_runcmd(location, ['ln', '-s', '-f', - version_label, + cliapp.ssh_runcmd(location, ['ln', '-sfn', version_label, default_path]) except Exception as e: try: -- cgit v1.2.1 From 89ad5f816fff7bd7897b2d4cb02ae5cc6b6799d2 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Sat, 15 Jun 2013 10:20:31 +0000 Subject: Unmount the remote mouting point instead of the root disk Unmounting the root disk as the side effect of turn it to be read only --- ssh-rsync.write | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'ssh-rsync.write') diff --git a/ssh-rsync.write b/ssh-rsync.write index 83091c4b..77266d33 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -101,7 +101,7 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): raise else: self.status(msg='Removing temporary mounts') - cliapp.ssh_runcmd(location, ['umount', root_disk]) + cliapp.ssh_runcmd(location, ['umount', remote_mnt]) cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) def update_remote_extlinux(self, location, remote_mnt, version_label): -- cgit v1.2.1 From d8a87880248ec754affc302fa8966bf5ebd83046 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Sun, 16 Jun 2013 11:41:32 +0000 Subject: Add a missing trailing slash to the source directory of rsync Accordingly the rsync manual: "A trailing slash on the source changes this behavior to avoid creating an additional directory level at the destination. You can think of a trailing / on a source as meaning "copy the contents of this directory" as opposed to "copy the directory by name". --- ssh-rsync.write | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'ssh-rsync.write') diff --git a/ssh-rsync.write b/ssh-rsync.write index 77266d33..b8d30e22 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -142,7 +142,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): old_orig, new_orig]) cliapp.runcmd(['rsync', '-as', '--checksum', '--numeric-ids', - '--delete', temp_root, '%s:%s' % (location, new_orig)]) + '--delete', temp_root + os.path.sep, + '%s:%s' % (location, new_orig)]) def get_old_orig(self, location, remote_mnt): '''Identify which subvolume to snapshot from''' -- cgit v1.2.1 From 4f630811332b7ebb21fc47551bccb8e14a456410 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Sat, 15 Jun 2013 15:48:01 +0000 Subject: Run the merge mode of baserock-system-config-sync when upgrading running systems. --- ssh-rsync.write | 9 +++++++++ 1 file changed, 9 insertions(+) (limited to 'ssh-rsync.write') diff --git a/ssh-rsync.write b/ssh-rsync.write index b8d30e22..9697e21b 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -71,6 +71,15 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', 'snapshot', orig_dir, run_dir]) + self.status(msg='Updating system configuration') + bscs_loc = os.path.join(run_dir, 'usr', 'bin', + 'baserock-system-config-sync') + try: + cliapp.ssh_runcmd(location, ['sh', bscs_loc, 'merge', + version_label]) + except: + self.status(msg='Updating system configuration failed') + self.install_remote_kernel(location, version_root, temp_root) default_path = os.path.join(remote_mnt, 'systems', 'default') if self.bootloader_is_wanted(): -- cgit v1.2.1 From ce80fe3e235ff747afbea9b20f992f5af41fe946 Mon Sep 17 00:00:00 2001 From: Tiago Gomes Date: Fri, 28 Jun 2013 15:00:36 +0000 Subject: Improvements to ssh-rsync extension --- ssh-rsync.write | 150 ++++++++++++++++++++++++++++++-------------------------- 1 file changed, 80 insertions(+), 70 deletions(-) (limited to 'ssh-rsync.write') diff --git a/ssh-rsync.write b/ssh-rsync.write index 9697e21b..211dbe5e 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -50,65 +50,74 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): root_disk = self.find_root_disk(location) version_label = os.environ.get('VERSION_LABEL') + self.status(msg='Creating remote mount point') + remote_mnt = cliapp.ssh_runcmd(location, ['mktemp', '-d']).strip() try: - self.status(msg='Creating remote mount point') - remote_mnt = cliapp.ssh_runcmd(location, ['mktemp', '-d']).strip() - self.status(msg='Mounting root disk') cliapp.ssh_runcmd(location, ['mount', root_disk, remote_mnt]) + except Exception as e: + try: + cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) + except: + pass + raise e + try: version_root = os.path.join(remote_mnt, 'systems', version_label) run_dir = os.path.join(version_root, 'run') orig_dir = os.path.join(version_root, 'orig') + + self.status(msg='Creating %s' % version_root) + cliapp.ssh_runcmd(location, ['mkdir', version_root]) + + self.create_remote_orig(location, version_root, remote_mnt, + temp_root) + + self.status(msg='Creating "run" subvolume') + cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', + 'snapshot', orig_dir, run_dir]) + + self.status(msg='Updating system configuration') + bscs_loc = os.path.join(run_dir, 'usr', 'bin', + 'baserock-system-config-sync') + + output = cliapp.ssh_runcmd(location, ['sh', '-c', + '"$1" merge "$2" &> /dev/null || echo -n cmdfailed', + '-', bscs_loc, version_label]) + if output == "cmdfailed": + self.status(msg='Updating system configuration failed') + + self.install_remote_kernel(location, version_root, temp_root) + default_path = os.path.join(remote_mnt, 'systems', 'default') + if self.bootloader_is_wanted(): + output = cliapp.ssh_runcmd(location, ['sh', '-c', + 'test -e "$1" && stat -c %F "$1" ' + '|| echo missing file', + '-', default_path]) + if output != "symbolic link": + # we are upgrading and old system that does + # not have an updated extlinux config file + self.update_remote_extlinux(location, remote_mnt, + version_label) + cliapp.ssh_runcmd(location, ['ln', '-sfn', version_label, + default_path]) + except Exception as e: + try: + cliapp.ssh_runcmd(location, + ['btrfs', 'subvolume', 'delete', run_dir]) + except: + pass + try: + cliapp.ssh_runcmd(location, + ['btrfs', 'subvolume', 'delete', orig_dir]) + except: + pass try: - self.status(msg='Creating %s' % version_root) - cliapp.ssh_runcmd(location, ['mkdir', version_root]) - - self.create_remote_orig(location, version_root, remote_mnt, - temp_root) - - self.status(msg='Creating "run" subvolume') - cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', - 'snapshot', orig_dir, run_dir]) - - self.status(msg='Updating system configuration') - bscs_loc = os.path.join(run_dir, 'usr', 'bin', - 'baserock-system-config-sync') - try: - cliapp.ssh_runcmd(location, ['sh', bscs_loc, 'merge', - version_label]) - except: - self.status(msg='Updating system configuration failed') - - self.install_remote_kernel(location, version_root, temp_root) - default_path = os.path.join(remote_mnt, 'systems', 'default') - if self.bootloader_is_wanted(): - output = cliapp.ssh_runcmd(location, ['sh', '-c', - 'test -e "$1" && stat -c %F "$1"' - ' || ' - 'echo missing file', - '-', default_path]) - if output != "symbolic link": - # we are upgrading and old system that does - # not have an updated extlinux config file - self.update_remote_extlinux(location, remote_mnt, - version_label) - cliapp.ssh_runcmd(location, ['ln', '-sfn', version_label, - default_path]) - except Exception as e: - try: - cliapp.ssh_runcmd(location, - ['btrfs', 'subvolume', 'delete', run_dir]) - cliapp.ssh_runcmd(location, - ['btrfs', 'subvolume', 'delete', orig_dir]) - cliapp.ssh_runcmd(location, ['rm', '-rf', version_root]) - except: - pass - raise e - - except: - raise - else: + cliapp.ssh_runcmd(location, ['rm', '-rf', version_root]) + except: + pass + raise e + finally: self.status(msg='Removing temporary mounts') cliapp.ssh_runcmd(location, ['umount', remote_mnt]) cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) @@ -118,8 +127,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Creating extlinux.conf') config = os.path.join(remote_mnt, 'extlinux.conf') - temp_file = tempfile.mkstemp()[1] - with open(temp_file, 'w') as f: + temp_fd, temp_path = tempfile.mkstemp() + with os.fdopen(temp_fd, 'w') as f: f.write('default linux\n') f.write('timeout 1\n') f.write('label linux\n') @@ -128,14 +137,13 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): 'rootflags=subvol=systems/default/run ' 'init=/sbin/init rw\n') - cliapp.ssh_runcmd(location, ['mv', config, config+'~']) - try: - cliapp.runcmd(['rsync', '-as', temp_file, - '%s:%s' % (location, config)]) + cliapp.runcmd(['rsync', '-as', temp_path, + '%s:%s~' % (location, config)]) + cliapp.ssh_runcmd(location, ['mv', config+'~', config]) except Exception as e: try: - cliapp.ssh_runcmd(location, ['mv', config+'~', config]) + cliapp.ssh_runcmd(location, ['rm', '-f', config+'~']) except: pass raise e @@ -168,19 +176,19 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): for line in contents.splitlines(): line_words = line.split() if (line_words[1] == '/' and line_words[0] != 'rootfs'): - return line_words[0] + return line_words[0] def install_remote_kernel(self, location, version_root, temp_root): '''Install the kernel in temp_root inside version_root on location''' self.status(msg='Installing kernel') - image_names = ['vmlinuz', 'zImage', 'uImage'] + image_names = ('vmlinuz', 'zImage', 'uImage') kernel_dest = os.path.join(version_root, 'kernel') for name in image_names: try_path = os.path.join(temp_root, 'boot', name) if os.path.exists(try_path): cliapp.runcmd(['rsync', '-as', try_path, - '%s:%s' % (location, kernel_dest)]) + '%s:%s' % (location, kernel_dest)]) def check_valid_target(self, location): try: @@ -189,14 +197,16 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): raise cliapp.AppException('%s does not respond to ssh:\n%s' % (location, e)) - try: - cliapp.ssh_runcmd(location, ['test', '-d', '/baserock']) - except: - raise cliapp.AppException('%s is not a baserock system' % location) - - try: - cliapp.ssh_runcmd(location, ['which', 'rsync']) - except: - raise cliapp.AppException('%s does not have rsync') + output = cliapp.ssh_runcmd(location, ['sh', '-c', + 'test -d /baserock || echo -n dirnotfound']) + if output == 'dirnotfound': + raise cliapp.AppException('%s is not a baserock system' + % location) + + output = cliapp.ssh_runcmd(location, ['sh', '-c', + 'type rsync &> /dev/null || echo -n cmdnotfound']) + if output == 'cmdnotfound': + raise cliapp.AppException('%s does not have rsync' + % location) SshRsyncWriteExtension().run() -- cgit v1.2.1 From e419d582cfd0ea873b393b2773c1b0670d16afe0 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Wed, 12 Feb 2014 19:03:05 +0000 Subject: deploy: Finish off the Btrfs system layout implementation The shared state directories defined in writeexts.py (/var, /home etc.) are now separate Btrfs subvolumes that are mounted in place using fstab. There are some warnings on mounting /var and /srv about the mountpoint not being empty. Not yet investigated. If a configure extension has already added / to the fstab, use the device it chose rather than assuming /dev/sda. This is required for the vdaboot.configure extension that we use for OpenStack deployments. Similarly, if a configure extension has added an entry for a state directory in /etc/fstab already, we don't replace it with a /state/xxx directory. That's only done as a default behaviour. --- ssh-rsync.write | 2 ++ 1 file changed, 2 insertions(+) (limited to 'ssh-rsync.write') diff --git a/ssh-rsync.write b/ssh-rsync.write index 211dbe5e..fe72bc9a 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -47,6 +47,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.upgrade_remote_system(location, temp_root) def upgrade_remote_system(self, location, temp_root): + self.complete_fstab_for_btrfs_layout(temp_root) + root_disk = self.find_root_disk(location) version_label = os.environ.get('VERSION_LABEL') -- cgit v1.2.1 From 9886dd3e919f7dc66b9099e3c8ab1be79404ae31 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Fri, 14 Feb 2014 12:08:33 +0000 Subject: deploy: Depend on client OS version manager to deploy upgrades We now have a OS version manager tool in Baserock (in tbdiff.git). The code to deploy a new base OS version should live there, to minimise duplication between write extensions. --- ssh-rsync.write | 132 ++++++++++++++++++-------------------------------------- 1 file changed, 41 insertions(+), 91 deletions(-) (limited to 'ssh-rsync.write') diff --git a/ssh-rsync.write b/ssh-rsync.write index fe72bc9a..4961ee4d 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -26,6 +26,14 @@ import tempfile import morphlib.writeexts + +def ssh_runcmd_ignore_failure(location, command, **kwargs): + try: + return cliapp.ssh_runcmd(location, command, **kwargs) + except cliapp.AppException: + pass + + class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): '''Upgrade a running baserock system with ssh and rsync. @@ -58,15 +66,11 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.status(msg='Mounting root disk') cliapp.ssh_runcmd(location, ['mount', root_disk, remote_mnt]) except Exception as e: - try: - cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) - except: - pass + ssh_runcmd_ignore_failure(location, ['rmdir', remote_mnt]) raise e try: version_root = os.path.join(remote_mnt, 'systems', version_label) - run_dir = os.path.join(version_root, 'run') orig_dir = os.path.join(version_root, 'orig') self.status(msg='Creating %s' % version_root) @@ -75,81 +79,32 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.create_remote_orig(location, version_root, remote_mnt, temp_root) - self.status(msg='Creating "run" subvolume') - cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', - 'snapshot', orig_dir, run_dir]) - - self.status(msg='Updating system configuration') - bscs_loc = os.path.join(run_dir, 'usr', 'bin', - 'baserock-system-config-sync') - - output = cliapp.ssh_runcmd(location, ['sh', '-c', - '"$1" merge "$2" &> /dev/null || echo -n cmdfailed', - '-', bscs_loc, version_label]) - if output == "cmdfailed": - self.status(msg='Updating system configuration failed') - - self.install_remote_kernel(location, version_root, temp_root) - default_path = os.path.join(remote_mnt, 'systems', 'default') - if self.bootloader_is_wanted(): - output = cliapp.ssh_runcmd(location, ['sh', '-c', - 'test -e "$1" && stat -c %F "$1" ' - '|| echo missing file', - '-', default_path]) - if output != "symbolic link": - # we are upgrading and old system that does - # not have an updated extlinux config file - self.update_remote_extlinux(location, remote_mnt, - version_label) - cliapp.ssh_runcmd(location, ['ln', '-sfn', version_label, - default_path]) + # Use the system-version-manager from the new system we just + # installed, so that we can upgrade from systems that don't have + # it installed. + self.status(msg='Calling system-version-manager to deploy upgrade') + deployment = os.path.join('/systems', version_label, 'orig') + system_config_sync = os.path.join( + remote_mnt, 'systems', version_label, 'orig', 'usr', 'bin', + 'baserock-system-config-sync') + system_version_manager = os.path.join( + remote_mnt, 'systems', version_label, 'orig', 'usr', 'bin', + 'system-version-manager') + cliapp.ssh_runcmd(location, + ['env', 'BASEROCK_SYSTEM_CONFIG_SYNC='+system_config_sync, + system_version_manager, 'deploy', deployment]) except Exception as e: - try: - cliapp.ssh_runcmd(location, - ['btrfs', 'subvolume', 'delete', run_dir]) - except: - pass - try: - cliapp.ssh_runcmd(location, - ['btrfs', 'subvolume', 'delete', orig_dir]) - except: - pass - try: - cliapp.ssh_runcmd(location, ['rm', '-rf', version_root]) - except: - pass + self.status(msg='Deployment failed') + ssh_runcmd_ignore_failure( + location, ['btrfs', 'subvolume', 'delete', orig_dir]) + ssh_runcmd_ignore_failure( + location, ['rm', '-rf', version_root]) raise e finally: self.status(msg='Removing temporary mounts') cliapp.ssh_runcmd(location, ['umount', remote_mnt]) cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) - def update_remote_extlinux(self, location, remote_mnt, version_label): - '''Install/reconfigure extlinux on location''' - - self.status(msg='Creating extlinux.conf') - config = os.path.join(remote_mnt, 'extlinux.conf') - temp_fd, temp_path = tempfile.mkstemp() - with os.fdopen(temp_fd, 'w') as f: - f.write('default linux\n') - f.write('timeout 1\n') - f.write('label linux\n') - f.write('kernel /systems/default/kernel\n') - f.write('append root=/dev/sda ' - 'rootflags=subvol=systems/default/run ' - 'init=/sbin/init rw\n') - - try: - cliapp.runcmd(['rsync', '-as', temp_path, - '%s:%s~' % (location, config)]) - cliapp.ssh_runcmd(location, ['mv', config+'~', config]) - except Exception as e: - try: - cliapp.ssh_runcmd(location, ['rm', '-f', config+'~']) - except: - pass - raise e - def create_remote_orig(self, location, version_root, remote_mnt, temp_root): '''Create the subvolume version_root/orig on location''' @@ -180,18 +135,6 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): if (line_words[1] == '/' and line_words[0] != 'rootfs'): return line_words[0] - def install_remote_kernel(self, location, version_root, temp_root): - '''Install the kernel in temp_root inside version_root on location''' - - self.status(msg='Installing kernel') - image_names = ('vmlinuz', 'zImage', 'uImage') - kernel_dest = os.path.join(version_root, 'kernel') - for name in image_names: - try_path = os.path.join(temp_root, 'boot', name) - if os.path.exists(try_path): - cliapp.runcmd(['rsync', '-as', try_path, - '%s:%s' % (location, kernel_dest)]) - def check_valid_target(self, location): try: cliapp.ssh_runcmd(location, ['true']) @@ -205,10 +148,17 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): raise cliapp.AppException('%s is not a baserock system' % location) - output = cliapp.ssh_runcmd(location, ['sh', '-c', - 'type rsync &> /dev/null || echo -n cmdnotfound']) - if output == 'cmdnotfound': - raise cliapp.AppException('%s does not have rsync' - % location) + def check_command_exists(command): + test = 'type %s > /dev/null 2>&1 || echo -n cmdnotfound' % command + output = cliapp.ssh_runcmd(location, ['sh', '-c', test]) + if output == 'cmdnotfound': + raise cliapp.AppException( + "%s does not have %s" % (location, command)) + + # The deploy requires baserock-system-config-sync and + # system-version-manager in the new system only. The old system doesn't + # need to have them at all. + check_command_exists('rsync') + SshRsyncWriteExtension().run() -- cgit v1.2.1 From 539eed7e4045ef39377f4c7fe6cc9314799b9f4d Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Tue, 4 Mar 2014 11:49:02 +0000 Subject: deploy: Always set new system as default --- ssh-rsync.write | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'ssh-rsync.write') diff --git a/ssh-rsync.write b/ssh-rsync.write index 4961ee4d..8dc0fe35 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -93,6 +93,11 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): cliapp.ssh_runcmd(location, ['env', 'BASEROCK_SYSTEM_CONFIG_SYNC='+system_config_sync, system_version_manager, 'deploy', deployment]) + + self.status(msg='Setting %s as the new default system' % + version_label) + cliapp.ssh_runcmd(location, + [system_version_manager, 'set-default', version_label]) except Exception as e: self.status(msg='Deployment failed') ssh_runcmd_ignore_failure( -- cgit v1.2.1 From 9293be701e6b8ae2a1017bc5df9b80c85c735173 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Mon, 17 Feb 2014 13:31:47 +0000 Subject: deploy: Honour AUTOSTART in ssh-rsync extension Now you can deploy an upgrade, set it to be the default version and reboot into it all with one call to `morph deploy`. --- ssh-rsync.write | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'ssh-rsync.write') diff --git a/ssh-rsync.write b/ssh-rsync.write index 8dc0fe35..509520ae 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -59,6 +59,7 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): root_disk = self.find_root_disk(location) version_label = os.environ.get('VERSION_LABEL') + autostart = self.get_environment_boolean('AUTOSTART') self.status(msg='Creating remote mount point') remote_mnt = cliapp.ssh_runcmd(location, ['mktemp', '-d']).strip() @@ -110,6 +111,10 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): cliapp.ssh_runcmd(location, ['umount', remote_mnt]) cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) + if autostart: + self.status(msg="Rebooting into new system ...") + ssh_runcmd_ignore_failure(location, ['reboot']) + def create_remote_orig(self, location, version_root, remote_mnt, temp_root): '''Create the subvolume version_root/orig on location''' -- cgit v1.2.1 From 6efbcd6ef631a79f73d2429622296ddfbde09003 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Tue, 20 May 2014 09:37:49 +0000 Subject: deploy: Do sanity checks earlier in ssh-rsync (upgrade) extension --- ssh-rsync.write | 26 -------------------------- 1 file changed, 26 deletions(-) (limited to 'ssh-rsync.write') diff --git a/ssh-rsync.write b/ssh-rsync.write index 509520ae..c139b6c0 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -51,7 +51,6 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): temp_root, location = args - self.check_valid_target(location) self.upgrade_remote_system(location, temp_root) def upgrade_remote_system(self, location, temp_root): @@ -145,30 +144,5 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): if (line_words[1] == '/' and line_words[0] != 'rootfs'): return line_words[0] - def check_valid_target(self, location): - try: - cliapp.ssh_runcmd(location, ['true']) - except Exception as e: - raise cliapp.AppException('%s does not respond to ssh:\n%s' - % (location, e)) - - output = cliapp.ssh_runcmd(location, ['sh', '-c', - 'test -d /baserock || echo -n dirnotfound']) - if output == 'dirnotfound': - raise cliapp.AppException('%s is not a baserock system' - % location) - - def check_command_exists(command): - test = 'type %s > /dev/null 2>&1 || echo -n cmdnotfound' % command - output = cliapp.ssh_runcmd(location, ['sh', '-c', test]) - if output == 'cmdnotfound': - raise cliapp.AppException( - "%s does not have %s" % (location, command)) - - # The deploy requires baserock-system-config-sync and - # system-version-manager in the new system only. The old system doesn't - # need to have them at all. - check_command_exists('rsync') - SshRsyncWriteExtension().run() -- cgit v1.2.1 From 4fff47462e598d475e930893383f9c27e6f2c381 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Wed, 1 Oct 2014 08:21:29 +0000 Subject: ssh-rsync: gett UUID of the disk before writing fstab With this patch, the fstab of the system to be deployed as an upgrade will be conifgured using the UUID of the disk. Now when doing an upgrade is not needed to specify the ROOT_DEVICE. --- ssh-rsync.write | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'ssh-rsync.write') diff --git a/ssh-rsync.write b/ssh-rsync.write index c139b6c0..468e5a1f 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -54,9 +54,12 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.upgrade_remote_system(location, temp_root) def upgrade_remote_system(self, location, temp_root): - self.complete_fstab_for_btrfs_layout(temp_root) - root_disk = self.find_root_disk(location) + uuid = cliapp.ssh_runcmd(location, ['blkid', '-s', 'UUID', '-o', + 'value', root_disk]).strip() + + self.complete_fstab_for_btrfs_layout(temp_root, uuid) + version_label = os.environ.get('VERSION_LABEL') autostart = self.get_environment_boolean('AUTOSTART') -- cgit v1.2.1 From bd6bae145a4f7064caa4ee49f7e815452d4469e8 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Wed, 8 Oct 2014 10:28:23 +0000 Subject: deploy: Make ssh-rsync upgrade extension handle unset VERSION_LABEL It now gives an error message. Previously it would fail with a backtrace like this: 2014-10-08 09:51:37 [systems/genivi-baseline-system-armv7lhf-jetson.morph][self]Removing temporary mounts Traceback (most recent call last): File "/usr/lib/python2.7/site-packages/cliapp/app.py", line 190, in _run self.process_args(args) File "/src/morph/morphlib/exts/ssh-rsync.write", line 54, in process_args self.upgrade_remote_system(location, temp_root) File "/src/morph/morphlib/exts/ssh-rsync.write", line 107, in upgrade_remote_system location, ['btrfs', 'subvolume', 'delete', orig_dir]) UnboundLocalError: local variable 'orig_dir' referenced before assignment --- ssh-rsync.write | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'ssh-rsync.write') diff --git a/ssh-rsync.write b/ssh-rsync.write index 468e5a1f..775619ec 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -60,7 +60,7 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self.complete_fstab_for_btrfs_layout(temp_root, uuid) - version_label = os.environ.get('VERSION_LABEL') + version_label = os.environ['VERSION_LABEL'] autostart = self.get_environment_boolean('AUTOSTART') self.status(msg='Creating remote mount point') -- cgit v1.2.1 From 7ec3d106f49fd4af2d193afc976c2230a9369a7e Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Thu, 9 Oct 2014 15:09:03 +0000 Subject: ssh-rsync: Don't delete version if it exists A quirk in the resource cleanup code meant that if you gave the same version label when deploying a new version, then it would fail, then remove the old version, as it had assumed that it was the one to create those directories. This patch fixes this issue by making short context managers for all the resource allocation, so cleanup is done by walking up the context managers, so only the mount and the temporary directory need to be cleaned up if the `mkdir "$VERSION_ROOT"` fails. I've tested this with a deploy of a version that doesn't already exist, and the version I'm currently running, so I can conclusively say it's fixed that problem. --- ssh-rsync.write | 182 ++++++++++++++++++++++++++++++++------------------------ 1 file changed, 105 insertions(+), 77 deletions(-) (limited to 'ssh-rsync.write') diff --git a/ssh-rsync.write b/ssh-rsync.write index 775619ec..2391d48c 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -18,6 +18,7 @@ '''A Morph deployment write extension for upgrading systems over ssh.''' +import contextlib import cliapp import os import sys @@ -45,107 +46,134 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): ''' - def process_args(self, args): - if len(args) != 2: - raise cliapp.AppException('Wrong number of command line args') - - temp_root, location = args - - self.upgrade_remote_system(location, temp_root) - - def upgrade_remote_system(self, location, temp_root): - root_disk = self.find_root_disk(location) - uuid = cliapp.ssh_runcmd(location, ['blkid', '-s', 'UUID', '-o', - 'value', root_disk]).strip() - - self.complete_fstab_for_btrfs_layout(temp_root, uuid) + def find_root_disk(self, location): + '''Read /proc/mounts on location to find which device contains "/"''' - version_label = os.environ['VERSION_LABEL'] - autostart = self.get_environment_boolean('AUTOSTART') + self.status(msg='Finding device that contains "/"') + contents = cliapp.ssh_runcmd(location, ['cat', '/proc/mounts']) + for line in contents.splitlines(): + line_words = line.split() + if (line_words[1] == '/' and line_words[0] != 'rootfs'): + return line_words[0] + @contextlib.contextmanager + def _remote_mount_point(self, location): self.status(msg='Creating remote mount point') remote_mnt = cliapp.ssh_runcmd(location, ['mktemp', '-d']).strip() try: - self.status(msg='Mounting root disk') - cliapp.ssh_runcmd(location, ['mount', root_disk, remote_mnt]) - except Exception as e: - ssh_runcmd_ignore_failure(location, ['rmdir', remote_mnt]) - raise e + yield remote_mnt + finally: + self.status(msg='Removing remote mount point') + cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) + @contextlib.contextmanager + def _remote_mount(self, location, root_disk, mountpoint): + self.status(msg='Mounting root disk') + cliapp.ssh_runcmd(location, ['mount', root_disk, mountpoint]) try: - version_root = os.path.join(remote_mnt, 'systems', version_label) - orig_dir = os.path.join(version_root, 'orig') - - self.status(msg='Creating %s' % version_root) - cliapp.ssh_runcmd(location, ['mkdir', version_root]) - - self.create_remote_orig(location, version_root, remote_mnt, - temp_root) - - # Use the system-version-manager from the new system we just - # installed, so that we can upgrade from systems that don't have - # it installed. - self.status(msg='Calling system-version-manager to deploy upgrade') - deployment = os.path.join('/systems', version_label, 'orig') - system_config_sync = os.path.join( - remote_mnt, 'systems', version_label, 'orig', 'usr', 'bin', - 'baserock-system-config-sync') - system_version_manager = os.path.join( - remote_mnt, 'systems', version_label, 'orig', 'usr', 'bin', - 'system-version-manager') - cliapp.ssh_runcmd(location, - ['env', 'BASEROCK_SYSTEM_CONFIG_SYNC='+system_config_sync, - system_version_manager, 'deploy', deployment]) - - self.status(msg='Setting %s as the new default system' % - version_label) - cliapp.ssh_runcmd(location, - [system_version_manager, 'set-default', version_label]) - except Exception as e: - self.status(msg='Deployment failed') - ssh_runcmd_ignore_failure( - location, ['btrfs', 'subvolume', 'delete', orig_dir]) - ssh_runcmd_ignore_failure( - location, ['rm', '-rf', version_root]) - raise e + yield finally: - self.status(msg='Removing temporary mounts') - cliapp.ssh_runcmd(location, ['umount', remote_mnt]) - cliapp.ssh_runcmd(location, ['rmdir', remote_mnt]) + self.status(msg='Unmounting root disk') + cliapp.ssh_runcmd(location, ['umount', mountpoint]) + + @contextlib.contextmanager + def _created_version_root(self, location, remote_mnt, version_label): + version_root = os.path.join(remote_mnt, 'systems', version_label) + self.status(msg='Creating %(root)s', root=version_root) + cliapp.ssh_runcmd(location, ['mkdir', version_root]) + try: + yield version_root + except BaseException as e: + # catch all, we always want to clean up + self.status(msg='Cleaning up %(root)s', root=version_root) + ssh_runcmd_ignore_failure(location, ['rmdir', version_root]) + raise - if autostart: - self.status(msg="Rebooting into new system ...") - ssh_runcmd_ignore_failure(location, ['reboot']) + def get_old_orig(self, location, remote_mnt): + '''Identify which subvolume to snapshot from''' - def create_remote_orig(self, location, version_root, remote_mnt, - temp_root): - '''Create the subvolume version_root/orig on location''' + # rawdisk upgrades use 'factory' + return os.path.join(remote_mnt, 'systems', 'factory', 'orig') + @contextlib.contextmanager + def _created_orig_subvolume(self, location, remote_mnt, version_root): self.status(msg='Creating "orig" subvolume') old_orig = self.get_old_orig(location, remote_mnt) new_orig = os.path.join(version_root, 'orig') cliapp.ssh_runcmd(location, ['btrfs', 'subvolume', 'snapshot', old_orig, new_orig]) + try: + yield new_orig + except BaseException as e: + ssh_runcmd_ignore_failure( + location, ['btrfs', 'subvolume', 'delete', new_orig]) + raise + def populate_remote_orig(self, location, new_orig, temp_root): + '''Populate the subvolume version_root/orig on location''' + + self.status(msg='Populating "orig" subvolume') cliapp.runcmd(['rsync', '-as', '--checksum', '--numeric-ids', '--delete', temp_root + os.path.sep, '%s:%s' % (location, new_orig)]) - def get_old_orig(self, location, remote_mnt): - '''Identify which subvolume to snapshot from''' + @contextlib.contextmanager + def _deployed_version(self, location, version_label, + system_config_sync, system_version_manager): + self.status(msg='Calling system-version-manager to deploy upgrade') + deployment = os.path.join('/systems', version_label, 'orig') + cliapp.ssh_runcmd(location, + ['env', 'BASEROCK_SYSTEM_CONFIG_SYNC='+system_config_sync, + system_version_manager, 'deploy', deployment]) + try: + yield deployment + except BaseException as e: + self.status(msg='Cleaning up failed version installation') + cliapp.ssh_runcmd(location, + [system_version_manager, 'remove', version_label]) + raise - # rawdisk upgrades use 'factory' - return os.path.join(remote_mnt, 'systems', 'factory', 'orig') + def upgrade_remote_system(self, location, temp_root): + root_disk = self.find_root_disk(location) + uuid = cliapp.ssh_runcmd(location, ['blkid', '-s', 'UUID', '-o', + 'value', root_disk]).strip() - def find_root_disk(self, location): - '''Read /proc/mounts on location to find which device contains "/"''' + self.complete_fstab_for_btrfs_layout(temp_root, uuid) - self.status(msg='Finding device that contains "/"') - contents = cliapp.ssh_runcmd(location, ['cat', '/proc/mounts']) - for line in contents.splitlines(): - line_words = line.split() - if (line_words[1] == '/' and line_words[0] != 'rootfs'): - return line_words[0] + version_label = os.environ['VERSION_LABEL'] + autostart = self.get_environment_boolean('AUTOSTART') + + with self._remote_mount_point(location) as remote_mnt, \ + self._remote_mount(location, root_disk, remote_mnt), \ + self._created_version_root(location, remote_mnt, + version_label) as version_root, \ + self._created_orig_subvolume(location, remote_mnt, + version_root) as orig: + self.populate_remote_orig(location, orig, temp_root) + system_config_sync = os.path.join( + remote_mnt, 'systems', version_label, 'orig', + 'usr', 'bin', 'baserock-system-config-sync') + system_version_manager = os.path.join( + remote_mnt, 'systems', version_label, 'orig', + 'usr', 'bin', 'system-version-manager') + with self._deployed_version(location, version_label, + system_config_sync, system_version_manager): + self.status(msg='Setting %(v)s as the new default system', + v=version_label) + cliapp.ssh_runcmd(location, [system_version_manager, + 'set-default', version_label]) + + if autostart: + self.status(msg="Rebooting into new system ...") + ssh_runcmd_ignore_failure(location, ['reboot']) + + def process_args(self, args): + if len(args) != 2: + raise cliapp.AppException('Wrong number of command line args') + + temp_root, location = args + + self.upgrade_remote_system(location, temp_root) SshRsyncWriteExtension().run() -- cgit v1.2.1 From aa7005c9b4207ac32621433a95a0c9b44007f6b4 Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Fri, 10 Oct 2014 13:09:25 +0000 Subject: Merge branch 'baserock/richardmaw/fix-ssh-rsync-destroying-versions' Reviewed-by: Sam Thursfield Reviewed-by: Jim MacArthur Reviewed-by: Richard Ipsum --- ssh-rsync.write | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'ssh-rsync.write') diff --git a/ssh-rsync.write b/ssh-rsync.write index 2391d48c..0ce89c7f 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -150,17 +150,17 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): self._created_orig_subvolume(location, remote_mnt, version_root) as orig: self.populate_remote_orig(location, orig, temp_root) - system_config_sync = os.path.join( - remote_mnt, 'systems', version_label, 'orig', - 'usr', 'bin', 'baserock-system-config-sync') - system_version_manager = os.path.join( - remote_mnt, 'systems', version_label, 'orig', - 'usr', 'bin', 'system-version-manager') + system_root = os.path.join(remote_mnt, 'systems', + version_label, 'orig') + config_sync = os.path.join(system_root, 'usr', 'bin', + 'baserock-system-config-sync') + version_manager = os.path.join(system_root, 'usr', 'bin', + 'system-version-manager') with self._deployed_version(location, version_label, - system_config_sync, system_version_manager): + config_sync, version_manager): self.status(msg='Setting %(v)s as the new default system', v=version_label) - cliapp.ssh_runcmd(location, [system_version_manager, + cliapp.ssh_runcmd(location, [version_manager, 'set-default', version_label]) if autostart: -- cgit v1.2.1 From ec477897b9d6afd3ab176ab512d12f36121618c1 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Date: Wed, 5 Nov 2014 10:24:07 +0000 Subject: Use the default symlink when creating the orig subvolume. This patch solves the issue caused by upgrading a system without a factory version. Currently we are only using the factory version to snapshot its orig subvolume to make faster the transfer of the new content (rsync won't have to send everything). The default symlink may not be present, but it can't be deleted easily using system-version-manager. This is a quick fix, but in the future we may want to not harcode the path from where we snapshot the orig subvolume. Or improve system-version-manager to make sure that the default symlink is always present. --- ssh-rsync.write | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'ssh-rsync.write') diff --git a/ssh-rsync.write b/ssh-rsync.write index 0ce89c7f..2d7258ba 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -92,8 +92,8 @@ class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): def get_old_orig(self, location, remote_mnt): '''Identify which subvolume to snapshot from''' - # rawdisk upgrades use 'factory' - return os.path.join(remote_mnt, 'systems', 'factory', 'orig') + # rawdisk upgrades use 'default' + return os.path.join(remote_mnt, 'systems', 'default', 'orig') @contextlib.contextmanager def _created_orig_subvolume(self, location, remote_mnt, version_root): -- cgit v1.2.1 From bfc74c3a2a1c5058e18653f4fb28e0390b66d520 Mon Sep 17 00:00:00 2001 From: Pete Fotheringham Date: Fri, 5 Dec 2014 15:34:07 +0000 Subject: Add a reference to write.help file --- ssh-rsync.write | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) (limited to 'ssh-rsync.write') diff --git a/ssh-rsync.write b/ssh-rsync.write index 2d7258ba..c4577026 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -37,14 +37,8 @@ def ssh_runcmd_ignore_failure(location, command, **kwargs): class SshRsyncWriteExtension(morphlib.writeexts.WriteExtension): - '''Upgrade a running baserock system with ssh and rsync. - - It assumes the system is baserock-based and has a btrfs partition. - - The location command line argument is the 'user@hostname' string - that will be passed to ssh and rsync - - ''' + '''See ssh-rsync.write.help for documentation''' + def find_root_disk(self, location): '''Read /proc/mounts on location to find which device contains "/"''' -- cgit v1.2.1 From ed741d8d090086e2380f7b9d68ddc3bd122acb9f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Javier=20Jard=C3=B3n?= Date: Fri, 13 Mar 2015 18:18:55 +0000 Subject: Use the modern way of the GPL copyright header: URL instead real address Change-Id: I992dc0c1d40f563ade56a833162d409b02be90a0 --- ssh-rsync.write | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) (limited to 'ssh-rsync.write') diff --git a/ssh-rsync.write b/ssh-rsync.write index c4577026..6d596500 100755 --- a/ssh-rsync.write +++ b/ssh-rsync.write @@ -1,5 +1,5 @@ #!/usr/bin/python -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -11,8 +11,7 @@ # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +# with this program. If not, see . '''A Morph deployment write extension for upgrading systems over ssh.''' -- cgit v1.2.1