Add distbuild-trove-nfsboot.write

The nfsboot.write deployment extension has been deprecated for a while
because it's not generally useful. It's only used for deploying
distbuild nodes to a Trove, as far as I know.

We still need to support setting up a bunch of machines that boot over
NFS from a Trove. But we can do this in a special-purpose .write
extension.

The new distbuild-trove-nfsboot.write is much more efficient than
the more generic nfsboot.write: instead of treating each system
individually (thus copying an almost identical ~2GB rootfs to the Trove
once per node) it copies the system image to the Trove once, and /then/
sets up a rootfs per node.

Upgrades are now supported, although the code assumes distbuild nodes
are stateless (as they should be) so nothing special is done for
upgrades, other than checking that there is already a version of the
given system in existance. The new extension does not create an
orig/ and run/ version of each system, because there is no need when the
deployed system is stateless.

There could be further gains in efficiency, but I don't have time to do
them right now. This write extension is full of compromises, its goal is
to better support the existing users who have a Trove and a distbuild
network deployed via NFS. It is specifically not intended to be useful
for other purposes.

Change-Id: I9a50c58b714ed272212d1d6c55b289aaa96051b1

1 files changed, 150 insertions, 0 deletions

diff --git a/distbuild-trove-nfsboot.check b/distbuild-trove-nfsboot.check
new file mode 100755
index 00000000..38c491e5
--- /dev/null
+++ b/distbuild-trove-nfsboot.check
@@ -0,0 +1,150 @@
+#!/usr/bin/python
+# Copyright (C) 2014-2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+'''Preparatory checks for Morph 'distbuild-trove-nfsboot' write extension'''
+
+import cliapp
+import logging
+import os
+
+import morphlib.writeexts
+
+
+class DistbuildTroveNFSBootCheckExtension(morphlib.writeexts.WriteExtension):
+
+    nfsboot_root = '/srv/nfsboot'
+    remote_user = 'root'
+
+    required_vars = [
+        'DISTBUILD_CONTROLLER',
+        'DISTBUILD_GIT_SERVER',
+        'DISTBUILD_SHARED_ARTIFACT_CACHE',
+        'DISTBUILD_TROVE_ID',
+        'DISTBUILD_WORKERS',
+        'DISTBUILD_WORKER_SSH_KEY',
+    ]
+
+    def system_path(self, system_name, version_label=None):
+        if version_label:
+            return os.path.join(self.nfsboot_root, system_name, 'systems',
+                                version_label, 'run')
+        else:
+            return os.path.join(self.nfsboot_root, system_name)
+
+    def process_args(self, args):
+        if len(args) != 1:
+            raise cliapp.AppException('Wrong number of command line args')
+
+        nfs_host = args[0]
+        nfs_netloc = '%s@%s' % (self.remote_user, nfs_host)
+
+        version_label = os.getenv('VERSION_LABEL', 'factory')
+
+        missing_vars = [var for var in self.required_vars
+                        if not var in os.environ]
+        if missing_vars:
+            raise cliapp.AppException(
+                'Please set: %s' % ', '.join(missing_vars))
+
+        controllers = os.getenv('DISTBUILD_CONTROLLER').split()
+        workers = os.getenv('DISTBUILD_WORKERS').split()
+
+        if len(controllers) != 1:
+            raise cliapp.AppException('Please specify exactly one controller.')
+
+        if len(workers) == 0:
+            raise cliapp.AppException('Please specify at least one worker.')
+
+        upgrade = self.get_environment_boolean('UPGRADE')
+
+        self.check_good_server(nfs_netloc)
+
+        system_names = set(controllers + workers)
+        for system_name in system_names:
+            if upgrade:
+                self.check_upgradeable(nfs_netloc, system_name, version_label)
+            else:
+                system_path = self.system_path(system_name)
+
+                if self.remote_directory_exists(nfs_netloc, system_path):
+                    if self.get_environment_boolean('OVERWRITE') == False:
+                        raise cliapp.AppException(
+                            'System %s already exists at %s:%s. Try `morph '
+                            'upgrade` instead of `morph deploy`.' % (
+                            system_name, nfs_netloc, system_path))
+
+    def check_good_server(self, netloc):
+        # FIXME: assumes root
+        self.check_ssh_connectivity(netloc.split('@')[-1])
+
+        # Is an NFS server
+        try:
+            cliapp.ssh_runcmd(
+                netloc, ['test', '-e', '/etc/exports'])
+        except cliapp.AppException:
+            raise cliapp.AppException('server %s is not an nfs server'
+                                      % netloc)
+        try:
+            cliapp.ssh_runcmd(
+                netloc, ['systemctl', 'is-enabled', 'nfs-server.service'])
+
+        except cliapp.AppException:
+            raise cliapp.AppException('server %s does not control its '
+                                      'nfs server by systemd' % netloc)
+
+        # TFTP server exports /srv/nfsboot/tftp
+        tftp_root = os.path.join(self.nfsboot_root, 'tftp')
+        try:
+            cliapp.ssh_runcmd(
+                netloc, ['test' , '-d', tftp_root])
+        except cliapp.AppException:
+            raise cliapp.AppException('server %s does not export %s' %
+                                     (netloc, tftp_root))
+
+    def check_upgradeable(self, nfs_netloc, system_name, version_label):
+        '''Check that there is already a version of the system present.
+
+        Distbuild nodes are stateless, so an upgrade is actually pretty much
+        the same as an initial deployment. This test is just a sanity check.
+
+        '''
+        system_path = self.system_path(system_name)
+        system_version_path = self.system_path(system_name, version_label)
+
+        if not self.remote_directory_exists(nfs_netloc, system_path):
+            raise cliapp.AppException(
+                'System %s not found at %s:%s, cannot deploy an upgrade.' % (
+                system_name, nfs_netloc, system_path))
+
+        if self.remote_directory_exists(nfs_netloc, system_version_path):
+            if self.get_environment_boolean('OVERWRITE'):
+                pass
+            else:
+                raise cliapp.AppException(
+                    'System %s version %s already exists at %s:%s.' % (
+                    system_name, version_label, nfs_netloc,
+                    system_version_path))
+
+    def remote_directory_exists(self, nfs_netloc, path):
+        try:
+            cliapp.ssh_runcmd(nfs_netloc, ['test', '-d', path])
+        except cliapp.AppException as e:
+            logging.debug('SSH exception: %s', e)
+            return False
+
+        return True
+
+
+DistbuildTroveNFSBootCheckExtension().run()