docker.write: First attempt

Uses netcat to transfer the image into the Docker commandline client.

This kind of sucks, and also there's no way of knowing when the remote
netcat has started so the send often fails because the receiver isn't
ready yet.

1 files changed, 196 insertions, 0 deletions

diff --git a/morphlib/exts/docker.write b/morphlib/exts/docker.write
new file mode 100644
index 00000000..508b3d9c
--- /dev/null
+++ b/morphlib/exts/docker.write
@@ -0,0 +1,196 @@
+#!/usr/bin/python
+# Copyright (C) 2014  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+'''A Morph deployment write extension for deploying to Docker hosts'''
+
+
+import cliapp
+import contextlib
+import logging
+import os
+import shutil
+import subprocess
+import sys
+import tarfile
+import threading
+import time
+import urlparse
+
+import morphlib.writeexts
+
+
+class DockerWriteExtension(morphlib.writeexts.WriteExtension):
+
+    '''Create a Docker image or container from a Morph deployment.
+
+    THIS IS A PROTOTYPE. IT USES NETCAT TO SEND THE IMAGE WHICH IS RATHER
+    FRAGILE AND SUCKY.
+
+    The location command line argument is a network location that should be
+    accessible over SSH, followed by the name of the image to be created.
+
+        docker://HOST/IMAGE
+
+    Docker image names commonly containly follow the form 'owner/name'. If
+    a VERSION_LABEL setting is supplied, this will be used to tag the image.
+
+    '''
+
+    def process_args(self, args):
+        if len(args) != 2:
+            raise cliapp.AppException('Wrong number of command line args')
+
+        temp_root, location = args
+
+        # config parameters:
+        #   image name
+
+        docker_host, image_name = self.parse_location(location)
+
+        self.import_image(temp_root, docker_host, image_name)
+
+        autostart = self.get_environment_boolean('AUTOSTART')
+
+        self.status(
+            msg='Docker image %(image_name)s has been created',
+            image_name=image_name)
+
+    def parse_location(self, location):
+        '''Parse the location argument to get relevant data.'''
+
+        x = urlparse.urlparse(location)
+        return x.netloc, x.path[1:]
+
+    def import_image(self, fs_root, ssh_host, image_name):
+        '''Transfer disk image to a Docker image on a remote host.
+
+        This is currently done using SSH and netcat, rather than the Docker
+        remote API. While the Docker daemon can be bound directly to a TCP
+        port, this socket provides root access on the host for anyone that
+        can access that port.
+
+        '''
+
+        self.status(msg='Transferring disk image')
+
+        port = '2222'
+
+        tarpipe_read, tarpipe_write = os.pipe()
+
+        def create_tar():
+            try:
+                # using tarfile.TarFile.gzopen() and passing compresslevel=1
+                # seems to result in compresslevel=9 anyway. That's completely
+                # unusable on ARM CPUs so it's important to force
+                # compresslevel=1 or something low.
+                import gzip
+                gzstream = gzip.GzipFile(
+                    mode='wb',
+                    compresslevel=1,
+                    fileobj=os.fdopen(tarpipe_write, 'w'))
+                tar = tarfile.TarFile.gzopen(
+                    name='docker.write-temp',
+                    mode='w',
+                    compresslevel=1,
+                    fileobj=gzstream)
+                logging.debug("Creating tar of rootfs")
+                tar.add(fs_root, recursive=True)
+                tar.close()
+                logging.debug('Tar complete')
+            except IOError as e:
+                # Most probably due to SIGPIPE due to the send process
+                # dying.
+                logging.debug('Writing image data failed due to %s', e)
+
+        @contextlib.contextmanager
+        def send():
+            hostname = ssh_host.split('@')[-1]
+            tarpipe_read_file = os.fdopen(tarpipe_read, 'r')
+            process = subprocess.Popen(
+                ['nc', hostname, port],
+                stdin=tarpipe_read_file,
+                stderr=subprocess.PIPE)
+            try:
+                yield process
+            except BaseException as e:
+                if process.poll() is None:
+                    logging.debug('Killing send process due to %s', e)
+                    process.kill()
+                raise
+            else:
+                process.terminate()
+            finally:
+                process.wait()
+                tarpipe_read_file.close()
+
+        @contextlib.contextmanager
+        def receive():
+            # Open subprocess to pipe the tar file using netcat
+            receiver_cmd = cliapp.shell_quote(
+                'nc -l -p %s > /tmp/sam-docker.img' % port)
+                #'nc -l -p %s | sudo docker import - %s' % (port, image_name))
+            logging.debug('Runcmd: %s', receiver_cmd)
+            # -t just so I can run sudo at the other end for now
+            process = subprocess.Popen(
+                ['ssh', '-oNumberOfPasswordPrompts=0', ssh_host, 'sh', '-c', receiver_cmd],
+                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            try:
+                yield process
+            except BaseException as e:
+                if process.poll() is None:
+                    logging.debug('Killing receive process due to %s', e)
+                    process.kill()
+                raise
+            finally:
+                process.wait()
+
+        with receive() as receive_process:
+            # Oh god! WAit for the netcat process to start ... we hope.
+            time.sleep(1)
+            with send() as send_process:
+                tar_thread = threading.Thread(
+                    name='tar-create', target=create_tar)
+                print "Starting to create the tar ..."
+                tar_thread.start()
+                while tar_thread.is_alive():
+                    time.sleep(1)
+                    print "Asleep"
+
+                    code = receive_process.poll()
+                    if code is not None:
+                        output = receive_process.stderr.read()
+                        output += '\n' + receive_process.stdout.read()
+                        raise cliapp.AppException(
+                            'Receive process exited with code %i, output %s' %
+                            (code, output))
+
+                    code = send_process.poll()
+                    if code is not None:
+                        output = send_process.stderr.read()
+                        raise cliapp.AppException(
+                            'Send process exited with code %i, output %s' %
+                            (code, output))
+
+        # FIXME: make path relative using filter
+        # You can't do this, of course, because the buffer of the pipe gets
+        # full ....
+        # What if you did a netcat locally to test??? That's a good idea!
+
+        print "OK!"
+
+
+DockerWriteExtension().run()