Use scripts/distbuild for the Yarn distbuild scenario

4 files changed, 101 insertions, 139 deletions

diff --git a/build-log-test.py b/scripts/distbuild
index d7f3e9e4..285b1524 100644..100755
--- a/build-log-test.py
+++ b/scripts/distbuild
@@ -1,11 +1,31 @@
-# Sam's ultra special distbuild build logs testing program test suite
+#!/usr/bin/env python
+# Copyright (C) 2015  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+# Run a local instance of a Morph distributed build network.
 
 
 import os
+import select
 import subprocess
 import tempfile
 import time
 
+import cliapp
+
+
 
 # Some hackz if you are running Morph from the source tree.
 os.environ['PYTHONPATH'] = '/src/morph'
@@ -138,13 +158,54 @@ class MorphWorkerDaemonProcess(MorphListenerProcess):
             self, name, ports, argv, settings, log_path=log_path)
 
 
-class DistbuildTestHarness(object):
+class DistbuildTestHarness(cliapp.Application):
     '''Harness for running a distbuild network on a single machine.'''
 
     def __init__(self):
         self.process = dict()
+        super(DistbuildTestHarness, self).__init__()
+
+    def add_settings(self):
+        self.settings.string(
+            ['datadir'],
+            'location to cache gits and artifacts, and write log files')
+        self.settings.string(
+            ['morph-instance'],
+            'Path to Morph program that will run worker-build and '
+            'serialise-artifact commands')
+        self.settings.string(
+            ['port-file'],
+            'write port used by initiator to FILE, when ready')
+
+    def cmd_run(self, args):
+        '''Run a distbuild network.'''
+
+        try:
+            datadir = self.settings['datadir'] or tempfile.mkdtemp()
+
+            worker_cache, shared_cache = self.start_cache_servers(datadir)
+
+            controller = self.start_distbuild_network(
+                datadir, worker_cache, shared_cache,
+                morph_instance=self.settings['morph-instance'],
+                n_workers=1)
+
+            if self.settings['port-file']:
+                with open(self.settings['port-file'], 'w') as f:
+                    f.write('%s' % controller.controller_initiator_port)
+
+            print('Distbuild initiator listening on port %i' %
+                controller.controller_initiator_port)
+            print('Data in %s' % datadir)
+
+            # Run until we get a TERM signal.
+            while True:
+                select.select([], [], [], 1)
+                self.check_processes_are_running()
+        finally:
+            self.terminate_processes()
 
-    def run(self):
+    def cmd_build(self):
         '''Start a distbuild network and run a single build on it.'''
         workdir = tempfile.mkdtemp()
         print('Working directory: %s' % workdir)
@@ -227,17 +288,18 @@ class DistbuildTestHarness(object):
         return worker_cache, shared_cache
 
     def start_distbuild_network(self, workdir, worker_cache, shared_cache,
-                                n_workers=4):
-
-        # This is a hack so we can pass arguments to the Morph used to run
-        # worker-build and serialise-artifact.
-        worker_morph = os.path.join(workdir, 'worker-morph')
-        with open(worker_morph, 'w') as f:
-            cache_dir = os.path.join(workdir, 'worker-cache')
-            log = os.path.join(workdir, 'morph.log')
-            f.write('#!/bin/sh\n')
-            f.write('%s --cachedir=%s --log=%s $@\n' % (MORPH, cache_dir, log))
-        os.chmod(worker_morph, 0755)
+                                morph_instance=None, n_workers=4):
+        if morph_instance is None:
+            # This is a hack so we can pass arguments to the Morph used to run
+            # worker-build and serialise-artifact.
+            worker_morph = os.path.join(workdir, 'worker-morph')
+            with open(worker_morph, 'w') as f:
+                cache_dir = os.path.join(workdir, 'worker-cache')
+                log = os.path.join(workdir, 'morph.log')
+                f.write('#!/bin/sh\n')
+                f.write('%s --cachedir=%s --log=%s $@\n' % (MORPH, cache_dir, log))
+            os.chmod(worker_morph, 0755)
+            morph_instance = worker_morph
 
         workers = []
         for n in range(0, n_workers):
@@ -266,7 +328,7 @@ class DistbuildTestHarness(object):
             argv=[MORPH, 'controller-daemon'],
             settings={
                 'controller-initiator-address': 'localhost',
-                'morph-instance': worker_morph,
+                'morph-instance': morph_instance,
                 'worker': ','.join(workers),
                 'worker-cache-server-port': worker_cache.port,
                 'writeable-cache-server': 'http://localhost:%s' % shared_cache.port,
diff --git a/yarns/building.yarn b/yarns/building.yarn
index b5e46b73..9889fce6 100644
--- a/yarns/building.yarn
+++ b/yarns/building.yarn
@@ -72,9 +72,7 @@ Distbuilding
     ASSUMING the morph-cache-server can be run
     GIVEN a workspace
     AND a git server
-    AND a communal cache server
-    AND a distbuild worker
-    AND a distbuild controller
+    AND a distbuild network
 
 Distbuilding works much the same way as regular building.
 
@@ -97,9 +95,7 @@ repos cached locally.
 >   THEN morph succeeded
 >   AND file workspace/master/test/morphs/test-system exists
 
-    FINALLY the distbuild controller is terminated
-    AND the distbuild worker is terminated
-    AND the communal cache server is terminated
+    FINALLY the distbuild network is terminated
     AND the git server is shut down
 
 Empty strata don't build
diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn
index 90168517..67d7769d 100644
--- a/yarns/implementations.yarn
+++ b/yarns/implementations.yarn
@@ -1054,66 +1054,9 @@ Distbuild
     IMPLEMENTS ASSUMING the morph-cache-server can be run
     "$SRCDIR/morph-cache-server" --version
 
-    IMPLEMENTS GIVEN a communal cache server
-    # The communal cache server has direct access to the git repositories
-    # and can have artifacts placed on it
-    artifact_dir="$DATADIR/shared-artifacts"
-    mkdir -p "$artifact_dir"
-
-    read_cache_server_port_file="$DATADIR/read-cache-server-port"
-    read_cache_server_pid_file="$DATADIR/read-cache-server-pid"
-    start_cache_server "$read_cache_server_port_file" \
-                       "$read_cache_server_pid_file" \
-                       "$artifact_dir"
-
-    write_cache_server_port_file="$DATADIR/write-cache-server-port"
-    write_cache_server_pid_file="$DATADIR/write-cache-server-pid"
-    start_cache_server "$write_cache_server_port_file" \
-                       "$write_cache_server_pid_file" \
-                       "$artifact_dir" --enable-writes
-
-    IMPLEMENTS FINALLY the communal cache server is terminated
-    stop_daemon "$DATADIR/read-cache-server-pid"
-    stop_daemon "$DATADIR/write-cache-server-pid"
-
-    IMPLEMENTS GIVEN a distbuild worker
-    # start worker cache server, so other workers can download results
-    worker_cachedir="$DATADIR/distbuild-worker-cache"
-    worker_artifacts="$worker_cachedir/artifacts"
-    mkdir -p "$worker_artifacts"
-    worker_cache_port_file="$DATADIR/worker-cache-server-port"
-    worker_cache_pid_file="$DATADIR/worker-cache-server-pid"
-    start_cache_server "$worker_cache_port_file" \
-                       "$worker_cache_pid_file" \
-                       "$worker_artifacts"
-
-    # start worker daemon
-    worker_daemon_port_file="$DATADIR/worker-daemon-port"
-    worker_daemon_pid_file="$DATADIR/worker-daemon-pid"
-    mkfifo "$worker_daemon_port_file"
-    communal_cache_port="$(cat "$DATADIR/read-cache-server-port")"
-    start-stop-daemon --start --pidfile="$worker_daemon_pid_file" \
-        --background --make-pidfile --verbose \
-        --startas="$SRCDIR/morph" -- worker-daemon \
-        --no-default-configs --config "$DATADIR/morph.conf" \
-        --worker-daemon-port=0 \
-        --worker-daemon-port-file="$worker_daemon_port_file" \
-        --cachedir="$worker_cachedir" \
-        --artifact-cache-server="http://localhost:$communal_cache_port/" \
-        --git-resolve-cache-server="http://localhost:$communal_cache_port/" \
-        --log="$DATADIR/worker-daemon-log" \
-        >"$DATADIR/worker-daemon-out" 2>"$DATADIR/worker-daemon-err"
-    worker_daemon_port="$(cat "$worker_daemon_port_file")"
-    rm "$worker_daemon_port_file"
-    echo "$worker_daemon_port" >"$worker_daemon_port_file"
-
-    # start worker helper
-    helper_pid_file="$DATADIR/worker-daemon-helper-pid"
-    start-stop-daemon --start --pidfile="$helper_pid_file" \
-        --background --make-pidfile --verbose \
-        --startas="$SRCDIR/distbuild-helper" -- \
-        --no-default-configs \
-        --parent-port="$worker_daemon_port"
+    IMPLEMENTS GIVEN a distbuild network
+    distbuild_port_file="$DATADIR/distbuild-port"
+    distbuild_pid_file="$DATADIR/distbuild-pid"
 
     # set up builder config
     install /dev/stdin <<'EOF' "$DATADIR/morph"
@@ -1121,65 +1064,22 @@ Distbuild
     exec "$SRCDIR/morph" --quiet \
         --cachedir-min-space=0 --tempdir-min-space=0 \
         --no-default-config --config "$DATADIR/morph.conf" \
-        --cachedir "$DATADIR/distbuild-worker-cache" "$@"
+        --cachedir "$DATADIR/distbuild/worker-cache" "$@"
     EOF
 
-    IMPLEMENTS FINALLY the distbuild worker is terminated
-    stop_daemon "$DATADIR/worker-cache-server-pid"
-    stop_daemon "$DATADIR/worker-daemon-pid"
-    stop_daemon "$DATADIR/worker-daemon-helper-pid"
-
-    IMPLEMENTS GIVEN a distbuild controller
-    worker_cache_port_file="$DATADIR/worker-cache-server-port"
-    worker_cache_server_port="$(cat "$worker_cache_port_file")"
-    worker_daemon_port_file="$DATADIR/worker-daemon-port"
-    worker_daemon_port="$(cat "$worker_daemon_port_file")"
-    communal_cache_read_port="$(cat "$DATADIR/read-cache-server-port")"
-    communal_cache_write_port="$(cat "$DATADIR/write-cache-server-port")"
-    initiator_port_file="$DATADIR/controller-initiator-port"
-    mkfifo "$initiator_port_file"
-    helper_port_file="$DATADIR/controller-helper-port"
-    mkfifo "$helper_port_file"
-    controller_pid_file="$DATADIR/controller-pid"
-    start-stop-daemon --start --pidfile="$controller_pid_file" \
-        --background --make-pidfile --verbose \
-        --startas="$SRCDIR/morph" -- controller-daemon \
-        --no-default-configs --config "$DATADIR/morph.conf" \
-        --worker="localhost:$worker_daemon_port" \
-        --worker-cache-server-port="$worker_cache_server_port" \
-        --artifact-cache-server="http://localhost:$communal_cache_read_port/" \
-        --git-resolve-cache-server="http://localhost:$communal_cache_read_port/" \
-        --writeable-cache-server="http://localhost:$communal_cache_write_port/" \
-        --controller-helper-port=0 \
-        --controller-helper-port-file="$helper_port_file" \
-        --controller-initiator-port=0 \
-        --controller-initiator-port-file="$initiator_port_file" \
-        --morph-instance="$DATADIR/morph" \
-        --log="$DATADIR/controller-daemon-log" \
-        >"$DATADIR/controller-daemon-out" 2>"$DATADIR/controller-daemon-err"
-    helper_port="$(cat "$helper_port_file")"
-    rm "$helper_port_file"
-    echo "$helper_port" >"$helper_port_file"
-    initiator_port="$(cat "$initiator_port_file")"
-    rm "$initiator_port_file"
-    echo "$initiator_port" >"$initiator_port_file"
-
-    # start worker helper
-    helper_pid_file="$DATADIR/controller-helper-pid"
-    start-stop-daemon --start --pidfile="$helper_pid_file" \
-        --background --make-pidfile --verbose \
-        --startas="$SRCDIR/distbuild-helper" -- \
-        --no-default-configs \
-        --parent-port="$helper_port"
+    start_distbuild "$distbuild_port_file" \
+                    "$distbuild_pid_file" \
+                    "$DATADIR/morph"
+
+    distbuild_port=$(cat "$distbuild_port_file")
 
     # make builds use distbuild
-    echo "controller-initiator-port = $initiator_port" \
+    echo "controller-initiator-port = $distbuild_port" \
         >>"$DATADIR/morph.conf"
     echo "controller-initiator-address = localhost" \
         >>"$DATADIR/morph.conf"
     echo "initiator-step-output-dir = $DATADIR" \
         >>"$DATADIR/morph.conf"
 
-    IMPLEMENTS FINALLY the distbuild controller is terminated
-    stop_daemon "$DATADIR/controller-helper-pid"
-    stop_daemon "$DATADIR/controller-pid"
+    IMPLEMENTS FINALLY the distbuild network is terminated
+    stop_daemon "$DATADIR/distbuild-pid"
diff --git a/yarns/morph.shell-lib b/yarns/morph.shell-lib
index d9c9eff6..206f0e38 100644
--- a/yarns/morph.shell-lib
+++ b/yarns/morph.shell-lib
@@ -189,15 +189,19 @@ slashify_colons()
     echo "$1" | sed s,:,/,g
 }
 
-start_cache_server(){
+start_distbuild(){
     mkfifo "$1"
     start-stop-daemon --start --pidfile="$2" \
         --background --make-pidfile --verbose \
-        --startas="$SRCDIR/morph-cache-server" -- \
-        --port-file="$1" --no-fcgi \
-        --repo-dir="$DATADIR/gits" --direct-mode \
-        --bundle-dir="$DATADIR/bundles" \
-        --artifact-dir="$3" "$@"
+        --startas="$SRCDIR/scripts/distbuild" -- \
+        run \
+        --port-file="$1" \
+        --datadir="$DATADIR/distbuild" \
+        --morph-instance="$3" "$@"
+
+    # Wait for the process to be ready by watching the FIFO. Beware that if the
+    # process doesn't start fully (maybe it crashes or exists with an error)
+    # then this will hang forever.
     port="$(cat "$1")"
     rm "$1"
     echo "$port" >"$1"