Use distbuild test harness in the Yarn distbuild scenariosam/distbuild-test-harness

3 files changed, 25 insertions, 125 deletions

diff --git a/yarns/building.yarn b/yarns/building.yarn
index 52f2b561..218dde4c 100644
--- a/yarns/building.yarn
+++ b/yarns/building.yarn
@@ -71,9 +71,7 @@ Distbuilding
     ASSUMING the morph-cache-server can be run
     GIVEN a workspace
     AND a git server
-    AND a communal cache server
-    AND a distbuild worker
-    AND a distbuild controller
+    AND a distbuild network
 
 Distbuilding works much the same way as regular building.
 
@@ -96,7 +94,5 @@ repos cached locally.
 >   THEN morph succeeded
 >   AND file workspace/master/test/morphs/test-system exists
 
-    FINALLY the distbuild controller is terminated
-    AND the distbuild worker is terminated
-    AND the communal cache server is terminated
+    FINALLY the distbuild network is terminated
     AND the git server is shut down
diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn
index 8b43286f..63f4e930 100644
--- a/yarns/implementations.yarn
+++ b/yarns/implementations.yarn
@@ -1029,66 +1029,9 @@ Distbuild
     IMPLEMENTS ASSUMING the morph-cache-server can be run
     "$SRCDIR/morph-cache-server" --version
 
-    IMPLEMENTS GIVEN a communal cache server
-    # The communal cache server has direct access to the git repositories
-    # and can have artifacts placed on it
-    artifact_dir="$DATADIR/shared-artifacts"
-    mkdir -p "$artifact_dir"
-
-    read_cache_server_port_file="$DATADIR/read-cache-server-port"
-    read_cache_server_pid_file="$DATADIR/read-cache-server-pid"
-    start_cache_server "$read_cache_server_port_file" \
-                       "$read_cache_server_pid_file" \
-                       "$artifact_dir"
-
-    write_cache_server_port_file="$DATADIR/write-cache-server-port"
-    write_cache_server_pid_file="$DATADIR/write-cache-server-pid"
-    start_cache_server "$write_cache_server_port_file" \
-                       "$write_cache_server_pid_file" \
-                       "$artifact_dir" --enable-writes
-
-    IMPLEMENTS FINALLY the communal cache server is terminated
-    stop_daemon "$DATADIR/read-cache-server-pid"
-    stop_daemon "$DATADIR/write-cache-server-pid"
-
-    IMPLEMENTS GIVEN a distbuild worker
-    # start worker cache server, so other workers can download results
-    worker_cachedir="$DATADIR/distbuild-worker-cache"
-    worker_artifacts="$worker_cachedir/artifacts"
-    mkdir -p "$worker_artifacts"
-    worker_cache_port_file="$DATADIR/worker-cache-server-port"
-    worker_cache_pid_file="$DATADIR/worker-cache-server-pid"
-    start_cache_server "$worker_cache_port_file" \
-                       "$worker_cache_pid_file" \
-                       "$worker_artifacts"
-
-    # start worker daemon
-    worker_daemon_port_file="$DATADIR/worker-daemon-port"
-    worker_daemon_pid_file="$DATADIR/worker-daemon-pid"
-    mkfifo "$worker_daemon_port_file"
-    communal_cache_port="$(cat "$DATADIR/read-cache-server-port")"
-    start-stop-daemon --start --pidfile="$worker_daemon_pid_file" \
-        --background --make-pidfile --verbose \
-        --startas="$SRCDIR/morph" -- worker-daemon \
-        --no-default-configs --config "$DATADIR/morph.conf" \
-        --worker-daemon-port=0 \
-        --worker-daemon-port-file="$worker_daemon_port_file" \
-        --cachedir="$worker_cachedir" \
-        --artifact-cache-server="http://localhost:$communal_cache_port/" \
-        --git-resolve-cache-server="http://localhost:$communal_cache_port/" \
-        --log="$DATADIR/worker-daemon-log" \
-        >"$DATADIR/worker-daemon-out" 2>"$DATADIR/worker-daemon-err"
-    worker_daemon_port="$(cat "$worker_daemon_port_file")"
-    rm "$worker_daemon_port_file"
-    echo "$worker_daemon_port" >"$worker_daemon_port_file"
-
-    # start worker helper
-    helper_pid_file="$DATADIR/worker-daemon-helper-pid"
-    start-stop-daemon --start --pidfile="$helper_pid_file" \
-        --background --make-pidfile --verbose \
-        --startas="$SRCDIR/distbuild-helper" -- \
-        --no-default-configs \
-        --parent-port="$worker_daemon_port"
+    IMPLEMENTS GIVEN a distbuild network
+    distbuild_port_file="$DATADIR/distbuild-port"
+    distbuild_pid_file="$DATADIR/distbuild-pid"
 
     # set up builder config
     install /dev/stdin <<'EOF' "$DATADIR/morph"
@@ -1096,65 +1039,22 @@ Distbuild
     exec "$SRCDIR/morph" --quiet \
         --cachedir-min-space=0 --tempdir-min-space=0 \
         --no-default-config --config "$DATADIR/morph.conf" \
-        --cachedir "$DATADIR/distbuild-worker-cache" "$@"
+        --cachedir "$DATADIR/distbuild/worker-cache" "$@"
     EOF
 
-    IMPLEMENTS FINALLY the distbuild worker is terminated
-    stop_daemon "$DATADIR/worker-cache-server-pid"
-    stop_daemon "$DATADIR/worker-daemon-pid"
-    stop_daemon "$DATADIR/worker-daemon-helper-pid"
-
-    IMPLEMENTS GIVEN a distbuild controller
-    worker_cache_port_file="$DATADIR/worker-cache-server-port"
-    worker_cache_server_port="$(cat "$worker_cache_port_file")"
-    worker_daemon_port_file="$DATADIR/worker-daemon-port"
-    worker_daemon_port="$(cat "$worker_daemon_port_file")"
-    communal_cache_read_port="$(cat "$DATADIR/read-cache-server-port")"
-    communal_cache_write_port="$(cat "$DATADIR/write-cache-server-port")"
-    initiator_port_file="$DATADIR/controller-initiator-port"
-    mkfifo "$initiator_port_file"
-    helper_port_file="$DATADIR/controller-helper-port"
-    mkfifo "$helper_port_file"
-    controller_pid_file="$DATADIR/controller-pid"
-    start-stop-daemon --start --pidfile="$controller_pid_file" \
-        --background --make-pidfile --verbose \
-        --startas="$SRCDIR/morph" -- controller-daemon \
-        --no-default-configs --config "$DATADIR/morph.conf" \
-        --worker="localhost:$worker_daemon_port" \
-        --worker-cache-server-port="$worker_cache_server_port" \
-        --artifact-cache-server="http://localhost:$communal_cache_read_port/" \
-        --git-resolve-cache-server="http://localhost:$communal_cache_read_port/" \
-        --writeable-cache-server="http://localhost:$communal_cache_write_port/" \
-        --controller-helper-port=0 \
-        --controller-helper-port-file="$helper_port_file" \
-        --controller-initiator-port=0 \
-        --controller-initiator-port-file="$initiator_port_file" \
-        --morph-instance="$DATADIR/morph" \
-        --log="$DATADIR/controller-daemon-log" \
-        >"$DATADIR/controller-daemon-out" 2>"$DATADIR/controller-daemon-err"
-    helper_port="$(cat "$helper_port_file")"
-    rm "$helper_port_file"
-    echo "$helper_port" >"$helper_port_file"
-    initiator_port="$(cat "$initiator_port_file")"
-    rm "$initiator_port_file"
-    echo "$initiator_port" >"$initiator_port_file"
-
-    # start worker helper
-    helper_pid_file="$DATADIR/controller-helper-pid"
-    start-stop-daemon --start --pidfile="$helper_pid_file" \
-        --background --make-pidfile --verbose \
-        --startas="$SRCDIR/distbuild-helper" -- \
-        --no-default-configs \
-        --parent-port="$helper_port"
+    start_distbuild "$distbuild_port_file" \
+                    "$distbuild_pid_file" \
+                    "$DATADIR/morph"
+
+    distbuild_port=$(cat "$distbuild_port_file")
 
     # make builds use distbuild
-    echo "controller-initiator-port = $initiator_port" \
+    echo "controller-initiator-port = $distbuild_port" \
         >>"$DATADIR/morph.conf"
     echo "controller-initiator-address = localhost" \
         >>"$DATADIR/morph.conf"
     echo "initiator-step-output-dir = $DATADIR" \
         >>"$DATADIR/morph.conf"
 
-    IMPLEMENTS FINALLY the distbuild controller is terminated
-    stop_daemon "$DATADIR/controller-helper-pid"
-    stop_daemon "$DATADIR/controller-pid"
+    IMPLEMENTS FINALLY the distbuild network is terminated
+    stop_daemon "$DATADIR/distbuild-pid"
diff --git a/yarns/morph.shell-lib b/yarns/morph.shell-lib
index d9c9eff6..c5de80df 100644
--- a/yarns/morph.shell-lib
+++ b/yarns/morph.shell-lib
@@ -3,7 +3,7 @@
 # The shell functions in this library are meant to make writing IMPLEMENTS
 # sections for yarn scenario tests easier.
 
-# Copyright (C) 2013-2014  Codethink Limited
+# Copyright (C) 2013-2015  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -189,15 +189,19 @@ slashify_colons()
     echo "$1" | sed s,:,/,g
 }
 
-start_cache_server(){
+start_distbuild(){
     mkfifo "$1"
     start-stop-daemon --start --pidfile="$2" \
         --background --make-pidfile --verbose \
-        --startas="$SRCDIR/morph-cache-server" -- \
-        --port-file="$1" --no-fcgi \
-        --repo-dir="$DATADIR/gits" --direct-mode \
-        --bundle-dir="$DATADIR/bundles" \
-        --artifact-dir="$3" "$@"
+        --startas="$SRCDIR/scripts/distbuild" -- \
+        run \
+        --port-file="$1" \
+        --datadir="$DATADIR/distbuild" \
+        --morph-instance="$3" "$@"
+
+    # Wait for the process to be ready by watching the FIFO. Beware that if the
+    # process doesn't start fully (maybe it crashes or exists with an error)
+    # then this will hang forever.
     port="$(cat "$1")"
     rm "$1"
     echo "$port" >"$1"