From 626d62feccefda2a57abd5f71da9c4fb164c604c Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Fri, 13 Feb 2015 17:25:39 +0000 Subject: Use distbuild test harness in the Yarn distbuild scenario --- yarns/building.yarn | 8 +-- yarns/implementations.yarn | 124 +++++---------------------------------------- yarns/morph.shell-lib | 18 ++++--- 3 files changed, 25 insertions(+), 125 deletions(-) diff --git a/yarns/building.yarn b/yarns/building.yarn index 52f2b561..218dde4c 100644 --- a/yarns/building.yarn +++ b/yarns/building.yarn @@ -71,9 +71,7 @@ Distbuilding ASSUMING the morph-cache-server can be run GIVEN a workspace AND a git server - AND a communal cache server - AND a distbuild worker - AND a distbuild controller + AND a distbuild network Distbuilding works much the same way as regular building. @@ -96,7 +94,5 @@ repos cached locally. > THEN morph succeeded > AND file workspace/master/test/morphs/test-system exists - FINALLY the distbuild controller is terminated - AND the distbuild worker is terminated - AND the communal cache server is terminated + FINALLY the distbuild network is terminated AND the git server is shut down diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index 8b43286f..63f4e930 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -1029,66 +1029,9 @@ Distbuild IMPLEMENTS ASSUMING the morph-cache-server can be run "$SRCDIR/morph-cache-server" --version - IMPLEMENTS GIVEN a communal cache server - # The communal cache server has direct access to the git repositories - # and can have artifacts placed on it - artifact_dir="$DATADIR/shared-artifacts" - mkdir -p "$artifact_dir" - - read_cache_server_port_file="$DATADIR/read-cache-server-port" - read_cache_server_pid_file="$DATADIR/read-cache-server-pid" - start_cache_server "$read_cache_server_port_file" \ - "$read_cache_server_pid_file" \ - "$artifact_dir" - - write_cache_server_port_file="$DATADIR/write-cache-server-port" - write_cache_server_pid_file="$DATADIR/write-cache-server-pid" - start_cache_server "$write_cache_server_port_file" \ - "$write_cache_server_pid_file" \ - "$artifact_dir" --enable-writes - - IMPLEMENTS FINALLY the communal cache server is terminated - stop_daemon "$DATADIR/read-cache-server-pid" - stop_daemon "$DATADIR/write-cache-server-pid" - - IMPLEMENTS GIVEN a distbuild worker - # start worker cache server, so other workers can download results - worker_cachedir="$DATADIR/distbuild-worker-cache" - worker_artifacts="$worker_cachedir/artifacts" - mkdir -p "$worker_artifacts" - worker_cache_port_file="$DATADIR/worker-cache-server-port" - worker_cache_pid_file="$DATADIR/worker-cache-server-pid" - start_cache_server "$worker_cache_port_file" \ - "$worker_cache_pid_file" \ - "$worker_artifacts" - - # start worker daemon - worker_daemon_port_file="$DATADIR/worker-daemon-port" - worker_daemon_pid_file="$DATADIR/worker-daemon-pid" - mkfifo "$worker_daemon_port_file" - communal_cache_port="$(cat "$DATADIR/read-cache-server-port")" - start-stop-daemon --start --pidfile="$worker_daemon_pid_file" \ - --background --make-pidfile --verbose \ - --startas="$SRCDIR/morph" -- worker-daemon \ - --no-default-configs --config "$DATADIR/morph.conf" \ - --worker-daemon-port=0 \ - --worker-daemon-port-file="$worker_daemon_port_file" \ - --cachedir="$worker_cachedir" \ - --artifact-cache-server="http://localhost:$communal_cache_port/" \ - --git-resolve-cache-server="http://localhost:$communal_cache_port/" \ - --log="$DATADIR/worker-daemon-log" \ - >"$DATADIR/worker-daemon-out" 2>"$DATADIR/worker-daemon-err" - worker_daemon_port="$(cat "$worker_daemon_port_file")" - rm "$worker_daemon_port_file" - echo "$worker_daemon_port" >"$worker_daemon_port_file" - - # start worker helper - helper_pid_file="$DATADIR/worker-daemon-helper-pid" - start-stop-daemon --start --pidfile="$helper_pid_file" \ - --background --make-pidfile --verbose \ - --startas="$SRCDIR/distbuild-helper" -- \ - --no-default-configs \ - --parent-port="$worker_daemon_port" + IMPLEMENTS GIVEN a distbuild network + distbuild_port_file="$DATADIR/distbuild-port" + distbuild_pid_file="$DATADIR/distbuild-pid" # set up builder config install /dev/stdin <<'EOF' "$DATADIR/morph" @@ -1096,65 +1039,22 @@ Distbuild exec "$SRCDIR/morph" --quiet \ --cachedir-min-space=0 --tempdir-min-space=0 \ --no-default-config --config "$DATADIR/morph.conf" \ - --cachedir "$DATADIR/distbuild-worker-cache" "$@" + --cachedir "$DATADIR/distbuild/worker-cache" "$@" EOF - IMPLEMENTS FINALLY the distbuild worker is terminated - stop_daemon "$DATADIR/worker-cache-server-pid" - stop_daemon "$DATADIR/worker-daemon-pid" - stop_daemon "$DATADIR/worker-daemon-helper-pid" - - IMPLEMENTS GIVEN a distbuild controller - worker_cache_port_file="$DATADIR/worker-cache-server-port" - worker_cache_server_port="$(cat "$worker_cache_port_file")" - worker_daemon_port_file="$DATADIR/worker-daemon-port" - worker_daemon_port="$(cat "$worker_daemon_port_file")" - communal_cache_read_port="$(cat "$DATADIR/read-cache-server-port")" - communal_cache_write_port="$(cat "$DATADIR/write-cache-server-port")" - initiator_port_file="$DATADIR/controller-initiator-port" - mkfifo "$initiator_port_file" - helper_port_file="$DATADIR/controller-helper-port" - mkfifo "$helper_port_file" - controller_pid_file="$DATADIR/controller-pid" - start-stop-daemon --start --pidfile="$controller_pid_file" \ - --background --make-pidfile --verbose \ - --startas="$SRCDIR/morph" -- controller-daemon \ - --no-default-configs --config "$DATADIR/morph.conf" \ - --worker="localhost:$worker_daemon_port" \ - --worker-cache-server-port="$worker_cache_server_port" \ - --artifact-cache-server="http://localhost:$communal_cache_read_port/" \ - --git-resolve-cache-server="http://localhost:$communal_cache_read_port/" \ - --writeable-cache-server="http://localhost:$communal_cache_write_port/" \ - --controller-helper-port=0 \ - --controller-helper-port-file="$helper_port_file" \ - --controller-initiator-port=0 \ - --controller-initiator-port-file="$initiator_port_file" \ - --morph-instance="$DATADIR/morph" \ - --log="$DATADIR/controller-daemon-log" \ - >"$DATADIR/controller-daemon-out" 2>"$DATADIR/controller-daemon-err" - helper_port="$(cat "$helper_port_file")" - rm "$helper_port_file" - echo "$helper_port" >"$helper_port_file" - initiator_port="$(cat "$initiator_port_file")" - rm "$initiator_port_file" - echo "$initiator_port" >"$initiator_port_file" - - # start worker helper - helper_pid_file="$DATADIR/controller-helper-pid" - start-stop-daemon --start --pidfile="$helper_pid_file" \ - --background --make-pidfile --verbose \ - --startas="$SRCDIR/distbuild-helper" -- \ - --no-default-configs \ - --parent-port="$helper_port" + start_distbuild "$distbuild_port_file" \ + "$distbuild_pid_file" \ + "$DATADIR/morph" + + distbuild_port=$(cat "$distbuild_port_file") # make builds use distbuild - echo "controller-initiator-port = $initiator_port" \ + echo "controller-initiator-port = $distbuild_port" \ >>"$DATADIR/morph.conf" echo "controller-initiator-address = localhost" \ >>"$DATADIR/morph.conf" echo "initiator-step-output-dir = $DATADIR" \ >>"$DATADIR/morph.conf" - IMPLEMENTS FINALLY the distbuild controller is terminated - stop_daemon "$DATADIR/controller-helper-pid" - stop_daemon "$DATADIR/controller-pid" + IMPLEMENTS FINALLY the distbuild network is terminated + stop_daemon "$DATADIR/distbuild-pid" diff --git a/yarns/morph.shell-lib b/yarns/morph.shell-lib index d9c9eff6..c5de80df 100644 --- a/yarns/morph.shell-lib +++ b/yarns/morph.shell-lib @@ -3,7 +3,7 @@ # The shell functions in this library are meant to make writing IMPLEMENTS # sections for yarn scenario tests easier. -# Copyright (C) 2013-2014 Codethink Limited +# Copyright (C) 2013-2015 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -189,15 +189,19 @@ slashify_colons() echo "$1" | sed s,:,/,g } -start_cache_server(){ +start_distbuild(){ mkfifo "$1" start-stop-daemon --start --pidfile="$2" \ --background --make-pidfile --verbose \ - --startas="$SRCDIR/morph-cache-server" -- \ - --port-file="$1" --no-fcgi \ - --repo-dir="$DATADIR/gits" --direct-mode \ - --bundle-dir="$DATADIR/bundles" \ - --artifact-dir="$3" "$@" + --startas="$SRCDIR/scripts/distbuild" -- \ + run \ + --port-file="$1" \ + --datadir="$DATADIR/distbuild" \ + --morph-instance="$3" "$@" + + # Wait for the process to be ready by watching the FIFO. Beware that if the + # process doesn't start fully (maybe it crashes or exists with an error) + # then this will hang forever. port="$(cat "$1")" rm "$1" echo "$port" >"$1" -- cgit v1.2.1