From f814cf6c4a8cae6b5a28349897626d42f5fd9f9e Mon Sep 17 00:00:00 2001 From: Richard Maw Date: Wed, 1 Oct 2014 21:01:40 +0000 Subject: Add distbuilding yarn Deploying after distbuild currently broken --- yarns/building.yarn | 33 ++++++++++ yarns/implementations.yarn | 146 ++++++++++++++++++++++++++++++++++++++++++--- yarns/morph.shell-lib | 20 +++++++ 3 files changed, 192 insertions(+), 7 deletions(-) diff --git a/yarns/building.yarn b/yarns/building.yarn index c708b5bb..5f79aa3c 100644 --- a/yarns/building.yarn +++ b/yarns/building.yarn @@ -8,3 +8,36 @@ Morph Building Tests AND the user creates an uncommitted system morphology called systems/base-system.morph for our architecture in system branch master THEN morph build the system systems/base-system.morph of the branch master FINALLY the git server is shut down + + SCENARIO distbuilding + GIVEN a workspace + AND a git server + AND a communal cache server + AND a distbuild worker + AND a distbuild controller + +Distbuilding works much the same way as regular building. + + WHEN the user checks out the system branch called master + AND the user creates an uncommitted system morphology called systems/base-system.morph for our architecture in system branch master + THEN morph distbuild the system systems/base-system.morph of the branch master + +Distbuilt systems can be deployed locally, since the building process +lets you download the artifacts for local use. + +Note: Currently broken. It's currently complaining about not having +repos cached locally. + +> GIVEN a cluster called test-cluster.morph in system branch master +> AND a system in cluster test-cluster.morph in branch master called test-system +> AND system test-system in cluster test-cluster.morph in branch master builds systems/base-system.morph +> AND system test-system in cluster test-cluster.morph in branch master has deployment type: sysroot +> AND system test-system in cluster test-cluster.morph in branch master has deployment location: test-system +> WHEN the user attempts to deploy the cluster test-cluster.morph in branch master +> THEN morph succeeded +> AND file workspace/master/test/morphs/test-system exists + + FINALLY the distbuild controller is terminated + AND the distbuild worker is terminated + AND the communal cache server is terminated + AND the git server is shut down diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index 86c3a9c4..71b8af60 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -683,10 +683,10 @@ Implementations for temporary build branch handling Implementation sections for building ==================================== - IMPLEMENTS WHEN the user (attempts to build|builds) the system (\S+) in branch (\S+) - cd "$DATADIR/workspace/$MATCH_3" - set build "$MATCH_2" - if [ $MATCH_1 == "builds" ]; then run_morph "$@" + IMPLEMENTS WHEN the user (attempts to )?((dist)?build)s? the system (\S+) in branch (\S+) + cd "$DATADIR/workspace/$MATCH_5" + set "$MATCH_2" "$MATCH_4" + if [ "$MATCH_1" != "attempts to " ]; then run_morph "$@" else attempt_morph "$@"; fi Implementation sections for cross-bootstraping @@ -872,9 +872,9 @@ variables in `$DATADIR/env`. We treat the value as a format string for Implementations for building systems ------------------------------------ - IMPLEMENTS THEN morph build the system (\S+) of the (branch|tag) (\S+) - cd "$DATADIR/workspace/$MATCH_3" - run_morph build "$MATCH_1" + IMPLEMENTS THEN morph ((dist)?build) the system (\S+) of the (branch|tag) (\S+) + cd "$DATADIR/workspace/$MATCH_5" + run_morph "$MATCH_1" "$MATCH_3" IMPLEMENTS WHEN the user builds (\S+) of the (\S+) (branch|tag) cd "$DATADIR/workspace/$MATCH_2" @@ -969,3 +969,135 @@ Altering morphologies in the workspace "$SRCDIR/scripts/edit-morph" cluster-system-set-deploy-variable \ "$DATADIR/workspace/$branch/test/morphs/$cluster" "$name" \ "$key" "$val" + + +Distbuild +========= + + IMPLEMENTS GIVEN a communal cache server + # The communal cache server has direct access to the git repositories + # and can have artifacts placed on it + artifact_dir="$DATADIR/shared-artifacts" + mkdir -p "$artifact_dir" + + read_cache_server_port_file="$DATADIR/read-cache-server-port" + read_cache_server_pid_file="$DATADIR/read-cache-server-pid" + start_cache_server "$read_cache_server_port_file" \ + "$read_cache_server_pid_file" \ + "$artifact_dir" + + write_cache_server_port_file="$DATADIR/write-cache-server-port" + write_cache_server_pid_file="$DATADIR/write-cache-server-pid" + start_cache_server "$write_cache_server_port_file" \ + "$write_cache_server_pid_file" \ + "$artifact_dir" --enable-writes + + IMPLEMENTS FINALLY the communal cache server is terminated + stop_daemon "$DATADIR/read-cache-server-pid" + stop_daemon "$DATADIR/write-cache-server-pid" + + IMPLEMENTS GIVEN a distbuild worker + # start worker cache server, so other workers can download results + worker_cachedir="$DATADIR/distbuild-worker-cache" + worker_artifacts="$worker_cachedir/artifacts" + mkdir -p "$worker_artifacts" + worker_cache_port_file="$DATADIR/worker-cache-server-port" + worker_cache_pid_file="$DATADIR/worker-cache-server-pid" + start_cache_server "$worker_cache_port_file" \ + "$worker_cache_pid_file" \ + "$worker_artifacts" + + # start worker daemon + worker_daemon_port_file="$DATADIR/worker-daemon-port" + worker_daemon_pid_file="$DATADIR/worker-daemon-pid" + mkfifo "$worker_daemon_port_file" + communal_cache_port="$(cat "$DATADIR/read-cache-server-port")" + start-stop-daemon --start --pidfile="$worker_daemon_pid_file" \ + --background --make-pidfile --verbose \ + --startas="$SRCDIR/morph" -- worker-daemon \ + --no-default-configs --config "$DATADIR/morph.conf" \ + --worker-daemon-port=0 \ + --worker-daemon-port-file="$worker_daemon_port_file" \ + --cachedir="$worker_cachedir" \ + --artifact-cache-server="http://localhost:$communal_cache_port/" \ + --git-resolve-cache-server="http://localhost:$communal_cache_port/" \ + --log="$DATADIR/worker-daemon-log" \ + >"$DATADIR/worker-daemon-out" 2>"$DATADIR/worker-daemon-err" + worker_daemon_port="$(cat "$worker_daemon_port_file")" + rm "$worker_daemon_port_file" + echo "$worker_daemon_port" >"$worker_daemon_port_file" + + # start worker helper + helper_pid_file="$DATADIR/worker-daemon-helper-pid" + start-stop-daemon --start --pidfile="$helper_pid_file" \ + --background --make-pidfile --verbose \ + --startas="$SRCDIR/distbuild-helper" -- \ + --no-default-configs \ + --parent-port="$worker_daemon_port" + + # set up builder config + install /dev/stdin <<'EOF' "$DATADIR/morph" + #!/bin/sh + exec "$SRCDIR/morph" --quiet \ + --cachedir-min-space=0 --tempdir-min-space=0 \ + --no-default-config --config "$DATADIR/morph.conf" \ + --cachedir "$DATADIR/distbuild-worker-cache" "$@" + EOF + + IMPLEMENTS FINALLY the distbuild worker is terminated + stop_daemon "$DATADIR/worker-cache-server-pid" + stop_daemon "$DATADIR/worker-daemon-pid" + stop_daemon "$DATADIR/worker-daemon-helper-pid" + + IMPLEMENTS GIVEN a distbuild controller + worker_cache_port_file="$DATADIR/worker-cache-server-port" + worker_cache_server_port="$(cat "$worker_cache_port_file")" + worker_daemon_port_file="$DATADIR/worker-daemon-port" + worker_daemon_port="$(cat "$worker_daemon_port_file")" + communal_cache_read_port="$(cat "$DATADIR/read-cache-server-port")" + communal_cache_write_port="$(cat "$DATADIR/write-cache-server-port")" + initiator_port_file="$DATADIR/controller-initiator-port" + mkfifo "$initiator_port_file" + helper_port_file="$DATADIR/controller-helper-port" + mkfifo "$helper_port_file" + controller_pid_file="$DATADIR/controller-pid" + start-stop-daemon --start --pidfile="$controller_pid_file" \ + --background --make-pidfile --verbose \ + --startas="$SRCDIR/morph" -- controller-daemon \ + --no-default-configs --config "$DATADIR/morph.conf" \ + --worker="localhost:$worker_daemon_port" \ + --worker-cache-server-port="$worker_cache_server_port" \ + --artifact-cache-server="http://localhost:$communal_cache_read_port/" \ + --git-resolve-cache-server="http://localhost:$communal_cache_read_port/" \ + --writeable-cache-server="http://localhost:$communal_cache_write_port/" \ + --controller-helper-port=0 \ + --controller-helper-port-file="$helper_port_file" \ + --controller-initiator-port=0 \ + --controller-initiator-port-file="$initiator_port_file" \ + --morph-instance="$DATADIR/morph" \ + --log="$DATADIR/controller-daemon-log" \ + >"$DATADIR/controller-daemon-out" 2>"$DATADIR/controller-daemon-err" + helper_port="$(cat "$helper_port_file")" + rm "$helper_port_file" + echo "$helper_port" >"$helper_port_file" + initiator_port="$(cat "$initiator_port_file")" + rm "$initiator_port_file" + echo "$initiator_port" >"$initiator_port_file" + + # start worker helper + helper_pid_file="$DATADIR/controller-helper-pid" + start-stop-daemon --start --pidfile="$helper_pid_file" \ + --background --make-pidfile --verbose \ + --startas="$SRCDIR/distbuild-helper" -- \ + --no-default-configs \ + --parent-port="$helper_port" + + # make builds use distbuild + echo "controller-initiator-port = $initiator_port" \ + >>"$DATADIR/morph.conf" + echo "controller-initiator-address = localhost" \ + >>"$DATADIR/morph.conf" + + IMPLEMENTS FINALLY the distbuild controller is terminated + stop_daemon "$DATADIR/controller-helper-pid" + stop_daemon "$DATADIR/controller-pid" diff --git a/yarns/morph.shell-lib b/yarns/morph.shell-lib index 9d67f2ab..9c13e449 100644 --- a/yarns/morph.shell-lib +++ b/yarns/morph.shell-lib @@ -184,3 +184,23 @@ slashify_colons() { echo "$1" | sed s,:,/,g } + +start_cache_server(){ + mkfifo "$1" + start-stop-daemon --start --pidfile="$2" \ + --background --make-pidfile --verbose \ + --startas="$SRCDIR/morph-cache-server" -- \ + --port-file="$1" --no-fcgi \ + --repo-dir="$DATADIR/gits" --direct-mode \ + --bundle-dir="$DATADIR/bundles" \ + --artifact-dir="$3" "$@" + port="$(cat "$1")" + rm "$1" + echo "$port" >"$1" +} + +stop_daemon(){ + if [ -e "$1" ]; then + start-stop-daemon --stop --pidfile "$1" --oknodo + fi +} -- cgit v1.2.1