summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorRichard Maw <richard.maw@gmail.com>2014-10-01 21:01:40 +0000
committerRichard Maw <richard.maw@codethink.co.uk>2014-10-02 13:09:31 +0000
commitf814cf6c4a8cae6b5a28349897626d42f5fd9f9e (patch)
tree1335f96af1a2fad7521bd5c3a0fdb95cf2edb915
parent5ae8a2eff6f179bdb566bb1556626e708cfa1a8e (diff)
downloadmorph-f814cf6c4a8cae6b5a28349897626d42f5fd9f9e.tar.gz
Add distbuilding yarn
Deploying after distbuild currently broken
-rw-r--r--yarns/building.yarn33
-rw-r--r--yarns/implementations.yarn146
-rw-r--r--yarns/morph.shell-lib20
3 files changed, 192 insertions, 7 deletions
diff --git a/yarns/building.yarn b/yarns/building.yarn
index c708b5bb..5f79aa3c 100644
--- a/yarns/building.yarn
+++ b/yarns/building.yarn
@@ -8,3 +8,36 @@ Morph Building Tests
AND the user creates an uncommitted system morphology called systems/base-system.morph for our architecture in system branch master
THEN morph build the system systems/base-system.morph of the branch master
FINALLY the git server is shut down
+
+ SCENARIO distbuilding
+ GIVEN a workspace
+ AND a git server
+ AND a communal cache server
+ AND a distbuild worker
+ AND a distbuild controller
+
+Distbuilding works much the same way as regular building.
+
+ WHEN the user checks out the system branch called master
+ AND the user creates an uncommitted system morphology called systems/base-system.morph for our architecture in system branch master
+ THEN morph distbuild the system systems/base-system.morph of the branch master
+
+Distbuilt systems can be deployed locally, since the building process
+lets you download the artifacts for local use.
+
+Note: Currently broken. It's currently complaining about not having
+repos cached locally.
+
+> GIVEN a cluster called test-cluster.morph in system branch master
+> AND a system in cluster test-cluster.morph in branch master called test-system
+> AND system test-system in cluster test-cluster.morph in branch master builds systems/base-system.morph
+> AND system test-system in cluster test-cluster.morph in branch master has deployment type: sysroot
+> AND system test-system in cluster test-cluster.morph in branch master has deployment location: test-system
+> WHEN the user attempts to deploy the cluster test-cluster.morph in branch master
+> THEN morph succeeded
+> AND file workspace/master/test/morphs/test-system exists
+
+ FINALLY the distbuild controller is terminated
+ AND the distbuild worker is terminated
+ AND the communal cache server is terminated
+ AND the git server is shut down
diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn
index 86c3a9c4..71b8af60 100644
--- a/yarns/implementations.yarn
+++ b/yarns/implementations.yarn
@@ -683,10 +683,10 @@ Implementations for temporary build branch handling
Implementation sections for building
====================================
- IMPLEMENTS WHEN the user (attempts to build|builds) the system (\S+) in branch (\S+)
- cd "$DATADIR/workspace/$MATCH_3"
- set build "$MATCH_2"
- if [ $MATCH_1 == "builds" ]; then run_morph "$@"
+ IMPLEMENTS WHEN the user (attempts to )?((dist)?build)s? the system (\S+) in branch (\S+)
+ cd "$DATADIR/workspace/$MATCH_5"
+ set "$MATCH_2" "$MATCH_4"
+ if [ "$MATCH_1" != "attempts to " ]; then run_morph "$@"
else attempt_morph "$@"; fi
Implementation sections for cross-bootstraping
@@ -872,9 +872,9 @@ variables in `$DATADIR/env`. We treat the value as a format string for
Implementations for building systems
------------------------------------
- IMPLEMENTS THEN morph build the system (\S+) of the (branch|tag) (\S+)
- cd "$DATADIR/workspace/$MATCH_3"
- run_morph build "$MATCH_1"
+ IMPLEMENTS THEN morph ((dist)?build) the system (\S+) of the (branch|tag) (\S+)
+ cd "$DATADIR/workspace/$MATCH_5"
+ run_morph "$MATCH_1" "$MATCH_3"
IMPLEMENTS WHEN the user builds (\S+) of the (\S+) (branch|tag)
cd "$DATADIR/workspace/$MATCH_2"
@@ -969,3 +969,135 @@ Altering morphologies in the workspace
"$SRCDIR/scripts/edit-morph" cluster-system-set-deploy-variable \
"$DATADIR/workspace/$branch/test/morphs/$cluster" "$name" \
"$key" "$val"
+
+
+Distbuild
+=========
+
+ IMPLEMENTS GIVEN a communal cache server
+ # The communal cache server has direct access to the git repositories
+ # and can have artifacts placed on it
+ artifact_dir="$DATADIR/shared-artifacts"
+ mkdir -p "$artifact_dir"
+
+ read_cache_server_port_file="$DATADIR/read-cache-server-port"
+ read_cache_server_pid_file="$DATADIR/read-cache-server-pid"
+ start_cache_server "$read_cache_server_port_file" \
+ "$read_cache_server_pid_file" \
+ "$artifact_dir"
+
+ write_cache_server_port_file="$DATADIR/write-cache-server-port"
+ write_cache_server_pid_file="$DATADIR/write-cache-server-pid"
+ start_cache_server "$write_cache_server_port_file" \
+ "$write_cache_server_pid_file" \
+ "$artifact_dir" --enable-writes
+
+ IMPLEMENTS FINALLY the communal cache server is terminated
+ stop_daemon "$DATADIR/read-cache-server-pid"
+ stop_daemon "$DATADIR/write-cache-server-pid"
+
+ IMPLEMENTS GIVEN a distbuild worker
+ # start worker cache server, so other workers can download results
+ worker_cachedir="$DATADIR/distbuild-worker-cache"
+ worker_artifacts="$worker_cachedir/artifacts"
+ mkdir -p "$worker_artifacts"
+ worker_cache_port_file="$DATADIR/worker-cache-server-port"
+ worker_cache_pid_file="$DATADIR/worker-cache-server-pid"
+ start_cache_server "$worker_cache_port_file" \
+ "$worker_cache_pid_file" \
+ "$worker_artifacts"
+
+ # start worker daemon
+ worker_daemon_port_file="$DATADIR/worker-daemon-port"
+ worker_daemon_pid_file="$DATADIR/worker-daemon-pid"
+ mkfifo "$worker_daemon_port_file"
+ communal_cache_port="$(cat "$DATADIR/read-cache-server-port")"
+ start-stop-daemon --start --pidfile="$worker_daemon_pid_file" \
+ --background --make-pidfile --verbose \
+ --startas="$SRCDIR/morph" -- worker-daemon \
+ --no-default-configs --config "$DATADIR/morph.conf" \
+ --worker-daemon-port=0 \
+ --worker-daemon-port-file="$worker_daemon_port_file" \
+ --cachedir="$worker_cachedir" \
+ --artifact-cache-server="http://localhost:$communal_cache_port/" \
+ --git-resolve-cache-server="http://localhost:$communal_cache_port/" \
+ --log="$DATADIR/worker-daemon-log" \
+ >"$DATADIR/worker-daemon-out" 2>"$DATADIR/worker-daemon-err"
+ worker_daemon_port="$(cat "$worker_daemon_port_file")"
+ rm "$worker_daemon_port_file"
+ echo "$worker_daemon_port" >"$worker_daemon_port_file"
+
+ # start worker helper
+ helper_pid_file="$DATADIR/worker-daemon-helper-pid"
+ start-stop-daemon --start --pidfile="$helper_pid_file" \
+ --background --make-pidfile --verbose \
+ --startas="$SRCDIR/distbuild-helper" -- \
+ --no-default-configs \
+ --parent-port="$worker_daemon_port"
+
+ # set up builder config
+ install /dev/stdin <<'EOF' "$DATADIR/morph"
+ #!/bin/sh
+ exec "$SRCDIR/morph" --quiet \
+ --cachedir-min-space=0 --tempdir-min-space=0 \
+ --no-default-config --config "$DATADIR/morph.conf" \
+ --cachedir "$DATADIR/distbuild-worker-cache" "$@"
+ EOF
+
+ IMPLEMENTS FINALLY the distbuild worker is terminated
+ stop_daemon "$DATADIR/worker-cache-server-pid"
+ stop_daemon "$DATADIR/worker-daemon-pid"
+ stop_daemon "$DATADIR/worker-daemon-helper-pid"
+
+ IMPLEMENTS GIVEN a distbuild controller
+ worker_cache_port_file="$DATADIR/worker-cache-server-port"
+ worker_cache_server_port="$(cat "$worker_cache_port_file")"
+ worker_daemon_port_file="$DATADIR/worker-daemon-port"
+ worker_daemon_port="$(cat "$worker_daemon_port_file")"
+ communal_cache_read_port="$(cat "$DATADIR/read-cache-server-port")"
+ communal_cache_write_port="$(cat "$DATADIR/write-cache-server-port")"
+ initiator_port_file="$DATADIR/controller-initiator-port"
+ mkfifo "$initiator_port_file"
+ helper_port_file="$DATADIR/controller-helper-port"
+ mkfifo "$helper_port_file"
+ controller_pid_file="$DATADIR/controller-pid"
+ start-stop-daemon --start --pidfile="$controller_pid_file" \
+ --background --make-pidfile --verbose \
+ --startas="$SRCDIR/morph" -- controller-daemon \
+ --no-default-configs --config "$DATADIR/morph.conf" \
+ --worker="localhost:$worker_daemon_port" \
+ --worker-cache-server-port="$worker_cache_server_port" \
+ --artifact-cache-server="http://localhost:$communal_cache_read_port/" \
+ --git-resolve-cache-server="http://localhost:$communal_cache_read_port/" \
+ --writeable-cache-server="http://localhost:$communal_cache_write_port/" \
+ --controller-helper-port=0 \
+ --controller-helper-port-file="$helper_port_file" \
+ --controller-initiator-port=0 \
+ --controller-initiator-port-file="$initiator_port_file" \
+ --morph-instance="$DATADIR/morph" \
+ --log="$DATADIR/controller-daemon-log" \
+ >"$DATADIR/controller-daemon-out" 2>"$DATADIR/controller-daemon-err"
+ helper_port="$(cat "$helper_port_file")"
+ rm "$helper_port_file"
+ echo "$helper_port" >"$helper_port_file"
+ initiator_port="$(cat "$initiator_port_file")"
+ rm "$initiator_port_file"
+ echo "$initiator_port" >"$initiator_port_file"
+
+ # start worker helper
+ helper_pid_file="$DATADIR/controller-helper-pid"
+ start-stop-daemon --start --pidfile="$helper_pid_file" \
+ --background --make-pidfile --verbose \
+ --startas="$SRCDIR/distbuild-helper" -- \
+ --no-default-configs \
+ --parent-port="$helper_port"
+
+ # make builds use distbuild
+ echo "controller-initiator-port = $initiator_port" \
+ >>"$DATADIR/morph.conf"
+ echo "controller-initiator-address = localhost" \
+ >>"$DATADIR/morph.conf"
+
+ IMPLEMENTS FINALLY the distbuild controller is terminated
+ stop_daemon "$DATADIR/controller-helper-pid"
+ stop_daemon "$DATADIR/controller-pid"
diff --git a/yarns/morph.shell-lib b/yarns/morph.shell-lib
index 9d67f2ab..9c13e449 100644
--- a/yarns/morph.shell-lib
+++ b/yarns/morph.shell-lib
@@ -184,3 +184,23 @@ slashify_colons()
{
echo "$1" | sed s,:,/,g
}
+
+start_cache_server(){
+ mkfifo "$1"
+ start-stop-daemon --start --pidfile="$2" \
+ --background --make-pidfile --verbose \
+ --startas="$SRCDIR/morph-cache-server" -- \
+ --port-file="$1" --no-fcgi \
+ --repo-dir="$DATADIR/gits" --direct-mode \
+ --bundle-dir="$DATADIR/bundles" \
+ --artifact-dir="$3" "$@"
+ port="$(cat "$1")"
+ rm "$1"
+ echo "$port" >"$1"
+}
+
+stop_daemon(){
+ if [ -e "$1" ]; then
+ start-stop-daemon --stop --pidfile "$1" --oknodo
+ fi
+}