diff options
Diffstat (limited to 'yarns/implementations.yarn')
| -rw-r--r-- | yarns/implementations.yarn | 485 |
1 files changed, 332 insertions, 153 deletions
diff --git a/yarns/implementations.yarn b/yarns/implementations.yarn index 52eee01d..c6d245d0 100644 --- a/yarns/implementations.yarn +++ b/yarns/implementations.yarn @@ -67,72 +67,47 @@ another to hold a chunk. # Create a directory for all the git repositories. mkdir "$DATADIR/gits" - # Create the chunk repository. - mkdir "$DATADIR/gits/test-chunk" - - run_in "$DATADIR/gits/test-chunk" git init . - cat > "$DATADIR/gits/test-chunk/test-bin" <<'EOF' - #!/bin/sh - echo Hello World - EOF - cat > "$DATADIR/gits/test-chunk/test.h" <<'EOF' - int foo(void); + # Create the bootstrap chunk repositories + mkdir "$DATADIR/gits/bootstrap-chunk" + cd "$DATADIR/gits/bootstrap-chunk" + git init . + git checkout -b bootstrap + cp "$SRCDIR/scripts/test-shell.c" sh.c + install /dev/stdin <<'EOF' configure + #!/bin/true EOF - cat > "$DATADIR/gits/test-chunk/test.pc" <<'EOF' - prefix=/usr - includedir=${prefix}/include - Name: test - Cflags: -I{includedir} - EOF - run_in "$DATADIR/gits/test-chunk" git add . - run_in "$DATADIR/gits/test-chunk" git commit --allow-empty -m Initial. + printf >Makefile ' + CFLAGS = -D_GNU_SOURCE -static - # Create a repo for the morphologies. + all: sh - mkdir "$DATADIR/gits/morphs" + install: sh + \tinstall -D -m755 sh $(DESTDIR)/bin/sh' + git add . + git commit -m "Add bootstrap shell" - arch=$(run_morph print-architecture) - install -m644 -D /dev/stdin << EOF "$DATADIR/gits/morphs/systems/test-system.morph" - name: test-system - kind: system - arch: $arch - strata: - - name: test-stratum - morph: strata/test-stratum.morph - EOF + git checkout --orphan master HEAD + # Commit a pre-built test-shell, as a compiler is too heavy to bootstrap + make sh + mkdir bin + mv sh bin/sh + git rm -f Makefile sh.c configure + git add bin/sh + git commit -m "Build bootstrap shell with bootstrap shell" - install -m644 -D /dev/stdin << EOF "$DATADIR/gits/morphs/strata/test-stratum.morph" - name: test-stratum - kind: stratum - chunks: - - name: test-chunk - repo: test:test-chunk - morph: test-chunk.morph - unpetrify-ref: master - ref: $(run_in "$DATADIR/gits/test-chunk" git rev-parse master) - build-mode: test - build-depends: [] - EOF + # Create the test chunk repository. + + mkdir "$DATADIR/gits/test-chunk" + cd "$DATADIR/gits/test-chunk" + git init . # To verify that chunk splitting works, we have a chunk that installs # dummy files in all the places that different kinds of files are # usually installed. e.g. executables in `/bin` and `/usr/bin` - install -m644 -D /dev/stdin << 'EOF' "$DATADIR/gits/morphs/test-chunk.morph" - name: test-chunk - kind: chunk - build-system: manual - - # `install-commands` is a list of shell commands to run. Commands - # may be on multiple lines, and indeed anything programmatic will - # benefit from doing so. Arguably we could have just one command, - # but it's split into multiple so that morph can inform us which - # command failed without us having to include a lot of status - # information in the command and look at the error message. - - install-commands: - + PREFIX=/usr + DESTDIR=. # It's important that we can test whether executables get # installed, so we install an empty script into `/usr/bin/test` and # `/usr/sbin/test`. @@ -144,10 +119,10 @@ another to hold a chunk. # that the file exists, rather than its contents, we can use /dev/null # as the source. - - | - for bindir in bin sbin; do - install -D /dev/null "$DESTDIR/$PREFIX/$bindir/test" - done + + for bindir in bin sbin; do + install -D /dev/null "$DESTDIR/$PREFIX/$bindir/test" + done # We need shared libraries too, sometimes they're libraries to support # the executables that a chunk provides, sometimes for other chunks. @@ -158,14 +133,13 @@ another to hold a chunk. # Shared libraries' file names start with lib and end with `.so` # for shared-object, with version numbers optionally suffixed. - - | - for libdir in lib lib32 lib64; do - dirpath="$DESTDIR/$PREFIX/$libdir" - install -D /dev/null "$dirpath/libtest.so" - ln -s libtest.so "$dirpath/libtest.so.0" - ln -s libtest.so.0 "$dirpath/libtest.so.0.0" - ln -s libtest.so.0.0 "$dirpath/libtest.so.0.0.0" - done + for libdir in lib lib32 lib64; do + dirpath="$DESTDIR/$PREFIX/$libdir" + install -D /dev/null "$dirpath/libtest.so" + ln -s libtest.so "$dirpath/libtest.so.0" + ln -s libtest.so.0 "$dirpath/libtest.so.0.0" + ln -s libtest.so.0.0 "$dirpath/libtest.so.0.0.0" + done # Shared objects aren't the only kind of library, some executable # binaries count as libraries, such as git's plumbing commands. @@ -173,8 +147,10 @@ another to hold a chunk. # In some distributions they go into /lib, in others, and the default # autotools configuration, they go into /libexec. - - | - install -D test-bin "$DESTDIR/$PREFIX/libexec/test-bin" + install -D /dev/stdin "$DESTDIR/$PREFIX/libexec/test-bin" <<'EOF' + #!/bin/sh + echo Hello World + EOF # As well as run-time libraries, there's development files. For C # this is headers, which describe the API of the libraries, which @@ -185,9 +161,9 @@ another to hold a chunk. # Header files go into `include` and end with `.h`. They are not # executable, so the install command changes the permissions with the # `-m` option. - - - | - install -D -m 644 test.h "$DESTDIR/$PREFIX/include/test.h" + install -D -m 644 /dev/stdin <<'EOF' "$DESTDIR/$PREFIX/include/test.h" + int foo(void); + EOF # `pkg-config` is a standard way to locate libraries and get the # compiler flags needed to build with the library. It's also used @@ -195,68 +171,148 @@ another to hold a chunk. # so as well as being found in `lib/pkgconfig`, it can be found in # `share/pkgconfig`, so we install dummy files to both. - - | - for pkgdir in lib lib32 lib64 share; do - install -D -m 644 test.pc \ - "$DESTDIR/$PREFIX/$pkgdir/pkgconfig/test.pc" - done + for pkgdir in lib lib32 lib64 share; do + install -D -m 644 /dev/stdin <<EOF \ + "$DESTDIR/$PREFIX/$pkgdir/pkgconfig/test.pc" + prefix=$PREFIX + includedir=\${prefix}/include + Name: test + Cflags: -I{includedir} + EOF + done # Static libraries can be used to build static binaries, which don't # require their dependencies to be installed. They are typically in # the form of `.a` archive and `.la` libtool archives. - - | - for libdir in lib lib32 lib64; do - for libname in libtest.a libtest.la; do - install -D -m 644 /dev/null "$DESTDIR/$PREFIX/$libdir/$libname" - done - done + for libdir in lib lib32 lib64; do + for libname in libtest.a libtest.la; do + install -D -m 644 /dev/null "$DESTDIR/$PREFIX/$libdir/$libname" + done + done # Packages may also install documentation, this comes in a variety # of formats, but info pages, man pages and html documentation are # the most common. - - | - for docfile in info/test.info.gz man/man3/test.3.gz doc/test/doc.html; do - install -D -m 644 /dev/null "$DESTDIR/$PREFIX/share/$docfile" - done + for docfile in info/test.info.gz man/man3/test.3.gz doc/test/doc.html; do + install -D -m 644 /dev/null "$DESTDIR/$PREFIX/share/$docfile" + done # Locale covers translations, timezones, keyboard layouts etc. in # all manner of strange file formats and locations. # Locale provides various translations for specific messages. - - | - install -D -m 644 /dev/null \ - "$DESTDIR/$PREFIX/share/locale/en_GB/LC_MESSAGES/test.mo" + install -D -m 644 /dev/null \ + "$DESTDIR/$PREFIX/share/locale/en_GB/LC_MESSAGES/test.mo" # Internationalisation (i18n) includes character maps and other data # such as currency. - - | - for localefile in i18n/locales/en_GB charmaps/UTF-8.gz; do - install -D -m 644 /dev/null "$DESTDIR/$PREFIX/share/$localefile" - done + for localefile in i18n/locales/en_GB charmaps/UTF-8.gz; do + install -D -m 644 /dev/null "$DESTDIR/$PREFIX/share/$localefile" + done # Timezones are another kind of localisation. - - | - install -D -m 644 /dev/null "$DESTDIR/$PREFIX/share/zoneinfo/UTC" + install -D -m 644 /dev/null "$DESTDIR/$PREFIX/share/zoneinfo/UTC" # We also need a catch rule for everything that doesn't fit into # the above categories, so to test that, we create some files that # don't belong in one. - - | - for cfgfile in test.conf README; do - install -D -m 644 /dev/null "$DESTDIR/etc/test.d/$cfgfile" - done + for cfgfile in test.conf README; do + install -D -m 644 /dev/null "$DESTDIR/etc/test.d/$cfgfile" + done + + git add . + git commit --allow-empty -m Initial. + + # Create a repo for the morphologies. + + mkdir "$DATADIR/gits/morphs" + cd "$DATADIR/gits/morphs" + git init . + arch=$(run_morph print-architecture) + install -m644 -D /dev/stdin << EOF "systems/test-system.morph" + name: test-system + kind: system + arch: $arch + strata: + - name: build-essential + morph: strata/build-essential.morph + - name: core + morph: strata/core.morph EOF - run_in "$DATADIR/gits/morphs" git init . - run_in "$DATADIR/gits/morphs" git add . - run_in "$DATADIR/gits/morphs" git commit -m Initial. - run_in "$DATADIR/gits/morphs" git tag -a "test-tag" -m "Tagging test-tag" + install -m644 -D /dev/stdin << EOF "strata/build-essential.morph" + name: build-essential + kind: stratum + chunks: + - name: stage1-chunk + repo: test:bootstrap-chunk + ref: $(run_in "$DATADIR/gits/bootstrap-chunk" git rev-parse bootstrap) + unpetrify-ref: nootstrap + build-mode: bootstrap + build-depends: [] + - name: stage2-chunk + morph: stage2-chunk.morph + repo: test:bootstrap-chunk + ref: $(run_in "$DATADIR/gits/bootstrap-chunk" git rev-parse master) + unpetrify-ref: master + build-depends: + - stage1-chunk + EOF + install -m644 -D /dev/stdin << EOF "strata/core.morph" + name: core + kind: stratum + build-depends: + - morph: strata/build-essential.morph + chunks: + - name: test-chunk + morph: test-chunk.morph + repo: test:test-chunk + unpetrify-ref: master + ref: $(run_in "$DATADIR/gits/test-chunk" git rev-parse master) + build-depends: [] + EOF + + install -m644 -D /dev/stdin << 'EOF' "test-chunk.morph" + name: test-chunk + kind: chunk + build-system: manual + # `build-commands` is a list of shell commands to run. Commands + # may be on multiple lines, and indeed anything programmatic will + # benefit from doing so. Arguably we could have just one command, + # but it's split into multiple so that morph can inform us which + # command failed without us having to include a lot of status + # information in the command and look at the error message. + # `build-commands` are passed MAKEFLAGS to specify build parallelism, + # and are expected to generate files to be installed in `install-files` + build-commands: + # All of morph's building needs to handle binary output, so we can echo + # that out on the command line. + # Trust me, this gets decoded to "echo " then a bunch of binary output. + - !!binary | + ZWNobyBQ1+k661ol3khrsF4VO4HNcuYzwN0LYxEWS8mPmhQiQ7Vu8CME2+gsnKQaoIRIFuUEiLCI + vfIj1GTdXG6cVTJfNQ== + + install-commands: + - copy files + EOF + + install -m644 -D /dev/stdin << 'EOF' "stage2-chunk.morph" + name: test-chunk + kind: chunk + build-system: manual + install-commands: + - copy files + EOF + + git add . + git commit -m Initial. + git tag -a "test-tag" -m "Tagging test-tag" # Start a git daemon to serve our git repositories port_file="$DATADIR/git-daemon-port" @@ -298,45 +354,6 @@ We need a consistent value for the architecture in some tests, so we have a morphology using the test architecture. IMPLEMENTS GIVEN a system called (\S+) for the test architecture in the git server - - cat << EOF > "$DATADIR/gits/morphs/stage1-chunk.morph" - name: stage1-chunk - kind: chunk - build-system: dummy - EOF - - run_in "$DATADIR/gits/morphs" git add . - run_in "$DATADIR/gits/morphs" git commit -m "Add chunk for $MATCH_1" - - - install -m644 -D /dev/stdin << EOF "$DATADIR/gits/morphs/strata/build-essential.morph" - name: build-essential - kind: stratum - chunks: - - name: stage1-chunk - morph: stage1-chunk.morph - repo: test:test-chunk - ref: $(run_in "$DATADIR/gits/test-chunk" git rev-parse master) - unpetrify-ref: master - build-mode: bootstrap - build-depends: [] - EOF - - install -m644 -D /dev/stdin << EOF "$DATADIR/gits/morphs/strata/core.morph" - name: core - kind: stratum - build-depends: - - morph: strata/build-essential.morph - chunks: - - name: test-chunk - morph: test-chunk.morph - repo: test:test-chunk - unpetrify-ref: master - ref: $(run_in "$DATADIR/gits/test-chunk" git rev-parse master) - build-mode: test - build-depends: [] - EOF - name="$(basename "${MATCH_1%.*}")" cat << EOF > "$DATADIR/gits/morphs/$MATCH_1" arch: testarch @@ -356,6 +373,25 @@ have a morphology using the test architecture. run_in "$DATADIR/gits/morphs" git add "$MATCH_1" run_in "$DATADIR/gits/morphs" git commit -m "Added $MATCH_1 and strata morphologies." +You need an architecture to build a system, we don't default to the host architecture. + + IMPLEMENTS GIVEN a system called (\S+) with no architecture in the git server + name="$(basename "${MATCH_1%.*}")" + cat << EOF > "$DATADIR/gits/morphs/$MATCH_1" + configuration-extensions: [] + description: A system called $name for test architecture + kind: system + name: $name + strata: + - name: build-essential + morph: strata/build-essential.morph + - name: core + morph: strata/core.morph + EOF + + run_in "$DATADIR/gits/morphs" git add "$MATCH_1" + run_in "$DATADIR/gits/morphs" git commit -m "Added $MATCH_1." + Implementation sections for system branch operations ---------------------------------------------------- @@ -375,12 +411,16 @@ Attempt to check out a system branch from a root that has no systems. cd "$DATADIR/workspace" attempt_morph checkout test:test-chunk master + IMPLEMENTS WHEN the user attempts to check out the system branch from (\S+) called (\S+) + cd "$DATADIR/workspace" + attempt_morph branch-from-image --metadata-dir "$DATADIR/$MATCH_1/baserock" "$MATCH_2" + We also need to verify that a system branch has been checked out. IMPLEMENTS THEN the system branch (\S+) is checked out is_dir "$DATADIR/workspace/$MATCH_1/test/morphs" is_file "$DATADIR/workspace/$MATCH_1/test/morphs/systems/test-system.morph" - is_file "$DATADIR/workspace/$MATCH_1/test/morphs/strata/test-stratum.morph" + is_file "$DATADIR/workspace/$MATCH_1/test/morphs/strata/core.morph" We can create a new branch, off master. @@ -487,8 +527,10 @@ print-architecture` to get a value appropriate for morph. kind: system name: $name strata: - - name: test-stratum - morph: strata/test-stratum.morph + - name: build-essential + morph: strata/build-essential.morph + - name: core + morph: strata/core.morph EOF Reporting status of checked out repositories: @@ -651,10 +693,10 @@ Implementations for temporary build branch handling Implementation sections for building ==================================== - IMPLEMENTS WHEN the user (attempts to build|builds) the system (\S+) in branch (\S+) - cd "$DATADIR/workspace/$MATCH_3" - set build "$MATCH_2" - if [ $MATCH_1 == "builds" ]; then run_morph "$@" + IMPLEMENTS WHEN the user (attempts to )?((dist)?build)s? the system (\S+) in branch (\S+) + cd "$DATADIR/workspace/$MATCH_5" + set "$MATCH_2" "$MATCH_4" + if [ "$MATCH_1" != "attempts to " ]; then run_morph "$@" else attempt_morph "$@"; fi Implementation sections for cross-bootstraping @@ -840,9 +882,9 @@ variables in `$DATADIR/env`. We treat the value as a format string for Implementations for building systems ------------------------------------ - IMPLEMENTS THEN morph build the system (\S+) of the (branch|tag) (\S+) - cd "$DATADIR/workspace/$MATCH_3" - run_morph build "$MATCH_1" + IMPLEMENTS THEN morph ((dist)?build) the system (\S+) of the (branch|tag) (\S+) + cd "$DATADIR/workspace/$MATCH_5" + run_morph "$MATCH_1" "$MATCH_3" IMPLEMENTS WHEN the user builds (\S+) of the (\S+) (branch|tag) cd "$DATADIR/workspace/$MATCH_2" @@ -937,3 +979,140 @@ Altering morphologies in the workspace "$SRCDIR/scripts/edit-morph" cluster-system-set-deploy-variable \ "$DATADIR/workspace/$branch/test/morphs/$cluster" "$name" \ "$key" "$val" + + +Distbuild +========= + + IMPLEMENTS ASSUMING the morph-cache-server can be run + "$SRCDIR/morph-cache-server" --version + + IMPLEMENTS GIVEN a communal cache server + # The communal cache server has direct access to the git repositories + # and can have artifacts placed on it + artifact_dir="$DATADIR/shared-artifacts" + mkdir -p "$artifact_dir" + + read_cache_server_port_file="$DATADIR/read-cache-server-port" + read_cache_server_pid_file="$DATADIR/read-cache-server-pid" + start_cache_server "$read_cache_server_port_file" \ + "$read_cache_server_pid_file" \ + "$artifact_dir" + + write_cache_server_port_file="$DATADIR/write-cache-server-port" + write_cache_server_pid_file="$DATADIR/write-cache-server-pid" + start_cache_server "$write_cache_server_port_file" \ + "$write_cache_server_pid_file" \ + "$artifact_dir" --enable-writes + + IMPLEMENTS FINALLY the communal cache server is terminated + stop_daemon "$DATADIR/read-cache-server-pid" + stop_daemon "$DATADIR/write-cache-server-pid" + + IMPLEMENTS GIVEN a distbuild worker + # start worker cache server, so other workers can download results + worker_cachedir="$DATADIR/distbuild-worker-cache" + worker_artifacts="$worker_cachedir/artifacts" + mkdir -p "$worker_artifacts" + worker_cache_port_file="$DATADIR/worker-cache-server-port" + worker_cache_pid_file="$DATADIR/worker-cache-server-pid" + start_cache_server "$worker_cache_port_file" \ + "$worker_cache_pid_file" \ + "$worker_artifacts" + + # start worker daemon + worker_daemon_port_file="$DATADIR/worker-daemon-port" + worker_daemon_pid_file="$DATADIR/worker-daemon-pid" + mkfifo "$worker_daemon_port_file" + communal_cache_port="$(cat "$DATADIR/read-cache-server-port")" + start-stop-daemon --start --pidfile="$worker_daemon_pid_file" \ + --background --make-pidfile --verbose \ + --startas="$SRCDIR/morph" -- worker-daemon \ + --no-default-configs --config "$DATADIR/morph.conf" \ + --worker-daemon-port=0 \ + --worker-daemon-port-file="$worker_daemon_port_file" \ + --cachedir="$worker_cachedir" \ + --artifact-cache-server="http://localhost:$communal_cache_port/" \ + --git-resolve-cache-server="http://localhost:$communal_cache_port/" \ + --log="$DATADIR/worker-daemon-log" \ + >"$DATADIR/worker-daemon-out" 2>"$DATADIR/worker-daemon-err" + worker_daemon_port="$(cat "$worker_daemon_port_file")" + rm "$worker_daemon_port_file" + echo "$worker_daemon_port" >"$worker_daemon_port_file" + + # start worker helper + helper_pid_file="$DATADIR/worker-daemon-helper-pid" + start-stop-daemon --start --pidfile="$helper_pid_file" \ + --background --make-pidfile --verbose \ + --startas="$SRCDIR/distbuild-helper" -- \ + --no-default-configs \ + --parent-port="$worker_daemon_port" + + # set up builder config + install /dev/stdin <<'EOF' "$DATADIR/morph" + #!/bin/sh + exec "$SRCDIR/morph" --quiet \ + --cachedir-min-space=0 --tempdir-min-space=0 \ + --no-default-config --config "$DATADIR/morph.conf" \ + --cachedir "$DATADIR/distbuild-worker-cache" "$@" + EOF + + IMPLEMENTS FINALLY the distbuild worker is terminated + stop_daemon "$DATADIR/worker-cache-server-pid" + stop_daemon "$DATADIR/worker-daemon-pid" + stop_daemon "$DATADIR/worker-daemon-helper-pid" + + IMPLEMENTS GIVEN a distbuild controller + worker_cache_port_file="$DATADIR/worker-cache-server-port" + worker_cache_server_port="$(cat "$worker_cache_port_file")" + worker_daemon_port_file="$DATADIR/worker-daemon-port" + worker_daemon_port="$(cat "$worker_daemon_port_file")" + communal_cache_read_port="$(cat "$DATADIR/read-cache-server-port")" + communal_cache_write_port="$(cat "$DATADIR/write-cache-server-port")" + initiator_port_file="$DATADIR/controller-initiator-port" + mkfifo "$initiator_port_file" + helper_port_file="$DATADIR/controller-helper-port" + mkfifo "$helper_port_file" + controller_pid_file="$DATADIR/controller-pid" + start-stop-daemon --start --pidfile="$controller_pid_file" \ + --background --make-pidfile --verbose \ + --startas="$SRCDIR/morph" -- controller-daemon \ + --no-default-configs --config "$DATADIR/morph.conf" \ + --worker="localhost:$worker_daemon_port" \ + --worker-cache-server-port="$worker_cache_server_port" \ + --artifact-cache-server="http://localhost:$communal_cache_read_port/" \ + --git-resolve-cache-server="http://localhost:$communal_cache_read_port/" \ + --writeable-cache-server="http://localhost:$communal_cache_write_port/" \ + --controller-helper-port=0 \ + --controller-helper-port-file="$helper_port_file" \ + --controller-initiator-port=0 \ + --controller-initiator-port-file="$initiator_port_file" \ + --morph-instance="$DATADIR/morph" \ + --log="$DATADIR/controller-daemon-log" \ + >"$DATADIR/controller-daemon-out" 2>"$DATADIR/controller-daemon-err" + helper_port="$(cat "$helper_port_file")" + rm "$helper_port_file" + echo "$helper_port" >"$helper_port_file" + initiator_port="$(cat "$initiator_port_file")" + rm "$initiator_port_file" + echo "$initiator_port" >"$initiator_port_file" + + # start worker helper + helper_pid_file="$DATADIR/controller-helper-pid" + start-stop-daemon --start --pidfile="$helper_pid_file" \ + --background --make-pidfile --verbose \ + --startas="$SRCDIR/distbuild-helper" -- \ + --no-default-configs \ + --parent-port="$helper_port" + + # make builds use distbuild + echo "controller-initiator-port = $initiator_port" \ + >>"$DATADIR/morph.conf" + echo "controller-initiator-address = localhost" \ + >>"$DATADIR/morph.conf" + echo "initiator-step-output-dir = $DATADIR" \ + >>"$DATADIR/morph.conf" + + IMPLEMENTS FINALLY the distbuild controller is terminated + stop_daemon "$DATADIR/controller-helper-pid" + stop_daemon "$DATADIR/controller-pid" |