diff options
author | Richard Maw <richard.maw@codethink.co.uk> | 2014-01-17 16:38:19 +0000 |
---|---|---|
committer | Richard Maw <richard.maw@codethink.co.uk> | 2014-01-17 16:38:19 +0000 |
commit | 18f24fbb0c35905af06f5af0915813fd1f0c22b3 (patch) | |
tree | 08c467976ca1292d1f7f11af450ab57396351943 /tests.build | |
parent | 11c4c8f8019457b27d9b4c1c7e7a928d6f87c321 (diff) | |
parent | 9901b78e48fddeda2ed7f6dbf954abcde8fa9a0f (diff) | |
download | morph-18f24fbb0c35905af06f5af0915813fd1f0c22b3.tar.gz |
Merge artifact splitting work
Rationale
=========
This patch series implements the concept of stratum splitting.
For a long time we've had code to split a chunk into multiple artifacts,
however there's not been a way to split strata up, or systems select a
subset of the produced stratum artifacts to be included in the system.
This patch series implements the ability to split strata and have systems
include them in a way which still has the same behaviour if no rules
are specified, but with default rules that split chunk artifacts up into
various components, strata into runtime and development versions and has
systems include everything by default, but can be told to include less.
The default rules have chunk foo split up into -bins, -libs, -devel,
-doc, -locale and -misc.
These rules can be overridden in the chunk morphology by adding the new
'products' field, which lists match rules like the following:
products:
- artifact: libudev
include:
- (usr/)?lib(32|64)?/lubg?udev\..*
- artifact: udev
include:
- (usr/)?s?bin/udev*
- (usr/)?lib(32|64|exec)?/systemd/systemd-udevd
Strata are by default split into -runtime and -devel. -devel by default
contains chunks ending with -devel and -doc, -runtime contains everything
else.
Extra match rules can be added to a stratum similarly to chunks, but
instead of matching file names, they match artifact names.
products:
- artifact: core-python
include:
- "cpython-.*" # lazy shortcut to put all of cpython in this stratum
- "python-.*" # lazy shortcut to include all python chunks in
Additionally, in chunk specs, chunk artifacts may be assigned to stratum
artifacts, this takes precedence over products match rules in the stratum
and the default match rules. Assigning the chunk to `null` will discard
the chunk.
chunks:
...
- name: systemd
...
artifacts:
libudev: foundation-runtime
udev: foundation-runtime
systemd-doc: null
By default a system includes every produced artifact of every stratum
listed. Instead a subset can be specified in the stratum spec as follows:
name: tiny-system
strata:
- name: build-essential
...
artifacts:
- build-essential-runtime
Diffstat (limited to 'tests.build')
21 files changed, 70 insertions, 92 deletions
diff --git a/tests.build/bootstrap-mode.script b/tests.build/bootstrap-mode.script index 923fb21f..0ac66220 100755 --- a/tests.build/bootstrap-mode.script +++ b/tests.build/bootstrap-mode.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -29,11 +29,15 @@ set -eu test:morphs-repo master hello-system cd "$DATADIR/cache/artifacts" -echo "build-essential stratum:" -stratum=$(ls *.stratum.build-essential) -cat $stratum | sed 's/[a-f0-9]\{64\}/xxxx/g' -echo +echo "build-essential strata:" +for stratum in $(find . -regex '.*\.stratum\.build-essential-[^.]*$' | sort) +do + echo "$stratum" + sed 's/[a-f0-9]\{64\}/xxxx/g' "$stratum" + echo +done echo echo "hello-system:" system=$(ls *hello-system-rootfs) -tar tf "$system" | LC_ALL=C sort | sed '/^\.\/./s:^\./::' +tar tf "$system" | LC_ALL=C sort -u | sed '/^\.\/./s:^\./::' \ + | grep -v '^baserock' diff --git a/tests.build/bootstrap-mode.stdout b/tests.build/bootstrap-mode.stdout index 329cdd78..b59d0029 100644 --- a/tests.build/bootstrap-mode.stdout +++ b/tests.build/bootstrap-mode.stdout @@ -1,12 +1,11 @@ -build-essential stratum: -["xxxx.chunk.cc"] +build-essential strata: +./5bbfd4cb94017e7b72e20ee4f91a76bed8085aa96176cf87ecce54ec71d5ddae.stratum.build-essential-devel +["xxxx.chunk.cc-devel", "xxxx.chunk.cc-doc"] +./c8fe8efd4f8c6edcd309bb2cf0a308c93dfd905bbff64be98c5b07f350951fde.stratum.build-essential-runtime +["xxxx.chunk.cc-bins", "xxxx.chunk.cc-libs", "xxxx.chunk.cc-locale", "xxxx.chunk.cc-misc"] hello-system: ./ -baserock/ -baserock/hello-stratum.meta -baserock/hello-system-rootfs.meta -baserock/hello.meta etc/ etc/fstab etc/os-release diff --git a/tests.build/build-stratum-with-submodules.script b/tests.build/build-stratum-with-submodules.script index c3c00578..c996e769 100755 --- a/tests.build/build-stratum-with-submodules.script +++ b/tests.build/build-stratum-with-submodules.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -71,5 +71,5 @@ EOF test:morphs-repo master hello-system system=$(ls "$DATADIR/cache/artifacts/"*hello-system-rootfs) -tar tf $system | LC_ALL=C sort | sed '/^\.\/./s:^\./::' +tar tf $system | LC_ALL=C sort | sed '/^\.\/./s:^\./::' | grep -v '^baserock/' diff --git a/tests.build/build-stratum-with-submodules.stdout b/tests.build/build-stratum-with-submodules.stdout index bf9836d7..6dda5049 100644 --- a/tests.build/build-stratum-with-submodules.stdout +++ b/tests.build/build-stratum-with-submodules.stdout @@ -1,8 +1,4 @@ ./ -baserock/ -baserock/hello-stratum.meta -baserock/hello-system-rootfs.meta -baserock/parent.meta etc/ etc/fstab etc/os-release diff --git a/tests.build/build-system-autotools.script b/tests.build/build-system-autotools.script index c2171750..ba5cd32f 100755 --- a/tests.build/build-system-autotools.script +++ b/tests.build/build-system-autotools.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -52,7 +52,5 @@ git commit --quiet -m "Convert hello to an autotools project" for chunk in "$DATADIR/cache/artifacts/"*.chunk.* do - echo "$chunk:" | sed 's/[^.]*//' - tar -tf "$chunk" | LC_ALL=C sort | sed '/^\.\/./s:^\./::' - echo -done + tar -tf "$chunk" +done | LC_ALL=C sort -u | sed '/^\.\/./s:^\./::' | grep -Ee '^(bin|etc)' diff --git a/tests.build/build-system-autotools.stdout b/tests.build/build-system-autotools.stdout index 8077cac2..683441c9 100644 --- a/tests.build/build-system-autotools.stdout +++ b/tests.build/build-system-autotools.stdout @@ -1,8 +1,3 @@ -.chunk.hello: -./ -baserock/ -baserock/hello.meta bin/ bin/hello etc/ - diff --git a/tests.build/build-system-cmake.script b/tests.build/build-system-cmake.script index 00b9ed23..ab5186d7 100755 --- a/tests.build/build-system-cmake.script +++ b/tests.build/build-system-cmake.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -53,7 +53,5 @@ git commit --quiet -m "Convert hello to a cmake project" for chunk in "$DATADIR/cache/artifacts/"*.chunk.* do - echo "$chunk:" | sed 's/[^.]*//' - tar -tf "$chunk" | LC_ALL=C sort | sed '/^\.\/./s:^\./::' - echo -done + tar -tf "$chunk" +done | LC_ALL=C sort -u | sed '/^\.\/./s:^\./::' | grep -Ee '^(usr/)?(bin|etc)' diff --git a/tests.build/build-system-cmake.stdout b/tests.build/build-system-cmake.stdout index ccf80a86..3410b113 100644 --- a/tests.build/build-system-cmake.stdout +++ b/tests.build/build-system-cmake.stdout @@ -1,8 +1,2 @@ -.chunk.hello: -./ -baserock/ -baserock/hello.meta -usr/ usr/bin/ usr/bin/hello - diff --git a/tests.build/build-system-cpan.script b/tests.build/build-system-cpan.script index b1823eb5..f66d4027 100755 --- a/tests.build/build-system-cpan.script +++ b/tests.build/build-system-cpan.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -81,7 +81,5 @@ git commit -q -m "Set custom install prefix for hello" for chunk in "$DATADIR/cache/artifacts/"*.chunk.* do - echo "$chunk:" | sed 's/[^.]*//' - tar -tf "$chunk" | LC_ALL=C sort | - sed -e '/^\.\/./s:^\./::' | grep -F "bin/hello" -done + tar -tf "$chunk" +done | LC_ALL=C sort | sed '/^\.\/./s:^\./::' | grep -F 'bin/hello' diff --git a/tests.build/build-system-cpan.stdout b/tests.build/build-system-cpan.stdout index 5cbe4c73..180e949b 100644 --- a/tests.build/build-system-cpan.stdout +++ b/tests.build/build-system-cpan.stdout @@ -1,2 +1 @@ -.chunk.hello: bin/hello diff --git a/tests.build/build-system-python-distutils.script b/tests.build/build-system-python-distutils.script index a0469528..e1dccb4b 100755 --- a/tests.build/build-system-python-distutils.script +++ b/tests.build/build-system-python-distutils.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -79,10 +79,8 @@ git commit -q -m "Set custom install prefix for hello" for chunk in "$DATADIR/cache/artifacts/"*.chunk.* do - echo "$chunk:" | sed 's/[^.]*//' - tar -tf "$chunk" | LC_ALL=C sort | sed '/^\.\/./s:^\./::' - echo -done | + tar -tf "$chunk" +done | LC_ALL=C sort -u | sed '/^\.\/./s:^\./::' | grep -Ee '^(bin|lib)' | sed -e 's:^local/::' \ -e 's:lib/python2.[6-9]/:lib/python2.x/:' \ -e 's:/hello-0\.0\.0[^/]*\.egg-info$:/hello.egg-info/:' \ diff --git a/tests.build/build-system-python-distutils.stdout b/tests.build/build-system-python-distutils.stdout index 4d4abdbb..4d4c3a1e 100644 --- a/tests.build/build-system-python-distutils.stdout +++ b/tests.build/build-system-python-distutils.stdout @@ -1,7 +1,3 @@ -.chunk.hello: -./ -baserock/ -baserock/hello.meta bin/ bin/hello lib/ diff --git a/tests.build/build-system.script b/tests.build/build-system.script index 75b9d0d0..56d80735 100755 --- a/tests.build/build-system.script +++ b/tests.build/build-system.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,4 +24,4 @@ set -eu test:morphs-repo master hello-system system=$(ls "$DATADIR/cache/artifacts/"*hello-system-rootfs) -tar tf $system | LC_ALL=C sort | sed '/^\.\/./s:^\./::' +tar tf $system | LC_ALL=C sort | sed '/^\.\/./s:^\./::' | grep -v '^baserock/' diff --git a/tests.build/build-system.stdout b/tests.build/build-system.stdout index 3d5201ee..2e8270dc 100644 --- a/tests.build/build-system.stdout +++ b/tests.build/build-system.stdout @@ -1,8 +1,4 @@ ./ -baserock/ -baserock/hello-stratum.meta -baserock/hello-system-rootfs.meta -baserock/hello.meta bin/ bin/hello etc/ diff --git a/tests.build/morphless-chunks.script b/tests.build/morphless-chunks.script index c9294c3e..9a8b41dd 100755 --- a/tests.build/morphless-chunks.script +++ b/tests.build/morphless-chunks.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2012-2013 Codethink Limited +# Copyright (C) 2012-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -44,7 +44,5 @@ git commit -q -m "Convert hello into an autodetectable chunk" for chunk in "$DATADIR/cache/artifacts/"*.chunk.* do - echo "$chunk:" | sed 's/[^.]*//' - tar -tf "$chunk" | LC_ALL=C sort | sed '/^\.\/./s:^\./::' - echo -done + tar -tf "$chunk" +done | cat >/dev/null # No files get installed apart from metadata diff --git a/tests.build/morphless-chunks.stdout b/tests.build/morphless-chunks.stdout index 22292c14..e69de29b 100644 --- a/tests.build/morphless-chunks.stdout +++ b/tests.build/morphless-chunks.stdout @@ -1,5 +0,0 @@ -.chunk.hello: -./ -baserock/ -baserock/hello.meta - diff --git a/tests.build/prefix.script b/tests.build/prefix.script index e9b8ecd2..ca9648c9 100755 --- a/tests.build/prefix.script +++ b/tests.build/prefix.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2013 Codethink Limited +# Copyright (C) 2013-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -82,6 +82,6 @@ git commit -q -m "Update stratum" test:morphs-repo master hello-system cd "$DATADIR/cache/artifacts" -first_chunk=$(ls -1 *.chunk.xyzzy | cut -c -64) -second_chunk=$(ls -1 *.chunk.plugh | cut -c -64) +first_chunk=$(ls -1 *.chunk.xyzzy-* | head -n1 | cut -c -64) +second_chunk=$(ls -1 *.chunk.plugh-* | head -n1 | cut -c -64) cat $first_chunk.build-log $second_chunk.build-log diff --git a/tests.build/rebuild-cached-stratum.script b/tests.build/rebuild-cached-stratum.script index 306c16f2..0014e545 100755 --- a/tests.build/rebuild-cached-stratum.script +++ b/tests.build/rebuild-cached-stratum.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -30,7 +30,7 @@ cache="$DATADIR/cache/artifacts" git checkout --quiet farrokh && git checkout --quiet -b rebuild-cached-stratum) -# Make a branch in the morphs repo and modify the stratum to refer to +# Make a branch in the morphs repo and modify the stratum to refer to # the new chunk branch. (cd "$DATADIR/morphs-repo" && git checkout --quiet -b rebuild-cached-stratum && @@ -42,18 +42,18 @@ cache="$DATADIR/cache/artifacts" "$SRCDIR/scripts/test-morph" build-morphology \ test:morphs-repo rebuild-cached-stratum hello-system echo "first build:" -(cd "$cache" && ls *.chunk.* *hello-stratum | sed 's/^[^.]*\./ /' | - LC_ALL=C sort) +(cd "$cache" && ls *.chunk.* *hello-stratum-* | sed 's/^[^.]*\./ /' | + LC_ALL=C sort -u) # Change the chunk. -(cd "$DATADIR/chunk-repo" && - echo >> hello.c && +(cd "$DATADIR/chunk-repo" && + echo >> hello.c && git commit --quiet -am change) # Rebuild. "$SRCDIR/scripts/test-morph" build-morphology \ test:morphs-repo rebuild-cached-stratum hello-system echo "second build:" -(cd "$cache" && ls *.chunk.* *hello-stratum | sed 's/^[^.]*\./ /' | - LC_ALL=C sort) +(cd "$cache" && ls *.chunk.* *hello-stratum-* | sed 's/^[^.]*\./ /' | + LC_ALL=C sort -u) diff --git a/tests.build/rebuild-cached-stratum.stdout b/tests.build/rebuild-cached-stratum.stdout index eee106f5..9c53ee60 100644 --- a/tests.build/rebuild-cached-stratum.stdout +++ b/tests.build/rebuild-cached-stratum.stdout @@ -1,8 +1,22 @@ first build: - chunk.hello - stratum.hello-stratum + chunk.hello-bins + chunk.hello-devel + chunk.hello-doc + chunk.hello-libs + chunk.hello-locale + chunk.hello-misc + stratum.hello-stratum-devel + stratum.hello-stratum-devel.meta + stratum.hello-stratum-runtime + stratum.hello-stratum-runtime.meta second build: - chunk.hello - chunk.hello - stratum.hello-stratum - stratum.hello-stratum + chunk.hello-bins + chunk.hello-devel + chunk.hello-doc + chunk.hello-libs + chunk.hello-locale + chunk.hello-misc + stratum.hello-stratum-devel + stratum.hello-stratum-devel.meta + stratum.hello-stratum-runtime + stratum.hello-stratum-runtime.meta diff --git a/tests.build/stratum-overlap-writes-overlap.script b/tests.build/stratum-overlap-writes-overlap.script index ca06454b..fe4ed4ee 100755 --- a/tests.build/stratum-overlap-writes-overlap.script +++ b/tests.build/stratum-overlap-writes-overlap.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011-2013 Codethink Limited +# Copyright (C) 2011-2014 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -27,9 +27,9 @@ cache="$DATADIR/cache/artifacts" "$SRCDIR/scripts/test-morph" build-morphology \ test:morphs-repo overlap hello-system > /dev/null "$SRCDIR/scripts/list-overlaps" groups \ - "$cache"/*.stratum.hello-stratum.overlaps | + "$cache"/*.stratum.hello-stratum-*.overlaps | while IFS='\n' read overlaps; do echo $overlaps "$SRCDIR/scripts/list-overlaps" list-files \ - "$cache"/*.stratum.hello-stratum.overlaps $overlaps + "$cache"/*.stratum.hello-stratum-*.overlaps $overlaps done diff --git a/tests.build/stratum-overlap-writes-overlap.stdout b/tests.build/stratum-overlap-writes-overlap.stdout index 40485659..1e36ca83 100644 --- a/tests.build/stratum-overlap-writes-overlap.stdout +++ b/tests.build/stratum-overlap-writes-overlap.stdout @@ -1,4 +1,4 @@ -overlap-foo-baz overlap-foobar overlap-fooqux +overlap-foo-baz-bins overlap-foobar-bins overlap-fooqux-bins bin/foo -overlap-foo-baz overlap-foobar +overlap-foo-baz-bins overlap-foobar-bins bin/bar |