From 834b129fd7458017f717ffbc1a53f98fc206767f Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Thu, 7 Mar 2013 10:59:35 +0000 Subject: Fix deploy to not assume artifacts are compressed --- morphlib/plugins/deploy_plugin.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/morphlib/plugins/deploy_plugin.py b/morphlib/plugins/deploy_plugin.py index b7d9189a..5bb2d9ec 100644 --- a/morphlib/plugins/deploy_plugin.py +++ b/morphlib/plugins/deploy_plugin.py @@ -117,8 +117,7 @@ class DeployPlugin(cliapp.Plugin): f = build_command.lac.get(artifact) else: f = build_command.rac.get(artifact) - ff = gzip.GzipFile(fileobj=f) - tf = tarfile.TarFile(fileobj=ff) + tf = tarfile.open(fileobj=f) tf.extractall(path=system_tree) self.app.status( -- cgit v1.2.1 From 192ca84cd2e6a6125b3c7cb2ed076580312d3471 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 6 Mar 2013 14:35:23 +0000 Subject: Do not compress chunk artifacts upon creation We already silently, automatically decompress if the artifact is compressed. It is thus safe to turn off compression. Compression saves disk space, but on ARM it takes up a lot extra time, and we have lots of disk space, so for speed, not compressing is a good idea. --- morphlib/bins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/morphlib/bins.py b/morphlib/bins.py index ba5f713f..0374c117 100644 --- a/morphlib/bins.py +++ b/morphlib/bins.py @@ -100,7 +100,7 @@ def create_chunk(rootdir, f, regexps, dump_memory_profile=None): dump_memory_profile('after walking') include = sorted(include) # get dirs before contents - tar = tarfile.open(fileobj=f, mode='w:gz') + tar = tarfile.open(fileobj=f, mode='w') for filename in include: # Normalize mtime for everything. tarinfo = tar.gettarinfo(filename, -- cgit v1.2.1 From 86235f28bb13c38e4e5ec0702aa7342c13f30d49 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Wed, 6 Mar 2013 14:37:39 +0000 Subject: Turn off rootfs-tarball compression --- morphlib/plugins/tarball-systembuilder_plugin.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/morphlib/plugins/tarball-systembuilder_plugin.py b/morphlib/plugins/tarball-systembuilder_plugin.py index 02622067..9f9cf02e 100644 --- a/morphlib/plugins/tarball-systembuilder_plugin.py +++ b/morphlib/plugins/tarball-systembuilder_plugin.py @@ -59,9 +59,7 @@ class RootfsTarballBuilder(SystemKindBuilder): # pragma: no cover unslashy_root) return info artiname = self.artifact.source.morphology['name'] - tar = tarfile.TarFile.gzopen(fileobj=handle, mode="w", - compresslevel=1, - name=artiname) + tar = tarfile.open(fileobj=handle, mode="w", name=artiname) self.app.status(msg='Constructing tarball of root filesystem', chatty=True) tar.add(fs_root, recursive=True, filter=uproot_info) -- cgit v1.2.1 From 6cc75e0a7e348c1ab92a0ec4de43cf20dd3e2c72 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Fri, 8 Mar 2013 14:49:25 +0000 Subject: Test that chunk artifacts are NOT compressed --- morphlib/bins_tests.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/morphlib/bins_tests.py b/morphlib/bins_tests.py index 3288207b..d7ed1656 100644 --- a/morphlib/bins_tests.py +++ b/morphlib/bins_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2012 Codethink Limited +# Copyright (C) 2011-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,6 +14,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import gzip import os import shutil import stat @@ -134,6 +135,11 @@ class ChunkTests(BinsTest): self.assertEqual([x for x, y in self.recursive_lstat(self.instdir)], ['.', 'lib', 'lib/libfoo.so']) + def test_does_not_compress_artifact(self): + self.create_chunk(['bin']) + with gzip.open(self.chunk_file) as f: + self.assertRaises(IOError, f.read) + class ExtractTests(unittest.TestCase): -- cgit v1.2.1 From d1b40f5b50941f617937d010da6e5fadcd782630 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Mon, 11 Mar 2013 13:45:34 +0000 Subject: Change test to also verify that artifact is not compressed --- tests.as-root/tarball-image-is-sensible.script | 7 +++++++ tests.as-root/tarball-image-is-sensible.stderr | 1 + 2 files changed, 8 insertions(+) create mode 100644 tests.as-root/tarball-image-is-sensible.stderr diff --git a/tests.as-root/tarball-image-is-sensible.script b/tests.as-root/tarball-image-is-sensible.script index 1b4d60ae..e36cdb5a 100755 --- a/tests.as-root/tarball-image-is-sensible.script +++ b/tests.as-root/tarball-image-is-sensible.script @@ -29,6 +29,13 @@ set -eu tar=$("$SRCDIR/scripts/test-morph" --find-system-artifact \ build-morphology test:morphs tarball-links hello-tarball) +# Verify that the tar archive is not compressed. +if gunzip -t "$tar" > /dev/null +then + echo "ERROR: $tar is gzip'd!" 1>&2 + exit 1 +fi + extracted="$DATADIR/extracted" mkdir -p "$extracted" cd "$extracted" diff --git a/tests.as-root/tarball-image-is-sensible.stderr b/tests.as-root/tarball-image-is-sensible.stderr new file mode 100644 index 00000000..9db65e02 --- /dev/null +++ b/tests.as-root/tarball-image-is-sensible.stderr @@ -0,0 +1 @@ +gunzip: invalid magic -- cgit v1.2.1 From 3e9c32fd0b4175bfcec7ddfe02334de5ceeb0a90 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Tue, 12 Mar 2013 14:22:26 +0000 Subject: Skip some tests in ./check by default Running all the checks takes over 20 minutes for me. This makes it likely that ./check doesn't get run very often. Dropping the slowest tests by default makes ./check more usefull. Add the --full option to enable the full test suite again. --- check | 41 ++++++++++++++++++++++++++++++++++++++--- 1 file changed, 38 insertions(+), 3 deletions(-) diff --git a/check b/check index e70de72a..5e16d23e 100755 --- a/check +++ b/check @@ -17,18 +17,53 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + set -e + +# Parse the command line. + +full=false +while [ "$#" -gt 0 ] +do + case "$1" in + --full) full=true; shift ;; + *) echo "ERROR: Unknown argument $1." 1>&2; exit 1 ;; + esac +done + + +# Set PYTHONPATH to start with the current directory so that we always +# find the right version of it for the test suite. + case "$PYTHONPATH" in '') PYTHONPATH="$(pwd)" ;; *) PYTHONPATH="$(pwd):$PYTHONPATH" ;; esac export PYTHONPATH + +# Clean up artifacts from previous (possibly failed) runs, build, +# and run the tests. + python setup.py clean check + cmdtest tests -cmdtest tests.branching -cmdtest tests.merging + +if $full +then + cmdtest tests.branching +else + echo "NOT RUNNING test.branching" +fi + +if $full +then + cmdtest tests.merging +else + echo "NOT RUNNING test.merging" +fi + cmdtest tests.deploy # Building systems requires the 'filter' parameter of tarfile.TarFile.add(): @@ -40,7 +75,7 @@ else fi # The as-root tests use YAML morphologies, so they require the PyYAML module. -if [ $(whoami) = root ] && command -v mkfs.btrfs > /dev/null && +if $full && [ $(whoami) = root ] && command -v mkfs.btrfs > /dev/null && python -c " import morphlib, sys if not morphlib.got_yaml: -- cgit v1.2.1 From 31b23c4460545bb41d5907524ff56fd272d44ffe Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Wed, 6 Feb 2013 11:59:15 +0000 Subject: Fix incorrect comment --- morphlib/source.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/morphlib/source.py b/morphlib/source.py index d4f1e119..b536cbdf 100644 --- a/morphlib/source.py +++ b/morphlib/source.py @@ -30,11 +30,6 @@ class Source(object): * ``tree`` -- the SHA1 of the tree corresponding to the commit * ``morphology`` -- the in-memory representation of the morphology we use * ``filename`` -- basename of the morphology filename - * ``dependencies`` -- list of Sources for build dependencies for us - * ``dependents`` -- list of Source for whom we are a build dependency - - The ``dependencies`` and ``dependents`` lists MUST be modified by - the ``add_dependencies`` and ``add_dependent`` methods only. ''' -- cgit v1.2.1 From 79a6f05bbf70a4fa10b1b426ef9f223d5cbaf3ee Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Wed, 20 Feb 2013 15:55:19 +0000 Subject: Remove old bootstrap mechanisms That means that bootstrapping Baserock is currently not possible with this branch of Morph, but there's no reason it cannot be bootstrapped using an older version of Morph instead. --- README | 91 -------- baserock-bootstrap | 537 --------------------------------------------- check | 2 - dump-build-times | 75 ------- run-bootstrap-in-chroot | 199 ----------------- scripts/assemble-stratum | 59 ----- scripts/bootstrap | 555 ----------------------------------------------- 7 files changed, 1518 deletions(-) delete mode 100755 baserock-bootstrap delete mode 100755 dump-build-times delete mode 100755 run-bootstrap-in-chroot delete mode 100755 scripts/assemble-stratum delete mode 100755 scripts/bootstrap diff --git a/README b/README index 36d33a43..3c9fa8ab 100644 --- a/README +++ b/README @@ -254,9 +254,6 @@ more consistently across machines and developers. See the `morphlib/builder.py` file, method `setup_env` for details. -Until Baserock is properly bootstrapped, it's awkward that the `PATH` -variable gets reset. To avoid that, use the `--keep-path` option. - Hacking morph ------------- @@ -271,94 +268,6 @@ and check out the `cmdtest` utility (from ). Run the checks before submitting a patch, please. -The Bootstrap Process ---------------------- - -The goal of the bootstrap process is to create an environment that has -only been built with morph. It is a long boring process that can take -a few hours. - -It can be started by running `./run-bootstrap-in-chroot $workingdir` -with some environment variables set. - -The variables are: -* `DEBIAN_MIRROR`; which must be a path suitable for passing to - debootstrap; which at the time of writing allowed http://, - file:// or ssh:// urls; **required** -* `GIT_TARBALLS`; which must be a url to a tarballs directory; - morph will try to fetch tarballs from here before trying the - git-base-url; **required** -* `snapshot`; which should be `false` (or any executable that - exits unsuccessfully) if creating a snapshot after each stage - of the bootstrap is not desirable; optional - -If snapshot is true or omitted then after each stage of the bootstrap -a tarball of the working directory is created, so that if a later stage -fails it can start from the last successful stage. - -The mirror variables allow the bootstrap to be run more quickly if the -required files are available locally. - -The mirror variables being required and snapshotting defaulting -to true are artefacts of the bootstrap process' development, where -only re-running the step that failed and using local mirrors were -essential for making fixes. - - -Bootstrap stages ----------------- - -The stages required for the bootstrap are to create a debian squeeze -chroot; build a skeleton system using the linux from scratch specified -tarballs; chroot into this system and build a system using morph with -bootstrap mode; then build the system again but instead using morph's ---staging-chroot mode and with the linux from scratch system removed. - -The squeeze chroot (stage 0) is used so that the bootstrap process does -not need to be supported on every available host system, but every host -system that can run debootstrap can be indirectly supported. - -In this stage debootstrap creates a chroot with all the packages required -to build the linux from scratch system, from then onwards every stage is -built by running baserock-bootstrap and optionally making a snapshot. - -Stage 1 builds everything in a subdirectory of the squeeze-chroot called -tree, and all the packages are built with --prefix=/tools, so the full -path for `cat` would be `$workingdir/squeeze-chroot/tree/tools/bin/cat`. - -This is done so that we have a known good system to work from, that won't -become entangled with the final system, which installed with --prefix=/usr. - -Stage 2 uses morph to build a system, while chrooted into -`$workingdir/squeeze-chroot/tree`, this naturally required PATH to include -/tools/bin. - -Morph is used to build the `foundation.morph` and `devel-bootstrap.morph` -strata in the `morphs` repository. `devel-bootstrap.morph` may not use the -same morphs as `devel.morph` because the bootstrap requires some hacks that -aren't required when a fully bootstrapped system is available. - -Morph also uses the `--bootstrap` flag to make the chunks be unpacked after -they have been built, this is required for build-dependencies to be met as -prerequisites are expected to be available on the system. - -After morph has finished a reasonably complete system should be available, -so `$workingdir/squeeze-chroot/tree/tools` is removed to save space and -make it obvious if stage 2 is missing anything needed for stage 3. - -Stage 3 builds `devel.morph` which does not need all the hacks required -in building during the bootstrap, so it should be much more like what -morph will finally build. - -Rather than using `--bootstrap`, stage 3 uses `--staging-chroot`, which -builds and extracts into a chroot, rather than using anything on the host -system. - -For this to be able to build anything `--staging-filler` is used, which -allows for specifying the environment that chunks are built in. Stage 3 -uses the strata that were build in stage 2. - - Legalese -------- diff --git a/baserock-bootstrap b/baserock-bootstrap deleted file mode 100755 index 619baeb4..00000000 --- a/baserock-bootstrap +++ /dev/null @@ -1,537 +0,0 @@ -#!/bin/bash - -set -e -set +h -set -u - -BASEDIR=$(dirname $(readlink -f $0)) - -LFS="$(pwd)/tree" -LFS="${LFS/\/\///}" -tools="$LFS/tools" - -buildwhat="$1" - -export LC_ALL=C - -case "$(uname -m)" in -*armv7*) - export LFS_TGT=$(uname -m)-lfs-linux-gnueabi - export TARGET_CFLAGS="-march=armv7-a -mfpu=vfpv3-d16 -mfloat-abi=softfp -O2" - export PASS3_STRATUM="bootstrap-pass3-imx53" - ;; -*armv5*) - export LFS_TGT=$(uname -m)-lfs-linux-gnueabi - export TARGET_CFLAGS="-march=armv5 -O2" - export PASS3_STRATUM="bootstrap-pass3-imx53" - ;; -*) - export LFS_TGT=$(uname -m)-lfs-linux-gnu - export TARGET_CFLAGS="-O2" - export PASS3_STRATUM="bootstrap-pass3-x86_64-generic" - ;; -esac - -HOST_CAT=`which cat` -HOST_CP=`which cp` -HOST_MKDIR=`which mkdir` -HOST_CHMOD=`which chmod` - -export CCACHE_DIR="/var/tmp/ccache" -export PATH="$tools/lib/ccache:$tools/bin:$tools/sbin:/usr/lib/ccache:/usr/bin:/bin:/usr/sbin:/sbin" - -pass1_directories() -{ - $HOST_MKDIR -p "$LFS" - - $HOST_MKDIR -p "$tools" - $HOST_MKDIR -p "$tools/bin" - $HOST_MKDIR -p "$tools/lib" - [ -h "$tools/sbin" ] || ln -sf "bin" "$tools/sbin" - [ -h "$tools/lib64" ] || ln -sf "lib" "$tools/lib64" - [ -h "$tools/libexec" ] || ln -sf "lib" "$tools/libexec" - - [ -e "$LFS/proc" ] || $HOST_MKDIR -p "$LFS/proc" - [ -e "$LFS/sys" ] || $HOST_MKDIR -p "$LFS/sys" - [ -e "$LFS/tmp" ] || $HOST_MKDIR -p "$LFS/tmp" - [ -e "$LFS/dev" ] || $HOST_MKDIR -p "$LFS/dev" - [ -e "$LFS/dev/console" ] || \ - mknod -m 600 "$LFS/dev/console" c 5 1 - [ -e "$LFS/dev/null" ] || mknod -m 666 "$LFS/dev/null" c 1 3 - [ -e "$LFS/dev/random" ] || mknod -m 644 "$LFS/dev/random" c 1 8 - [ -e "$LFS/dev/urandom" ] || \ - mknod -m 644 "$LFS/dev/urandom" c 1 9 -} - -pass1_get_sources_with_morph() -{ - cd $LFS/baserock/gits/morph - $HOST_MKDIR -p $LFS/baserock/cache - python ./morph --verbose update-gits \ - baserock:baserock/morphs master bootstrap-pass1 \ - --cachedir=$LFS/baserock/cache \ - --log=$LFS/baserock/morph.log \ - --dump-memory-profile=none \ - --keep-path \ - --tarball-server="$GIT_TARBALLS" -} - -pass1_build_with_morph() -{ - cd $LFS/baserock/gits/morph - $HOST_MKDIR -p $LFS/baserock/cache - $HOST_MKDIR -p $LFS/tmp - python ./morph --verbose build-morphology \ - baserock:baserock/morphs master bootstrap-pass1 \ - --no-git-update \ - --no-distcc \ - --no-ccache \ - --bootstrap \ - --cachedir=$LFS/baserock/cache \ - --tempdir=$LFS/tmp \ - --log=$LFS/baserock/morph.log \ - --dump-memory-profile=none \ - --keep-path \ - --prefix="$tools" \ - --toolchain-target="$LFS_TGT" \ - --target-cflags="$TARGET_CFLAGS" -} - -pass1_dump_build_times() -{ - echo "Dumping pass 1 build times" - cd $LFS/baserock/gits/morph - python ./dump-build-times $LFS/baserock/cache/artifacts -} - -pass2_get_morph() -{ - echo "Get morph" - if [ ! -e "$LFS/baserock/gits" ] - then - $HOST_MKDIR -p "$LFS/baserock/gits" - cp -al "$HOME/baserock/gits/morph" "$LFS/baserock/gits/morph" - fi -} - - -pass2_prepare_for_chroot() -{ - echo "Preparing $LFS for chroot" - cd "$LFS" - if [ ! -h "$LFS/$LFS" ] - then - $HOST_MKDIR -p "$LFS/$LFS" - /bin/rmdir "$LFS/$LFS" - /bin/ln -s / "$LFS/$LFS" - fi - - if [ ! -d "$LFS/etc" ] - then - mkdir -p "$LFS/etc" - chmod 777 "$LFS/etc" - touch "$LFS/etc/ld.so.conf" - chmod 666 "$LFS/etc/ld.so.conf" - cat < "$LFS/etc/ld.so.conf" -/lib64 -/lib -/usr/lib64 -/usr/lib -EOF - fi - -# $HOST_MKDIR -p "$LFS/etc" - [ -e "$LFS/etc/passwd" ] || \ - echo 'root::0:0:root:/root:/bin/bash' > "$LFS/etc/passwd" - [ -e "$LFS/etc/group" ] || echo 'root::0:' > "$LFS/etc/group" - -# [ -e "$LFS/etc/hostname" ] || echo 'baserock-boot' | -# /usr/bin/tee "$LFS/etc/hostname" > /dev/null - - # Add symlinks for common locations of specific tools - # These are needed for #! lines in scripts - [ -e "$LFS/bin" ] || $HOST_MKDIR -p "$LFS/bin" - [ -e "$LFS/bin/sh" ] || ln -sf ../tools/bin/bash "$LFS/bin/sh" - [ -e "$LFS/bin/bash" ] || \ - ln -sf ../tools/bin/bash "$LFS/bin/bash" - [ -e "$LFS/bin/pwd" ] || ln -sf ../tools/bin/pwd "$LFS/bin/pwd" - [ -e "$LFS/bin/echo" ] || \ - ln -sf ../tools/bin/echo "$LFS/bin/echo" - if [ ! -e "$LFS/usr/bin/perl" ]; then - $HOST_MKDIR -p $LFS/usr/bin - ln -sf ../../tools/bin/perl "$LFS/usr/bin/perl" - fi -} - - -pass2_get_sources_with_morph_in_chroot() -{ - echo "Getting sources with morph" - cat < "$LFS/baserock/build.sh" -#!/tools/bin/bash -set -e -set -x - -cd /baserock/gits/morph -mkdir -p /baserock/cache -export PATH="/usr/bin:/bin:$tools/bin:$tools/sbin" -python ./morph --verbose update-gits \ - baserock:baserock/morphs master bootstrap-pass2 \ - --bootstrap \ - --cachedir=/baserock/cache \ - --log=/baserock/morph.log \ - --dump-memory-profile=none \ - --keep-path \ - --tarball-server="$GIT_TARBALLS" -EOF - $HOST_CHMOD +x "$LFS/baserock/build.sh" - local do_chroot="$BASEDIR/do-chroot.bash" - $HOST_CAT <"$do_chroot" -#!/bin/bash -trap "umount $LFS/proc $LFS/sys || true" INT TERM EXIT -set -e -set -x -if ! mount | grep "$LFS/proc" >/dev/null -then - mount -t proc proc "$LFS/proc" -fi -if ! mount | grep "$LFS/sys" >/dev/null -then - mount -t sysfs sysfs "$LFS/sys" -fi -$HOST_CP -f /etc/resolv.conf "$LFS/etc/resolv.conf" -/usr/sbin/chroot "$LFS" \\ - /tools/bin/env -i HOME=/baserock TERM=\$TERM \\ - PATH="/bin:/usr/bin:/sbin:/usr/sbin:/tools/bin" \\ - BOOTSTRAP_TOOLS="$LFS/tools" \\ - "\${@-\$SHELL}" -EOF - $HOST_CHMOD +x "$do_chroot" - "$do_chroot" /baserock/build.sh -} - - -pass2_build_with_morph_in_chroot() -{ - echo "Building Baserock with morph" - cat < "$LFS/baserock/build.sh" -#!/tools/bin/bash -set -e -set -x - -cd /baserock/gits/morph -mkdir -p /baserock/cache -export PATH="/usr/bin:/bin:$tools/bin:$tools/sbin" -export LD_LIBRARY_PATH="/usr/lib:/lib:/lib64:/tools/lib:/tools/lib64" -python ./morph --verbose build-morphology \ - baserock:baserock/morphs master bootstrap-pass2 \ - --bootstrap \ - --no-git-update \ - --cachedir=/baserock/cache \ - --log=/baserock/morph.log \ - --dump-memory-profile=none \ - --keep-path \ - --tarball-server="$GIT_TARBALLS" \ - --target-cflags="$TARGET_CFLAGS" -EOF - $HOST_CHMOD +x "$LFS/baserock/build.sh" - local do_chroot="$BASEDIR/do-chroot.bash" - $HOST_CAT <"$do_chroot" -#!/bin/bash -trap "umount $LFS/proc $LFS/sys || true" INT TERM EXIT -set -e -set -x -if ! mount | grep "$LFS/proc" >/dev/null -then - mount -t proc proc "$LFS/proc" -fi -if ! mount | grep "$LFS/sys" >/dev/null -then - mount -t sysfs sysfs "$LFS/sys" -fi -$HOST_CP -f /etc/resolv.conf "$LFS/etc/resolv.conf" -/usr/sbin/chroot "$LFS" \\ - /tools/bin/env -i HOME=/baserock TERM=\$TERM \\ - PATH="/bin:/usr/bin:/sbin:/usr/sbin:/tools/bin" \\ - BOOTSTRAP_TOOLS="$LFS/tools" \\ - "\${@-\$SHELL}" -EOF - $HOST_CHMOD +x "$do_chroot" - "$do_chroot" /baserock/build.sh -} - -pass2_dump_build_times() -{ - echo "Dumping pass 2 build times" - cat < "$LFS/baserock/dump-pass2-build-times.sh" -#!/tools/bin/bash -set -e -set -x - -cd /baserock/gits/morph -python ./dump-build-times /baserock/cache/artifacts -EOF - $HOST_CHMOD +x "$LFS/baserock/dump-pass2-build-times.sh" - local do_chroot="$BASEDIR/do-chroot.bash" - $HOST_CAT <"$do_chroot" -#!/bin/bash -trap "umount $LFS/proc $LFS/sys || true" INT TERM EXIT -set -e -set -x -if ! mount | grep "$LFS/proc" >/dev/null -then - mount -t proc proc "$LFS/proc" -fi -if ! mount | grep "$LFS/sys" >/dev/null -then - mount -t sysfs sysfs "$LFS/sys" -fi -$HOST_CP -f /etc/resolv.conf "$LFS/etc/resolv.conf" -/usr/sbin/chroot "$LFS" \\ - /tools/bin/env -i HOME=/baserock TERM=\$TERM \\ - PATH="/bin:/usr/bin:/sbin:/usr/sbin:/tools/bin" \\ - BOOTSTRAP_TOOLS="$LFS/tools" \\ - "\${@-\$SHELL}" -EOF - $HOST_CHMOD +x "$do_chroot" - "$do_chroot" /baserock/dump-pass2-build-times.sh -} - -pass2a_cleanup_at_end() -{ - echo "Remove unnecessary stuff at the end of pass2a" - rm -f "$LFS/baserock/cache/gits/"*.bndl -} - - -pass2b_cleanup_at_end() -{ - echo "Remove unnecessary stuff at the end of pass2b" - rm -rf "$LFS/tools" -} - - -pass3_remove_tools() -{ - echo "Removing $LFS/tools" - rm -rf "$LFS/tools" -} - -pass3_get_sources_with_morph_in_chroot() -{ - echo "Getting sources with morph" - - cd "$LFS/baserock/gits/morph" - cat < "$LFS/baserock/build.sh" -#!/bin/bash -set -e -set -x - -cd /baserock/gits/morph -python ./morph --verbose update-gits \ - baserock:baserock/morphs master $PASS3_STRATUM \ - --cachedir=/baserock/cache \ - --log=/baserock/morph.log \ - --dump-memory-profile=none \ - --keep-path \ - --tarball-server="$GIT_TARBALLS" -EOF - $HOST_CHMOD +x "$LFS/baserock/build.sh" - local do_chroot="$BASEDIR/do-chroot.bash" - $HOST_CAT <"$do_chroot" -#!/bin/bash -trap "umount $LFS/proc $LFS/sys || true" INT TERM EXIT -set -e -set -x -if ! mount | grep "$LFS/proc" >/dev/null -then - mount -t proc proc "$LFS/proc" -fi -if ! mount | grep "$LFS/sys" >/dev/null -then - mount -t sysfs sysfs "$LFS/sys" -fi -$HOST_CP -f /etc/resolv.conf "$LFS/etc/resolv.conf" -/usr/sbin/chroot "$LFS" \\ - /usr/bin/env -i HOME=/baserock TERM=\$TERM \\ - PATH="/bin:/usr/bin:/sbin:/usr/sbin" \\ - "\${@-\$SHELL}" -EOF - $HOST_CHMOD +x "$do_chroot" - "$do_chroot" /baserock/build.sh -} - -pass3_build_with_morph_in_chroot() -{ - echo "Building Baserock with morph" - - cat <<'EOF' >"$LFS/usr/bin/linux-user-chroot" -#!/bin/sh - -CHDIR=. - -while true; do - case "$1" in - --help) - echo 'See "man linux-user-chroot"' - exit 0 - ;; - --version) - echo 'Fake' - exit 0 - ;; - --mount-bind) - # swallow option and arguments - shift 3 - ;; - --mount-readonly) - shift 2 - ;; - --mount-proc|--unshare-ipc|--unshare-pid) - # swallow configure flag - shift - ;; - --chdir) - CHDIR="$2" - shift 2 - ;; - *) - # terminate arg processing - ROOTDIR="$1" - shift - break - ;; - esac -done - -exec chroot "$ROOTDIR" sh -c 'cd "$1" && shift && exec "$@"' -- "$CHDIR" "$@" -EOF - chmod +x "$LFS/usr/bin/linux-user-chroot" - - cat < "$LFS/baserock/build_pass3.sh" -#!/bin/bash -set -e -set -x - -export PATH="/usr/bin:/bin:/usr/sbin:/sbin" -cd /baserock/gits/morph - -filler=\$(mktemp) -scripts/assemble-stratum --cachedir /baserock/cache \ - "$LFS/baserock/cache/artifacts/"*.stratum.bootstrap-pass2 \ - "\$filler" bootstrap-pass2 - -python ./morph --verbose build-morphology \ - baserock:baserock/morphs master $PASS3_STRATUM \ - --staging-chroot \ - --staging-filler "\$filler" \ - --no-git-update \ - --cachedir=/baserock/cache \ - --log=/baserock/morph.log \ - --dump-memory-profile=none \ - --keep-path \ - --tarball-server="$GIT_TARBALLS" \ - --target-cflags="$TARGET_CFLAGS" -rm "\$filler" -EOF - $HOST_CHMOD +x "$LFS/baserock/build_pass3.sh" - local do_chroot="$BASEDIR/do-chroot.bash" - $HOST_CAT >"$do_chroot" </dev/null -then - mount -t proc proc "$LFS/proc" -fi -if ! mount | grep "$LFS/sys" >/dev/null -then - mount -t sysfs sysfs "$LFS/sys" -fi -$HOST_CP -f /etc/resolv.conf "$LFS/etc/resolv.conf" -/usr/sbin/chroot "$LFS" \\ - /usr/bin/env -i HOME=/baserock TERM=\$TERM \\ - PATH="/bin:/usr/bin:/sbin:/usr/sbin" \\ - "\${@-\$SHELL}" -EOF - $HOST_CHMOD +x "$do_chroot" - "$do_chroot" /baserock/build_pass3.sh -} - -pass3_dump_build_times() -{ - echo "Dumping pass 3 build times" - - cat < "$LFS/baserock/dump-pass3-build-times.sh" -#!/bin/bash -set -e -set -x - -export PATH="/usr/bin:/bin" -cd /baserock/gits/morph -python ./dump-build-times /baserock/cache/artifacts -EOF - $HOST_CHMOD +x "$LFS/baserock/dump-pass3-build-times.sh" - local do_chroot="$BASEDIR/do-chroot.bash" - $HOST_CAT >"$do_chroot" </dev/null -then - mount -t proc proc "$LFS/proc" -fi -if ! mount | grep "$LFS/sys" >/dev/null -then - mount -t sysfs sysfs "$LFS/sys" -fi -$HOST_CP -f /etc/resolv.conf "$LFS/etc/resolv.conf" -/usr/sbin/chroot "$LFS" \\ - /usr/bin/env -i HOME=/baserock TERM=\$TERM \\ - PATH="/bin:/usr/bin:/sbin:/usr/sbin" \\ - "\${@-\$SHELL}" -EOF - $HOST_CHMOD +x "$do_chroot" - "$do_chroot" /baserock/dump-pass3-build-times.sh -} -echo "Bootstrapping Baserock development environment" -echo "LFS_TGT=$LFS_TGT" - -time pass1_directories - -case "$buildwhat" in - pass1a) - time pass1_get_sources_with_morph - ;; - pass1b) - time pass1_build_with_morph - time pass1_dump_build_times - ;; - pass2a) - time pass2_get_morph - time pass2_prepare_for_chroot - time pass2_get_sources_with_morph_in_chroot - time pass2a_cleanup_at_end - ;; - pass2b) - time pass2_build_with_morph_in_chroot - time pass2_dump_build_times - time pass2b_cleanup_at_end - ;; - pass3a) - time pass3_remove_tools - time pass3_get_sources_with_morph_in_chroot - ;; - pass3b) - time pass3_build_with_morph_in_chroot - time pass3_dump_build_times - ;; - *) echo "Usage! (sorry, I'm unhelpful)" 1>&2 - exit 1 - ;; -esac - -echo "$buildwhat finished OK" - diff --git a/check b/check index 5e16d23e..6e02ef6b 100755 --- a/check +++ b/check @@ -107,8 +107,6 @@ then fi case "$x" in - baserock-bootstrap) - ;; *) if awk 'length > 79' "$x" | grep . > /dev/null then diff --git a/dump-build-times b/dump-build-times deleted file mode 100755 index 48fdca65..00000000 --- a/dump-build-times +++ /dev/null @@ -1,75 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import cliapp -import glob -import json -import os -import re -import StringIO - - -class ExtractBuildTimes(cliapp.Application): - - '''Extracts build times of chunks in a morph cache directory. - - Given a morph cache directory as the first argument, this app finds all - cached chunks, loads their meta data and prints their build times. - - ''' - - def process_args(self, args): - cachedir = args[0] - - def chunk_hash(chunk): - short = re.split('\.', chunk) - return os.path.basename(short[-3]) - - def chunk_name(chunk): - short = re.split('\.', chunk) - return short[-1] - - chunks = glob.glob(os.path.join(cachedir, '*.chunk.*')) - items = [] - - for chunk in chunks: - hash = chunk_hash(chunk) - metafile = os.path.join(cachedir, '%s.meta' % hash) - with open(metafile) as f: - data = f.read() - io = StringIO.StringIO(data) - metainfo = json.load(io) - time = metainfo['build-times']['overall-build']['delta'] - minutes = float(time) / 60.0 - items.append((chunk_name(chunk), minutes)) - - items = sorted(items, key=lambda x: x[1], reverse=True) - print '%s' % (43 * '-') - print 'Build times of cached chunks in' - print '%s' % cachedir - print '%s' % (43 * '-') - sum = 0.0 - for name, time in items: - print '%30s: %6.1f mins' % (name, time) - sum += time - print '%s' % (43 * '-') - print '%30s: %6.1f mins' % ('total', sum) - - -if __name__ == '__main__': - ExtractBuildTimes().run() diff --git a/run-bootstrap-in-chroot b/run-bootstrap-in-chroot deleted file mode 100755 index fbe1e9f5..00000000 --- a/run-bootstrap-in-chroot +++ /dev/null @@ -1,199 +0,0 @@ -#!/bin/bash - -set -e - -unmount_virtual() -{ - umount "$1/proc" || true - umount "$1/sys" || true - umount "$1/tree/proc" || true - umount "$1/tree/sys" || true -} - -pass_snapshot() -{ - echo -n "$snapshotdir/$1-snapshot.tar.gz" -} - -has_pass() -{ - if [ -e $(pass_snapshot "$1") ] - then - return 0 - else - return 1 - fi -} - -update_morph() -{ - local dir="$1" - cp baserock-bootstrap "$dir/." # update bootstrap script - rm -rf "$dir/tree/baserock/gits/morph" - mkdir -p "$dir/tree/baserock/gits/morph" - - # Copy everything except the target directory into the target directory. - # The point is to be able to keep the working area for a bootstrap inside - # the morph source directory. This is useful for Jenkins jobs. - local base=$(basename $(basename "$dir")) - find . -mindepth 1 -maxdepth 1 ! -name "$base" ! -name '*.tar.gz' \ - -exec cp -a '{}' "$dir/tree/baserock/gits/morph" ';' -} - -run_pass() -{ - local dir="$1" - local passname="$2" - local tarball=$(pass_snapshot "$passname") - - if "$snapshot" && has_pass "$passname" - then - tar -C "$dir" -xhf "$tarball" - update_morph "$dir" - else - update_morph "$dir" - "./do-squeeze-chroot" bash -x baserock-bootstrap "$passname" || exit 1 - if "$snapshot" - then - tar -C "$dir" -caf "$tarball" . - fi - fi -} - -export LC_ALL=C - -if [ "x$1" = x ] -then - echo "Usage: $0 chroot-dir" 1>&2 - exit 1 -fi - -mkdir -p "$1" -dir="$1/squeeze-chroot" -: ${snapshot:=true} -snapshotdir="$1" - -cat >"./do-squeeze-chroot" <&2) || - ([ "x$GIT_TARBALLS" = x ] && echo GIT_TARBALLS is unspecified >&2) -then - echo You have to set DEBIAN_MIRROR and other environment variables 1>&2 - exit 1 -fi - -if ! which dpkg 2> /dev/null; then - echo "Warning: dpkg not found -- should debootstrap fail to autodetect " - echo "your architecture, set ARCH in the environment (probably to amd64)" -fi - -# prepare the ccache directory in the chroot, if necessary -if [ "x$CCACHE_HOST_DIR" = "x" ]; then - # print a warning if the CCACHE_HOST_DIR is not set - echo "CCACHE_HOST_DIR is unspecified, but that's ok" >&2 -fi - -unmount_virtual "$dir" -rm -rf "$dir" -mkdir "$dir" - -chrootsnapshot="$snapshotdir/squeeze.tar.gz" -if ! "$snapshot" || ! has_pass pass1a; then - if "$snapshot" && [ -e "$chrootsnapshot" ] - then - tar -C "$dir" -xhf "$chrootsnapshot" - else - EXTRAPACKAGES="build-essential,gawk,bison,flex,python,autoconf" - EXTRAPACKAGES="$EXTRAPACKAGES,autopoint,automake,gettext,libtool" - EXTRAPACKAGES="$EXTRAPACKAGES,help2man,texinfo,sudo,ccache,gperf" - EXTRAPACKAGES="$EXTRAPACKAGES,python-pip,python-simplejson,python-yaml" - - EXTRAARGS= - if [ -n $ARCH]; then - EXTRAARGS=$ARCH - fi - - debootstrap $EXTRAARGS --include="$EXTRAPACKAGES" squeeze \ - "$dir" "$DEBIAN_MIRROR" - - mkdir -p "$dir/etc" - hostname > "$dir/etc/hostname" - cat < "$dir/etc/hosts" -127.0.0.1 localhost -127.0.1.1 `hostname` -EOF - - # create the directory for cached ccache object files - mkdir -p "$dir/var/tmp/ccache" - - # manually build and install cliapp - "./do-squeeze-chroot" mkdir -p /src - "./do-squeeze-chroot" \ - git clone git://roadtrain.codethink.co.uk/delta/cliapp /src/cliapp - "./do-squeeze-chroot" \ - sh -c 'cd /src/cliapp && "$@"' -- python setup.py build - "./do-squeeze-chroot" \ - sh -c 'cd /src/cliapp && "$@"' -- python setup.py install - - "./do-squeeze-chroot" pip install ordereddict - - if "$snapshot" - then - tar -caf "$chrootsnapshot" -C "$dir" . - fi - fi -fi - -# Unpack existing snapshot, or run pass1 of bootstrap and then make snapshot. - -if ! "$snapshot" || ! has_pass pass1b -then - run_pass "$dir" pass1a -fi - -if ! "$snapshot" || ! has_pass pass2a -then - run_pass "$dir" pass1b -fi - -if ! "$snapshot" || ! has_pass pass2b -then - run_pass "$dir" pass2a -fi - -if ! "$snapshot" || ! has_pass pass3a -then - run_pass "$dir" pass2b -fi - -if ! "$snapshot" || ! has_pass pass3b -then - run_pass "$dir" pass3a -fi - -snapshot=false run_pass "$dir" pass3b - -echo "Passes 1, 2, and 3 of bootstrap done (possibly cached)." - diff --git a/scripts/assemble-stratum b/scripts/assemble-stratum deleted file mode 100755 index 9addd36d..00000000 --- a/scripts/assemble-stratum +++ /dev/null @@ -1,59 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2011-2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -# This is a program to convert the json dump of the overlaps between artifacts -# in a format more suited to shell programs, or human reading - -import json -import tarfile -import os - -import cliapp - - -class AssembleStratum(cliapp.Application): - - def add_settings(self): - self.settings.string(['cachedir'], - 'Where the cache basedir is') - self.settings.string(['tarformat'], - 'What format to write tar to', - default='') - - def process_args(self, args): - chunklist = json.load(open(args[0])) - tarformat = 'w' - if self.settings['tarformat'] != "": - tarformat += self.settings['tarformat'] - outfile = tarfile.open(args[1], tarformat) - # concatenate chunk tarballs - for chunk in chunklist: - path = os.path.join(self.settings['cachedir'], 'artifacts', chunk) - chunktar = tarfile.open(path, mode='r:*') - for tarinfo in chunktar: - if tarinfo.isfile(): - outfile.addfile(tarinfo, chunktar.extractfile(tarinfo)) - else: - outfile.addfile(tarinfo) - chunktar.close() - # add the stratum's metadata - if os.path.exists(args[0] + '.meta'): - outfile.add(args[0] + '.meta', - os.path.join('baserock', '%s.meta' % args[2])) - outfile.close() - -AssembleStratum().run() diff --git a/scripts/bootstrap b/scripts/bootstrap deleted file mode 100755 index 37ce6952..00000000 --- a/scripts/bootstrap +++ /dev/null @@ -1,555 +0,0 @@ -#!/bin/sh - -# Build an intermediate build-essential stratum using the host's tools -# -- everything is installed into $PREFIX inside a chroot -# -- a "cross-compiler" toolchain is built first, as suggested by LFS, and -# this builds the actual build-essential tools. -# -- unlike LFS, the initial toolchain goes into a separate prefix. This is -# because programs in the target chroot cannot be executed outside of the -# chroot without creating a symlink in the root directory of the host -# system, which is not pleasant. -# -- this also means we effectively always cross-compile, because nothing in -# the chroot is executable during the build process. This is good, because -# build-essential must be cross-compilable so that we can port Baserock -# to new architectures. -# -- we can't build g++, because since it's effectively a cross build we can't -# bootstrap the compiler, and although theoretically we could build one in -# the temporary toolchain, that doesn't actually work. Since the next step -# is a native build we can get a c++ compiler in stage 2 of the bootstrap. - -# In future Morph will handle the build of build-essential that is currently -# done by this script. - -# Disable hashing, so that newly built tools are picked up in PATH immediately -set +h - -set -eu - -## Configuration - -# Installing to a different sysroot (e.g. /tools) is supported, but in order -# to use the result as a staging filler Morph needs to know to create /bin and -# /lib{64} symlinks in the staging area if not present (creating the symlinks -# in the filler itself will conflict with the fhs-dirs chunk) -PREFIX=/usr - -export MAKEFLAGS="-j 4" -export CFLAGS="-O2" - -# i686 -#TARGET="i686-baserock-linux-gnu" -#TARGET_GCC_CONFIG= -#export ARCH=i386 - -# x86_64 -TARGET="x86_64-baserock-linux-gnu" -TARGET_GCC_CONFIG= -export ARCH="x86_64" - -# Little-endian ARM -#TARGET="armv7l-baserock-linux-gnueabi" -#TARGET_GCC_CONFIG="--with-arch=armv7-a" -#export ARCH="arm" - -# Big-endian ARM -# No stable GCC release as of January 2013 can default to big-endian linking. -# You must backport a patch from trunk to make this work. See: -# http://gcc.gnu.org/bugzilla/show_bug.cgi?id=16350 -# Correct linking also depends on the correct --with-arch= flag being given. -#TARGET="armv7leb-baserock-linux-gnueabi" -#TARGET_GCC_CONFIG="--with-arch=armv7-a" -#export ARCH="arm" - - -## Setup - -if [ "$#" -ne "1" ]; then - echo "Usage: $0 BASE_DIR" - echo - echo " Compiles a build-essential chroot in BASE_DIR. The directory" - echo " BASE_DIR/gits is expected to contain checkouts of all source" - echo " code required for the builds; you will currently have to" - echo " assemble this yourself based on the source code of this script." - exit 1 -fi - -BASE_DIR="$1" - -if [ ! -d "$BASE_DIR" ]; then - echo "$BASE_DIR does not exist" - exit 1 -fi - -BASE_DIR="$(readlink -f $BASE_DIR)" - -TOOLCHAIN_DIR="$BASE_DIR"/toolchain -CHROOT_DIR="$BASE_DIR"/chroot - -if [ ! -d "$BASE_DIR" ]; then - echo "$BASE_DIR does not exist" - exit 1 -fi - -if [ ! -d "$BASE_DIR/gits" ]; then - echo "$BASE_DIR/gits does not exist" - exit 1 -fi - -if [ ! -d "$TOOLCHAIN_DIR" ]; then - mkdir -p "$TOOLCHAIN_DIR" -fi - -if [ ! -d "$CHROOT_DIR" ]; then - mkdir -p "$CHROOT_DIR" -fi - - -## Architecture-specific hacks - -fix_chroot_for_target() { - case "$TARGET" in - x86_64*) - # eglibc 2.15 is belligerant about putting things into /lib64, - # especially when prefix is /usr. Currently if PREFIX is NOT /usr - # we force ld.so to be in PREFIX/lib/ld.so; but eglibc installs it - # in /lib64 anyway. - if [ ! -e $CHROOT_DIR/lib64/ld-linux-x86-64.so.2 ]; then - mkdir -p $CHROOT_DIR$PREFIX/lib64 - ln -sf $CHROOT_DIR$PREFIX/lib/ld-linux-x86-64.so.2 \ - $CHROOT_DIR$PREFIX/lib64 - fi ;; - esac -} - -## Build process - -assert_branch() { - branch="$1" - if which git > /dev/null && - [ $(git rev-parse HEAD) != $(git rev-parse "$branch") ]; then - echo "Expected to be building '$branch' branch in $(basename $(pwd))" \ - >&2 - exit 1 - fi -} - -touch_tree() { - # git doesn't preserve mtimes, but this causes rebuilds of things that - # haven't changed which breaks builds if the correct tools are not - # available - #date=$(date '+%C%y%m%d%H%M.%S') - #find -type f -exec touch -m -t $date \{} \; - return 0 -} - -# build-essential - -build_binutils() { - pass="$1" - - source_dir="$BASE_DIR"/gits/binutils-redhat - build_dir="$BASE_DIR"/builds/binutils-"$pass" - - cd "$source_dir" - assert_branch "baserock/build-essential" - touch_tree - - rm -Rf "$build_dir" && mkdir -p "$build_dir" - cd "$build_dir" - - # Note for shell escape purists: quotes around --with-lib-path's argument - # do not work correctly - if [ "$pass" == "pass1" ]; then - extra_config="--target=$TARGET --with-sysroot=\"$CHROOT_DIR\" \ - --with-lib-path=$CHROOT_DIR$PREFIX/lib" - elif [ "$pass" == "pass2" ]; then - extra_config="--host=$TARGET --with-lib-path=$PREFIX/lib " - else - echo "Invalid pass for binutils: $pass" - exit 1 - fi - - # Note that the root configure script's --help doesn't display the - # arguments for sub-configure scripts, but it does pass on arguments to - # them - "$source_dir"/configure \ - --prefix=$PREFIX --disable-nls --disable-werror \ - $extra_config - make - make DESTDIR="$DESTDIR" install -} - -build_busybox() { - source_dir="$BASE_DIR"/gits/busybox - build_dir="$BASE_DIR"/builds/busybox - - cd "$source_dir" - assert_branch "baserock/build-essential" - touch_tree - - rm -Rf "$build_dir" && mkdir -p "$build_dir" - cd "$build_dir" - - # Busybox's default config includes basically everything - make KBUILD_SRC="$source_dir" -f "$source_dir"/Makefile defconfig - - if [ "$PREFIX" != "/usr" ]; then - # Install everything into DESTDIR instead of putting some of it into - # DESTDIR/usr/. For compatibility with the old Baserock staging - # fillers, if the prefix is /usr we still do the traditional /bin vs. - # /usr/bin split for now. - sed -e 's/.*CONFIG_INSTALL_NO_USR.*/CONFIG_INSTALL_NO_USR=y/' -i \ - .config - fi - - sed -e 's/.*CONFIG_STATIC.*/CONFIG_STATIC=y/' -i .config - - # Requires stuff that was removed after eglibc 2.14 - sed -e 's/.*CONFIG_INETD.*/# CONFIG_INETD is not set/' -i .config - - # Disable all module tools. BusyBox's depmod isn't sufficient for Linux - # builds, but Linux will build OK if depmod isn't present at all. Also, we - # have kmod in Baserock which can do all this stuff anyway. - sed -e 's/.*MODPROBE_SMALL=.*/# CONFIG_MODPROBE_SMALL is not set/' \ - -i .config - sed -e 's/.*INSMOD=.*/CONFIG_INSMOD=y/' -i .config - sed -e 's/.*RMMOD=.*/CONFIG_RMMOD=y/' -i .config - sed -e 's/.*LSMOD=.*/CONFIG_LSMOD=y/' -i .config - sed -e 's/.*DEPMOD=.*/CONFIG_DEPMOD=y/' -i .config - sed -e 's/.*MODPROBE=.*/CONFIG_MODPROBE=y/' -i .config - - sed -e 's/.*CONFIG_INETD.*/# CONFIG_INETD is not set/' -i .config - - - make CROSS_COMPILE="$TARGET-" - - if [ "$PREFIX" != "/usr" ]; then - make CROSS_COMPILE="$TARGET-" CONFIG_PREFIX="$DESTDIR$PREFIX" install - else - make CROSS_COMPILE="$TARGET-" CONFIG_PREFIX="$DESTDIR" install - fi -} - -build_eglibc() { - source_dir="$BASE_DIR"/gits/eglibc2 - build_dir="$BASE_DIR"/builds/eglibc - - # Necessary for ARM port. - if [ ! -e "$source_dir/libc/ports" ]; then - ln -s "$source_dir/ports" "$source_dir/libc/ports" - fi - - cd "$source_dir" - assert_branch "baserock/2.15-build-essential" - touch_tree - - rm -Rf "$build_dir" && mkdir -p "$build_dir" - cd "$build_dir" - - # If prefix is set to /usr, eglibc otherwise decides to install its - # libraries in /usr/lib64 on some Linux - extra_config="--libdir=$PREFIX/lib" - - # Suggested by Linux From Scratch. Is the default really to build with - # profiling?? - extra_config="--disable-profile $extra_config" - - # Minimum kernel version that this eglibc will be usable with. - extra_config="--enable-kernel=2.6.25 $extra_config" - - # Location of headers - extra_config="--with-headers=$CHROOT_DIR$PREFIX/include $extra_config" - - # Force configure flags of certain things that can't be detected in a - # cross-compile. - extra_config="$extra_config \ - libc_cv_c_cleanup=yes libc_cv_ctors_header=yes \ - libc_cv_forced_unwind=yes libc_cv_ssp=no" - - # + --without-fp for ARM - "$source_dir"/libc/configure --prefix=$PREFIX --host="$TARGET" \ - --build=$("$source_dir"/libc/scripts/config.guess) \ - --enable-add-ons=nptl,ports $extra_config - make - make "install_root=$DESTDIR" install -} - -build_gcc() { - pass="$1" - source_dir="$BASE_DIR"/gits/gcc-tarball - build_dir="$BASE_DIR"/builds/gcc-"$pass" - - cd "$source_dir" - assert_branch "baserock/build-essential" - touch_tree - - # This hack is to prevent the multilib configuration of the host OS - # leaking into Baserock, which does not use multilib at all. Without this, - # the pass2 gcc will install its libraries into $PREFIX/lib64 instead of - # $PREFIX/lib, because its configure script decides ${toolexeclibdir} - # based on the output of gcc -print-multi-os-directory - if [ "$(echo $TARGET | cut -c -6)" = "x86_64" ]; then - sed -i "$source_dir/gcc/config/i386/t-linux64" \ - -e "/^MULTILIB_OSDIRNAMES/ c\ - MULTILIB_OSDIRNAMES = ." - fi - - rm -Rf "$build_dir" && mkdir -p "$build_dir" - cd "$build_dir" - - if [ "$pass" == "pass1" ]; then - # In pass 1 we build a crosscompiler for $TARGET. - extra_config="--target=$TARGET" - - # The pass 1 compiler needs to find the libraries we build in pass 2. - # Include path must be set explicility, because it defaults to - # $CHROOT_DIR/usr/include. - extra_config="$extra_config \ - --with-sysroot="$CHROOT_DIR" \ - --with-native-system-header-dir=\"$CHROOT_DIR$PREFIX/include\"" - - # Disable stuff that doesn't work when building a cross compiler - # without an existing libc, and generally try to keep this build as - # simple as possible. - extra_config="$extra_config \ - --enable-languages=c \ - --disable-decimal-float --disable-libmudflap \ - --disable-libquadmath --disable-libssp --disable-shared \ - --disable-threads --disable-target-libiberty \ - --disable-target-zlib --without-headers --with-newlib \ - --with-system-zlib" - - # way too slow with this, but it maybe a good idea / required - extra_config="--disable-bootstrap $extra_config" - elif [ "$pass" == "pass2" ]; then - # Pass 1 gcc's fixincludes process created a limits.h before there was - # a real limits.h available for the target. This step (taken from - # Linux From Scratch) creates a better one so gcc can compile. - libgcc_dir=$(dirname $($TARGET-gcc -print-libgcc-file-name)) - cat "$source_dir/gcc/limitx.h" "$source_dir/gcc/glimits.h" \ - "$source_dir/gcc/limity.h" \ - > "$libgcc_dir/include-fixed/limits.h" - - # This time we are creating a native compiler running on $TARGET - extra_config="--host=$TARGET" - - # I'm not sure why this is needed. target should default to host, - # but if we don't pass this explicitly some of the files that should - # go in $PREFIX/lib/gcc/$TARGET/4.6.2 end up in - # $PREFIX/lib/gcc/$TARGET/4.6.3 instead. Weird. - extra_config="--target=$TARGET $extra_config" - - # The two-step compiler process means we don't need to bootstrap now - # (and couldn't anyway, since build != host for pass 2). - extra_config="--disable-bootstrap $extra_config" - - # C++ doesn't build without a C++ compiler, I think, so we still - # can't have that yet. We don't need one anyway. - extra_config="$extra_config \ - --enable-clocale=gnu \ - --enable-languages=c --enable-shared \ - --enable-threads=posix" - else - echo "Invalid pass for gcc: $pass" - exit 1 - fi - - # An attempt to stop anything going in $PREFIX/lib64 - extra_config=" --libdir=$PREFIX/lib $extra_config" - - # Disable searching /usr/local/include for headers - extra_config="--with-local-prefix=$PREFIX $extra_config" - - # It makes no sense for Baserock to use multilib. - extra_config="--disable-multilib $extra_config" - - # General stuff that we don't need / won't work right now - extra_config="$extra_config \ - --disable-libgomp --without-cloog --without-ppl" - - # mpfr is built as part of gcc, but we need to point latter components - # to the result of this build. - extra_config="$extra_config \ - --with-mpfr-include="$source_dir"/mpfr/src \ - --with-mpfr-lib="$build_dir"/mpfr/src/.libs" - - "$source_dir"/configure --prefix=$PREFIX --disable-nls \ - $TARGET_GCC_CONFIG $extra_config - make - make DESTDIR="$DESTDIR" install -} - -build_linux_api_headers() { - source_dir="$BASE_DIR"/gits/linux - build_dir="$BASE_DIR"/builds/linux-api-headers - - cd "$source_dir" - assert_branch "baserock/build-essential" - touch_tree - - rm -Rf "$build_dir" && mkdir -p "$build_dir" - - # We don't achieve a real out of tree build here :( - make O="$build_dir" mrproper - #make O="$build_dir" headers_check - make O="$build_dir" INSTALL_HDR_PATH="$DESTDIR$PREFIX" headers_install -} - -# build-essential-plus - -build_gawk() { - source_dir="$BASE_DIR"/gits/gawk - build_dir="$BASE_DIR"/builds/gawk - - cd "$source_dir" - assert_branch "baserock/build-essential" - touch_tree - - rm -Rf "$build_dir" && mkdir -p "$build_dir" - cd "$build_dir" - - $source_dir/configure --prefix=$PREFIX --host=$TARGET --disable-nls - make - make DESTDIR="$DESTDIR" install -} - -build_make() { - source_dir="$BASE_DIR"/gits/make - build_dir="$BASE_DIR"/builds/make - - cd "$source_dir" - assert_branch "baserock/build-essential" - touch_tree - - rm -Rf "$build_dir" && mkdir -p "$build_dir" - cd "$build_dir" - - $source_dir/configure --prefix=$PREFIX --host=$TARGET --disable-nls - make - make DESTDIR="$DESTDIR" install -} - - -## 1. Build a "cross-compiler" -# -# We always do two builds of the compiler for now; hopefully the rebootstrap -# process in Baserock will allow us to optimise this out when not -# cross-compiling (we will never be cross-compiling except when bootstrapping -# a new architecture). - -DESTDIR="$TOOLCHAIN_DIR" -export CC="gcc" -export PATH="$TOOLCHAIN_DIR$PREFIX/bin":"$PATH" - -build_binutils pass1 -build_gcc pass1 - -if ! [ -x "$(which $TARGET-gcc)" ]; then - echo "Missing $TARGET-gcc in PATH: something went wrong" - exit 1 -fi - -# A hack so that we can build eglibc with a no-shared-libs gcc; libgcc_eh is -# referenced in the eglibc build, but its static version contains all of the -# necessary symbols anyway. -for f in `find "$TOOLCHAIN_DIR" -name libgcc.a`; do \ - EH="`echo "$f" | sed 's/libgcc/&_eh/'`" && if [ ! -e "$EH" ]; then - ln -s libgcc.a "$EH"; - fi; -done - -## 2. Build the actual intermediate build-essential chroot -## -DESTDIR="$CHROOT_DIR" -export CC= - -# We don't want to customise the specs until we actually have eglibc installed. -specs_dir="$(dirname $($TARGET-gcc --print-libgcc-file-name))" -rm -f "$specs_dir/specs" - -build_linux_api_headers -build_eglibc - -# Passing --with-lib-path to our pass1 linker gives it the correct library -# search path, but this doesn't work for the startup files. The startup files -# (crt*.o) are searched for by gcc itself, and passed to ld as absolute paths -# if found or just filenames if not (in which case, ld will not search for -# them either because they are .o files, not libraries). -# -# One alternative to this hack is to pass -B $CHROOT_DIR$PREFIX/lib to gcc, but -# that option often gets eaten by libtool and by gcc's nested configure -# scripts so it's not fully effective. -$TARGET-gcc -dumpspecs | \ -sed -e "s@\(crt1\|gcrt1\|Scrt1\|crti\|crtn\)\.o@$CHROOT_DIR$PREFIX/lib/&@g" \ - > "$specs_dir/specs" - -if [ "$PREFIX" != "/usr" ]; then - # eglibc (in sysdeps/unix/sysv/linux/configure) puts some files in /lib64 - # if PREFIX is /usr, so for now we go with this and avoid doing any fixups. - # For other prefixes, everything is installed correctly into the sysroot. - # - # In the future we should fix eglibc to put all files into PREFIX/lib at - # which point this fix will always be necessary, until GCC grows a better - # way to specify the location of ld.so. - sed -i "$specs_dir/specs" -e "s@/lib\(64\)\?/ld@$PREFIX/&@g" -fi - -fix_chroot_for_target - -echo -echo "Testing pass 1 compiler and chroot ..." - -cat < $CHROOT_DIR/build-test.c -#include -int main() { printf("OK"); return 0; } -EOF -$TARGET-gcc $CFLAGS "$CHROOT_DIR/build-test.c" -o "$CHROOT_DIR/build-test" - -if [ "$PREFIX" != "/usr" ]; then - # Check that we successfully forced use of the ld.so inside the sysroot - if ! readelf -l "$CHROOT_DIR"/build-test | grep -q "interpreter: $PREFIX" ; - then - echo "Wrong interpreter in output of pass 1 C compiler" - exit 1 - fi -fi - -rm "$CHROOT_DIR/build-test" "$CHROOT_DIR/build-test.c" - -export CC=$TARGET-gcc -export AR=$TARGET-ar -export RANLIB=$TARGET-ranlib - -build_binutils pass2 -build_busybox - -# This is hardcoded into gcc as NATIVE_SYSTEM_HEADERS. It's used only when -# running fixincludes, which only makes sense when cross-compiling anyway. -# Compilation fails if this directory is missing anyway, unfortunately. -created_native_system_header_dir="no" -if [ ! -d /usr/include ]; then - mkdir -p /usr/include - created_native_system_header_dir="yes" -fi - -build_gcc pass2 - -if [ ! -e "$DESTDIR$PREFIX/bin/cc" ]; then - ln -s gcc "$DESTDIR$PREFIX/bin/cc" -fi - -if [ "$created_native_system_header_dir" == "yes" ]; then - rm /usr/include - rm /usr -fi - -build_gawk -build_make - -echo -echo "Complete! You now have a build-essential for $TARGET in " -echo "$CHROOT_DIR." -echo -echo "Before it can be used as a staging filler for Morph, you need to " -echo "manually run the install-commands from the 'fhs-dirs' chunk as root to " -echo "set up the necessary files and device nodes." -- cgit v1.2.1 From b7f69986034afef8a792839fac601caa0f20d55a Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Mon, 28 Jan 2013 14:57:39 +0000 Subject: Remove features used by old bootstrap method That means that bootstrapping Baserock is currently not possible with this branch of Morph, but there's no reason it cannot be bootstrapped using an older version of Morph instead. --- morphlib/app.py | 56 ++++++++++++++++++++++---------------- morphlib/buildcommand.py | 26 ++++++------------ morphlib/buildenvironment.py | 14 ++-------- morphlib/buildenvironment_tests.py | 32 ++-------------------- morphlib/builder2.py | 3 ++ morphlib/source.py | 2 +- tests.as-root/setup | 1 - tests.branching/setup | 3 +- tests.deploy/setup | 1 - tests.merging/setup | 3 +- tests/setup | 3 +- 11 files changed, 53 insertions(+), 91 deletions(-) diff --git a/morphlib/app.py b/morphlib/app.py index 87710ee5..c8b9b911 100755 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -105,16 +105,7 @@ class Morph(cliapp.Application): default=None, group=group_advanced) - # Build Options group_build = 'Build Options' - self.settings.boolean(['bootstrap'], - 'build stuff in bootstrap mode; this is ' - 'DANGEROUS and will install stuff on your ' - 'system', - group=group_build) - self.settings.boolean(['keep-path'], - 'do not touch the PATH environment variable', - group=group_build) self.settings.integer(['max-jobs'], 'run at most N parallel jobs with make (default ' 'is to a value based on the number of CPUs ' @@ -135,20 +126,6 @@ class Morph(cliapp.Application): 'always push temporary build branches to the ' 'remote repository', group=group_build) - self.settings.boolean(['staging-chroot'], - 'build things in an isolated chroot ' - '(default: true)', - group=group_build) - self.settings.string_list(['staging-filler'], - 'use FILE as contents of build chroot', - metavar='FILE', - group=group_build) - self.settings.string(['target-cflags'], - 'inject additional CFLAGS into the environment ' - 'that is used to build chunks', - metavar='CFLAGS', - default='', - group=group_build) self.settings.string(['tempdir'], 'temporary directory to use for builds ' '(this is separate from just setting $TMPDIR ' @@ -159,13 +136,36 @@ class Morph(cliapp.Application): metavar='DIR', default=os.environ.get('TMPDIR'), group=group_build) + + # Would be better to have a separate tool to cross-bootstrap + # because it's completely outside normal Morph usage + group_bootstrap = 'Bootstrap Options' + self.settings.string(['target-cflags'], + 'inject additional CFLAGS into the environment ' + 'that is used to build chunks', + metavar='CFLAGS', + default='', + group=group_bootstrap) self.settings.string(['toolchain-target'], 'set the TOOLCHAIN_TARGET variable which is used ' 'in some chunks to determine which architecture ' 'to build tools for', metavar='TOOLCHAIN_TARGET', default=defaults['toolchain-target'], - group=group_build) + group=group_bootstrap) + + # These cannot be removed just yet because existing morph.conf files + # would fail to parse. + group_obsolete = 'Obsolete Options' + self.settings.boolean(['staging-chroot'], + 'build things in an isolated chroot ' + '(default: true)', + default=True, + group=group_obsolete) + self.settings.string_list(['staging-filler'], + 'use FILE as contents of build chroot', + metavar='FILE', + group=group_obsolete) def check_time(self): # Check that the current time is not far in the past. @@ -179,6 +179,14 @@ class Morph(cliapp.Application): def process_args(self, args): self.check_time() + # Handle obsolete options + if self.settings['staging-chroot'] is not True: + raise cliapp.AppException( + 'The "staging-chroot" option has been set to False. This ' + 'option is obsolete and should be left as the default (True).') + if self.settings['staging-filler'] is not None: + logging.warning('Use of a staging filler is deprecated.') + # Combine the aliases into repo-alias before passing on to normal # command processing. This means everything from here on down can # treat settings['repo-alias'] as the sole source of prefixes for git diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index dfacd760..4200dc0e 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -219,18 +219,14 @@ class BuildCommand(object): deps = self.get_recursive_deps(artifact) self.cache_artifacts_locally(deps) staging_area = self.create_staging_area(artifact) - if self.app.settings['staging-chroot']: - if artifact.source.morphology.needs_staging_area: - self.install_fillers(staging_area) - self.install_chunk_artifacts(staging_area, - deps, artifact) - morphlib.builder2.ldconfig(self.app.runcmd, - staging_area.tempdir) + if artifact.source.morphology.needs_staging_area: + self.install_fillers(staging_area) + self.install_chunk_artifacts(staging_area, + deps, artifact) + morphlib.builder2.ldconfig(self.app.runcmd, + staging_area.tempdir) self.build_and_cache(staging_area, artifact) - if self.app.settings['bootstrap']: - self.install_chunk_artifacts(staging_area, - (artifact,)) self.remove_staging_area(staging_area) self.app.status(msg='%(kind)s %(name)s is cached at %(cachepath)s', kind=artifact.source.morphology['kind'], @@ -319,12 +315,8 @@ class BuildCommand(object): def create_staging_area(self, artifact): '''Create the staging area for building a single artifact.''' - if self.app.settings['staging-chroot']: - staging_root = tempfile.mkdtemp(dir=self.app.settings['tempdir']) - staging_temp = staging_root - else: - staging_root = '/' - staging_temp = tempfile.mkdtemp(dir=self.app.settings['tempdir']) + staging_root = tempfile.mkdtemp(dir=self.app.settings['tempdir']) + staging_temp = staging_root self.app.status(msg='Creating staging area') staging_area = morphlib.stagingarea.StagingArea(self.app, @@ -387,5 +379,5 @@ class BuildCommand(object): setup_mounts = self.app.settings['staging-chroot'] builder = morphlib.builder2.Builder( self.app, staging_area, self.lac, self.rac, self.lrc, - self.build_env, self.app.settings['max-jobs'], setup_mounts) + self.build_env, self.app.settings['max-jobs'], True) return builder.build_and_cache(artifact) diff --git a/morphlib/buildenvironment.py b/morphlib/buildenvironment.py index d9e3210f..57270414 100644 --- a/morphlib/buildenvironment.py +++ b/morphlib/buildenvironment.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -44,7 +44,6 @@ class BuildEnvironment(): # copy a set of white-listed variables from the original env copied_vars = dict.fromkeys([ - 'BOOTSTRAP_TOOLS', 'DISTCC_HOSTS', 'LD_PRELOAD', 'LD_LIBRARY_PATH', @@ -62,10 +61,6 @@ class BuildEnvironment(): if copied_vars[name] is not None: env[name] = copied_vars[name] - if settings['bootstrap'] or not settings['staging-chroot']: - if 'TMPDIR' in self._osenv: - env['TMPDIR'] = self._osenv['TMPDIR'] - env['TERM'] = self._override_term env['SHELL'] = self._override_shell env['USER'] = \ @@ -74,15 +69,12 @@ class BuildEnvironment(): env['LC_ALL'] = self._override_locale env['HOME'] = self._override_home - if settings['keep-path'] or settings['bootstrap']: - env['PATH'] = path - else: - env['PATH'] = self._default_path + env['PATH'] = self._default_path env['TOOLCHAIN_TARGET'] = settings['toolchain-target'] env['CFLAGS'] = settings['target-cflags'] env['PREFIX'] = settings['prefix'] - env['BOOTSTRAP'] = 'true' if settings['bootstrap'] else 'false' + env['BOOTSTRAP'] = 'false' if not settings['no-ccache']: env['PATH'] = ('%s:%s' % (self._ccache_path, env['PATH'])) # FIXME: we should set CCACHE_BASEDIR so any objects that refer to their diff --git a/morphlib/buildenvironment_tests.py b/morphlib/buildenvironment_tests.py index 61844c19..a55cd5ac 100644 --- a/morphlib/buildenvironment_tests.py +++ b/morphlib/buildenvironment_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -24,14 +24,11 @@ class BuildEnvironmentTests(unittest.TestCase): def setUp(self): self.settings = { - 'keep-path': False, - 'bootstrap': False, 'toolchain-target': '%s-baserock-linux-gnu' % morphlib.util.arch(), 'target-cflags': '', 'prefix': '/usr', 'no-ccache': True, - 'no-distcc': True, - 'staging-chroot': False, + 'no-distcc': True } self.fake_env = { 'PATH': '/fake_bin', @@ -48,39 +45,16 @@ class BuildEnvironmentTests(unittest.TestCase): self.assertEqual(buildenv.arch, 'noarch') def test_sets_default_path(self): - self.settings['keep-path'] = False - self.settings['bootstrap'] = False olddefaultpath = buildenvironment.BuildEnvironment._default_path buildenvironment.BuildEnvironment._default_path = self.default_path buildenv = buildenvironment.BuildEnvironment(self.settings) buildenvironment.BuildEnvironment._default_path = olddefaultpath self.assertTrue(self.default_path in buildenv.env['PATH']) - def test_uses_env_path_with_keep_path(self): - self.settings['keep-path'] = True - - old_osenv = buildenvironment.BuildEnvironment._osenv - buildenvironment.BuildEnvironment._osenv = self.fake_env - buildenv = buildenvironment.BuildEnvironment(self.settings) - buildenvironment.BuildEnvironment._osenv = old_osenv - - self.assertEqual(buildenv.env['PATH'], self.fake_env['PATH']) - - def test_uses_env_path_with_bootstrap(self): - self.settings['bootstrap'] = True - - old_osenv = buildenvironment.BuildEnvironment._osenv - buildenvironment.BuildEnvironment._osenv = self.fake_env - buildenv = buildenvironment.BuildEnvironment(self.settings) - buildenvironment.BuildEnvironment._osenv = old_osenv - - self.assertEqual(buildenv.env['PATH'], self.fake_env['PATH']) - def test_copies_whitelist_vars(self): env = self.fake_env safe = { 'DISTCC_HOSTS': 'example.com:example.co.uk', - 'TMPDIR': '/buildenv/tmp/dir', 'LD_PRELOAD': '/buildenv/lib/libbuildenv.so', 'LD_LIBRARY_PATH': '/buildenv/lib:/buildenv/lib64', 'FAKEROOTKEY': 'b011de73', @@ -120,8 +94,6 @@ class BuildEnvironmentTests(unittest.TestCase): self.settings['target-cflags']) self.assertEqual(buildenv.env['PREFIX'], self.settings['prefix']) - self.assertEqual(buildenv.env['BOOTSTRAP'], - 'true' if self.settings['bootstrap'] else 'false') def test_ccache_vars_set(self): self.settings['no-ccache'] = False diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 73745d66..6f7836e3 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -54,6 +54,9 @@ def ldconfig(runcmd, rootdir): # pragma: no cover ''' + # FIXME: use the version in ROOTDIR, since even in + # bootstrap it will now always exist due to being part of build-essential + conf = os.path.join(rootdir, 'etc', 'ld.so.conf') if os.path.exists(conf): logging.debug('Running ldconfig for %s' % rootdir) diff --git a/morphlib/source.py b/morphlib/source.py index b536cbdf..99b0a993 100644 --- a/morphlib/source.py +++ b/morphlib/source.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/tests.as-root/setup b/tests.as-root/setup index 03a438e0..a85507e0 100755 --- a/tests.as-root/setup +++ b/tests.as-root/setup @@ -200,7 +200,6 @@ cat < "$DATADIR/morph.conf" repo-alias = test=file://$DATADIR/#file://$DATADIR/ cachedir = $DATADIR/cache log = $DATADIR/morph.log -keep-path = true no-distcc = true quiet = true log = /tmp/morph.log diff --git a/tests.branching/setup b/tests.branching/setup index c0dfd969..1fa5b8e3 100755 --- a/tests.branching/setup +++ b/tests.branching/setup @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -32,7 +32,6 @@ cat < "$DATADIR/morph.conf" repo-alias = test=file://$DATADIR/%s#file://$DATADIR/%s cachedir = $DATADIR/workspace/.morph/cache log = $DATADIR/morph.log -keep-path = true no-distcc = true quiet = true EOF diff --git a/tests.deploy/setup b/tests.deploy/setup index 1065849d..b0054fa5 100755 --- a/tests.deploy/setup +++ b/tests.deploy/setup @@ -196,7 +196,6 @@ cat < "$DATADIR/morph.conf" repo-alias = test=file://$DATADIR/#file://$DATADIR/ cachedir = $DATADIR/cache log = $DATADIR/morph.log -keep-path = true no-distcc = true quiet = true log = /tmp/morph.log diff --git a/tests.merging/setup b/tests.merging/setup index c0dfd969..1fa5b8e3 100755 --- a/tests.merging/setup +++ b/tests.merging/setup @@ -1,5 +1,5 @@ #!/bin/bash -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -32,7 +32,6 @@ cat < "$DATADIR/morph.conf" repo-alias = test=file://$DATADIR/%s#file://$DATADIR/%s cachedir = $DATADIR/workspace/.morph/cache log = $DATADIR/morph.log -keep-path = true no-distcc = true quiet = true EOF diff --git a/tests/setup b/tests/setup index 25472dc7..50fd5bd4 100755 --- a/tests/setup +++ b/tests/setup @@ -10,7 +10,7 @@ # The stratum repository contains a single branch, "master", with a # stratum and a system morphology that include the chunk above. # -# Copyright (C) 2011, 2012 Codethink Limited +# Copyright (C) 2011-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -128,7 +128,6 @@ cat < "$DATADIR/morph.conf" repo-alias = test=file://$DATADIR/%s#file://$DATADIR/%s cachedir = $DATADIR/cache log = $DATADIR/morph.log -keep-path = true no-distcc = true quiet = true EOF -- cgit v1.2.1 From 6a61dd9cc1fe8a3ccd2128fb628ed929fd496ad0 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Fri, 1 Feb 2013 16:20:10 +0000 Subject: Remove support for bootstrap mode in staging area This involved having a staging area with split personalities and was generally a bit ugly. --- morphlib/buildcommand.py | 26 ++++++-------------------- morphlib/stagingarea.py | 20 ++++++++------------ morphlib/stagingarea_tests.py | 9 ++------- 3 files changed, 16 insertions(+), 39 deletions(-) diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 4200dc0e..c7c650d3 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -224,8 +224,7 @@ class BuildCommand(object): self.install_chunk_artifacts(staging_area, deps, artifact) morphlib.builder2.ldconfig(self.app.runcmd, - staging_area.tempdir) - + staging_area.dirname) self.build_and_cache(staging_area, artifact) self.remove_staging_area(staging_area) self.app.status(msg='%(kind)s %(name)s is cached at %(cachepath)s', @@ -315,32 +314,19 @@ class BuildCommand(object): def create_staging_area(self, artifact): '''Create the staging area for building a single artifact.''' - staging_root = tempfile.mkdtemp(dir=self.app.settings['tempdir']) - staging_temp = staging_root - self.app.status(msg='Creating staging area') - staging_area = morphlib.stagingarea.StagingArea(self.app, - staging_root, - staging_temp) + staging_dir = tempfile.mkdtemp(dir=self.app.settings['tempdir']) + staging_area = morphlib.stagingarea.StagingArea(self.app, staging_dir) return staging_area def remove_staging_area(self, staging_area): '''Remove the staging area.''' - if staging_area.dirname != '/': - self.app.status(msg='Removing staging area') - staging_area.remove() - temp_path = staging_area.tempdir - if temp_path != '/' and os.path.exists(temp_path): - self.app.status(msg='Removing temporary staging directory') - shutil.rmtree(temp_path) + self.app.status(msg='Removing staging area') + staging_area.remove() def install_fillers(self, staging_area): - '''Install staging fillers into the staging area. - - This must not be called in bootstrap mode. - - ''' + '''Install staging fillers into the staging area.''' logging.debug('Pre-populating staging area %s' % staging_area.dirname) logging.debug('Fillers: %s' % diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index ae9e7e39..24d4ebf9 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012,2013 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -27,21 +27,17 @@ class StagingArea(object): '''Represent the staging area for building software. - The build dependencies of what will be built will be installed in the - staging area. The staging area may be a dedicated part of the - filesystem, used with chroot, or it can be the actual root of the - filesystem, which is needed when bootstrap building Baserock. The - caller chooses this by providing the root directory of the staging - area when the object is created. The directory must already exist. - - The staging area can also install build artifacts. + The staging area is a temporary directory. In normal operation the build + dependencies of the artifact being built are installed into the staging + area and then 'chroot' is used to isolate the build processes from the host + system. Chunks built in 'test' or 'build-essential' mode have an empty + staging area and are allowed to use the tools of the host. ''' - def __init__(self, app, dirname, tempdir): + def __init__(self, app, dirname): self._app = app self.dirname = dirname - self.tempdir = tempdir self.builddirname = None self.destdirname = None self.mounted = None @@ -52,7 +48,7 @@ class StagingArea(object): os.mkdir(dirname) def _dir_for_source(self, source, suffix): - dirname = os.path.join(self.tempdir, + dirname = os.path.join(self.dirname, '%s.%s' % (source.morphology['name'], suffix)) self._mkdir(dirname) return dirname diff --git a/morphlib/stagingarea_tests.py b/morphlib/stagingarea_tests.py index 313226d2..5c547f6e 100644 --- a/morphlib/stagingarea_tests.py +++ b/morphlib/stagingarea_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012,2013 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -57,8 +57,7 @@ class StagingAreaTests(unittest.TestCase): self.staging = os.path.join(self.tempdir, 'staging') self.created_dirs = [] self.sa = morphlib.stagingarea.StagingArea( - FakeApplication(self.cachedir, self.tempdir), - self.staging, self.staging) + FakeApplication(self.cachedir, self.tempdir), self.staging) def tearDown(self): shutil.rmtree(self.tempdir) @@ -89,10 +88,6 @@ class StagingAreaTests(unittest.TestCase): def test_remembers_specified_directory(self): self.assertEqual(self.sa.dirname, self.staging) - def test_accepts_root_directory(self): - sa = morphlib.stagingarea.StagingArea(object(), '/', '/tmp') - self.assertEqual(sa.dirname, '/') - def test_creates_build_directory(self): source = FakeSource() self.sa._mkdir = self.fake_mkdir -- cgit v1.2.1 From ece7f823de6bd61a0676edf71a9525697848824e Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Fri, 22 Feb 2013 19:07:13 +0000 Subject: Refactor build process Reorganise the build_artifact() and build_artifacts() functions to allow more complex work when setting up chunk builds in build_artifact(). The staging area now holds the BuildEnvironment object (the environment variables that should be set during build). This makes sense because all build commands should be run inside the staging area and therefore through the StagingArea object. The BuildEnvironment object is now considered immutable after it is created. The environment is used in cache key computation when computing what artifacts are required; if it changes after that point we risk either computing different artifact keys for the same artifact or missing data in the cache key that should be included in the hash. Better to force changes into a separate 'extra_env' variable. --- morphlib/artifact.py | 1 - morphlib/buildcommand.py | 92 +++++++++++++++++--------------------- morphlib/buildenvironment.py | 20 ++++----- morphlib/buildenvironment_tests.py | 33 +++++--------- morphlib/builder2.py | 21 ++++----- morphlib/builder2_tests.py | 9 ++-- morphlib/cachekeycomputer.py | 7 ++- morphlib/cachekeycomputer_tests.py | 11 ++--- morphlib/stagingarea.py | 52 ++++++++++++++------- morphlib/stagingarea_tests.py | 17 ++++++- morphlib/util.py | 1 - 11 files changed, 130 insertions(+), 134 deletions(-) diff --git a/morphlib/artifact.py b/morphlib/artifact.py index aef48d76..5f084384 100644 --- a/morphlib/artifact.py +++ b/morphlib/artifact.py @@ -83,4 +83,3 @@ class Artifact(object): yield a return list(depth_first(self)) - diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index c7c650d3..56c7fae8 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -181,19 +181,38 @@ class BuildCommand(object): assert len(maybe) == 1 return maybe.pop() - def build_in_order(self, artifact): + def build_in_order(self, root_artifact): '''Build everything specified in a build order.''' - self.app.status(msg='Building according to build ordering', - chatty=True) - artifacts = artifact.walk() + self.app.status(msg='Building a set of artifacts', chatty=True) + artifacts = root_artifact.walk() old_prefix = self.app.status_prefix for i, a in enumerate(artifacts): self.app.status_prefix = ( old_prefix + '[Build %d/%d] ' % ((i+1), len(artifacts))) - self.build_artifact(a) + + self.app.status(msg='Checking if %(kind)s %(name)s needs building', + kind=a.source.morphology['kind'], name=a.name) + + if self.is_built(a): + self.app.status(msg='The %(kind)s %(name)s is already built', + kind=a.source.morphology['kind'], name=a.name) + self.cache_artifacts_locally([a]) + else: + self.app.status(msg='Building %(kind)s %(name)s', + kind=a.source.morphology['kind'], name=a.name) + self.build_artifact(a) + + self.app.status(msg='%(kind)s %(name)s is cached at %(cachepath)s', + kind=a.source.morphology['kind'], name=a.name, + cachepath=self.lac.artifact_filename(a), + chatty=(a.source.morphology['kind'] != "system")) self.app.status_prefix = old_prefix + def is_built(self, artifact): + '''Does either cache already have the artifact?''' + return self.lac.has(artifact) or (self.rac and self.rac.has(artifact)) + def build_artifact(self, artifact): '''Build one artifact. @@ -201,52 +220,20 @@ class BuildCommand(object): in either the local or remote cache already. ''' - - self.app.status(msg='Checking if %(kind)s %(name)s needs building', - kind=artifact.source.morphology['kind'], - name=artifact.name) - - if self.is_built(artifact): - self.app.status(msg='The %(kind)s %(name)s is already built', - kind=artifact.source.morphology['kind'], - name=artifact.name) - self.cache_artifacts_locally([artifact]) - else: - self.app.status(msg='Building %(kind)s %(name)s', - kind=artifact.source.morphology['kind'], - name=artifact.name) - self.get_sources(artifact) - deps = self.get_recursive_deps(artifact) - self.cache_artifacts_locally(deps) - staging_area = self.create_staging_area(artifact) - if artifact.source.morphology.needs_staging_area: - self.install_fillers(staging_area) - self.install_chunk_artifacts(staging_area, - deps, artifact) - morphlib.builder2.ldconfig(self.app.runcmd, - staging_area.dirname) - self.build_and_cache(staging_area, artifact) - self.remove_staging_area(staging_area) - self.app.status(msg='%(kind)s %(name)s is cached at %(cachepath)s', - kind=artifact.source.morphology['kind'], - name=artifact.name, - cachepath=self.lac.artifact_filename(artifact), - chatty=(artifact.source.morphology['kind'] != - "system")) - - def is_built(self, artifact): - '''Does either cache already have the artifact?''' - return self.lac.has(artifact) or (self.rac and self.rac.has(artifact)) + self.get_sources(artifact) + deps = self.get_recursive_deps(artifact) + self.cache_artifacts_locally(deps) + staging_area = self.create_staging_area() + if artifact.source.morphology.needs_staging_area: + self.install_fillers(staging_area) + self.install_chunk_artifacts(staging_area, deps, artifact) + morphlib.builder2.ldconfig(self.app.runcmd, + staging_area.dirname) + self.build_and_cache(staging_area, artifact) + self.remove_staging_area(staging_area) def get_recursive_deps(self, artifact): - done = set() - todo = set((artifact,)) - while todo: - for a in todo.pop().dependencies: - if a not in done: - done.add(a) - todo.add(a) - return done + return artifact.walk()[:-1] def get_sources(self, artifact): '''Update the local git repository cache with the sources.''' @@ -311,12 +298,13 @@ class BuildCommand(object): copy(self.rac.get_artifact_metadata(artifact, 'meta'), self.lac.put_artifact_metadata(artifact, 'meta')) - def create_staging_area(self, artifact): + def create_staging_area(self): '''Create the staging area for building a single artifact.''' self.app.status(msg='Creating staging area') staging_dir = tempfile.mkdtemp(dir=self.app.settings['tempdir']) - staging_area = morphlib.stagingarea.StagingArea(self.app, staging_dir) + staging_area = morphlib.stagingarea.StagingArea( + self.app, staging_dir, self.build_env, False, {}) return staging_area def remove_staging_area(self, staging_area): @@ -365,5 +353,5 @@ class BuildCommand(object): setup_mounts = self.app.settings['staging-chroot'] builder = morphlib.builder2.Builder( self.app, staging_area, self.lac, self.rac, self.lrc, - self.build_env, self.app.settings['max-jobs'], True) + self.app.settings['max-jobs'], True) return builder.build_and_cache(artifact) diff --git a/morphlib/buildenvironment.py b/morphlib/buildenvironment.py index 57270414..fce98da4 100644 --- a/morphlib/buildenvironment.py +++ b/morphlib/buildenvironment.py @@ -21,17 +21,18 @@ import morphlib class BuildEnvironment(): def __init__(self, settings, arch=None): + self.extra_path = [] + self.arch = morphlib.util.arch() if arch is None else arch self.env = self._clean_env(settings) _osenv = os.environ - _default_path = '/sbin:/usr/sbin:/bin:/usr/bin' - _override_term = 'dumb' + _ccache_path = '/usr/lib/ccache' + _override_home = '/tmp' + _override_locale = 'C' _override_shell = '/bin/sh' + _override_term = 'dumb' _override_username = 'tomjon' - _override_locale = 'C' - _override_home = '/tmp' - _ccache_path = '/usr/lib/ccache' def _clean_env(self, settings): '''Create a fresh set of environment variables for a clean build. @@ -40,8 +41,6 @@ class BuildEnvironment(): ''' - path = self._osenv['PATH'] - # copy a set of white-listed variables from the original env copied_vars = dict.fromkeys([ 'DISTCC_HOSTS', @@ -69,14 +68,11 @@ class BuildEnvironment(): env['LC_ALL'] = self._override_locale env['HOME'] = self._override_home - env['PATH'] = self._default_path - - env['TOOLCHAIN_TARGET'] = settings['toolchain-target'] - env['CFLAGS'] = settings['target-cflags'] env['PREFIX'] = settings['prefix'] env['BOOTSTRAP'] = 'false' if not settings['no-ccache']: - env['PATH'] = ('%s:%s' % (self._ccache_path, env['PATH'])) + self.extra_path.append(self._ccache_path) + # FIXME: we should set CCACHE_BASEDIR so any objects that refer to their # current directory get corrected. This improve the cache hit rate # env['CCACHE_BASEDIR'] = self.tempdir.dirname diff --git a/morphlib/buildenvironment_tests.py b/morphlib/buildenvironment_tests.py index a55cd5ac..2af3f605 100644 --- a/morphlib/buildenvironment_tests.py +++ b/morphlib/buildenvironment_tests.py @@ -14,6 +14,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import copy import unittest import morphlib @@ -24,8 +25,6 @@ class BuildEnvironmentTests(unittest.TestCase): def setUp(self): self.settings = { - 'toolchain-target': '%s-baserock-linux-gnu' % morphlib.util.arch(), - 'target-cflags': '', 'prefix': '/usr', 'no-ccache': True, 'no-distcc': True @@ -44,13 +43,6 @@ class BuildEnvironmentTests(unittest.TestCase): arch='noarch') self.assertEqual(buildenv.arch, 'noarch') - def test_sets_default_path(self): - olddefaultpath = buildenvironment.BuildEnvironment._default_path - buildenvironment.BuildEnvironment._default_path = self.default_path - buildenv = buildenvironment.BuildEnvironment(self.settings) - buildenvironment.BuildEnvironment._default_path = olddefaultpath - self.assertTrue(self.default_path in buildenv.env['PATH']) - def test_copies_whitelist_vars(self): env = self.fake_env safe = { @@ -62,15 +54,15 @@ class BuildEnvironmentTests(unittest.TestCase): 'FAKEROOT_FD_BASE': '-1', } env.update(safe) - old_osenv = buildenvironment.BuildEnvironment._osenv buildenvironment.BuildEnvironment._osenv = env - buildenv = buildenvironment.BuildEnvironment(self.settings) - buildenvironment.BuildEnvironment._osenv = old_osenv + buildenv = buildenvironment.BuildEnvironment(self.settings) self.assertEqual(sorted(safe.items()), sorted([(k, buildenv.env[k]) for k in safe.keys()])) + buildenvironment.BuildEnvironment._osenv = old_osenv + def test_user_spellings_equal(self): buildenv = buildenvironment.BuildEnvironment(self.settings) self.assertTrue(buildenv.env['USER'] == buildenv.env['USERNAME'] == @@ -88,16 +80,13 @@ class BuildEnvironmentTests(unittest.TestCase): def test_environment_settings_set(self): buildenv = buildenvironment.BuildEnvironment(self.settings) - self.assertEqual(buildenv.env['TOOLCHAIN_TARGET'], - self.settings['toolchain-target']) - self.assertEqual(buildenv.env['CFLAGS'], - self.settings['target-cflags']) - self.assertEqual(buildenv.env['PREFIX'], - self.settings['prefix']) + #self.assertEqual(buildenv.env['TOOLCHAIN_TARGET'], + # self.settings['toolchain-target']) def test_ccache_vars_set(self): - self.settings['no-ccache'] = False - self.settings['no-distcc'] = False - buildenv = buildenvironment.BuildEnvironment(self.settings) - self.assertTrue(buildenv._ccache_path in buildenv.env['PATH']) + new_settings = copy.copy(self.settings) + new_settings['no-ccache'] = False + new_settings['no-distcc'] = False + buildenv = buildenvironment.BuildEnvironment(new_settings) + self.assertTrue(buildenv._ccache_path in buildenv.extra_path) self.assertEqual(buildenv.env['CCACHE_PREFIX'], 'distcc') diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 6f7836e3..f6a1bafa 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -155,15 +155,14 @@ class BuilderBase(object): '''Base class for building artifacts.''' def __init__(self, app, staging_area, local_artifact_cache, - remote_artifact_cache, artifact, repo_cache, - build_env, max_jobs, setup_mounts): + remote_artifact_cache, artifact, repo_cache, max_jobs, + setup_mounts): self.app = app self.staging_area = staging_area self.local_artifact_cache = local_artifact_cache self.remote_artifact_cache = remote_artifact_cache self.artifact = artifact self.repo_cache = repo_cache - self.build_env = build_env self.max_jobs = max_jobs self.build_watch = morphlib.stopwatch.Stopwatch() self.setup_mounts = setup_mounts @@ -253,7 +252,6 @@ class BuilderBase(object): return a def runcmd(self, *args, **kwargs): - kwargs['env'] = self.build_env.env return self.staging_area.runcmd(*args, **kwargs) @@ -378,7 +376,7 @@ class ChunkBuilder(BuilderBase): relative_builddir = self.staging_area.relative(builddir) relative_destdir = self.staging_area.relative(destdir) - self.build_env.env['DESTDIR'] = relative_destdir + extra_env = { 'DESTDIR': relative_destdir } steps = [ ('pre-configure', False), @@ -406,9 +404,9 @@ class ChunkBuilder(BuilderBase): max_jobs = self.artifact.source.morphology['max-jobs'] if max_jobs is None: max_jobs = self.max_jobs - self.build_env.env['MAKEFLAGS'] = '-j%s' % max_jobs + extra_env['MAKEFLAGS'] = '-j%s' % max_jobs else: - self.build_env.env['MAKEFLAGS'] = '-j1' + extra_env['MAKEFLAGS'] = '-j1' try: # flushing is needed because writes from python and # writes from being the output in Popen have different @@ -416,6 +414,7 @@ class ChunkBuilder(BuilderBase): logfile.write('# # %s\n' % cmd) logfile.flush() self.runcmd(['sh', '-c', cmd], + extra_env=extra_env, cwd=relative_builddir, stdout=logfile, stderr=subprocess.STDOUT) @@ -670,14 +669,12 @@ class Builder(object): # pragma: no cover } def __init__(self, app, staging_area, local_artifact_cache, - remote_artifact_cache, repo_cache, build_env, max_jobs, - setup_mounts): + remote_artifact_cache, repo_cache, max_jobs, setup_mounts): self.app = app self.staging_area = staging_area self.local_artifact_cache = local_artifact_cache self.remote_artifact_cache = remote_artifact_cache self.repo_cache = repo_cache - self.build_env = build_env self.max_jobs = max_jobs self.setup_mounts = setup_mounts @@ -686,8 +683,8 @@ class Builder(object): # pragma: no cover o = self.classes[kind](self.app, self.staging_area, self.local_artifact_cache, self.remote_artifact_cache, artifact, - self.repo_cache, self.build_env, - self.max_jobs, self.setup_mounts) + self.repo_cache, self.max_jobs, + self.setup_mounts) logging.debug('Builder.build: artifact %s with %s' % (artifact.name, repr(o))) built_artifacts = o.build_and_cache() diff --git a/morphlib/builder2_tests.py b/morphlib/builder2_tests.py index df07a59f..c0da3cd9 100644 --- a/morphlib/builder2_tests.py +++ b/morphlib/builder2_tests.py @@ -35,8 +35,9 @@ class FakeApp(object): class FakeStagingArea(object): - def __init__(self, runcmd): + def __init__(self, runcmd, build_env): self.runcmd = runcmd + self.env = build_env.env class FakeSource(object): @@ -146,7 +147,7 @@ class BuilderBaseTests(unittest.TestCase): def setUp(self): self.commands_run = [] self.app = FakeApp(self.fake_runcmd) - self.staging_area = FakeStagingArea(self.fake_runcmd) + self.staging_area = FakeStagingArea(self.fake_runcmd, FakeBuildEnv()) self.artifact_cache = FakeArtifactCache() self.artifact = FakeArtifact('le-artifact') self.repo_cache = None @@ -158,7 +159,6 @@ class BuilderBaseTests(unittest.TestCase): None, self.artifact, self.repo_cache, - self.build_env, self.max_jobs, False) @@ -252,8 +252,7 @@ class ChunkBuilderTests(unittest.TestCase): def setUp(self): self.app = FakeApp() self.build = morphlib.builder2.ChunkBuilder(self.app, None, None, - None, None, None, None, 1, - False) + None, None, None, 1, False) def test_uses_morphology_commands_when_given(self): m = {'build-commands': ['build-it']} diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index a4ea10ed..15dc5ae9 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -27,9 +27,8 @@ class CacheKeyComputer(object): self._calculated = {} def _filterenv(self, env): - return dict([(k, env[k]) for k in ("USER", "USERNAME", "LOGNAME", - "TOOLCHAIN_TARGET", "PREFIX", - "BOOTSTRAP", "CFLAGS")]) + keys = ["BOOTSTRAP", "LOGNAME", "PREFIX", "USER", "USERNAME"] + return dict([(k, env[k]) for k in keys]) def compute_key(self, artifact): logging.debug('computing cache key for artifact %s from source ' diff --git a/morphlib/cachekeycomputer_tests.py b/morphlib/cachekeycomputer_tests.py index 411ad3f5..cc0b3ab8 100644 --- a/morphlib/cachekeycomputer_tests.py +++ b/morphlib/cachekeycomputer_tests.py @@ -14,6 +14,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import copy import unittest import morphlib @@ -155,14 +156,8 @@ class CacheKeyComputerTests(unittest.TestCase): def test_different_env_gives_different_key(self): artifact = self._find_artifact('system-rootfs') oldsha = self.ckc.compute_key(artifact) - build_env = DummyBuildEnvironment({ - "USER": "foouser", - "USERNAME": "foouser", - "LOGNAME": "foouser", - "TOOLCHAIN_TARGET": "dummy-baserock-linux-gnu", - "PREFIX": "/baserock", - "BOOTSTRAP": "false", - "CFLAGS": "-Os"}) + build_env = copy.deepcopy(self.build_env) + build_env.env["USER"] = "brian" ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) self.assertNotEqual(oldsha, ckc.compute_key(artifact)) diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index 24d4ebf9..ee3e444f 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -35,7 +35,9 @@ class StagingArea(object): ''' - def __init__(self, app, dirname): + _base_path = ['/sbin', '/usr/sbin', '/bin', '/usr/bin'] + + def __init__(self, app, dirname, build_env, use_chroot=True, extra_env={}): self._app = app self.dirname = dirname self.builddirname = None @@ -43,6 +45,17 @@ class StagingArea(object): self.mounted = None self._bind_readonly_mount = None + self.use_chroot = use_chroot + self.env = build_env.env + self.env.update(extra_env) + + if use_chroot: + path = build_env.extra_path + self._base_path + else: + full_path = [self.relative(p) for p in build_env.extra_path] + path = full_path + os.environ['PATH'].split(':') + self.env['PATH'] = ':'.join(path) + # Wrapper to be overridden by unit tests. def _mkdir(self, dirname): # pragma: no cover os.mkdir(dirname) @@ -75,6 +88,9 @@ class StagingArea(object): def relative(self, filename): '''Return a filename relative to the staging area.''' + if not self.use_chroot: + return filename + dirname = self.dirname if not dirname.endswith('/'): dirname += '/' @@ -190,8 +206,7 @@ class StagingArea(object): if not os.path.isdir(ccache_repodir): os.mkdir(ccache_repodir) # Get the destination path - ccache_destdir= os.path.join(self.tempdir, - 'tmp', 'ccache') + ccache_destdir= os.path.join(self.dirname, 'tmp', 'ccache') # Make sure that the destination exists. We'll create /tmp if necessary # to avoid breaking when faced with an empty staging area. if not os.path.isdir(ccache_destdir): @@ -247,14 +262,20 @@ class StagingArea(object): def runcmd(self, argv, **kwargs): # pragma: no cover '''Run a command in a chroot in the staging area.''' + assert 'env' not in kwargs + kwargs['env'] = self.env + if 'extra_env' in kwargs: + kwargs['env'].update(kwargs['extra_env']) + del kwargs['extra_env'] + + if self.use_chroot: + cwd = kwargs.get('cwd') or '/' + if 'cwd' in kwargs: + cwd = kwargs['cwd'] + del kwargs['cwd'] + else: + cwd = '/' - cwd = kwargs.get('cwd') or '/' - if 'cwd' in kwargs: - cwd = kwargs['cwd'] - del kwargs['cwd'] - else: - cwd = '/' - if self._app.settings['staging-chroot']: not_readonly_dirs = [self.builddirname, self.destdirname, 'dev', 'proc', 'tmp'] dirs = os.listdir(self.dirname) @@ -265,12 +286,11 @@ class StagingArea(object): for entry in dirs: real_argv += ['--mount-readonly', '/'+entry] - real_argv += [self.dirname] - else: - real_argv = ['chroot', '/'] - real_argv += ['sh', '-c', 'cd "$1" && shift && exec "$@"', '--', cwd] - real_argv += argv + real_argv += ['sh', '-c', 'cd "$1" && shift && exec "$@"', '--', cwd] + real_argv += argv - return self._app.runcmd(real_argv, **kwargs) + return self._app.runcmd(real_argv, **kwargs) + else: + return self._app.runcmd(argv, **kwargs) diff --git a/morphlib/stagingarea_tests.py b/morphlib/stagingarea_tests.py index 5c547f6e..35174f3b 100644 --- a/morphlib/stagingarea_tests.py +++ b/morphlib/stagingarea_tests.py @@ -24,6 +24,13 @@ import unittest import morphlib +class FakeBuildEnvironment(object): + + def __init__(self): + self.env = { + } + self.extra_path = ['/extra-path'] + class FakeSource(object): def __init__(self): @@ -56,8 +63,10 @@ class StagingAreaTests(unittest.TestCase): os.mkdir(os.path.join(self.cachedir, 'artifacts')) self.staging = os.path.join(self.tempdir, 'staging') self.created_dirs = [] + self.build_env = FakeBuildEnvironment() self.sa = morphlib.stagingarea.StagingArea( - FakeApplication(self.cachedir, self.tempdir), self.staging) + FakeApplication(self.cachedir, self.tempdir), self.staging, + self.build_env) def tearDown(self): shutil.rmtree(self.tempdir) @@ -118,3 +127,9 @@ class StagingAreaTests(unittest.TestCase): self.sa.install_artifact(f) self.sa.remove() self.assertFalse(os.path.exists(self.staging)) + + def test_supports_non_isolated_mode(self): + sa = morphlib.stagingarea.StagingArea( + object(), self.staging, self.build_env, use_chroot=False) + filename = os.path.join(self.staging, 'foobar') + self.assertEqual(sa.relative(filename), filename) diff --git a/morphlib/util.py b/morphlib/util.py index b4e06092..16063f45 100644 --- a/morphlib/util.py +++ b/morphlib/util.py @@ -42,7 +42,6 @@ def arch(): '''Return the CPU architecture of the current host.''' return os.uname()[4] - def indent(string, spaces=4): '''Return ``string`` indented by ``spaces`` spaces. -- cgit v1.2.1 From 92d98f57eaa6e0871b70d0d0ab1db879cf3ea47a Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Mon, 25 Feb 2013 11:45:30 +0000 Subject: Set environment variables defining target for build-essential In the future we will allow this to be modified to provide a cross-bootstrap mode, but for now we use the same target as the host compiler. --- morphlib/app.py | 18 ------------------ morphlib/buildcommand.py | 7 +++++-- morphlib/buildenvironment.py | 22 +++++++++++++++++++--- morphlib/buildenvironment_tests.py | 27 +++++++++++++++++---------- morphlib/cachekeycomputer.py | 3 ++- morphlib/cachekeycomputer_tests.py | 12 ++++++------ morphlib/util.py | 11 +++++++++++ 7 files changed, 60 insertions(+), 40 deletions(-) diff --git a/morphlib/app.py b/morphlib/app.py index c8b9b911..d3136b43 100755 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -42,7 +42,6 @@ defaults = { 'cachedir': os.path.expanduser('~/.cache/morph'), 'max-jobs': morphlib.util.make_concurrency(), 'prefix': '/usr', - 'toolchain-target': '%s-baserock-linux-gnu' % os.uname()[4], 'build-ref-prefix': 'baserock/builds' } @@ -137,23 +136,6 @@ class Morph(cliapp.Application): default=os.environ.get('TMPDIR'), group=group_build) - # Would be better to have a separate tool to cross-bootstrap - # because it's completely outside normal Morph usage - group_bootstrap = 'Bootstrap Options' - self.settings.string(['target-cflags'], - 'inject additional CFLAGS into the environment ' - 'that is used to build chunks', - metavar='CFLAGS', - default='', - group=group_bootstrap) - self.settings.string(['toolchain-target'], - 'set the TOOLCHAIN_TARGET variable which is used ' - 'in some chunks to determine which architecture ' - 'to build tools for', - metavar='TOOLCHAIN_TARGET', - default=defaults['toolchain-target'], - group=group_bootstrap) - # These cannot be removed just yet because existing morph.conf files # would fail to parse. group_obsolete = 'Obsolete Options' diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 56c7fae8..ca097145 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -33,12 +33,14 @@ class BuildCommand(object): ''' def __init__(self, app): + self.supports_local_build = True + self.target = morphlib.util.target(app.runcmd) + self.app = app self.build_env = self.new_build_env() self.ckc = self.new_cache_key_computer(self.build_env) self.lac, self.rac = self.new_artifact_caches() self.lrc, self.rrc = self.new_repo_caches() - self.supports_local_build = True def build(self, args): '''Build triplets specified on command line.''' @@ -55,7 +57,8 @@ class BuildCommand(object): def new_build_env(self): '''Create a new BuildEnvironment instance.''' - return morphlib.buildenvironment.BuildEnvironment(self.app.settings) + return morphlib.buildenvironment.BuildEnvironment(self.app.settings, + self.target) def new_cache_key_computer(self, build_env): '''Create a new cache key computer.''' diff --git a/morphlib/buildenvironment.py b/morphlib/buildenvironment.py index fce98da4..29561220 100644 --- a/morphlib/buildenvironment.py +++ b/morphlib/buildenvironment.py @@ -13,6 +13,7 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import cliapp import os import morphlib @@ -20,9 +21,11 @@ import morphlib class BuildEnvironment(): - def __init__(self, settings, arch=None): - self.extra_path = [] + def __init__(self, settings, target, arch=None): + '''Create a new BuildEnvironment object''' + self.extra_path = [] + self.target = target self.arch = morphlib.util.arch() if arch is None else arch self.env = self._clean_env(settings) @@ -34,6 +37,15 @@ class BuildEnvironment(): _override_term = 'dumb' _override_username = 'tomjon' + def get_bootstrap_target(self, target): + '''Set 'vendor' field of the given machine triplet as 'bootstrap' ''' + + parts = target.split('-') + if len(parts) < 2: + raise morphlib.Error('Failed to parse machine triplet returned by ' + 'host compiler: %s' % target) + return '-'.join([parts[0], 'bootstrap'] + parts[2:]) + def _clean_env(self, settings): '''Create a fresh set of environment variables for a clean build. @@ -69,7 +81,11 @@ class BuildEnvironment(): env['HOME'] = self._override_home env['PREFIX'] = settings['prefix'] - env['BOOTSTRAP'] = 'false' + env['BUILD'] = self.target + env['TARGET'] = self.target + env['TARGET_STAGE1'] = self.get_bootstrap_target(self.target) + env['TARGET_GCC_CONFIG'] = '' + if not settings['no-ccache']: self.extra_path.append(self._ccache_path) diff --git a/morphlib/buildenvironment_tests.py b/morphlib/buildenvironment_tests.py index 2af3f605..03799190 100644 --- a/morphlib/buildenvironment_tests.py +++ b/morphlib/buildenvironment_tests.py @@ -29,20 +29,28 @@ class BuildEnvironmentTests(unittest.TestCase): 'no-ccache': True, 'no-distcc': True } + self.target = '%s-baserock-linux-gnu' % morphlib.util.arch() self.fake_env = { 'PATH': '/fake_bin', } self.default_path = 'no:such:path' + def new_build_env(self, settings=None, target=None, **kws): + settings = settings or self.settings + target = target or self.target + return buildenvironment.BuildEnvironment(settings, target, **kws) + def test_arch_defaults_to_host(self): - buildenv = buildenvironment.BuildEnvironment(self.settings) + buildenv = self.new_build_env() self.assertEqual(buildenv.arch, morphlib.util.arch()) def test_arch_overridable(self): - buildenv = buildenvironment.BuildEnvironment(self.settings, - arch='noarch') + buildenv = self.new_build_env(arch='noarch') self.assertEqual(buildenv.arch, 'noarch') + def test_target_always_valid(self): + self.assertRaises(morphlib.Error, self.new_build_env, target="invalid") + def test_copies_whitelist_vars(self): env = self.fake_env safe = { @@ -57,19 +65,19 @@ class BuildEnvironmentTests(unittest.TestCase): old_osenv = buildenvironment.BuildEnvironment._osenv buildenvironment.BuildEnvironment._osenv = env - buildenv = buildenvironment.BuildEnvironment(self.settings) + buildenv = self.new_build_env() self.assertEqual(sorted(safe.items()), sorted([(k, buildenv.env[k]) for k in safe.keys()])) buildenvironment.BuildEnvironment._osenv = old_osenv def test_user_spellings_equal(self): - buildenv = buildenvironment.BuildEnvironment(self.settings) + buildenv = self.new_build_env() self.assertTrue(buildenv.env['USER'] == buildenv.env['USERNAME'] == buildenv.env['LOGNAME']) def test_environment_overrides(self): - buildenv = buildenvironment.BuildEnvironment(self.settings) + buildenv = self.new_build_env() self.assertEqual(buildenv.env['TERM'], buildenv._override_term) self.assertEqual(buildenv.env['SHELL'], buildenv._override_shell) self.assertEqual(buildenv.env['USER'], buildenv._override_username) @@ -79,14 +87,13 @@ class BuildEnvironmentTests(unittest.TestCase): self.assertEqual(buildenv.env['HOME'], buildenv._override_home) def test_environment_settings_set(self): - buildenv = buildenvironment.BuildEnvironment(self.settings) - #self.assertEqual(buildenv.env['TOOLCHAIN_TARGET'], - # self.settings['toolchain-target']) + buildenv = self.new_build_env() + self.assertEqual(buildenv.env['TARGET'], self.target) def test_ccache_vars_set(self): new_settings = copy.copy(self.settings) new_settings['no-ccache'] = False new_settings['no-distcc'] = False - buildenv = buildenvironment.BuildEnvironment(new_settings) + buildenv = self.new_build_env(settings=new_settings) self.assertTrue(buildenv._ccache_path in buildenv.extra_path) self.assertEqual(buildenv.env['CCACHE_PREFIX'], 'distcc') diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index 15dc5ae9..d9ad5762 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -27,7 +27,8 @@ class CacheKeyComputer(object): self._calculated = {} def _filterenv(self, env): - keys = ["BOOTSTRAP", "LOGNAME", "PREFIX", "USER", "USERNAME"] + keys = ["LOGNAME", "PREFIX", "TARGET", "TARGET_STAGE1", + "TARGET_GCC_CONFIG", "USER", "USERNAME"] return dict([(k, env[k]) for k in keys]) def compute_key(self, artifact): diff --git a/morphlib/cachekeycomputer_tests.py b/morphlib/cachekeycomputer_tests.py index cc0b3ab8..1dba80c2 100644 --- a/morphlib/cachekeycomputer_tests.py +++ b/morphlib/cachekeycomputer_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -104,13 +104,13 @@ class CacheKeyComputerTests(unittest.TestCase): elif m['kind'] == 'chunk': m.builds_artifacts = [m['name']] self.build_env = DummyBuildEnvironment({ - "USER": "foouser", - "USERNAME": "foouser", "LOGNAME": "foouser", - "TOOLCHAIN_TARGET": "dummy-baserock-linux-gnu", "PREFIX": "/baserock", - "BOOTSTRAP": "false", - "CFLAGS": "-O4"}) + "TARGET": "dummy-baserock-linux-gnu", + "TARGET_STAGE1": "dummy-baserock-linux-gnu", + "TARGET_GCC_CONFIG": "", + "USER": "foouser", + "USERNAME": "foouser"}) self.artifact_resolver = morphlib.artifactresolver.ArtifactResolver() self.artifacts = self.artifact_resolver.resolve_artifacts( self.source_pool) diff --git a/morphlib/util.py b/morphlib/util.py index 16063f45..c3a7ac9f 100644 --- a/morphlib/util.py +++ b/morphlib/util.py @@ -42,6 +42,17 @@ def arch(): '''Return the CPU architecture of the current host.''' return os.uname()[4] + +def target(runcmd): # pragma: no cover + '''Returns the machine triplet of the current host''' + try: + target = runcmd(['cc', '-dumpmachine']).strip() + except cliapp.AppException as e: + raise morphlib.Error( + 'Failed to execute host compiler \'cc\': %s' % e) + return target + + def indent(string, spaces=4): '''Return ``string`` indented by ``spaces`` spaces. -- cgit v1.2.1 From 84807d4d7c23f45d4f0a0f87e6c7ba7ba7470936 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Wed, 6 Feb 2013 10:23:57 +0000 Subject: Add 'build-mode' field for chunks in a stratum Allowed values: staging: build with a staging chroot (default) test: build with the host's tools bootstrap: build with the host's tools, and do not include this chunk in the final stratum artifact In the past, 'normal mode' has been used to describe building a chunk with the host's tools. We don't want that mode to ever be used, because it is a huge hole in reproducability, but we need to keep it around to avoid making Morph's cmdtest suite depend on Baserock. Hopefully naming it 'test' should discourage potential abusers. It is unfortunate that the build tests now take a separate code path compared to real-world usage of Morph. However, this is necessary to avoid a circular dependency between Morph's test suite and the build-essential stratum in Baserock. We do whole-build testing of Baserock, too, so the 'staging' code path is still tested outside of Morph. However, testing a staging area requires populating it with at minimum a working shell, and this is a bit too complex to go in Morph's test suite. --- morphlib/artifactresolver.py | 5 +++- morphlib/buildcommand.py | 38 ++++++++++++++++-------- morphlib/buildenvironment.py | 12 +++++++- morphlib/cachekeycomputer.py | 1 + morphlib/morph2.py | 4 ++- morphlib/morphologyfactory.py | 3 -- morphlib/morphologyfactory_tests.py | 12 -------- morphlib/plugins/trebuchet_plugin.py | 3 +- morphlib/stagingarea.py | 6 ++-- scripts/setup-3rd-party-strata | 3 +- tests.as-root/build-with-external-strata.script | 1 + tests.as-root/setup | 3 ++ tests.as-root/system-overlap.script | 6 +++- tests.as-root/tarball-image-is-sensible.setup | 1 + tests.branching/workflow-petrify.stdout | 6 ++++ tests.build/build-stratum-with-submodules.script | 3 +- tests.build/setup | 1 + tests.build/stratum-overlap-warns.setup | 12 +++++--- tests.deploy/setup | 6 ++-- 19 files changed, 84 insertions(+), 42 deletions(-) diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index 4b7956e0..76178b35 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -220,6 +220,9 @@ class ArtifactResolver(object): for other_stratum in strata: chunk_artifact.add_dependency(other_stratum) + # Resolve now to avoid a search for the parent morphology later + chunk_source.build_mode = info['build-mode'] + build_depends = info.get('build-depends', None) if build_depends is None: diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index ca097145..2de71d8f 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -226,13 +226,24 @@ class BuildCommand(object): self.get_sources(artifact) deps = self.get_recursive_deps(artifact) self.cache_artifacts_locally(deps) - staging_area = self.create_staging_area() - if artifact.source.morphology.needs_staging_area: + + setup_mounts = False + if artifact.source.morphology['kind'] == 'chunk': + build_mode = artifact.source.build_mode + + if build_mode not in ['bootstrap', 'staging', 'test']: + raise morphlib.Error( + 'Unknown build mode for chunk %s: %s' % + (artifact.name, build_mode)) + + use_chroot = build_mode=='staging' + staging_area = self.create_staging_area(use_chroot) self.install_fillers(staging_area) - self.install_chunk_artifacts(staging_area, deps, artifact) - morphlib.builder2.ldconfig(self.app.runcmd, - staging_area.dirname) - self.build_and_cache(staging_area, artifact) + self.install_dependencies(staging_area, deps, artifact) + else: + staging_area = self.create_staging_area() + + self.build_and_cache(staging_area, artifact, setup_mounts) self.remove_staging_area(staging_area) def get_recursive_deps(self, artifact): @@ -301,13 +312,13 @@ class BuildCommand(object): copy(self.rac.get_artifact_metadata(artifact, 'meta'), self.lac.put_artifact_metadata(artifact, 'meta')) - def create_staging_area(self): + def create_staging_area(self, use_chroot=True): '''Create the staging area for building a single artifact.''' self.app.status(msg='Creating staging area') staging_dir = tempfile.mkdtemp(dir=self.app.settings['tempdir']) staging_area = morphlib.stagingarea.StagingArea( - self.app, staging_dir, self.build_env, False, {}) + self.app, staging_dir, self.build_env, use_chroot, {}) return staging_area def remove_staging_area(self, staging_area): @@ -328,7 +339,7 @@ class BuildCommand(object): filename=filename) staging_area.install_artifact(f) - def install_chunk_artifacts(self, staging_area, artifacts, parent_art): + def install_dependencies(self, staging_area, artifacts, target_artifact): '''Install chunk artifacts into staging area. We only ever care about chunk artifacts as build dependencies, @@ -343,12 +354,15 @@ class BuildCommand(object): if artifact.source.morphology['kind'] != 'chunk': continue self.app.status(msg='[%(name)s] Installing chunk %(chunk_name)s', - name=parent_art.name, + name=target_artifact.name, chunk_name=artifact.name) handle = self.lac.get(artifact) staging_area.install_artifact(handle) - def build_and_cache(self, staging_area, artifact): + if target_artifact.source.build_mode == 'staging': + morphlib.builder2.ldconfig(self.app.runcmd, staging_area.dirname) + + def build_and_cache(self, staging_area, artifact, setup_mounts): '''Build an artifact and put it into the local artifact cache.''' self.app.status(msg='Starting actual build: %(name)s', @@ -356,5 +370,5 @@ class BuildCommand(object): setup_mounts = self.app.settings['staging-chroot'] builder = morphlib.builder2.Builder( self.app, staging_area, self.lac, self.rac, self.lrc, - self.app.settings['max-jobs'], True) + self.app.settings['max-jobs'], setup_mounts) return builder.build_and_cache(artifact) diff --git a/morphlib/buildenvironment.py b/morphlib/buildenvironment.py index 29561220..6ba950ff 100644 --- a/morphlib/buildenvironment.py +++ b/morphlib/buildenvironment.py @@ -13,6 +13,7 @@ # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import copy import cliapp import os @@ -21,6 +22,16 @@ import morphlib class BuildEnvironment(): + '''Represents the build environment for an artifact + + This should be as consistent as possible across builds, but some + artifacts will require tweaks. The intention of this object is + to create one once and call populate() to create an initial state + and when changes are required, call clone() to get another instance + which can be modified. + + ''' + def __init__(self, settings, target, arch=None): '''Create a new BuildEnvironment object''' @@ -88,7 +99,6 @@ class BuildEnvironment(): if not settings['no-ccache']: self.extra_path.append(self._ccache_path) - # FIXME: we should set CCACHE_BASEDIR so any objects that refer to their # current directory get corrected. This improve the cache hit rate # env['CCACHE_BASEDIR'] = self.tempdir.dirname diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index d9ad5762..4573ad0d 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -87,6 +87,7 @@ class CacheKeyComputer(object): kind = artifact.source.morphology['kind'] if kind == 'chunk': + keys['build-mode'] = artifact.source.build_mode keys['tree'] = artifact.source.tree elif kind in ('system', 'stratum'): morphology = artifact.source.morphology diff --git a/morphlib/morph2.py b/morphlib/morph2.py index 3cdf49a9..728fa533 100644 --- a/morphlib/morph2.py +++ b/morphlib/morph2.py @@ -52,7 +52,7 @@ class Morphology(object): 'stratum': [ ('chunks', []), ('description', ''), - ('build-depends', None) + ('build-depends', None), ], 'system': [ ('strata', []), @@ -157,6 +157,8 @@ class Morphology(object): self._set_default_value(source, 'morph', source['name']) if 'build-depends' not in source: self._set_default_value(source, 'build-depends', None) + if 'build-mode' not in source: + self._set_default_value(source, 'build-mode', 'staging') def _parse_size(self, size): if isinstance(size, basestring): diff --git a/morphlib/morphologyfactory.py b/morphlib/morphologyfactory.py index 76905eb9..7ae68697 100644 --- a/morphlib/morphologyfactory.py +++ b/morphlib/morphologyfactory.py @@ -113,7 +113,6 @@ class MorphologyFactory(object): name = morphology['name'] morphology.builds_artifacts = [name + '-rootfs'] - morphology.needs_staging_area = False morphology.needs_artifact_metadata_cached = False def _check_and_tweak_stratum(self, morphology, reponame, sha1, filename): @@ -129,7 +128,6 @@ class MorphologyFactory(object): (filename, name)) morphology.builds_artifacts = [morphology['name']] - morphology.needs_staging_area = False morphology.needs_artifact_metadata_cached = True def _check_and_tweak_chunk(self, morphology, reponame, sha1, filename): @@ -140,5 +138,4 @@ class MorphologyFactory(object): else: morphology.builds_artifacts = [morphology['name']] - morphology.needs_staging_area = True morphology.needs_artifact_metadata_cached = False diff --git a/morphlib/morphologyfactory_tests.py b/morphlib/morphologyfactory_tests.py index 6e17df48..30cfb8fb 100644 --- a/morphlib/morphologyfactory_tests.py +++ b/morphlib/morphologyfactory_tests.py @@ -223,18 +223,6 @@ class MorphologyFactoryTests(unittest.TestCase): morph = self.mf.get_morphology('reponame', 'sha1', 'system.morph') self.assertEqual(morph.builds_artifacts, ['system-rootfs']) - def test_sets_needs_staging_for_chunk(self): - morph = self.mf.get_morphology('reponame', 'sha1', 'chunk.morph') - self.assertEqual(morph.needs_staging_area, True) - - def test_does_not_set_needs_staging_for_stratum(self): - morph = self.mf.get_morphology('reponame', 'sha1', 'stratum.morph') - self.assertEqual(morph.needs_staging_area, False) - - def test_does_not_set_needs_staging_for_system(self): - morph = self.mf.get_morphology('reponame', 'sha1', 'system.morph') - self.assertEqual(morph.needs_staging_area, False) - def test_does_not_set_needs_artifact_metadata_cached_for_chunk(self): morph = self.mf.get_morphology('reponame', 'sha1', 'chunk.morph') self.assertEqual(morph.needs_artifact_metadata_cached, False) diff --git a/morphlib/plugins/trebuchet_plugin.py b/morphlib/plugins/trebuchet_plugin.py index 1ebffbf4..742d23c8 100644 --- a/morphlib/plugins/trebuchet_plugin.py +++ b/morphlib/plugins/trebuchet_plugin.py @@ -46,7 +46,8 @@ class TrebuchetPlugin(cliapp.Plugin): repo_name2, ref2, filename2 = args[4:7] app = self.app - build_env = morphlib.buildenvironment.BuildEnvironment(app.settings) + build_env = morphlib.buildenvironment.BuildEnvironment( + app.settings, morphlib.util.target(self.app.runcmd)) ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) lac, rac = morphlib.util.new_artifact_caches(app.settings) lrc, rrc = morphlib.util.new_repo_caches(app) diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index ee3e444f..24b72867 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -37,7 +37,8 @@ class StagingArea(object): _base_path = ['/sbin', '/usr/sbin', '/bin', '/usr/bin'] - def __init__(self, app, dirname, build_env, use_chroot=True, extra_env={}): + def __init__(self, app, dirname, build_env, use_chroot=True, extra_env={}, + extra_path=[]): self._app = app self.dirname = dirname self.builddirname = None @@ -52,7 +53,8 @@ class StagingArea(object): if use_chroot: path = build_env.extra_path + self._base_path else: - full_path = [self.relative(p) for p in build_env.extra_path] + rel_path = build_env.extra_path + full_path = [os.path.normpath(dirname + p) for p in rel_path] path = full_path + os.environ['PATH'].split(':') self.env['PATH'] = ':'.join(path) diff --git a/scripts/setup-3rd-party-strata b/scripts/setup-3rd-party-strata index a2a8b1e5..d1cc320d 100644 --- a/scripts/setup-3rd-party-strata +++ b/scripts/setup-3rd-party-strata @@ -1,5 +1,5 @@ #!/bin/sh -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -55,6 +55,7 @@ cat < "$1/$2.morph" "name": "hello", "repo": "test:$2-hello", "ref": "master", + "build-mode": "test", "build-depends": [] } ] diff --git a/tests.as-root/build-with-external-strata.script b/tests.as-root/build-with-external-strata.script index 2d5d0fed..fd021399 100755 --- a/tests.as-root/build-with-external-strata.script +++ b/tests.as-root/build-with-external-strata.script @@ -38,6 +38,7 @@ cat <> stratum2.morph "name": "linux", "repo": "test:kernel-repo", "ref": "master", + "build-mode": "test", "build-depends": [] } ] diff --git a/tests.as-root/setup b/tests.as-root/setup index a85507e0..b9d5d477 100755 --- a/tests.as-root/setup +++ b/tests.as-root/setup @@ -110,6 +110,7 @@ chunks: - name: hello repo: test:chunk-repo ref: farrokh + build-mode: test build-depends: [] EOF git add hello-stratum.morph @@ -125,6 +126,7 @@ chunks: - name: tools repo: test:tools-repo ref: master + build-mode: test build-depends: [] EOF git add tools-stratum.morph @@ -153,6 +155,7 @@ chunks: - name: linux repo: test:kernel-repo ref: master + build-mode: test build-depends: [] EOF git add linux-stratum.morph diff --git a/tests.as-root/system-overlap.script b/tests.as-root/system-overlap.script index cc308536..b8888491 100755 --- a/tests.as-root/system-overlap.script +++ b/tests.as-root/system-overlap.script @@ -1,6 +1,6 @@ #!/bin/sh # -# Copyright (C) 2011, 2012 Codethink Limited +# Copyright (C) 2011-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -57,12 +57,14 @@ cat <foo-baz-stratum.morph "name": "overlap-foo-baz", "repo": "test:chunk-repo", "ref": "overlap", + "build-mode": "test", "build-depends": [] }, { "name": "linux", "repo": "test:kernel-repo", "ref": "master", + "build-mode": "test", "build-depends": ["overlap-foo-baz"] } ] @@ -77,12 +79,14 @@ cat <foo-barqux-stratum.morph "name": "overlap-foobar", "repo": "test:chunk-repo", "ref": "overlap", + "build-mode": "test", "build-depends": [] }, { "name": "overlap-fooqux", "repo": "test:chunk-repo", "ref": "overlap", + "build-mode": "test", "build-depends": ["overlap-foobar"] } ] diff --git a/tests.as-root/tarball-image-is-sensible.setup b/tests.as-root/tarball-image-is-sensible.setup index e159070c..fa904c2c 100755 --- a/tests.as-root/tarball-image-is-sensible.setup +++ b/tests.as-root/tarball-image-is-sensible.setup @@ -84,6 +84,7 @@ cat < link-stratum.morph "name": "links", "repo": "test:chunk-repo", "ref": "tarball-links", + "build-mode": "test", "build-depends": [] } ] diff --git a/tests.branching/workflow-petrify.stdout b/tests.branching/workflow-petrify.stdout index b572ed67..c86afa2e 100644 --- a/tests.branching/workflow-petrify.stdout +++ b/tests.branching/workflow-petrify.stdout @@ -31,6 +31,7 @@ test/petrify after petrifying: "name": "hello", "repo": "test:stratum2-hello", "ref": "f4730636e429149bb923fa16be3aa9802d484b23", + "build-mode": "test", "build-depends": [], "unpetrify-ref": "master" } @@ -44,6 +45,7 @@ test/petrify after petrifying: "name": "hello", "repo": "test:stratum3-hello", "ref": "f4730636e429149bb923fa16be3aa9802d484b23", + "build-mode": "test", "build-depends": [], "unpetrify-ref": "master" } @@ -83,6 +85,7 @@ test/petrify after editing a chunk: "name": "hello", "repo": "test:stratum2-hello", "ref": "test/petrify", + "build-mode": "test", "build-depends": [] } ] @@ -95,6 +98,7 @@ test/petrify after editing a chunk: "name": "hello", "repo": "test:stratum3-hello", "ref": "f4730636e429149bb923fa16be3aa9802d484b23", + "build-mode": "test", "build-depends": [], "unpetrify-ref": "master" } @@ -134,6 +138,7 @@ test/unpetrify after unpetrifying: "name": "hello", "repo": "test:stratum2-hello", "ref": "test/petrify", + "build-mode": "test", "build-depends": [] } ] @@ -146,6 +151,7 @@ test/unpetrify after unpetrifying: "name": "hello", "repo": "test:stratum3-hello", "ref": "master", + "build-mode": "test", "build-depends": [] } ] diff --git a/tests.build/build-stratum-with-submodules.script b/tests.build/build-stratum-with-submodules.script index f64ba9f6..c3c00578 100755 --- a/tests.build/build-stratum-with-submodules.script +++ b/tests.build/build-stratum-with-submodules.script @@ -55,7 +55,8 @@ cat < "$morphs/hello-stratum.morph" "name": "parent", "repo": "test:parent-repo", "ref": "master", - "build-depends": [] + "build-depends": [], + "build-mode": "test" } ] } diff --git a/tests.build/setup b/tests.build/setup index 935e388b..499dbb21 100755 --- a/tests.build/setup +++ b/tests.build/setup @@ -95,6 +95,7 @@ cat < hello-stratum.morph "name": "hello", "repo": "test:chunk-repo", "ref": "farrokh", + "build-mode": "test", "build-depends": [] } ] diff --git a/tests.build/stratum-overlap-warns.setup b/tests.build/stratum-overlap-warns.setup index 520a37a1..626f2094 100755 --- a/tests.build/stratum-overlap-warns.setup +++ b/tests.build/stratum-overlap-warns.setup @@ -34,25 +34,29 @@ cat <hello-stratum.morph "name": "dirs", "repo": "test:chunk-repo", "ref": "overlap", - "build-depends": [] + "build-depends": [], + "build-mode": "test" }, { "name": "overlap-foobar", "repo": "test:chunk-repo", "ref": "overlap", - "build-depends": ["dirs"] + "build-depends": ["dirs"], + "build-mode": "test" }, { "name": "overlap-fooqux", "repo": "test:chunk-repo", "ref": "overlap", - "build-depends": ["overlap-foobar"] + "build-depends": ["overlap-foobar"], + "build-mode": "test" }, { "name": "overlap-foo-baz", "repo": "test:chunk-repo", "ref": "overlap", - "build-depends": ["overlap-fooqux"] + "build-depends": ["overlap-fooqux"], + "build-mode": "test" } ] } diff --git a/tests.deploy/setup b/tests.deploy/setup index b0054fa5..584ce039 100755 --- a/tests.deploy/setup +++ b/tests.deploy/setup @@ -100,7 +100,8 @@ cat < hello-stratum.morph "name": "hello", "repo": "test:chunk-repo", "ref": "farrokh", - "build-depends": [] + "build-depends": [], + "build-mode": "test" } ] } @@ -140,7 +141,8 @@ cat < linux-stratum.morph "name": "linux", "repo": "test:kernel-repo", "ref": "master", - "build-depends": [] + "build-depends": [], + "build-mode": "test" } ] } -- cgit v1.2.1 From 0cc3de60d6282d22108ea59f169a0749be5e59ea Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Mon, 25 Feb 2013 10:13:34 +0000 Subject: Add 'prefix' property for chunks within strata Morph no longer supports setting the prefix using the --prefix argument / setting. This was only used in tests and during bootstrap. If a chunk build-depends on a chunk within a stratum which has a custom prefix, that prefix is appended to the PATH in the build environment. --- morphlib/app.py | 5 -- morphlib/artifact.py | 14 ++++ morphlib/artifact_tests.py | 12 +++- morphlib/artifactresolver.py | 1 + morphlib/buildcommand.py | 13 +++- morphlib/buildenvironment.py | 1 - morphlib/buildenvironment_tests.py | 6 +- morphlib/cachekeycomputer.py | 5 +- morphlib/cachekeycomputer_tests.py | 1 - morphlib/morph2.py | 2 + morphlib/stagingarea.py | 4 +- tests.build/build-system-cpan.script | 24 ++++++- tests.build/build-system-python-distutils.script | 25 ++++++- tests.build/prefix.script | 87 ++++++++++++++++++++++++ tests.build/prefix.stdout | 8 +++ 15 files changed, 190 insertions(+), 18 deletions(-) create mode 100755 tests.build/prefix.script create mode 100644 tests.build/prefix.stdout diff --git a/morphlib/app.py b/morphlib/app.py index d3136b43..b5379c83 100755 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -41,7 +41,6 @@ defaults = { ], 'cachedir': os.path.expanduser('~/.cache/morph'), 'max-jobs': morphlib.util.make_concurrency(), - 'prefix': '/usr', 'build-ref-prefix': 'baserock/builds' } @@ -117,10 +116,6 @@ class Morph(cliapp.Application): self.settings.boolean(['no-distcc'], 'do not use distcc (default: true)', group=group_build, default=True) - self.settings.string(['prefix'], - 'build chunks with prefix PREFIX', - metavar='PREFIX', default=defaults['prefix'], - group=group_build) self.settings.boolean(['push-build-branches'], 'always push temporary build branches to the ' 'remote repository', diff --git a/morphlib/artifact.py b/morphlib/artifact.py index 5f084384..82680709 100644 --- a/morphlib/artifact.py +++ b/morphlib/artifact.py @@ -61,6 +61,20 @@ class Artifact(object): self.name, metadata_name) + def get_dependency_prefix_set(self): + '''Collects all install prefixes of this artifact's build dependencies + + If any of the build dependencies of a chunk artifact are installed + to non-standard prefixes, we need to add those prefixes to the + PATH of the current artifact. + + ''' + result = set() + for d in self.dependencies: + if d.source.morphology['kind'] == 'chunk': + result.add(d.source.prefix) + return result + def __str__(self): # pragma: no cover return '%s|%s' % (self.source, self.name) diff --git a/morphlib/artifact_tests.py b/morphlib/artifact_tests.py index 1d9e6cca..8edbbde2 100644 --- a/morphlib/artifact_tests.py +++ b/morphlib/artifact_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2012 Codethink Limited +# Copyright (C) 2012-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -14,6 +14,7 @@ # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. +import copy import unittest import morphlib @@ -78,3 +79,12 @@ class ArtifactTests(unittest.TestCase): self.assertEqual(self.artifact.dependencies, [self.other]) self.assertEqual(self.other.dependents, [self.artifact]) self.assertTrue(self.artifact.depends_on(self.other)) + + def test_get_dependency_prefix(self): + self.artifact.add_dependency(self.other) + self.artifact.source.prefix = '/bar' + self.other.source = copy.copy(self.artifact.source) + self.other.source.prefix = '/foo' + + prefix_set = self.artifact.get_dependency_prefix_set() + self.assertEqual(prefix_set, set(['/foo'])) diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index 76178b35..186d5357 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -222,6 +222,7 @@ class ArtifactResolver(object): # Resolve now to avoid a search for the parent morphology later chunk_source.build_mode = info['build-mode'] + chunk_source.prefix = info['prefix'] build_depends = info.get('build-depends', None) diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 2de71d8f..d602b5ea 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -230,6 +230,10 @@ class BuildCommand(object): setup_mounts = False if artifact.source.morphology['kind'] == 'chunk': build_mode = artifact.source.build_mode + extra_env = {'PREFIX': artifact.source.prefix} + + dep_prefix_set = artifact.get_dependency_prefix_set() + extra_path = [os.path.join(d, 'bin') for d in dep_prefix_set] if build_mode not in ['bootstrap', 'staging', 'test']: raise morphlib.Error( @@ -237,7 +241,8 @@ class BuildCommand(object): (artifact.name, build_mode)) use_chroot = build_mode=='staging' - staging_area = self.create_staging_area(use_chroot) + staging_area = self.create_staging_area( + use_chroot, extra_env=extra_env, extra_path=extra_path) self.install_fillers(staging_area) self.install_dependencies(staging_area, deps, artifact) else: @@ -312,13 +317,15 @@ class BuildCommand(object): copy(self.rac.get_artifact_metadata(artifact, 'meta'), self.lac.put_artifact_metadata(artifact, 'meta')) - def create_staging_area(self, use_chroot=True): + def create_staging_area(self, use_chroot=True, extra_env={}, + extra_path=[]): '''Create the staging area for building a single artifact.''' self.app.status(msg='Creating staging area') staging_dir = tempfile.mkdtemp(dir=self.app.settings['tempdir']) staging_area = morphlib.stagingarea.StagingArea( - self.app, staging_dir, self.build_env, use_chroot, {}) + self.app, staging_dir, self.build_env, use_chroot, extra_env, + extra_path) return staging_area def remove_staging_area(self, staging_area): diff --git a/morphlib/buildenvironment.py b/morphlib/buildenvironment.py index 6ba950ff..e6dccb04 100644 --- a/morphlib/buildenvironment.py +++ b/morphlib/buildenvironment.py @@ -91,7 +91,6 @@ class BuildEnvironment(): env['LC_ALL'] = self._override_locale env['HOME'] = self._override_home - env['PREFIX'] = settings['prefix'] env['BUILD'] = self.target env['TARGET'] = self.target env['TARGET_STAGE1'] = self.get_bootstrap_target(self.target) diff --git a/morphlib/buildenvironment_tests.py b/morphlib/buildenvironment_tests.py index 03799190..1995923b 100644 --- a/morphlib/buildenvironment_tests.py +++ b/morphlib/buildenvironment_tests.py @@ -33,7 +33,11 @@ class BuildEnvironmentTests(unittest.TestCase): self.fake_env = { 'PATH': '/fake_bin', } - self.default_path = 'no:such:path' + + def new_build_env(self, settings=None, target=None, **kws): + settings = settings or self.settings + target = target or self.target + return buildenvironment.BuildEnvironment(settings, target, **kws) def new_build_env(self, settings=None, target=None, **kws): settings = settings or self.settings diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index 4573ad0d..244257a0 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -27,8 +27,8 @@ class CacheKeyComputer(object): self._calculated = {} def _filterenv(self, env): - keys = ["LOGNAME", "PREFIX", "TARGET", "TARGET_STAGE1", - "TARGET_GCC_CONFIG", "USER", "USERNAME"] + keys = ["LOGNAME", "TARGET", "TARGET_STAGE1", "TARGET_GCC_CONFIG", + "USER", "USERNAME"] return dict([(k, env[k]) for k in keys]) def compute_key(self, artifact): @@ -88,6 +88,7 @@ class CacheKeyComputer(object): kind = artifact.source.morphology['kind'] if kind == 'chunk': keys['build-mode'] = artifact.source.build_mode + keys['prefix'] = artifact.source.prefix keys['tree'] = artifact.source.tree elif kind in ('system', 'stratum'): morphology = artifact.source.morphology diff --git a/morphlib/cachekeycomputer_tests.py b/morphlib/cachekeycomputer_tests.py index 1dba80c2..ec4c9d22 100644 --- a/morphlib/cachekeycomputer_tests.py +++ b/morphlib/cachekeycomputer_tests.py @@ -105,7 +105,6 @@ class CacheKeyComputerTests(unittest.TestCase): m.builds_artifacts = [m['name']] self.build_env = DummyBuildEnvironment({ "LOGNAME": "foouser", - "PREFIX": "/baserock", "TARGET": "dummy-baserock-linux-gnu", "TARGET_STAGE1": "dummy-baserock-linux-gnu", "TARGET_GCC_CONFIG": "", diff --git a/morphlib/morph2.py b/morphlib/morph2.py index 728fa533..a8e1d7d3 100644 --- a/morphlib/morph2.py +++ b/morphlib/morph2.py @@ -159,6 +159,8 @@ class Morphology(object): self._set_default_value(source, 'build-depends', None) if 'build-mode' not in source: self._set_default_value(source, 'build-mode', 'staging') + if 'prefix' not in source: + self._set_default_value(source, 'prefix', '/usr') def _parse_size(self, size): if isinstance(size, basestring): diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index 24b72867..de29eede 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -51,9 +51,9 @@ class StagingArea(object): self.env.update(extra_env) if use_chroot: - path = build_env.extra_path + self._base_path + path = extra_path + build_env.extra_path + self._base_path else: - rel_path = build_env.extra_path + rel_path = extra_path + build_env.extra_path full_path = [os.path.normpath(dirname + p) for p in rel_path] path = full_path + os.environ['PATH'].split(':') self.env['PATH'] = ':'.join(path) diff --git a/tests.build/build-system-cpan.script b/tests.build/build-system-cpan.script index 19e45e42..b1823eb5 100755 --- a/tests.build/build-system-cpan.script +++ b/tests.build/build-system-cpan.script @@ -54,7 +54,29 @@ git add hello.morph git commit --quiet -m 'convert hello into a perl cpan project' -"$SRCDIR/scripts/test-morph" build-morphology --prefix=/ \ +# Set 'prefix' of hello to something custom +cd "$DATADIR/morphs-repo" +cat < hello-stratum.morph +{ + "name": "hello-stratum", + "kind": "stratum", + "chunks": [ + { + "name": "hello", + "repo": "test:chunk-repo", + "ref": "farrokh", + "build-depends": [], + "build-mode": "test", + "prefix": "/" + } + ] +} +EOF +git add hello-stratum.morph +git commit -q -m "Set custom install prefix for hello" + + +"$SRCDIR/scripts/test-morph" build-morphology \ test:morphs-repo master hello-system for chunk in "$DATADIR/cache/artifacts/"*.chunk.* diff --git a/tests.build/build-system-python-distutils.script b/tests.build/build-system-python-distutils.script index cebb9f84..a0469528 100755 --- a/tests.build/build-system-python-distutils.script +++ b/tests.build/build-system-python-distutils.script @@ -51,7 +51,30 @@ git add hello.morph git commit --quiet -m 'convert hello into a python project' -"$SRCDIR/scripts/test-morph" build-morphology --prefix= \ + +# Set 'prefix' of hello to something custom +cd "$DATADIR/morphs-repo" +cat < hello-stratum.morph +{ + "name": "hello-stratum", + "kind": "stratum", + "chunks": [ + { + "name": "hello", + "repo": "test:chunk-repo", + "ref": "farrokh", + "build-depends": [], + "build-mode": "test", + "prefix": "" + } + ] +} +EOF +git add hello-stratum.morph +git commit -q -m "Set custom install prefix for hello" + + +"$SRCDIR/scripts/test-morph" build-morphology \ test:morphs-repo master hello-system for chunk in "$DATADIR/cache/artifacts/"*.chunk.* diff --git a/tests.build/prefix.script b/tests.build/prefix.script new file mode 100755 index 00000000..e9b8ecd2 --- /dev/null +++ b/tests.build/prefix.script @@ -0,0 +1,87 @@ +#!/bin/sh +# +# Copyright (C) 2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +## Honour 'prefix' attribute for chunks within stratum morphs + +set -eu + +# Create two chunks which print out PATH and PREFIX from their environment. +cd "$DATADIR/chunk-repo" +git checkout -q master +cat <<\EOF > xyzzy.morph +{ + "name": "xyzzy", + "kind": "chunk", + "configure-commands": [ + "echo First chunk: prefix $PREFIX" + ] +} +EOF + +cat <<\EOF > plugh.morph +{ + "name": "plugh", + "kind": "chunk", + "configure-commands": [ + "echo Second chunk: prefix $PREFIX", + "echo Path: $(echo $PATH | grep -o '/plover')" + ] +} +EOF + +git add xyzzy.morph +git add plugh.morph +git commit -q -m "Add chunks" + +# Change stratum to include those two chunks, and use a custom install prefix +cd "$DATADIR/morphs-repo" +cat < hello-stratum.morph +{ + "name": "hello-stratum", + "kind": "stratum", + "chunks": [ + { + "name": "xyzzy", + "repo": "test:chunk-repo", + "ref": "master", + "build-depends": [], + "build-mode": "test", + "prefix": "/plover" + }, + { + "name": "plugh", + "repo": "test:chunk-repo", + "ref": "master", + "build-mode": "test", + "build-depends": [ + "xyzzy" + ] + } + ] +} +EOF +git add hello-stratum.morph +git commit -q -m "Update stratum" + +"$SRCDIR/scripts/test-morph" build-morphology \ + test:morphs-repo master hello-system + +cd "$DATADIR/cache/artifacts" +first_chunk=$(ls -1 *.chunk.xyzzy | cut -c -64) +second_chunk=$(ls -1 *.chunk.plugh | cut -c -64) +cat $first_chunk.build-log $second_chunk.build-log diff --git a/tests.build/prefix.stdout b/tests.build/prefix.stdout new file mode 100644 index 00000000..80c18fae --- /dev/null +++ b/tests.build/prefix.stdout @@ -0,0 +1,8 @@ +# configure +# # echo First chunk: prefix $PREFIX +First chunk: prefix /plover +# configure +# # echo Second chunk: prefix $PREFIX +Second chunk: prefix /usr +# # echo Path: $(echo $PATH | grep -o '/plover') +Path: /plover -- cgit v1.2.1 From 18b162cd4556eb8c63767916cd376d87630db1b7 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Thu, 21 Feb 2013 16:23:40 +0000 Subject: Don't include bootstrap chunks in output strata This is necessary so that the build-essential stratum doesn't end up containing three different compilers. --- morphlib/builder2.py | 11 +++++-- tests.build/bootstrap-mode.script | 65 +++++++++++++++++++++++++++++++++++++++ tests.build/bootstrap-mode.stdout | 10 ++++++ 3 files changed, 83 insertions(+), 3 deletions(-) create mode 100755 tests.build/bootstrap-mode.script create mode 100644 tests.build/bootstrap-mode.stdout diff --git a/morphlib/builder2.py b/morphlib/builder2.py index f6a1bafa..f8f4ea88 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -461,13 +461,18 @@ class StratumBuilder(BuilderBase): '''Build stratum artifacts.''' + def is_constituent(self, artifact): # pragma: no cover + '''True if artifact should be included in the stratum artifact''' + return (artifact.source.morphology['kind'] == 'chunk' and \ + artifact.source.build_mode != 'bootstrap') + def build_and_cache(self): # pragma: no cover with self.build_watch('overall-build'): - constituents = [dependency - for dependency in self.artifact.dependencies - if dependency.source.morphology['kind'] == 'chunk'] + constituents = [d for d in self.artifact.dependencies + if self.is_constituent(d)] if len(constituents) == 0: logging.warning('Stratum %s is empty' % self.artifact.name) + # the only reason the StratumBuilder has to download chunks is to # check for overlap now that strata are lists of chunks with self.build_watch('check-chunks'): diff --git a/tests.build/bootstrap-mode.script b/tests.build/bootstrap-mode.script new file mode 100755 index 00000000..02c74dfe --- /dev/null +++ b/tests.build/bootstrap-mode.script @@ -0,0 +1,65 @@ +#!/bin/sh +# +# Copyright (C) 2011-2013 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +## 'bootstrap' mode is similar to 'test' mode, but they should not be included +## in the final stratum artifact. This feature is used to bootstrap the +## build-essential stratum using the toolchain of the host. + +set -eu + +# Add a chunk in hello-stratum that is built in bootstrap mode + +cd "$DATADIR/morphs-repo" +cat < "hello-stratum.morph" +{ + "name": "hello-stratum", + "kind": "stratum", + "chunks": [ + { + "name": "stage1-hello", + "repo": "test:chunk-repo", + "ref": "farrokh", + "build-depends": [], + "build-mode": "bootstrap" + }, + { + "name": "hello", + "repo": "test:chunk-repo", + "ref": "farrokh", + "build-depends": [ + "stage1-hello" + ], + "build-mode": "test" + } + ] +} +EOF +git add hello-stratum.morph +git commit -q -m "Add bootstrap chunk in hello-stratum" + +cd "$DATADIR/chunk-repo" +git checkout -q farrokh +sed -e "s/\"hello\"/\"stage1-hello\"/g" hello.morph > stage1-hello.morph +git add stage1-hello.morph +git commit -q -m "Add stage1-hello morphology for bootstrapping test" + +"$SRCDIR/scripts/test-morph" build-morphology \ + test:morphs-repo master hello-system + +system=$(ls "$DATADIR/cache/artifacts/"*hello-system-rootfs) +tar tf "$system" | LC_ALL=C sort | sed '/^\.\/./s:^\./::' diff --git a/tests.build/bootstrap-mode.stdout b/tests.build/bootstrap-mode.stdout new file mode 100644 index 00000000..3d5201ee --- /dev/null +++ b/tests.build/bootstrap-mode.stdout @@ -0,0 +1,10 @@ +./ +baserock/ +baserock/hello-stratum.meta +baserock/hello-system-rootfs.meta +baserock/hello.meta +bin/ +bin/hello +etc/ +etc/fstab +etc/os-release -- cgit v1.2.1 From a20a6bdf6ed75c6bd5c06a25961e0c6bec93bdd6 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Fri, 22 Feb 2013 11:45:16 +0000 Subject: Don't install bootstrapped chunks when building artifacts in other strata When building a stratum artifact it's easy to only include chunks that were built in 'staging' mode. Constructing the staging area doesn't use that code path, though, so we need an extra hack to filter out those artifacts while building. It may be neater to express stratum build-depends as actual dependencies on the stratum artifact, rather than each of its constituent chunks as we do now. --- morphlib/buildcommand.py | 12 ++++ tests.build/bootstrap-mode.script | 127 ++++++++++++++++++++++++++++++++------ tests.build/bootstrap-mode.stdout | 9 ++- 3 files changed, 128 insertions(+), 20 deletions(-) diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index d602b5ea..9f6a0470 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -346,6 +346,15 @@ class BuildCommand(object): filename=filename) staging_area.install_artifact(f) + # Nasty hack to avoid installing chunks built in 'bootstrap' mode in a + # different stratum when constructing staging areas. + def is_stratum(self, a): + return a.source.morphology['kind'] == 'stratum' + + def in_same_stratum(self, a, b): + return len(filter(self.is_stratum, a.dependencies)) == \ + len(filter(self.is_stratum, b.dependencies)) + def install_dependencies(self, staging_area, artifacts, target_artifact): '''Install chunk artifacts into staging area. @@ -360,6 +369,9 @@ class BuildCommand(object): for artifact in artifacts: if artifact.source.morphology['kind'] != 'chunk': continue + if artifact.source.build_mode == 'bootstrap': + if not self.in_same_stratum(artifact, target_artifact): + continue self.app.status(msg='[%(name)s] Installing chunk %(chunk_name)s', name=target_artifact.name, chunk_name=artifact.name) diff --git a/tests.build/bootstrap-mode.script b/tests.build/bootstrap-mode.script index 02c74dfe..f4ff0a36 100755 --- a/tests.build/bootstrap-mode.script +++ b/tests.build/bootstrap-mode.script @@ -22,44 +22,135 @@ set -eu -# Add a chunk in hello-stratum that is built in bootstrap mode +# Create a fake 'compiler' chunk to go into build-essential stratum +mkdir -p "$DATADIR/cc-repo" +cd "$DATADIR/cc-repo" + +cat < "morph-test-cc" +#!/bin/sh +echo "I'm a compiler!" +EOF +chmod +x morph-test-cc + +cat < "stage1-cc.morph" +{ + "name": "stage1-cc", + "kind": "chunk", + "install-commands": [ + "install -d \"\$DESTDIR\$PREFIX/bin\"", + "install -m 755 morph-test-cc \"\$DESTDIR\$PREFIX/bin/morph-test-cc\"" + ] +} +EOF + +cat < "cc.morph" +{ + "name": "cc", + "kind": "chunk", + "configure-commands": [ + "[ -e ../tools/bin/morph-test-cc ]" + ], + "install-commands": [ + "install -d \"\$DESTDIR\$PREFIX/bin\"", + "install -m 755 morph-test-cc \"\$DESTDIR\$PREFIX/bin/morph-test-cc\"" + ] +} +EOF + +git init -q +git add morph-test-cc cc.morph stage1-cc.morph +git commit -q -m "Create compiler chunk" + +# Require 'cc' in hello-chunk. We should have the second version available +# but *not* the first one. +cd "$DATADIR/chunk-repo" +git checkout -q farrokh +cat < "hello.morph" +{ + "name": "hello", + "kind": "chunk", + "configure-commands": [ + "[ ! -e ../tools/bin/morph-test-cc ]", + "[ -e ../usr/bin/morph-test-cc ]" + ], + "build-commands": [ + "../usr/bin/morph-test-cc > hello" + ], + "install-commands": [ + "install -d \"\$DESTDIR\$PREFIX/bin\"", + "install hello \"\$DESTDIR\$PREFIX/bin/hello\"" + ] +} +EOF +git add hello.morph +git commit -q -m "Make 'hello' require our mock compiler" + +# Add 'build-essential' stratum and make hello-stratum depend upon it. Only +# the *second* 'cc' chunk should make it into the build-essential stratum +# artifact, and neither should make it into the system. cd "$DATADIR/morphs-repo" -cat < "hello-stratum.morph" +cat < "build-essential.morph" { - "name": "hello-stratum", + "name": "build-essential", "kind": "stratum", "chunks": [ { - "name": "stage1-hello", - "repo": "test:chunk-repo", - "ref": "farrokh", + "name": "stage1-cc", + "repo": "test:cc-repo", + "ref": "master", "build-depends": [], - "build-mode": "bootstrap" + "build-mode": "bootstrap", + "prefix": "/tools" }, + { + "name": "cc", + "repo": "test:cc-repo", + "ref": "master", + "build-depends": [ + "stage1-cc" + ], + "build-mode": "test" + } + ] +} +EOF + +cat < "hello-stratum.morph" +{ + "name": "hello-stratum", + "kind": "stratum", + "build-depends": [ + { + "morph": "build-essential", + "repo": "test:morphs-repo", + "ref": "master" + } + ], + "chunks": [ { "name": "hello", "repo": "test:chunk-repo", "ref": "farrokh", - "build-depends": [ - "stage1-hello" - ], + "build-depends": [], "build-mode": "test" } ] } EOF -git add hello-stratum.morph -git commit -q -m "Add bootstrap chunk in hello-stratum" -cd "$DATADIR/chunk-repo" -git checkout -q farrokh -sed -e "s/\"hello\"/\"stage1-hello\"/g" hello.morph > stage1-hello.morph -git add stage1-hello.morph -git commit -q -m "Add stage1-hello morphology for bootstrapping test" +git add build-essential.morph hello-stratum.morph hello-system.morph +git commit -q -m "Add fake build-essential stratum" "$SRCDIR/scripts/test-morph" build-morphology \ test:morphs-repo master hello-system -system=$(ls "$DATADIR/cache/artifacts/"*hello-system-rootfs) +cd "$DATADIR/cache/artifacts" +echo "build-essential stratum:" +stratum=$(ls *.stratum.build-essential) +cat $stratum | sed 's/[a-f0-9]\{64\}/xxxx/g' +echo +echo +echo "hello-system:" +system=$(ls *hello-system-rootfs) tar tf "$system" | LC_ALL=C sort | sed '/^\.\/./s:^\./::' diff --git a/tests.build/bootstrap-mode.stdout b/tests.build/bootstrap-mode.stdout index 3d5201ee..329cdd78 100644 --- a/tests.build/bootstrap-mode.stdout +++ b/tests.build/bootstrap-mode.stdout @@ -1,10 +1,15 @@ +build-essential stratum: +["xxxx.chunk.cc"] + +hello-system: ./ baserock/ baserock/hello-stratum.meta baserock/hello-system-rootfs.meta baserock/hello.meta -bin/ -bin/hello etc/ etc/fstab etc/os-release +usr/ +usr/bin/ +usr/bin/hello -- cgit v1.2.1 From bd3e607439a0d3fe03bfac0eeea4701ff606b1e1 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Mon, 4 Mar 2013 16:58:59 +0000 Subject: Only bind mount necessary directories in chroot If for example /bin is a symlink, we shouldn't and can't mount the host system's /bin over the top. --- morphlib/stagingarea.py | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index de29eede..418ef15d 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -278,16 +278,14 @@ class StagingArea(object): else: cwd = '/' - not_readonly_dirs = [self.builddirname, self.destdirname, + do_not_mount_dirs = [self.builddirname, self.destdirname, 'dev', 'proc', 'tmp'] - dirs = os.listdir(self.dirname) - for excluded_dir in not_readonly_dirs: - dirs.remove(excluded_dir) real_argv = ['linux-user-chroot'] - - for entry in dirs: - real_argv += ['--mount-readonly', '/'+entry] + for d in os.listdir(self.dirname): + if d not in do_not_mount_dirs: + if os.path.isdir(os.path.join(self.dirname, d)): + real_argv += ['--mount-readonly', '/'+d] real_argv += [self.dirname] real_argv += ['sh', '-c', 'cd "$1" && shift && exec "$@"', '--', cwd] -- cgit v1.2.1 From d2eb452ee48253d6e90f7aa4f6c41b6fd8cdf009 Mon Sep 17 00:00:00 2001 From: Sam Thursfield Date: Wed, 13 Mar 2013 11:59:17 +0000 Subject: Allow unknown values of build-mode through with just a warning This gives us some forward compatibility leeway to introduce new build modes, as long as they are compatible with staging. As suggested by Richard Maw. --- morphlib/buildcommand.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 9f6a0470..6fe46e0b 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -236,9 +236,10 @@ class BuildCommand(object): extra_path = [os.path.join(d, 'bin') for d in dep_prefix_set] if build_mode not in ['bootstrap', 'staging', 'test']: - raise morphlib.Error( - 'Unknown build mode for chunk %s: %s' % - (artifact.name, build_mode)) + logging.warning('Unknown build mode %s for chunk %s. ' + 'Defaulting to staging mode.' % + (build_mode, artifact.name)) + build_mode = 'staging' use_chroot = build_mode=='staging' staging_area = self.create_staging_area( -- cgit v1.2.1