diff options
| -rw-r--r-- | morphlib/builder2.py | 47 | ||||
| -rwxr-xr-x | tests.as-root/make-patch.script | 19 | ||||
| -rwxr-xr-x | tests.as-root/setup | 18 | ||||
| -rwxr-xr-x | tests.as-root/system-overlap.script | 123 | ||||
| -rw-r--r-- | tests.as-root/system-overlap.stdout | 30 | ||||
| -rwxr-xr-x | tests/stratum-overlap.script | 101 | ||||
| -rw-r--r-- | tests/stratum-overlap.stdout | 20 |
7 files changed, 335 insertions, 23 deletions
diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 0e5c469f..2a7ca2d2 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -19,6 +19,8 @@ import logging import os import shutil import time +from collections import defaultdict +import tarfile import morphlib @@ -60,6 +62,31 @@ def ldconfig(ex, rootdir): # pragma: no cover logging.debug('No %s, not running ldconfig' % conf) +def check_overlap(artifact, constituents, lac): #pragma: no cover + # check whether strata overlap + installed = defaultdict(set) + for dep in constituents: + handle = lac.get(dep) + tar = tarfile.open(fileobj=handle) + for member in tar.getmembers(): + if member.type is not tarfile.DIRTYPE: + installed[member.name].add(dep) + tar.close() + handle.close() + overlaps = defaultdict(set) + for filename, artifacts in installed.iteritems(): + if len(artifacts) > 1: + overlaps[frozenset(artifacts)].add(filename) + if len(overlaps) > 0: + logging.warning('Overlaps in artifact %s detected' % artifact.name) + for overlapping, files in sorted(overlaps.iteritems()): + logging.warning(' Artifacts %s overlap with files:' % + ', '.join(a.name for a in sorted(overlapping)) + ) + for filename in sorted(files): + logging.warning(' %s' % filename) + + class BuilderBase(object): '''Base class for building artifacts.''' @@ -319,15 +346,21 @@ class StratumBuilder(BuilderBase): for dependency in self.artifact.dependencies if dependency.source.morphology['kind'] == 'chunk'] with self.build_watch('unpack-chunks'): + # download the chunk artifact if necessary for chunk_artifact in constituents: - # download the chunk artifact if necessary if not self.local_artifact_cache.has(chunk_artifact): source = self.remote_artifact_cache.get(chunk_artifact) target = self.local_artifact_cache.put(chunk_artifact) shutil.copyfileobj(source, target) target.close() source.close() - # unpack it from the local artifact cache + + # check for chunk overlaps + check_overlap(self.artifact, constituents, + self.local_artifact_cache) + + # unpack it from the local artifact cache + for chunk_artifact in constituents: logging.debug('unpacking chunk %s into stratum %s' % (chunk_artifact.basename(), self.artifact.basename())) @@ -430,15 +463,21 @@ class SystemBuilder(BuilderBase): # pragma: no cover def _unpack_strata(self, path): logging.debug('Unpacking strata to %s' % path) with self.build_watch('unpack-strata'): + # download the stratum artifact if necessary for stratum_artifact in self.artifact.dependencies: - # download the stratum artifact if necessary if not self.local_artifact_cache.has(stratum_artifact): source = self.remote_artifact_cache.get(stratum_artifact) target = self.local_artifact_cache.put(stratum_artifact) shutil.copyfileobj(source, target) target.close() source.close() - # unpack it from the local artifact cache + + # check whether the strata overlap + check_overlap(self.artifact, self.artifact.dependencies, + self.local_artifact_cache) + + # unpack it from the local artifact cache + for stratum_artifact in self.artifact.dependencies: f = self.local_artifact_cache.get(stratum_artifact) morphlib.bins.unpack_binary_from_file(f, path) f.close() diff --git a/tests.as-root/make-patch.script b/tests.as-root/make-patch.script index 0151ef6c..172f3e07 100755 --- a/tests.as-root/make-patch.script +++ b/tests.as-root/make-patch.script @@ -19,25 +19,6 @@ set -eu -# Make a dummy kernel chunk. -mkdir "$DATADIR/kernel-repo" -cat <<EOF > "$DATADIR/kernel-repo/linux.morph" -{ - "name": "linux", - "kind": "chunk", - "install-commands": [ - "mkdir -p \"\$DESTDIR/boot\"", - "touch \"\$DESTDIR\"/extlinux.conf", - "touch \"\$DESTDIR\"/boot/vmlinuz", - "touch \"\$DESTDIR\"/boot/System.map" - ] -} -EOF -"$SRCDIR/scripts/run-git-in" "$DATADIR/kernel-repo" init --quiet -"$SRCDIR/scripts/run-git-in" "$DATADIR/kernel-repo" add . -"$SRCDIR/scripts/run-git-in" "$DATADIR/kernel-repo" commit --quiet -m foo \ - > /dev/null - # Make a stratum that include hello and kernel chunks. cat <<EOF > "$DATADIR/morphs-repo/hello-stratum.morph" { diff --git a/tests.as-root/setup b/tests.as-root/setup index 0b78ef9f..59e4a944 100755 --- a/tests.as-root/setup +++ b/tests.as-root/setup @@ -115,6 +115,24 @@ git add hello-system.morph git commit --quiet -m "add morphs" +# Make a dummy kernel chunk. +mkdir "$DATADIR/kernel-repo" +cat <<EOF > "$DATADIR/kernel-repo/linux.morph" +{ + "name": "linux", + "kind": "chunk", + "install-commands": [ + "mkdir -p \"\$DESTDIR/boot\"", + "touch \"\$DESTDIR\"/extlinux.conf", + "touch \"\$DESTDIR\"/boot/vmlinuz", + "touch \"\$DESTDIR\"/boot/System.map" + ] +} +EOF +"$SRCDIR/scripts/run-git-in" "$DATADIR/kernel-repo" init --quiet +"$SRCDIR/scripts/run-git-in" "$DATADIR/kernel-repo" add . +"$SRCDIR/scripts/run-git-in" "$DATADIR/kernel-repo" commit --quiet -m foo \ + > /dev/null # Create a morph configuration file. cat <<EOF > "$DATADIR/morph.conf" diff --git a/tests.as-root/system-overlap.script b/tests.as-root/system-overlap.script new file mode 100755 index 00000000..066f39cf --- /dev/null +++ b/tests.as-root/system-overlap.script @@ -0,0 +1,123 @@ +#!/bin/sh +# +# If a system has multiple strata that have the same files in them, +# then this should be notified +# +# Copyright (C) 2011, 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +set -eu + +cache="$DATADIR/cache/artifacts" +chunkrepo="$DATADIR/chunk-repo" +morphsrepo="$DATADIR/morphs-repo" +log="$DATADIR/morph.log" + +cd "$morphsrepo" +git checkout --quiet -b overlap master +cat <<EOF >overlap-system.morph +{ + "name": "overlap-system", + "kind": "system", + "disk-size": "1G", + "strata": [ + "foo-baz-stratum", + "foo-barqux-stratum" + ] +} +EOF +cat <<EOF >foo-baz-stratum.morph +{ + "name": "foo-baz-stratum", + "kind": "stratum", + "sources": [ + { + "name": "overlap-foo-baz", + "repo": "test:chunk-repo", + "ref": "overlap" + }, + { + "name": "linux", + "repo": "test:kernel-repo", + "ref": "master" + } + ] +} +EOF +cat <<EOF >foo-barqux-stratum.morph +{ + "name": "foo-barqux-stratum", + "kind": "stratum", + "sources": [ + { + "name": "overlap-foobar", + "repo": "test:chunk-repo", + "ref": "overlap" + }, + { + "name": "overlap-fooqux", + "repo": "test:chunk-repo", + "ref": "overlap" + } + ] +} +EOF +git add overlap-system.morph foo-baz-stratum.morph foo-barqux-stratum.morph +git commit --quiet -m "add overlapping system" + +cd "$chunkrepo" +git checkout --quiet -b overlap master +cat <<EOF >overlap-foo-baz.morph +{ + "name": "overlap-foo-baz", + "kind": "chunk", + "install-commands": [ + "mkdir -p \$DESTDIR/bin", + "for f in foo bar baz; do echo echo \$f >\$DESTDIR/bin/\$f; done" + ] +} +EOF + +cat <<EOF >overlap-foobar.morph +{ + "name": "overlap-foobar", + "kind": "chunk", + "install-commands": [ + "mkdir -p \$DESTDIR/usr/bin \$DESTDIR/bin", + "echo echo foobar >\$DESTDIR/usr/bin/foobar", + "ln -s /usr/bin/foobar \$DESTDIR/bin/foo", + "ln -s /usr/bin/foobar \$DESTDIR/bin/bar" + ] +} +EOF + +cat <<EOF >overlap-fooqux.morph +{ + "name": "overlap-fooqux", + "kind": "chunk", + "install-commands": [ + "mkdir -p \$DESTDIR/usr/bin \$DESTDIR/bin", + "for f in qux fooqux; do echo echo \$f >\$DESTDIR/usr/bin/\$f; done", + "ln -s /usr/bin/fooqux \$DESTDIR/bin/foo" + ] +} +EOF +git add overlap-*.morph + +git commit --quiet -m 'Add overlapping chunks' + +"$SRCDIR/scripts/test-morph" \ + build --verbose --log=$log test:morphs-repo overlap overlap-system.morph +grep "WARNING\s" $log | sed 's/^.*WARNING/WARNING/' diff --git a/tests.as-root/system-overlap.stdout b/tests.as-root/system-overlap.stdout new file mode 100644 index 00000000..c984ab16 --- /dev/null +++ b/tests.as-root/system-overlap.stdout @@ -0,0 +1,30 @@ +Build starts +Building test:morphs-repo overlap overlap-system.morph +Figuring out the right build order +Cloning/updating test:chunk-repo +Cloning/updating test:chunk-repo +Cloning/updating test:kernel-repo +Cloning/updating test:chunk-repo +Cloning/updating test:morphs-repo +Cloning/updating test:morphs-repo +Cloning/updating test:morphs-repo +Building overlap-foo-baz +Building overlap-foobar +Building linux +Building overlap-fooqux +Building foo-baz-stratum +Building foo-barqux-stratum +Building overlap-system +WARNING Overlaps in artifact foo-barqux-stratum detected +WARNING Artifacts overlap-foobar, overlap-fooqux overlap with files: +WARNING bin/foo +WARNING Overlaps in artifact overlap-system detected +WARNING Artifacts foo-baz-stratum, foo-barqux-stratum overlap with files: +WARNING ./bin/bar +WARNING ./bin/foo + chunk.linux + chunk.overlap-foo-baz + chunk.overlap-foobar + chunk.overlap-fooqux + stratum.foo-barqux-stratum + stratum.foo-baz-stratum diff --git a/tests/stratum-overlap.script b/tests/stratum-overlap.script new file mode 100755 index 00000000..442704a8 --- /dev/null +++ b/tests/stratum-overlap.script @@ -0,0 +1,101 @@ +#!/bin/sh +# +# If a stratum has multiple chunks that have the same files in them, +# then this should be notified +# +# Copyright (C) 2011, 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +set -eu + +cache="$DATADIR/cache/artifacts" +chunkrepo="$DATADIR/chunk-repo" +morphsrepo="$DATADIR/morphs-repo" +log="$DATADIR/morph.log" + +cd "$morphsrepo" +git checkout --quiet -b overlap master +cat <<EOF >overlap-stratum.morph +{ + "name": "overlap-stratum", + "kind": "stratum", + "sources": [ + { + "name": "overlap-foo-baz", + "repo": "test:chunk-repo", + "ref": "overlap" + }, + { + "name": "overlap-foobar", + "repo": "test:chunk-repo", + "ref": "overlap" + }, + { + "name": "overlap-fooqux", + "repo": "test:chunk-repo", + "ref": "overlap" + } + ] +} +EOF +git add overlap-stratum.morph +git commit --quiet -m "add overlapping stratum" + +cd "$chunkrepo" +git checkout --quiet -b overlap master +cat <<EOF >overlap-foo-baz.morph +{ + "name": "overlap-foo-baz", + "kind": "chunk", + "install-commands": [ + "mkdir -p \$DESTDIR/bin", + "for f in foo bar baz; do echo echo \$f >\$DESTDIR/bin/\$f; done" + ] +} +EOF + +cat <<EOF >overlap-foobar.morph +{ + "name": "overlap-foobar", + "kind": "chunk", + "install-commands": [ + "mkdir -p \$DESTDIR/usr/bin \$DESTDIR/bin", + "echo echo foobar >\$DESTDIR/usr/bin/foobar", + "ln -s /usr/bin/foobar \$DESTDIR/bin/foo", + "ln -s /usr/bin/foobar \$DESTDIR/bin/bar" + ] +} +EOF + +cat <<EOF >overlap-fooqux.morph +{ + "name": "overlap-fooqux", + "kind": "chunk", + "install-commands": [ + "mkdir -p \$DESTDIR/usr/bin \$DESTDIR/bin", + "for f in qux fooqux; do echo echo \$f >\$DESTDIR/usr/bin/\$f; done", + "ln -s /usr/bin/fooqux \$DESTDIR/bin/foo" + ] +} +EOF +git add overlap-*.morph + +git commit --quiet -m 'Add overlapping chunks' + +"$SRCDIR/scripts/test-morph" \ + build --verbose --log=$log test:morphs-repo overlap overlap-stratum.morph +grep WARNING $log | sed 's/^.*WARNING/WARNING/' +(cd "$cache" && ls *.chunk.* *.stratum.* | sed 's/^[^.]*\./ /' | + LC_ALL=C sort) diff --git a/tests/stratum-overlap.stdout b/tests/stratum-overlap.stdout new file mode 100644 index 00000000..90913dd4 --- /dev/null +++ b/tests/stratum-overlap.stdout @@ -0,0 +1,20 @@ +Build starts +Building test:morphs-repo overlap overlap-stratum.morph +Figuring out the right build order +Cloning/updating test:chunk-repo +Cloning/updating test:chunk-repo +Cloning/updating test:chunk-repo +Cloning/updating test:morphs-repo +Building overlap-foo-baz +Building overlap-foobar +Building overlap-fooqux +Building overlap-stratum +WARNING Overlaps in artifact overlap-stratum detected +WARNING Artifacts overlap-foo-baz, overlap-foobar overlap with files: +WARNING bin/bar +WARNING Artifacts overlap-foo-baz, overlap-foobar, overlap-fooqux overlap with files: +WARNING bin/foo + chunk.overlap-foo-baz + chunk.overlap-foobar + chunk.overlap-fooqux + stratum.overlap-stratum |