diff options
24 files changed, 378 insertions, 116 deletions
diff --git a/morphlib/__init__.py b/morphlib/__init__.py index baf4c253..4d9ad22e 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -26,6 +26,7 @@ class Error(cliapp.AppException): import artifact +import artifactcachereference import artifactresolver import bins import buildenvironment diff --git a/morphlib/artifactcachereference.py b/morphlib/artifactcachereference.py new file mode 100644 index 00000000..169bf96d --- /dev/null +++ b/morphlib/artifactcachereference.py @@ -0,0 +1,37 @@ +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +class ArtifactCacheReference(object): + + '''Represent the information needed to retrieve an artifact + + The artifact cache doesn't need to know the dependencies or the + morphology of an artifact, it just needs to know the basename + + The basename could be generated, from the name, cache_key and kind, + but if the algorithm changes then morph wouldn't be able to find + old artifacts with a saved ArtifactCacheReference. + + Conversely if it generated the basename then old strata wouldn't be + able to refer to new chunks, but strata change more often than the chunks. + ''' + def __init__(self, basename): + self._basename = basename + + def basename(self): + return self._basename + + def metadata_basename(self, metadata_name): + return '%s.%s' % (self._basename, metadata_name) diff --git a/morphlib/bins.py b/morphlib/bins.py index 93aa7b15..0c9ecadf 100644 --- a/morphlib/bins.py +++ b/morphlib/bins.py @@ -95,14 +95,6 @@ def create_chunk(rootdir, f, regexps, dump_memory_profile=None): dump_memory_profile('after removing in create_chunks') -def create_stratum(rootdir, f): - '''Create a stratum from the contents of a directory.''' - logging.debug('Creating stratum file %s from %s' % (f.name, rootdir)) - tar = tarfile.open(fileobj=f, mode='w:gz') - tar.add(rootdir, arcname='.') - tar.close() - - def unpack_binary_from_file(f, dirname): # pragma: no cover '''Unpack a binary into a directory. diff --git a/morphlib/bins_tests.py b/morphlib/bins_tests.py index 2da5d047..544e9013 100644 --- a/morphlib/bins_tests.py +++ b/morphlib/bins_tests.py @@ -132,30 +132,3 @@ class ChunkTests(BinsTest): self.assertEqual([x for x,y in self.recursive_lstat(self.instdir)], ['.', 'lib', 'lib/libfoo.so']) - -class StratumTests(BinsTest): - - def setUp(self): - self.tempdir = tempfile.mkdtemp() - self.instdir = os.path.join(self.tempdir, 'inst') - self.stratum_file = os.path.join(self.tempdir, 'stratum') - self.stratum_f = open(self.stratum_file, 'wb') - self.unpacked = os.path.join(self.tempdir, 'unpacked') - - def tearDown(self): - self.stratum_f.close() - shutil.rmtree(self.tempdir) - - def populate_instdir(self): - os.mkdir(self.instdir) - os.mkdir(os.path.join(self.instdir, 'bin')) - - def test_creates_and_unpacks_stratum_exactly(self): - self.populate_instdir() - morphlib.bins.create_stratum(self.instdir, self.stratum_f) - self.stratum_f.flush() - os.mkdir(self.unpacked) - morphlib.bins.unpack_binary(self.stratum_file, self.unpacked) - self.assertEqual(self.recursive_lstat(self.instdir), - self.recursive_lstat(self.unpacked)) - diff --git a/morphlib/builder2.py b/morphlib/builder2.py index caf0d8e1..4191c32a 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -23,6 +23,7 @@ from collections import defaultdict import tarfile import morphlib +from morphlib.artifactcachereference import ArtifactCacheReference def ldconfig(runcmd, rootdir): # pragma: no cover @@ -60,31 +61,74 @@ def ldconfig(runcmd, rootdir): # pragma: no cover else: logging.debug('No %s, not running ldconfig' % conf) - -def check_overlap(artifact, constituents, lac): #pragma: no cover +def download_depends(constituents, lac, rac, metadatas=None): + for constituent in constituents: + if not lac.has(constituent): + source = rac.get(constituent) + target = lac.put(constituent) + shutil.copyfileobj(source, target) + target.close() + source.close() + if metadatas is not None: + for metadata in metadatas: + if (not lac.has_artifact_metadata(constituent, metadata) + and rac.has_artifact_metadata(constituent, metadata)): + src = rac.get_artifact_metadata(constituent, metadata) + dst = lac.put_artifact_metadata(constituent, metadata) + shutil.copyfileobj(src, dst) + dst.close() + src.close() + +def get_chunk_files(f): # pragma: no cover + tar = tarfile.open(fileobj=f) + for member in tar.getmembers(): + if member.type is not tarfile.DIRTYPE: + yield member.name + tar.close() + +def get_stratum_files(f, lac): # pragma: no cover + for ca in (ArtifactCacheReference(a) for a in json.load(f)): + cf = lac.get(ca) + for filename in get_chunk_files(cf): + yield filename + cf.close() + +def get_overlaps(artifact, constituents, lac): #pragma: no cover # check whether strata overlap installed = defaultdict(set) for dep in constituents: handle = lac.get(dep) - tar = tarfile.open(fileobj=handle) - for member in tar.getmembers(): - if member.type is not tarfile.DIRTYPE: - installed[member.name].add(dep) - tar.close() + if artifact.source.morphology['kind'] == 'stratum': + for filename in get_chunk_files(handle): + installed[filename].add(dep) + elif artifact.source.morphology['kind'] == 'system': + for filename in get_stratum_files(handle, lac): + installed[filename].add(dep) handle.close() overlaps = defaultdict(set) for filename, artifacts in installed.iteritems(): if len(artifacts) > 1: overlaps[frozenset(artifacts)].add(filename) - if len(overlaps) > 0: - logging.warning('Overlaps in artifact %s detected' % artifact.name) - for overlapping, files in sorted(overlaps.iteritems()): - logging.warning(' Artifacts %s overlap with files:' % - ', '.join(sorted(a.name for a in overlapping)) - ) - for filename in sorted(files): - logging.warning(' %s' % filename) - + return overlaps + +def log_overlaps(overlaps): #pragma: no cover + for overlapping, files in sorted(overlaps.iteritems()): + logging.warning(' Artifacts %s overlap with files:' % + ', '.join(sorted(a.name for a in overlapping)) + ) + for filename in sorted(files): + logging.warning(' %s' % filename) + +def write_overlap_metadata(artifact, overlaps, lac): #pragma: no cover + f = lac.put_artifact_metadata(artifact, 'overlaps') + # the big list comprehension is because json can't serialize + # artifacts, sets or dicts with non-string keys + json.dump([ + [ + [a.name for a in afs], list(files) + ] for afs, files in overlaps.iteritems() + ], f, indent=4) + f.close() class BuilderBase(object): @@ -339,40 +383,35 @@ class StratumBuilder(BuilderBase): def build_and_cache(self): # pragma: no cover with self.build_watch('overall-build'): - destdir = self.staging_area.destdir(self.artifact.source) - constituents = [dependency for dependency in self.artifact.dependencies if dependency.source.morphology['kind'] == 'chunk'] - with self.build_watch('unpack-chunks'): + # the only reason the StratumBuilder has to download chunks is to + # check for overlap now that strata are lists of chunks + with self.build_watch('check-chunks'): # download the chunk artifact if necessary - for chunk_artifact in constituents: - if not self.local_artifact_cache.has(chunk_artifact): - source = self.remote_artifact_cache.get(chunk_artifact) - target = self.local_artifact_cache.put(chunk_artifact) - shutil.copyfileobj(source, target) - target.close() - source.close() - + download_depends(constituents, + self.local_artifact_cache, + self.remote_artifact_cache) # check for chunk overlaps - check_overlap(self.artifact, constituents, - self.local_artifact_cache) - - # unpack it from the local artifact cache - for chunk_artifact in constituents: - logging.debug('unpacking chunk %s into stratum %s' % - (chunk_artifact.basename(), - self.artifact.basename())) - f = self.local_artifact_cache.get(chunk_artifact) - morphlib.bins.unpack_binary_from_file(f, destdir) - f.close() - - with self.build_watch('create-binary'): + overlaps = get_overlaps(self.artifact, constituents, + self.local_artifact_cache) + if len(overlaps) > 0: + logging.warning('Overlaps in stratum artifact %s detected' + % self.artifact.name) + log_overlaps(overlaps) + write_overlap_metadata(self.artifact, overlaps, + self.local_artifact_cache) + + with self.build_watch('create-chunk-list'): + lac = self.local_artifact_cache artifact_name = self.artifact.source.morphology['name'] - self.write_metadata(destdir, artifact_name) artifact = self.new_artifact(artifact_name) + meta = self.create_metadata(artifact_name) + with lac.put_artifact_metadata(artifact, 'meta') as f: + json.dump(meta, f, indent=4, sort_keys=True) with self.local_artifact_cache.put(artifact) as f: - morphlib.bins.create_stratum(destdir, f) + json.dump([c.basename() for c in constituents], f) self.save_build_times() @@ -481,24 +520,48 @@ class SystemBuilder(BuilderBase): # pragma: no cover def _unpack_strata(self, path): logging.debug('Unpacking strata to %s' % path) with self.build_watch('unpack-strata'): - # download the stratum artifact if necessary + # download the stratum artifacts if necessary + download_depends(self.artifact.dependencies, + self.local_artifact_cache, + self.remote_artifact_cache, + ('meta',)) + + # download the chunk artifacts if necessary for stratum_artifact in self.artifact.dependencies: - if not self.local_artifact_cache.has(stratum_artifact): - source = self.remote_artifact_cache.get(stratum_artifact) - target = self.local_artifact_cache.put(stratum_artifact) - shutil.copyfileobj(source, target) - target.close() - source.close() + f = self.local_artifact_cache.get(stratum_artifact) + chunks = [ArtifactCacheReference(a) for a in json.load(f)] + download_depends(chunks, + self.local_artifact_cache, + self.remote_artifact_cache) + f.close() # check whether the strata overlap - check_overlap(self.artifact, self.artifact.dependencies, - self.local_artifact_cache) + overlaps = get_overlaps(self.artifact, self.artifact.dependencies, + self.local_artifact_cache) + if len(overlaps) > 0: + logging.warning('Overlaps in system artifact %s detected' % + self.artifact.name) + log_overlaps(overlaps) + write_overlap_metadata(self.artifact, overlaps, + self.local_artifact_cache) # unpack it from the local artifact cache for stratum_artifact in self.artifact.dependencies: f = self.local_artifact_cache.get(stratum_artifact) - morphlib.bins.unpack_binary_from_file(f, path) + for chunk in (ArtifactCacheReference(a) for a in json.load(f)): + chunk_handle = self.local_artifact_cache.get(chunk) + morphlib.bins.unpack_binary_from_file(chunk_handle, path) + chunk_handle.close() f.close() + meta = self.local_artifact_cache.get_artifact_metadata( + stratum_artifact, 'meta') + dst = morphlib.savefile.SaveFile( + os.path.join(path, 'baserock', + '%s.meta' % stratum_artifact.name), 'w') + shutil.copyfileobj(meta, dst) + dst.close() + meta.close() + ldconfig(self.app.runcmd, path) def _create_fstab(self, path): diff --git a/morphlib/builder2_tests.py b/morphlib/builder2_tests.py index 9730b59e..6214891d 100644 --- a/morphlib/builder2_tests.py +++ b/morphlib/builder2_tests.py @@ -80,9 +80,15 @@ class FakeFileHandle(object): def __enter__(self): return self - def __exit__(self, type, value, traceback): + def _writeback(self): self._cache._cached[self._key] = self._string + def __exit__(self, type, value, traceback): + self._writeback() + + def close(self): + self._writeback() + def write(self, string): self._string += string @@ -209,6 +215,34 @@ class BuilderBaseTests(unittest.TestCase): self.assertEqual(sorted(events), sorted(meta['build-times'].keys())) + def test_downloads_depends(self): + lac = FakeArtifactCache() + rac = FakeArtifactCache() + afacts = [FakeArtifact(name) for name in ('a', 'b', 'c')] + for a in afacts: + fh = rac.put(a) + fh.write(a.name) + fh.close() + morphlib.builder2.download_depends(afacts, lac, rac) + self.assertTrue(all(lac.has(a) for a in afacts)) + + def test_downloads_depends_metadata(self): + lac = FakeArtifactCache() + rac = FakeArtifactCache() + afacts = [FakeArtifact(name) for name in ('a', 'b', 'c')] + for a in afacts: + fh = rac.put(a) + fh.write(a.name) + fh.close() + fh = rac.put_artifact_metadata(a, 'meta') + fh.write('metadata') + fh.close() + morphlib.builder2.download_depends(afacts, lac, rac, ('meta',)) + self.assertTrue(all(lac.has(a) for a in afacts)) + self.assertTrue(all(lac.has_artifact_metadata(a, 'meta') + for a in afacts)) + + class ChunkBuilderTests(unittest.TestCase): def setUp(self): diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index e8cbed2c..720ec4c2 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -94,6 +94,8 @@ class CacheKeyComputer(object): checksum = hashlib.sha1() self._hash_thing(checksum, le_dict) keys['morphology-sha1'] = checksum.hexdigest() + if kind == 'stratum': + keys['stratum-format-version'] = 1 return keys diff --git a/scripts/assemble-stratum b/scripts/assemble-stratum new file mode 100755 index 00000000..ef81c7f6 --- /dev/null +++ b/scripts/assemble-stratum @@ -0,0 +1,58 @@ +#!/usr/bin/env python +# +# Copyright (C) 2011-2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# This is a program to convert the json dump of the overlaps between artifacts +# in a format more suited to shell programs, or human reading + +import json +import tarfile +import os + +import cliapp + +class AssembleStratum(cliapp.Application): + + def add_settings(self): + self.settings.string(['cachedir'], + 'Where the cache basedir is') + self.settings.string(['tarformat'], + 'What format to write tar to', + default='') + + def process_args(self, args): + chunklist = json.load(open(args[0])) + tarformat = 'w' + if self.settings['tarformat'] != "": + tarformat += self.settings['tarformat'] + outfile = tarfile.open(args[1], tarformat) + # concatenate chunk tarballs + for chunk in chunklist: + path = os.path.join(self.settings['cachedir'], 'artifacts', chunk) + chunktar = tarfile.open(path, mode='r:*') + for tarinfo in chunktar: + if tarinfo.isfile(): + outfile.addfile(tarinfo, chunktar.extractfile(tarinfo)) + else: + outfile.addfile(tarinfo) + chunktar.close() + # add the stratum's metadata + if os.path.exists(args[0] + '.meta'): + outfile.add(args[0] + '.meta', + os.path.join('baserock', '%s.meta' % args[2])) + outfile.close() + +AssembleStratum().run() diff --git a/scripts/list-overlaps b/scripts/list-overlaps new file mode 100755 index 00000000..3d92d4ad --- /dev/null +++ b/scripts/list-overlaps @@ -0,0 +1,44 @@ +#!/usr/bin/env python +# +# Copyright (C) 2011-2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +# This is a program to convert the json dump of the overlaps between artifacts +# in a format more suited to shell programs, or human reading + +import json + +import cliapp + +class ListOverlaps(cliapp.Application): + + @staticmethod + def _load_overlap(filename): + data = json.load(open(filename)) + overlaps = dict((frozenset(pair[0]), set(pair[1])) for pair in data) + return overlaps + + def cmd_groups(self, args): + overlaps = ListOverlaps._load_overlap(args[0]) + for group in overlaps: + print(' '.join(sorted(group))) + + def cmd_list_files(self, args): + overlaps = self._load_overlap(args[0]) + group = frozenset(args[1:]) + for filename in overlaps[group]: + print filename + +ListOverlaps().run() diff --git a/tests.as-root/make-patch.script b/tests.as-root/make-patch.script index 172f3e07..d312cb55 100755 --- a/tests.as-root/make-patch.script +++ b/tests.as-root/make-patch.script @@ -44,7 +44,7 @@ EOF # Build first image. Remember the stratum. "$SRCDIR/scripts/test-morph" build test:morphs-repo master hello-system.morph img1=$(find "$DATADIR/cache/artifacts" -maxdepth 1 -name '*.system.*') -stratum1=$(find "$DATADIR/cache/artifacts" -maxdepth 1 -name '*.stratum.*') +stratum1=$(find "$DATADIR/cache/artifacts" -maxdepth 1 -name '*.hello-stratum') # Modify the chunk, in a new branch. "$SRCDIR/scripts/run-git-in" "$DATADIR/chunk-repo" checkout --quiet farrokh @@ -74,7 +74,9 @@ patch="$DATADIR/patch" # Unpack the original stratum and run the program. mkdir "$DATADIR/unpacked" cd "$DATADIR/unpacked" -tar -xf "$stratum1" +"$SRCDIR/scripts/assemble-stratum" --cachedir "$DATADIR/cache" "$stratum1" \ + "$DATADIR/stratum.tar" hello-stratum +tar -xf "$DATADIR/stratum.tar" echo "old version:" ./bin/hello diff --git a/tests.as-root/system-overlap.script b/tests.as-root/system-overlap.script index 868d3e61..28307301 100755 --- a/tests.as-root/system-overlap.script +++ b/tests.as-root/system-overlap.script @@ -23,7 +23,6 @@ set -eu cache="$DATADIR/cache/artifacts" chunkrepo="$DATADIR/chunk-repo" morphsrepo="$DATADIR/morphs-repo" -log="$DATADIR/morph.log" cd "$morphsrepo" git checkout --quiet -b overlap master @@ -119,6 +118,11 @@ git add overlap-*.morph git commit --quiet -m 'Add overlapping chunks' "$SRCDIR/scripts/test-morph" \ - build --log=$log test:morphs-repo overlap overlap-system.morph -grep "WARNING\s" $log | sed 's/^.*WARNING */WARNING /' | - sed 's/ \.\// /' + build test:morphs-repo overlap overlap-system.morph +"$SRCDIR/scripts/list-overlaps" groups \ + "$cache"/*.system.overlap-system.overlaps | +while IFS='\n' read overlaps; do + echo $overlaps + "$SRCDIR/scripts/list-overlaps" list-files \ + "$cache"/*.system.overlap-system.overlaps $overlaps +done diff --git a/tests.as-root/system-overlap.stdout b/tests.as-root/system-overlap.stdout index 33b70636..fe106ad9 100644 --- a/tests.as-root/system-overlap.stdout +++ b/tests.as-root/system-overlap.stdout @@ -1,7 +1,3 @@ -WARNING Overlaps in artifact foo-barqux-stratum detected -WARNING Artifacts overlap-foobar, overlap-fooqux overlap with files: -WARNING bin/foo -WARNING Overlaps in artifact overlap-system detected -WARNING Artifacts foo-barqux-stratum, foo-baz-stratum overlap with files: -WARNING bin/bar -WARNING bin/foo +foo-barqux-stratum foo-baz-stratum +bin/foo +bin/bar diff --git a/tests/build-stratum-with-submodules.script b/tests/build-stratum-with-submodules.script index 9f226068..7d44caf6 100755 --- a/tests/build-stratum-with-submodules.script +++ b/tests/build-stratum-with-submodules.script @@ -65,6 +65,8 @@ EOF # No build and verify we got a stratum. "$SRCDIR/scripts/test-morph" build test:morphs-repo master submod-stratum.morph -tar -tf "$DATADIR/cache/artifacts/"*.stratum.* | LC_ALL=C sort \ - | sed '/^\.\/./s:^\./::' +"$SRCDIR/scripts/assemble-stratum" --cachedir "$DATADIR/cache" \ + "$DATADIR/cache/artifacts/"*submod-stratum \ + "$DATADIR/stratum.tar" submod-stratum +tar -tf "$DATADIR/stratum.tar" | LC_ALL=C sort | sed '/^\.\/./s:^\./::' diff --git a/tests/build-stratum.script b/tests/build-stratum.script index 994c4077..f4fa15a1 100755 --- a/tests/build-stratum.script +++ b/tests/build-stratum.script @@ -20,6 +20,8 @@ set -eu "$SRCDIR/scripts/test-morph" build test:morphs-repo master hello-stratum.morph -tar -tf "$DATADIR/cache/artifacts/"*.stratum.* | LC_ALL=C sort \ - | sed '/^\.\/./s:^\./::' +"$SRCDIR/scripts/assemble-stratum" --cachedir "$DATADIR/cache" \ + "$DATADIR/cache/artifacts/"*.hello-stratum \ + "$DATADIR/stratum.tar" hello-stratum +tar -tf "$DATADIR/stratum.tar" | LC_ALL=C sort | sed '/^\.\/./s:^\./::' diff --git a/tests/name-clash.script b/tests/name-clash.script index 669949ed..dcc04a8a 100755 --- a/tests/name-clash.script +++ b/tests/name-clash.script @@ -56,5 +56,8 @@ EOF # unpack it and check the contents INSTDIR="$DATADIR"/unpack mkdir -p "$INSTDIR" -tar -C "$INSTDIR" -xhf "$DATADIR/cache/artifacts/"*hello-stratum* +"$SRCDIR/scripts/assemble-stratum" --cachedir "$DATADIR/cache" \ + "$DATADIR/cache/artifacts/"*hello-stratum \ + "$DATADIR/stratum.tar" hello-stratum +tar -C "$INSTDIR" -xhf "$DATADIR/stratum.tar" test "$("$INSTDIR"/bin/hello)" = "hello, world" diff --git a/tests/rebuild-cached-stratum.script b/tests/rebuild-cached-stratum.script index c0022ba6..1bea1ae6 100755 --- a/tests/rebuild-cached-stratum.script +++ b/tests/rebuild-cached-stratum.script @@ -40,7 +40,7 @@ cache="$DATADIR/cache/artifacts" "$SRCDIR/scripts/test-morph" \ build test:morphs-repo rebuild-cached-stratum hello-stratum.morph echo "first build:" -(cd "$cache" && ls *.chunk.* *.stratum.* | sed 's/^[^.]*\./ /' | +(cd "$cache" && ls *.chunk.* *hello-stratum | sed 's/^[^.]*\./ /' | LC_ALL=C sort) # Change the chunk. @@ -52,6 +52,6 @@ echo "first build:" "$SRCDIR/scripts/test-morph" \ build test:morphs-repo rebuild-cached-stratum hello-stratum.morph echo "second build:" -(cd "$cache" && ls *.chunk.* *.stratum.* | sed 's/^[^.]*\./ /' | +(cd "$cache" && ls *.chunk.* *hello-stratum | sed 's/^[^.]*\./ /' | LC_ALL=C sort) diff --git a/tests/stratum-overlap-keeps-directory-links.script b/tests/stratum-overlap-keeps-directory-links.script index a995194f..2c5916aa 100755 --- a/tests/stratum-overlap-keeps-directory-links.script +++ b/tests/stratum-overlap-keeps-directory-links.script @@ -27,6 +27,8 @@ mkdir -p "$extracted" "$SRCDIR/scripts/test-morph" \ build test:morphs-repo overlap overlap-stratum.morph for f in "$cache"/*.stratum.overlap-stratum; do - tar -xf "$f" -C "$extracted" + "$SRCDIR/scripts/assemble-stratum" --cachedir "$DATADIR/cache" \ + "$f" "$DATADIR/stratum.tar" overlap-stratum + tar -xf "$DATADIR/stratum.tar" -C "$extracted" done test -h "$extracted/usr" diff --git a/tests/stratum-overlap-stomps-file-links.script b/tests/stratum-overlap-stomps-file-links.script index 0d429bbf..202e1f09 100755 --- a/tests/stratum-overlap-stomps-file-links.script +++ b/tests/stratum-overlap-stomps-file-links.script @@ -27,6 +27,8 @@ mkdir -p "$extracted" "$SRCDIR/scripts/test-morph" \ build test:morphs-repo overlap overlap-stratum.morph for f in "$cache"/*.stratum.overlap-stratum; do - tar -xf "$f" -C "$extracted" + "$SRCDIR/scripts/assemble-stratum" --cachedir "$DATADIR/cache" \ + "$f" "$DATADIR/stratum.tar" overlap-stratum + tar -xf "$DATADIR/stratum.tar" -C "$extracted" done test -f "$extracted/bin/foo" diff --git a/tests/stratum-overlap-warns.script b/tests/stratum-overlap-warns.script index edb1d855..2e9965fe 100755 --- a/tests/stratum-overlap-warns.script +++ b/tests/stratum-overlap-warns.script @@ -21,8 +21,18 @@ set -eu log="$DATADIR/morph.log" +warnings="$DATADIR/warnings" cache="$DATADIR/cache/artifacts" +warning_mentions(){ + grep -F "$1" <"$warnings" >/dev/null 2>/dev/null +} + "$SRCDIR/scripts/test-morph" \ build --log=$log test:morphs-repo overlap overlap-stratum.morph -grep WARNING $log | sed 's/^.*WARNING/WARNING/' +grep WARNING "$log" >"$warnings" +for str in overlap-stratum \ + overlap-foo-baz overlap-foobar bin/bar \ + overlap-fooqux bin/foo; do + warning_mentions 'overlap-stratum' || exit $? +done diff --git a/tests/stratum-overlap-warns.stdout b/tests/stratum-overlap-warns.stdout deleted file mode 100644 index eaddf9c1..00000000 --- a/tests/stratum-overlap-warns.stdout +++ /dev/null @@ -1,5 +0,0 @@ -WARNING Overlaps in artifact overlap-stratum detected -WARNING Artifacts overlap-foo-baz, overlap-foobar overlap with files: -WARNING bin/bar -WARNING Artifacts overlap-foo-baz, overlap-foobar, overlap-fooqux overlap with files: -WARNING bin/foo diff --git a/tests/stratum-overlap-writes-overlap.script b/tests/stratum-overlap-writes-overlap.script new file mode 100755 index 00000000..28d3d044 --- /dev/null +++ b/tests/stratum-overlap-writes-overlap.script @@ -0,0 +1,34 @@ +#!/bin/sh +# +# If a stratum has multiple chunks that have the same files in them, +# then the overlaps must be written to the cache +# +# Copyright (C) 2011, 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +set -eu + +cache="$DATADIR/cache/artifacts" + + +"$SRCDIR/scripts/test-morph" \ + build test:morphs-repo overlap overlap-stratum.morph +"$SRCDIR/scripts/list-overlaps" groups \ + "$cache"/*.stratum.overlap-stratum.overlaps | +while IFS='\n' read overlaps; do + echo $overlaps + "$SRCDIR/scripts/list-overlaps" list-files \ + "$cache"/*.stratum.overlap-stratum.overlaps $overlaps +done diff --git a/tests/stratum-overlap-writes-overlap.setup b/tests/stratum-overlap-writes-overlap.setup new file mode 120000 index 00000000..255e9a74 --- /dev/null +++ b/tests/stratum-overlap-writes-overlap.setup @@ -0,0 +1 @@ +stratum-overlap-warns.setup
\ No newline at end of file diff --git a/tests/stratum-overlap-writes-overlap.stdout b/tests/stratum-overlap-writes-overlap.stdout new file mode 100644 index 00000000..40485659 --- /dev/null +++ b/tests/stratum-overlap-writes-overlap.stdout @@ -0,0 +1,4 @@ +overlap-foo-baz overlap-foobar overlap-fooqux +bin/foo +overlap-foo-baz overlap-foobar +bin/bar diff --git a/without-test-modules b/without-test-modules index d91cc77b..936e0b07 100644 --- a/without-test-modules +++ b/without-test-modules @@ -1,4 +1,5 @@ morphlib/__init__.py +morphlib/artifactcachereference.py morphlib/builddependencygraph.py morphlib/tester.py morphlib/git.py |