diff options
| -rw-r--r-- | morphlib/builder2.py | 64 | ||||
| -rw-r--r-- | morphlib/plugins/artifact_inspection_plugin.py | 32 | ||||
| -rwxr-xr-x | scripts/list-overlaps | 45 | ||||
| -rwxr-xr-x | tests.as-root/system-overlap.script | 113 | ||||
| -rw-r--r-- | tests.as-root/system-overlap.stdout | 3 | ||||
| -rwxr-xr-x | tests.build/stratum-overlap-warns.script | 39 | ||||
| -rwxr-xr-x | tests.build/stratum-overlap-warns.setup | 102 | ||||
| -rwxr-xr-x | tests.build/stratum-overlap-writes-overlap.script | 35 | ||||
| l--------- | tests.build/stratum-overlap-writes-overlap.setup | 1 | ||||
| -rw-r--r-- | tests.build/stratum-overlap-writes-overlap.stdout | 4 |
10 files changed, 0 insertions, 438 deletions
diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 20cae225..9cd3a074 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -162,46 +162,6 @@ def get_stratum_files(f, lac): # pragma: no cover cf.close() -def get_overlaps(source, constituents, lac): # pragma: no cover - # check whether strata overlap - installed = defaultdict(set) - for dep in constituents: - handle = lac.get(dep) - if source.morphology['kind'] == 'stratum': - for filename in get_chunk_files(handle): - installed[filename].add(dep) - elif source.morphology['kind'] == 'system': - for filename in get_stratum_files(handle, lac): - installed[filename].add(dep) - handle.close() - overlaps = defaultdict(set) - for filename, artifacts in installed.iteritems(): - if len(artifacts) > 1: - overlaps[frozenset(artifacts)].add(filename) - return overlaps - - -def log_overlaps(overlaps): # pragma: no cover - for overlapping, files in sorted(overlaps.iteritems()): - logging.warning(' Artifacts %s overlap with files:' % - ', '.join(sorted(a.name for a in overlapping))) - for filename in sorted(files): - logging.warning(' %s' % filename) - - -def write_overlap_metadata(artifact, overlaps, lac): # pragma: no cover - f = lac.put_artifact_metadata(artifact, 'overlaps') - # the big list comprehension is because json can't serialize - # artifacts, sets or dicts with non-string keys - json.dump( - [ - [ - [a.name for a in afs], list(files) - ] for afs, files in overlaps.iteritems() - ], f, indent=4, encoding='unicode-escape') - f.close() - - class BuilderBase(object): '''Base class for building artifacts.''' @@ -559,18 +519,6 @@ class StratumBuilder(BuilderBase): download_depends(constituents, self.local_artifact_cache, self.remote_artifact_cache) - # check for chunk overlaps - overlaps = get_overlaps(self.source, constituents, - self.local_artifact_cache) - if len(overlaps) > 0: - logging.warning( - 'Overlaps in stratum artifact %s detected' %a_name) - log_overlaps(overlaps) - self.app.status(msg='Overlaps in stratum artifact ' - '%(stratum_name)s detected', - stratum_name=a_name, error=True) - write_overlap_metadata(a, overlaps, - self.local_artifact_cache) with self.build_watch('create-chunk-list'): lac = self.local_artifact_cache @@ -674,18 +622,6 @@ class SystemBuilder(BuilderBase): # pragma: no cover self.remote_artifact_cache) f.close() - # check whether the strata overlap - overlaps = get_overlaps(self.source, self.source.dependencies, - self.local_artifact_cache) - if len(overlaps) > 0: - self.app.status(msg='Overlaps in system artifact ' - '%(artifact_name)s detected', - artifact_name=a_name, - error=True) - log_overlaps(overlaps) - write_overlap_metadata(a, overlaps, - self.local_artifact_cache) - # unpack it from the local artifact cache for stratum_artifact in self.source.dependencies: self.unpack_one_stratum(stratum_artifact, path) diff --git a/morphlib/plugins/artifact_inspection_plugin.py b/morphlib/plugins/artifact_inspection_plugin.py index d47de334..74645f41 100644 --- a/morphlib/plugins/artifact_inspection_plugin.py +++ b/morphlib/plugins/artifact_inspection_plugin.py @@ -257,9 +257,6 @@ class ManifestGenerator(object): class ArtifactInspectionPlugin(cliapp.Plugin): def enable(self): - self.app.add_subcommand('run-in-artifact', - self.run_in_artifact, - arg_synopsis='ARTIFACT CMD') self.app.add_subcommand('generate-manifest', self.generate_manifest, arg_synopsis='SYSTEM-ARTIFACT') @@ -267,35 +264,6 @@ class ArtifactInspectionPlugin(cliapp.Plugin): def disable(self): pass - def run_in_artifact(self, args): - '''Run a command inside an extracted/mounted chunk or system. - - Command line arguments: - - * `ARTIFACT` is a filename for the artifact. - * `CMD` is the command to run. - - run-in-artifact unpacks an artifact, and runs a command in - the temporary directory it was unpacked to. - - The command must be given to Morph as a single argument, so - use shell quoting appropriately. - - ''' - - if len(args) < 2: - raise cliapp.AppException( - 'run-in-artifact requires arguments: a chunk or ' - 'system artifact and a a shell command') - - artifact, cmd = args[0], args[1:] - - def run_command_in_dir(dirname): - output = self.app.runcmd(cmd, cwd=dirname) - self.app.output.write(output) - - call_in_artifact_directory(self.app, artifact, run_command_in_dir) - def generate_manifest(self, args): '''Generate a content manifest for a system image. diff --git a/scripts/list-overlaps b/scripts/list-overlaps deleted file mode 100755 index d092ba75..00000000 --- a/scripts/list-overlaps +++ /dev/null @@ -1,45 +0,0 @@ -#!/usr/bin/env python -# -# Copyright (C) 2011-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -# This is a program to convert the json dump of the overlaps between artifacts -# in a format more suited to shell programs, or human reading - -import json - -import cliapp - - -class ListOverlaps(cliapp.Application): - - @staticmethod - def _load_overlap(filename): - data = json.load(open(filename), encoding='unicode-escape') - overlaps = dict((frozenset(pair[0]), set(pair[1])) for pair in data) - return overlaps - - def cmd_groups(self, args): - overlaps = ListOverlaps._load_overlap(args[0]) - for group in overlaps: - print(' '.join(sorted(group))) - - def cmd_list_files(self, args): - overlaps = self._load_overlap(args[0]) - group = frozenset(args[1:]) - for filename in overlaps[group]: - print filename - -ListOverlaps().run() diff --git a/tests.as-root/system-overlap.script b/tests.as-root/system-overlap.script deleted file mode 100755 index 9be6df13..00000000 --- a/tests.as-root/system-overlap.script +++ /dev/null @@ -1,113 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2011-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -# If a system has multiple strata that have the same files in them, -# then this should be noted. - -set -eu - -cache="$DATADIR/cache/artifacts" -chunkrepo="$DATADIR/chunk-repo" -morphsrepo="$DATADIR/morphs" - -cd "$morphsrepo" -git checkout --quiet -b overlap master -cat <<EOF >overlap-system.morph -name: overlap-system -kind: system -arch: $("$SRCDIR/scripts/test-morph" print-architecture) -strata: - - morph: foo-baz-stratum - - morph: foo-barqux-stratum -EOF -cat <<EOF >foo-baz-stratum.morph -name: foo-baz-stratum -kind: stratum -chunks: - - name: overlap-foo-baz - repo: test:chunk-repo - ref: overlap - build-mode: test - build-depends: [] - - name: linux - repo: test:kernel-repo - ref: master - build-mode: test - build-depends: - - overlap-foo-baz -EOF -cat <<EOF >foo-barqux-stratum.morph -name: foo-barqux-stratum -kind: stratum -chunks: - - name: overlap-foobar - repo: test:chunk-repo - ref: overlap - build-mode: test - build-depends: [] - - name: overlap-fooqux - repo: test:chunk-repo - ref: overlap - build-mode: test - build-depends: - - overlap-foobar -EOF -git add overlap-system.morph foo-baz-stratum.morph foo-barqux-stratum.morph -git commit --quiet -m "add overlapping system" - -cd "$chunkrepo" -git checkout --quiet -b overlap master -cat <<EOF >overlap-foo-baz.morph -name: overlap-foo-baz -kind: chunk -install-commands: - - mkdir -p "\$DESTDIR"/bin - - for f in foo bar baz; do echo echo \$f >"\$DESTDIR"/bin/\$f; done -EOF - -cat <<EOF >overlap-foobar.morph -name: overlap-foobar -kind: chunk -install-commands: - - mkdir -p "\$DESTDIR"/usr/bin "\$DESTDIR"/bin - - echo echo foobar >"\$DESTDIR"/usr/bin/foobar - - ln -s /usr/bin/foobar "\$DESTDIR"/bin/foo - - ln -s /usr/bin/foobar "\$DESTDIR"/bin/bar -EOF - -cat <<EOF >overlap-fooqux.morph -name: overlap-fooqux -kind: chunk -install-commands: - - mkdir -p "\$DESTDIR"/usr/bin "\$DESTDIR"/bin - - for f in qux fooqux; do echo echo \$f >"\$DESTDIR"/usr/bin/\$f; done - - ln -s /usr/bin/fooqux "\$DESTDIR"/bin/foo -EOF -git add overlap-*.morph - -git commit --quiet -m 'Add overlapping chunks' - -"$SRCDIR/scripts/test-morph" \ - build-morphology test:morphs overlap overlap-system > /dev/null -"$SRCDIR/scripts/list-overlaps" groups \ - "$cache"/*.system.overlap-system*.overlaps | -while IFS='\n' read overlaps; do - echo $overlaps - "$SRCDIR/scripts/list-overlaps" list-files \ - "$cache"/*.system.overlap-system*.overlaps $overlaps -done diff --git a/tests.as-root/system-overlap.stdout b/tests.as-root/system-overlap.stdout deleted file mode 100644 index f67d54c8..00000000 --- a/tests.as-root/system-overlap.stdout +++ /dev/null @@ -1,3 +0,0 @@ -foo-barqux-stratum-runtime foo-baz-stratum-runtime -bin/foo -bin/bar diff --git a/tests.build/stratum-overlap-warns.script b/tests.build/stratum-overlap-warns.script deleted file mode 100755 index 2a3b06e1..00000000 --- a/tests.build/stratum-overlap-warns.script +++ /dev/null @@ -1,39 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2011-2013 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## If a stratum has multiple chunks that have the same files in them, -## then this should be noted. - -set -eu - -log="$DATADIR/morph.log" -warnings="$DATADIR/warnings" -cache="$DATADIR/cache/artifacts" - -warning_mentions(){ - grep -F "$1" <"$warnings" >/dev/null 2>/dev/null -} - -"$SRCDIR/scripts/test-morph" build-morphology --log=$log \ - test:morphs-repo overlap hello-system > /dev/null -grep WARNING "$log" >"$warnings" -for str in hello-stratum \ - overlap-foo-baz overlap-foobar bin/bar \ - overlap-fooqux bin/foo; do - warning_mentions 'hello-stratum' || exit $? -done diff --git a/tests.build/stratum-overlap-warns.setup b/tests.build/stratum-overlap-warns.setup deleted file mode 100755 index b969822d..00000000 --- a/tests.build/stratum-overlap-warns.setup +++ /dev/null @@ -1,102 +0,0 @@ -#!/bin/sh -# -# If a stratum has multiple chunks that have the same files in them, -# then this should be notified -# -# Copyright (C) 2011-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -set -eu - -chunkrepo="$DATADIR/chunk-repo" -morphsrepo="$DATADIR/morphs-repo" - -cd "$morphsrepo" -git checkout --quiet -b overlap master -cat <<EOF >hello-stratum.morph -name: hello-stratum -kind: stratum -chunks: - - name: dirs - repo: test:chunk-repo - ref: overlap - build-depends: [] - build-mode: test - - name: overlap-foobar - repo: test:chunk-repo - ref: overlap - build-depends: - - dirs - build-mode: test - - name: overlap-fooqux - repo: test:chunk-repo - ref: overlap - build-depends: - - overlap-foobar - build-mode: test - - name: overlap-foo-baz - repo: test:chunk-repo - ref: overlap - build-depends: - - overlap-fooqux - build-mode: test -EOF -sed -i 's/master/overlap/' hello-system.morph -git add hello-stratum.morph hello-system.morph -git commit --quiet -m "Make hello stratum contain overlaps" - -cd "$chunkrepo" -git checkout --quiet -b overlap master - -cat <<EOF >dirs.morph -name: dirs -kind: chunk -install-commands: - - mkdir -p "\$DESTDIR/bin" - - ln -s .. "\$DESTDIR/usr" -EOF -git add dirs.morph - -cat <<EOF >overlap-foo-baz.morph -name: overlap-foo-baz -kind: chunk -install-commands: - - mkdir -p "\$DESTDIR/bin" - - for f in foo bar baz; do echo echo \$f >"\$DESTDIR/bin/\$f"; done -EOF -git add overlap-foo-baz.morph - -cat <<EOF >overlap-foobar.morph -name: overlap-foobar -kind: chunk -install-commands: - - mkdir -p "\$DESTDIR/usr/bin" "\$DESTDIR/bin" - - echo echo foobar >"\$DESTDIR/usr/bin/foobar" - - ln -s /usr/bin/foobar "\$DESTDIR/bin/foo" - - ln -s /usr/bin/foobar "\$DESTDIR/bin/bar" -EOF -git add overlap-foobar.morph - -cat <<EOF >overlap-fooqux.morph -name: overlap-fooqux -kind: chunk -install-commands: - - mkdir -p "\$DESTDIR/usr/bin" "\$DESTDIR/bin" - - for f in qux fooqux; do echo echo \$f >"\$DESTDIR/usr/bin/\$f"; done - - ln -s /usr/bin/fooqux "\$DESTDIR/bin/foo" -EOF -git add overlap-fooqux.morph - -git commit --quiet -m 'Add overlapping chunks' diff --git a/tests.build/stratum-overlap-writes-overlap.script b/tests.build/stratum-overlap-writes-overlap.script deleted file mode 100755 index fe4ed4ee..00000000 --- a/tests.build/stratum-overlap-writes-overlap.script +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/sh -# -# Copyright (C) 2011-2014 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -## If a stratum has multiple chunks that have the same files in them, -## then the overlaps must be written to the cache - -set -eu - -cache="$DATADIR/cache/artifacts" - - -"$SRCDIR/scripts/test-morph" build-morphology \ - test:morphs-repo overlap hello-system > /dev/null -"$SRCDIR/scripts/list-overlaps" groups \ - "$cache"/*.stratum.hello-stratum-*.overlaps | -while IFS='\n' read overlaps; do - echo $overlaps - "$SRCDIR/scripts/list-overlaps" list-files \ - "$cache"/*.stratum.hello-stratum-*.overlaps $overlaps -done diff --git a/tests.build/stratum-overlap-writes-overlap.setup b/tests.build/stratum-overlap-writes-overlap.setup deleted file mode 120000 index 255e9a74..00000000 --- a/tests.build/stratum-overlap-writes-overlap.setup +++ /dev/null @@ -1 +0,0 @@ -stratum-overlap-warns.setup
\ No newline at end of file diff --git a/tests.build/stratum-overlap-writes-overlap.stdout b/tests.build/stratum-overlap-writes-overlap.stdout deleted file mode 100644 index 1e36ca83..00000000 --- a/tests.build/stratum-overlap-writes-overlap.stdout +++ /dev/null @@ -1,4 +0,0 @@ -overlap-foo-baz-bins overlap-foobar-bins overlap-fooqux-bins -bin/foo -overlap-foo-baz-bins overlap-foobar-bins -bin/bar |