diff options
author | Richard Maw <richard.maw@codethink.co.uk> | 2012-01-18 14:17:30 +0000 |
---|---|---|
committer | Richard Maw <richard.maw@codethink.co.uk> | 2012-01-18 14:17:30 +0000 |
commit | b1022b4807b923de9d2df25583d8e28a70ac981e (patch) | |
tree | 6057766ff6255359347d1937d66427b51fc1c2c7 | |
parent | 23a74cc5d9ee42972a57b15b40f91e4797274410 (diff) | |
parent | 0a603422096852ec09e1e7b89ecc79c66d3986b1 (diff) | |
download | morph-b1022b4807b923de9d2df25583d8e28a70ac981e.tar.gz |
Merge branch 'master' into richardmaw/merge
-rw-r--r-- | .gitignore | 6 | ||||
-rw-r--r-- | README | 36 | ||||
-rwxr-xr-x | baserock-bootstrap | 98 | ||||
-rwxr-xr-x | check | 2 | ||||
-rwxr-xr-x | morph | 45 | ||||
-rw-r--r-- | morphlib/__init__.py | 4 | ||||
-rw-r--r-- | morphlib/bins.py | 10 | ||||
-rw-r--r-- | morphlib/bins_tests.py | 6 | ||||
-rw-r--r-- | morphlib/builddependencygraph.py | 291 | ||||
-rw-r--r-- | morphlib/builddependencygraph_tests.py | 71 | ||||
-rw-r--r-- | morphlib/builder.py | 304 | ||||
-rw-r--r-- | morphlib/execute.py | 31 | ||||
-rw-r--r-- | morphlib/execute_tests.py | 6 | ||||
-rw-r--r-- | morphlib/morphology.py | 7 | ||||
-rw-r--r-- | morphlib/morphology_tests.py | 10 | ||||
-rw-r--r-- | morphlib/morphologyloader.py | 57 | ||||
-rw-r--r-- | morphlib/morphologyloader_tests.py | 47 | ||||
-rw-r--r-- | morphlib/stopwatch.py | 15 | ||||
-rw-r--r-- | morphlib/stopwatch_tests.py | 17 | ||||
-rwxr-xr-x | run-bootstrap-in-chroot | 36 | ||||
-rw-r--r-- | tests/build-system.stdout | 1 | ||||
-rwxr-xr-x | tests/show-dependencies.script (renamed from tests/build-system.script) | 25 | ||||
-rwxr-xr-x | tests/show-dependencies.setup | 314 | ||||
-rw-r--r-- | tests/show-dependencies.stdout | 132 | ||||
-rw-r--r-- | wget-list | 2 |
25 files changed, 1326 insertions, 247 deletions
@@ -1 +1,7 @@ *.pyc +*.swp +*.swo +*.stdout-actual +*.stdout-diff +*.stderr-actual +*.stderr-diff @@ -20,11 +20,15 @@ used to actually run the build. The usual workflow is this: * put the morphology for an upstream project with its source code * put other morphologies in the `morphs` (note plural) repository -* run `morph` to build stuff +* run `fakeroot morph` or `sudo morph` to build stuff Eventually, `morph` will get a manual page. Meanwhile, a short usage: - morph --keep-path -v build morphs master base-system.morph + fakeroot morph --keep-path -v build morphs master foundation.morph + sudo morph --keep-path -v build morphs master base-system.morph + +(Run with `fakeroot`, when building anything but a system image; run +with `sudo` when building a system image.) Run `morph --help` to get a list of all options and short descriptions. @@ -103,13 +107,25 @@ For chunks, use the following fields: For strata, use the following fields: +* `build-depends`: a list of strings, each of which refers to another + stratum that the current stratum depends on. This list may be omitted + or empty if the stratum does not depend on anything else. * `sources`: a list of key/value mappings, where each mapping corresponds to a chunk to be included in the stratum; the mappings may use the following keys: `name` is the chunk's name (may be different from the morphology name), `repo` is the repository in which to find (defaults to chunk name), `ref` identifies the commit to use (typically a branch name, but any tree-ish git accepts is ok), and `morph` is the name - of the morphology to use; optional + of the morphology to use and is optional. In addition to these keys, + each of the sources may specify a list of build dependencies using the + `build-depends` field. This field may be omitted to make the source + depend on all other chunks that are listed earlier in the `sources` + list. The field may be an empty list to indicate that the chunk does + not depend on anything else in the same stratum. To specify one ore + more chunk dependencies, `build-depends` needs to be set to a list + that contains the names of chunks that the source depends on in the + same stratum. These names correspond to the values of the `name` + fields of the other sources. For systems, use the following fields: @@ -150,15 +166,23 @@ Example stratum: { "name": "linux-api-headers", "repo": "linux", - "ref": "baserock/morph" + "ref": "baserock/morph", + "build-depends": [] }, { "name": "eglibc", - "ref": "baserock/bootstrap" + "ref": "baserock/bootstrap", + "build-depends": [ + "linux-api-headers" + ] }, { "name": "busybox", - "ref": "baserock/bootstrap" + "ref": "baserock/bootstrap", + "build-depends": [ + "fhs-dirs", + "linux-api-headers" + ] } ] } diff --git a/baserock-bootstrap b/baserock-bootstrap index 019101b3..83e35958 100755 --- a/baserock-bootstrap +++ b/baserock-bootstrap @@ -12,9 +12,8 @@ sources="$LFS/sources" tools="$LFS/tools" SNAPSHOT="${1:-'no'}" -JOBS=$((echo -n '1.5*'; grep -c '^processor' /proc/cpuinfo) | - bc -lq | - sed 's/\..*//') +CPUS=$(grep -c '^processor' /proc/cpuinfo) +JOBS=$(expr 2 '*' $CPUS) export LC_ALL=C export LFS_TGT=$(uname -m)-lfs-linux-gnu @@ -157,31 +156,30 @@ pass1_linux_api_headers() } -pass1_eglibc() +pass1_glibc() { - echo "Building eglibc" + echo "Building glibc" if [ ! -e "$tools/lib/libc.so.6" ] then - unpack eglibc_2.13 - cd "$sources/eglibc-2.13" - (cd libc && ln -s ../ports ports) + unpack glibc-2.14.1 + cd "$sources/glibc-2.14.1" + patch -Np1 -i ../glibc-2.14.1-gcc_fix-1.patch + patch -Np1 -i ../glibc-2.14.1-cpuid-1.patch - $HOST_MKDIR "$sources/eglibc-build" - cd "$sources/eglibc-build" - echo "CFLAGS += -O2 -U_FORTIFY_SOURCE -fno-stack-protector" > configparms + $HOST_MKDIR "$sources/glibc-build" + cd "$sources/glibc-build" case `uname -m` in - i?86) echo "CFLAGS += -march=i486 -mtune=native" >> configparms ;; + i?86) echo "CFLAGS += -march=i486 -mtune=native" > configparms ;; esac - ../eglibc-2.13/libc/configure --prefix="$tools" \ - --host=$LFS_TGT --build=$(../eglibc-2.13/scripts/config.guess) \ + ../glibc-2.14.1/configure --prefix="$tools" \ + --host=$LFS_TGT --build=$(../glibc-2.14.1/scripts/config.guess) \ --disable-profile --enable-add-ons \ --enable-kernel=2.6.25 --with-headers="$tools/include" \ --without-selinux --without-cvs \ libc_cv_forced_unwind=yes libc_cv_c_cleanup=yes libc_cv_ssp=no - make -j$JOBS make install vardbdir="$tools/var/db" - rm -rf "$sources/eglibc-2.13" + rm -rf "$sources/glibc-2.14.1" fi } @@ -674,14 +672,14 @@ pass1_cliapp() echo "Building cliapp" if [ ! -e "$tools/lib/python2.7/site-packages/cliapp" ] then - cp "$sources/python-cliapp_0.22.orig.tar.gz" \ - "$sources/cliapp-0.22.tar.gz" - unpack cliapp-0.22 - cd "$sources/cliapp-0.22" + cp "$sources/python-cliapp_0.23.orig.tar.gz" \ + "$sources/cliapp-0.23.tar.gz" + unpack cliapp-0.23 + cd "$sources/cliapp-0.23" $HOST_SED -i '/^import cliapp/d' setup.py - $HOST_SED -i 's/cliapp.__version__/"0.22"/g' setup.py + $HOST_SED -i 's/cliapp.__version__/"0.23"/g' setup.py python setup.py install --prefix="$tools" - rm -rf "$sources/cliapp-0.22" + rm -rf "$sources/cliapp-0.23" fi } @@ -780,10 +778,13 @@ pass2_build_with_morph_in_chroot() #!/tools/bin/bash set -e set -x + /tools/bin/ldconfig -f /etc/ld.so.conf -C /etc/ld.so.cache cd /baserock/gits/morph mkdir -p /baserock/cache python ./morph --verbose build \ + file:///baserock/gits/morphs/ master linux-stratum.morph \ + file:///baserock/gits/morphs/ master foundation.morph \ file:///baserock/gits/morphs/ master devel.morph \ --bootstrap \ --cachedir=/baserock/cache \ @@ -818,18 +819,65 @@ EOF } +pass2_build_devel_system_outside_chroot() +{ + cd "$LFS/.." + img="devsys.img" + + $HOST_SUDO qemu-img create -f raw "$img" 1G + $HOST_SUDO parted -s "$img" mklabel msdos + $HOST_SUDO parted -s "$img" mkpart primary 0% 100% + $HOST_SUDO parted -s "$img" set 1 boot on + $HOST_SUDO install-mbr "$img" + part=/dev/mapper/$($HOST_SUDO kpartx -av "$img" | awk '/^add map/ { print $3 }' | head -n1) + trap "$HOST_SUDO kpartx -dv $img" EXIT + $HOST_SUDO mkfs -t ext4 "$part" + mp="$(mktemp -d)" + $HOST_SUDO mount "$part" "$mp" + + for stratum in "$LFS"/baserock/cache/*.stratum.{foundation,linux-stratum,devel} + do + $HOST_SUDO tar -C "$mp" -xf "$stratum" + done + + cat <<EOF | $HOST_SUDO tee "$mp/etc/fstab" +proc /proc proc defaults 0 0 +sysfs /sys sysfs defaults 0 0 +/dev/sda1 / ext4 errors=remount-ro 0 1 +EOF + + cat <<EOF | $HOST_SUDO tee "$mp/extlinux.conf" +default linux +timeout 1 + +label linux +kernel /vmlinuz +append root=/dev/sda1 init=/sbin/init quiet rw +EOF + + $HOST_SUDO extlinux --install "$mp" + sync + sleep 2 + + $HOST_SUDO umount "$mp" +} + + echo "Bootstrapping Baserock development environment" echo "LFS_TGT=$LFS_TGT" pass1_directories -$HOST_CP -alu "$allsources/." "$LFS/sources/." || true +if [ -e "$allsources" ] +then + $HOST_CP -au "$allsources/." "$LFS/sources/." +fi download_all pass1_binutils_1 pass1_gcc_1 pass1_linux_api_headers -pass1_eglibc +pass1_glibc pass1_adjust_gcc_specs pass1_sanity_check pass1_binutils_2 @@ -873,4 +921,4 @@ fi pass2_get_sources pass2_prepare_for_chroot pass2_build_with_morph_in_chroot - +pass2_build_devel_system_outside_chroot @@ -20,4 +20,4 @@ set -e python setup.py clean check -cmdtest -c ./morph tests +fakeroot cmdtest -c ./morph tests @@ -25,6 +25,7 @@ import logging import os import shutil import tempfile +import urlparse import morphlib @@ -79,7 +80,7 @@ class Morph(cliapp.Application): tempdir = morphlib.tempdir.Tempdir() builder = morphlib.builder.Builder(tempdir, self) - if not os.path.exists(self.settings['cachedir']) and os.getuid() != 0: + if not os.path.exists(self.settings['cachedir']): os.mkdir(self.settings['cachedir']) ret = [] @@ -91,7 +92,7 @@ class Morph(cliapp.Application): # we may not have permission to tempdir.remove() ex = morphlib.execute.Execute('.', lambda msg: None) - ex.runv(["rm", "-rf", tempdir.dirname], as_root=True) + ex.runv(["rm", "-rf", tempdir.dirname]) if args: raise cliapp.AppException('Extra args on command line: %s' % args) @@ -149,6 +150,46 @@ class Morph(cliapp.Application): self.msg('not testing %s %s (not a system)' % (morph.kind, morph.name)) + def cmd_show_dependencies(self, args): + '''Dumps the dependency tree of all input morphologies.''' + + while len(args) >= 3: + # read the build tuple from the command line + repo, ref, filename = args[:3] + args = args[3:] + + # resolve the URL to the repository + base_url = self.settings['git-base-url'] + if not base_url.endswith('/'): + base_url += '/' + repo = urlparse.urljoin(base_url, repo) + + # load the morphology corresponding to the build tuple + loader = morphlib.morphologyloader.MorphologyLoader(self.settings) + morphology = loader.load(repo, ref, filename) + + # create a dependency graph for the morphology + graph = \ + morphlib.builddependencygraph.BuildDependencyGraph(loader, + morphology) + graph.resolve() + + # print the graph + self.output.write('dependency tree:\n') + for node in graph.nodes: + self.output.write(' %s\n' % node) + for dep in node.dependencies: + self.output.write(' -> %s\n' % dep) + + # compute a build order from the graph + order = graph.build_order() + self.output.write('build order:\n') + for group in order: + self.output.write(' group:\n') + for morphology in group: + self.output.write(' %s (%s)\n' % (morphology.name, + morphology.kind)) + def msg(self, msg): '''Show a message to the user about what is going on.''' logging.debug(msg) diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 5d8568b4..35939c45 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011 Codethink Limited +# Copyright (C) 2011-2012 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -21,11 +21,13 @@ __version__ = '0.0' import bins +import builddependencygraph import builder import cachedir import execute import git import morphology +import morphologyloader import stopwatch import tempdir import tester diff --git a/morphlib/bins.py b/morphlib/bins.py index 90d96d82..cd77a037 100644 --- a/morphlib/bins.py +++ b/morphlib/bins.py @@ -101,20 +101,16 @@ def create_stratum(rootdir, stratum_filename, ex): '''Create a stratum from the contents of a directory.''' logging.debug('Creating stratum file %s from %s' % (stratum_filename, rootdir)) - ex.runv(['tar', '-C', rootdir, '-caf', stratum_filename, '.'], - as_fakeroot=True) + ex.runv(['tar', '-C', rootdir, '-caf', stratum_filename, '.']) -def unpack_binary(filename, dirname, ex, as_fakeroot=False, as_root=False): +def unpack_binary(filename, dirname, ex): '''Unpack a binary into a directory. The directory must exist already. - If the binary will be packed up again by tar with the same Execute - object then as_fakeroot will suffice - If it will be creating a system image as_root will be needed ''' logging.debug('Unpacking %s into %s' % (filename, dirname)) - ex.runv(['tar', '-C', dirname, '-xvf', filename], as_fakeroot=as_fakeroot, as_root=as_root) + ex.runv(['tar', '-C', dirname, '-xvf', filename]) diff --git a/morphlib/bins_tests.py b/morphlib/bins_tests.py index 6fe48aa7..8a54033b 100644 --- a/morphlib/bins_tests.py +++ b/morphlib/bins_tests.py @@ -92,8 +92,7 @@ class ChunkTests(unittest.TestCase): morphlib.bins.create_chunk(self.instdir, self.chunk_file, ['.'], self.ex) os.mkdir(self.unpacked) - morphlib.bins.unpack_binary(self.chunk_file, self.unpacked, self.ex, - as_fakeroot=True) + morphlib.bins.unpack_binary(self.chunk_file, self.unpacked, self.ex) self.assertEqual(orig_files, recursive_lstat(self.unpacked)) def test_uses_only_matching_names(self): @@ -127,8 +126,7 @@ class StratumTests(unittest.TestCase): self.populate_instdir() morphlib.bins.create_stratum(self.instdir, self.stratum_file, self.ex) os.mkdir(self.unpacked) - morphlib.bins.unpack_binary(self.stratum_file, self.unpacked, self.ex, - as_fakeroot=True) + morphlib.bins.unpack_binary(self.stratum_file, self.unpacked, self.ex) self.assertEqual(recursive_lstat(self.instdir), recursive_lstat(self.unpacked)) diff --git a/morphlib/builddependencygraph.py b/morphlib/builddependencygraph.py new file mode 100644 index 00000000..7a432005 --- /dev/null +++ b/morphlib/builddependencygraph.py @@ -0,0 +1,291 @@ +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import collections +import copy +import os + +import morphlib + + +class Node(object): + + '''Node in the dependency graph.''' + + def __init__(self, morphology): + self.morphology = morphology + self.dependents = [] + self.dependencies = [] + + def add_dependency(self, other): + if other not in self.dependencies: + self.dependencies.append(other) + if self not in other.dependents: + other.dependents.append(self) + + def remove_dependency(self, other): + if other in self.dependencies: + self.dependencies.remove(other) + if self in other.dependents: + other.dependents.remove(self) + + def depends_on(self, other): + return other in self.dependencies + + def __str__(self): # pragma: no cover + return '%s (%s)' % (self.morphology.name, self.morphology.kind) + + def __deepcopy__(self, memo): # pragma: no cover + return Node(self.morphology) + + +class NodeList(list): + + def __deepcopy__(self, memo): # pragma: no cover + nodes = NodeList() + + old_to_new = {} + + for node in self: + node_copy = copy.deepcopy(node) + old_to_new[node] = node_copy + nodes.append(node_copy) + + for node in self: + node_copy = old_to_new[node] + for dep in node.dependencies: + dep_copy = old_to_new[dep] + node_copy.add_dependency(dep_copy) + + return nodes + + +class BuildDependencyGraph(object): # pragma: no cover + + def __init__(self, loader, morphology): + self.loader = loader + self.morphology = morphology + self.nodes = None + + def add(self, morphology): + node = Node(morphology) + if node not in self.nodes: + self.nodes.append(node) + + def resolve(self): + self.cached_morphologies = {} + self._resolve_strata() + self._resolve_chunks() + + def build_order(self): + sorting = self._compute_topological_sorting() + + #print + #print 'sorting: %s' % [str(x) for x in sorting] + #print + + groups = [] + group_nodes = {} + + group = [] + groups.append(group) + + for node in sorting: + create_group = False + for dep in node.dependencies: + if dep in group: + create_group = True + + if create_group: + group = [] + groups.append(group) + + group.append(node) + + morphology_groups = [] + for group in groups: + morphology_group = [] + for node in group: + morphology_group.append(node.morphology) + morphology_groups.append(morphology_group) + + return morphology_groups + + def _resolve_strata(self): + self.nodes = NodeList() + + if self.morphology.kind == 'stratum': + root = Node(self.morphology) + self.nodes.append(root) + + queue = collections.deque() + queue.append(root) + while len(queue) > 0: + node = queue.popleft() + + if not node.morphology.build_depends: + continue + + if isinstance(node.morphology.build_depends, list): + for other_name in node.morphology.build_depends: + # prepare a tuple for loading the morphology + repo = node.morphology.repo + ref = node.morphology.ref + filename = '%s.morph' % other_name + info = (repo, ref, filename) + + # look up or create a node for the morphology + other_node = self.cached_morphologies.get(info, None) + if not other_node: + #print other_name + morphology = self.loader.load(repo, ref, filename) + other_node = Node(morphology) + + # cache the node for this morphology + self.cached_morphologies[info] = other_node + + # add the morphology node to the graph + node.add_dependency(other_node) + self.nodes.append(other_node) + queue.append(other_node) + else: + raise Exception('%s uses an invalid "build-depends" format' % + os.path.basename(stratum_node.morphology.filename)) + + #print 'strata: %s' % [str(x) for x in self.nodes] + + def _resolve_chunks(self): + strata_nodes = list(self.nodes) + for stratum_node in strata_nodes: + self._resolve_stratum_chunks(stratum_node) + + def _morphology_node(self, repo, ref, filename): + info = (repo, ref, filename) + + if info in self.cached_morphologies: + return self.cached_morphologies[info] + else: + morphology = self.loader.load(repo, ref, filename) + node = Node(morphology) + self.nodes.append(node) + self.cached_morphologies[info] = node + return node + + def _resolve_stratum_chunks(self, stratum_node): + stratum_chunk_nodes = [] + chunk_lookup = {} + + # second, create nodes for all chunks in the stratum + for i in xrange(0, len(stratum_node.morphology.sources)): + source = stratum_node.morphology.sources[i] + + # construct the build tuple + repo = source['repo'] + ref = source['ref'] + filename = '%s.morph' % (source['morph'] + if 'morph' in source + else source['name']) + + chunk_node = self._morphology_node(repo, ref, filename) + + chunk_lookup[source['name']] = chunk_node + + stratum_chunk_nodes.append(chunk_node) + + # read the build-depends information + build_depends = (source['build-depends'] + if 'build-depends' in source + else None) + + # turn build-depends into proper dependencies in the graph + if build_depends is None: + for j in xrange(0, i): + chunk_node.add_dependency(stratum_chunk_nodes[j]) + elif isinstance(build_depends, list): + for depname in build_depends: + if depname in chunk_lookup: + depnode = chunk_lookup[depname] + chunk_node.add_dependency(depnode) + else: + raise Exception('%s: source "%s" references "%s" ' + 'before it is defined' % + (os.path.basename(stratum_node.morphology.filename), + source['name'], + depname)) + else: + raise Exception('%s: source "%s" uses an invalid ' + '"build-depends" format' % + (os.path.basename(stratum_node.morphology.filename), + source['name'])) + + # make the chunk nodes in this stratum depend on all strata + # that need to be built first + for chunk_node in stratum_chunk_nodes: + for stratum_dep in stratum_node.dependencies: + chunk_node.add_dependency(stratum_dep) + + # clear the dependencies of the stratum + stratum_node.dependencies = [] + + # make the stratum node depend on all its chunk nodes + for chunk_node in stratum_chunk_nodes: + stratum_node.add_dependency(chunk_node) + + def _compute_topological_sorting(self): + nodes = copy.deepcopy(self.nodes) + + original_node = {} + for node in self.nodes: + for node_copy in nodes: + if node.morphology == node_copy.morphology: + original_node[node_copy] = node + + #print 'compute topological sorting:' + #print ' nodes: %s' % [str(x) for x in nodes] + + sorting = [] + leafs = collections.deque() + + for node in nodes: + if len(node.dependencies) is 0: + leafs.append(node) + + #print ' leafs: %s' % [str(x) for x in leafs] + + while len(leafs) > 0: + leaf = leafs.popleft() + sorting.append(leaf) + + #print ' visit %s' % leaf + + #print ' visit %s' % leaf + + for parent in list(leaf.dependents): + #print ' parent %s' % parent + #print ' parent %s dependencies: %s' % (parent, [str(x) for x in parent.dependencies]) + parent.remove_dependency(leaf) + #print ' parent %s dependencies: %s' % (parent, [str(x) for x in parent.dependencies]) + if len(parent.dependencies) == 0: + #print ' add %s' % parent + leafs.append(parent) + + #print [str(node) for node in sorting] + if len(sorting) < len(nodes): + raise Exception('Cyclic dependencies found in the dependency ' + 'graph of "%s"' % self.morphology) + + #return sorting + return [original_node[node] for node in sorting] diff --git a/morphlib/builddependencygraph_tests.py b/morphlib/builddependencygraph_tests.py new file mode 100644 index 00000000..dec4e49c --- /dev/null +++ b/morphlib/builddependencygraph_tests.py @@ -0,0 +1,71 @@ +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import os +import StringIO +import unittest + +import morphlib + + +class BuildDependencyGraphTests(unittest.TestCase): + + def test_create_node_with_morphology(self): + fake_morphology = "fake morphology" + + node = morphlib.builddependencygraph.Node(fake_morphology) + self.assertEqual(node.morphology, fake_morphology) + + def test_node_add_remove_dependency(self): + node1 = morphlib.builddependencygraph.Node(None) + node2 = morphlib.builddependencygraph.Node(None) + + node1.add_dependency(node2) + + assert(node2 in node1.dependencies) + assert(node1 in node2.dependents) + + assert(node1.depends_on(node2)) + + node2.add_dependency(node1) + + assert(node2 in node1.dependencies) + assert(node1 in node2.dependents) + assert(node1 in node2.dependencies) + assert(node2 in node1.dependents) + + assert(node1.depends_on(node2)) + assert(node2.depends_on(node1)) + + node1.remove_dependency(node2) + + assert(node2 not in node1.dependencies) + assert(node1 not in node2.dependents) + assert(node1 in node2.dependencies) + assert(node2 in node1.dependents) + + assert(not node1.depends_on(node2)) + assert(node2.depends_on(node1)) + + node2.remove_dependency(node1) + + assert(node2 not in node1.dependencies) + assert(node1 not in node2.dependents) + assert(node1 not in node2.dependencies) + assert(node2 not in node1.dependents) + + assert(not node1.depends_on(node2)) + assert(not node2.depends_on(node1)) diff --git a/morphlib/builder.py b/morphlib/builder.py index 1add2835..88a46633 100644 --- a/morphlib/builder.py +++ b/morphlib/builder.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011 Codethink Limited +# Copyright (C) 2011-2012 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -25,6 +25,35 @@ import urlparse import morphlib +def ldconfig(ex, rootdir): + '''Run ldconfig for the filesystem below ``rootdir``. + + Essentially, ``rootdir`` specifies the root of a new system. + Only directories below it are considered. + + ``etc/ld.so.conf`` below ``rootdir`` is assumed to exist and + be populated by the right directories, and should assume + the root directory is ``rootdir``. Example: if ``rootdir`` + is ``/tmp/foo``, then ``/tmp/foo/etc/ld.so.conf`` should + contain ``/lib``, not ``/tmp/foo/lib``. + + The ldconfig found via ``$PATH`` is used, not the one in ``rootdir``, + since in bootstrap mode that might not yet exist, the various + implementations should be compatible enough. + + ''' + + conf = os.path.join(rootdir, 'etc', 'ld.so.conf') + if os.path.exists(conf): + logging.debug('Running ldconfig for %s' % rootdir) + cache = os.path.join(rootdir, 'etc', 'ld.so.cache') + old_path = ex.env['PATH'] + ex.env['PATH'] = '%s:/sbin:/usr/sbin:/usr/local/sbin' % old_path + ex.runv(['ldconfig', '-f', conf, '-C', cache, '-r', rootdir]) + ex.env['PATH'] = old_path + else: + logging.debug('No %s, not running ldconfig' % conf) + class BinaryBlob(object): def __init__(self, morph, repo, ref): @@ -210,15 +239,16 @@ class Chunk(BinaryBlob): def create_source_and_tarball(self): self.msg('Creating source tree and tarball') - self.build_watch.start('create-source-tarball') - self.dump_memory_profile('before creating source and tarball for chunk') - tarball = self.cache_prefix + '.src.tar' - morphlib.git.export_sources(self.repo, self.ref, tarball) - self.dump_memory_profile('after exporting sources') - os.mkdir(self.builddir) - self.ex.runv(['tar', '-C', self.builddir, '-xf', tarball]) - self.dump_memory_profile('after creating source and tarball for chunk') - self.build_watch.stop('create-source-tarball') + with self.build_watch('create-source-tarball'): + self.dump_memory_profile('before creating source and tarball ' + 'for chunk') + tarball = self.cache_prefix + '.src.tar' + morphlib.git.export_sources(self.repo, self.ref, tarball) + self.dump_memory_profile('after exporting sources') + os.mkdir(self.builddir) + self.ex.runv(['tar', '-C', self.builddir, '-xf', tarball]) + self.dump_memory_profile('after creating source and tarball ' + 'for chunk') def build_using_buildsystem(self): bs_name = self.morph.build_system @@ -227,48 +257,44 @@ class Chunk(BinaryBlob): self.run_sequentially('configure', bs['configure-commands']) self.run_in_parallel('build', bs['build-commands']) self.run_sequentially('test', bs['test-commands']) - self.run_sequentially('install', bs['install-commands'], - as_fakeroot=True) + self.run_sequentially('install', bs['install-commands']) def build_using_commands(self): self.msg('Building using explicit commands') self.run_sequentially('configure', self.morph.configure_commands) self.run_in_parallel('build', self.morph.build_commands) self.run_sequentially('test', self.morph.test_commands) - self.run_sequentially('install', self.morph.install_commands, - as_fakeroot=True) + self.run_sequentially('install', self.morph.install_commands) def run_in_parallel(self, what, commands): self.msg('commands: %s' % what) - self.build_watch.start(what) - self.ex.run(commands) - self.build_watch.stop(what) + with self.build_watch(what): + self.ex.run(commands) - def run_sequentially(self, what, commands, as_fakeroot=False, as_root=False): + def run_sequentially(self, what, commands): self.msg ('commands: %s' % what) - self.build_watch.start(what) - flags = self.ex.env['MAKEFLAGS'] - self.ex.env['MAKEFLAGS'] = '-j1' - logging.debug('Setting MAKEFLAGS=%s' % self.ex.env['MAKEFLAGS']) - self.ex.run(commands, as_fakeroot=as_fakeroot, as_root=as_root) - self.ex.env['MAKEFLAGS'] = flags - logging.debug('Restore MAKEFLAGS=%s' % self.ex.env['MAKEFLAGS']) - self.build_watch.stop(what) + with self.build_watch(what): + flags = self.ex.env['MAKEFLAGS'] + self.ex.env['MAKEFLAGS'] = '-j1' + logging.debug('Setting MAKEFLAGS=%s' % self.ex.env['MAKEFLAGS']) + self.ex.run(commands) + self.ex.env['MAKEFLAGS'] = flags + logging.debug('Restore MAKEFLAGS=%s' % self.ex.env['MAKEFLAGS']) def create_chunks(self, chunks): ret = {} - self.build_watch.start('create-chunks') - for chunk_name in chunks: - self.msg('Creating chunk %s' % chunk_name) - self.prepare_binary_metadata(chunk_name) - patterns = chunks[chunk_name] - patterns += [r'baserock/%s\.' % chunk_name] - filename = self.filename(chunk_name) - self.msg('Creating binary for %s' % chunk_name) - morphlib.bins.create_chunk(self.destdir, filename, patterns, - self.ex, self.dump_memory_profile) - ret[chunk_name] = filename - self.build_watch.stop('create-chunks') + with self.build_watch('create-chunks'): + for chunk_name in chunks: + self.msg('Creating chunk %s' % chunk_name) + self.prepare_binary_metadata(chunk_name) + patterns = chunks[chunk_name] + patterns += [r'baserock/%s\.' % chunk_name] + filename = self.filename(chunk_name) + self.msg('Creating binary for %s' % chunk_name) + morphlib.bins.create_chunk(self.destdir, filename, patterns, + self.ex, self.dump_memory_profile) + ret[chunk_name] = filename + files = os.listdir(self.destdir) if files: raise Exception('DESTDIR %s is not empty: %s' % @@ -294,18 +320,15 @@ class Stratum(BinaryBlob): def build(self): os.mkdir(self.destdir) ex = morphlib.execute.Execute(self.destdir, self.msg) - self.build_watch.start('unpack-chunks') - for chunk_name, filename in self.built: - self.msg('Unpacking chunk %s' % chunk_name) - morphlib.bins.unpack_binary(filename, self.destdir, ex, - as_fakeroot=True) - self.build_watch.stop('unpack-chunks') - self.prepare_binary_metadata(self.morph.name) - self.build_watch.start('create-binary') - self.msg('Creating binary for %s' % self.morph.name) - filename = self.filename(self.morph.name) - morphlib.bins.create_stratum(self.destdir, filename, ex) - self.build_watch.stop('create-binary') + with self.build_watch('unpack-chunks'): + for chunk_name, filename in self.built: + self.msg('Unpacking chunk %s' % chunk_name) + morphlib.bins.unpack_binary(filename, self.destdir, ex) + with self.build_watch('create-binary'): + self.prepare_binary_metadata(self.morph.name) + self.msg('Creating binary for %s' % self.morph.name) + filename = self.filename(self.morph.name) + morphlib.bins.create_stratum(self.destdir, filename, ex) return { self.morph.name: filename } @@ -323,118 +346,106 @@ class System(BinaryBlob): self.ex = morphlib.execute.Execute(self.tempdir.dirname, self.msg) # Create image. - self.build_watch.start('create-image') - image_name = self.tempdir.join('%s.img' % self.morph.name) - self.ex.runv(['qemu-img', 'create', '-f', 'raw', image_name, - self.morph.disk_size]) - self.build_watch.stop('create-image') + with self.build_watch('create-image'): + image_name = self.tempdir.join('%s.img' % self.morph.name) + self.ex.runv(['qemu-img', 'create', '-f', 'raw', image_name, + self.morph.disk_size]) # Partition it. - self.build_watch.start('partition-image') - self.ex.runv(['parted', '-s', image_name, 'mklabel', 'msdos'], - as_root=True) - self.ex.runv(['parted', '-s', image_name, 'mkpart', 'primary', - '0%', '100%'], as_root=True) - self.ex.runv(['parted', '-s', image_name, 'set', '1', 'boot', 'on'], - as_root=True) - self.build_watch.stop('partition-image') + with self.build_watch('partition-image'): + self.ex.runv(['parted', '-s', image_name, 'mklabel', 'msdos']) + self.ex.runv(['parted', '-s', image_name, 'mkpart', 'primary', + '0%', '100%']) + self.ex.runv(['parted', '-s', image_name, 'set', '1', 'boot', 'on']) # Install first stage boot loader into MBR. - self.build_watch.start('install-mbr') - self.ex.runv(['install-mbr', image_name], as_root=True) - self.build_watch.stop('install-mbr') + with self.build_watch('install-mbr'): + self.ex.runv(['install-mbr', image_name]) # Setup device mapper to access the partition. - self.build_watch.start('setup-device-mapper') - out = self.ex.runv(['kpartx', '-av', image_name], as_root=True) - devices = [line.split()[2] - for line in out.splitlines() - if line.startswith('add map ')] - partition = '/dev/mapper/%s' % devices[0] - self.build_watch.stop('setup-device-mapper') + with self.build_watch('setup-device-mapper'): + out = self.ex.runv(['kpartx', '-av', image_name]) + devices = [line.split()[2] + for line in out.splitlines() + if line.startswith('add map ')] + partition = '/dev/mapper/%s' % devices[0] mount_point = None try: # Create filesystem. - self.build_watch.start('create-filesystem') - self.ex.runv(['mkfs', '-t', 'ext3', partition], as_root=True) - self.build_watch.stop('create-filesystem') + with self.build_watch('create-filesystem'): + self.ex.runv(['mkfs', '-t', 'ext3', partition]) # Mount it. - self.build_watch.start('mount-filesystem') - mount_point = self.tempdir.join('mnt') - os.mkdir(mount_point) - self.ex.runv(['mount', partition, mount_point], as_root=True) - self.build_watch.stop('mount-filesystem') + with self.build_watch('mount-filesystem'): + mount_point = self.tempdir.join('mnt') + os.mkdir(mount_point) + self.ex.runv(['mount', partition, mount_point]) # Unpack all strata into filesystem. - self.build_watch.start('unpack-strata') - for name, filename in self.built: - self.msg('unpack %s from %s' % (name, filename)) - self.ex.runv(['tar', '-C', mount_point, '-xf', filename], - as_root=True) - self.build_watch.stop('unpack-strata') + # Also, run ldconfig. + with self.build_watch('unpack-strata'): + for name, filename in self.built: + self.msg('unpack %s from %s' % (name, filename)) + self.ex.runv(['tar', '-C', mount_point, '-xf', filename]) + ldconfig(ex, mount_point) # Create fstab. - self.build_watch.start('create-fstab') - fstab = self.tempdir.join('mnt/etc/fstab') - # sorry about the hack, I wish I knew a better way - self.ex.runv(['tee', fstab], feed_stdin=''' + with self.build_watch('create-fstab'): + fstab = self.tempdir.join('mnt/etc/fstab') + # sorry about the hack, I wish I knew a better way + self.ex.runv(['tee', fstab], feed_stdin=''' proc /proc proc defaults 0 0 sysfs /sys sysfs defaults 0 0 /dev/sda1 / ext4 errors=remount-ro 0 1 -''', as_root=True, stdout=open(os.devnull,'w')) - self.build_watch.stop('create-fstab') +''', stdout=open(os.devnull,'w')) # Install extlinux bootloader. - self.build_watch.start('install-bootloader') - conf = os.path.join(mount_point, 'extlinux.conf') - logging.debug('configure extlinux %s' % conf) - self.ex.runv(['tee', conf], feed_stdin=''' + with self.build_watch('install-bootloader'): + conf = os.path.join(mount_point, 'extlinux.conf') + logging.debug('configure extlinux %s' % conf) + self.ex.runv(['tee', conf], feed_stdin=''' default linux timeout 1 label linux kernel /vmlinuz append root=/dev/sda1 init=/sbin/init quiet rw -''', as_root=True, stdout=open(os.devnull, 'w')) +''', stdout=open(os.devnull, 'w')) - self.ex.runv(['extlinux', '--install', mount_point], as_root=True) - - # Weird hack that makes extlinux work. There is a bug somewhere. - self.ex.runv(['sync']) - import time; time.sleep(2) - self.build_watch.stop('install-bootloader') + self.ex.runv(['extlinux', '--install', mount_point]) + + # Weird hack that makes extlinux work. + # FIXME: There is a bug somewhere. + self.ex.runv(['sync']) + import time; time.sleep(2) # Unmount. - self.build_watch.start('unmount-filesystem') - self.ex.runv(['umount', mount_point], as_root=True) - self.build_watch.stop('unmount-filesystem') + with self.build_watch('unmount-filesystem'): + self.ex.runv(['umount', mount_point]) except BaseException, e: # Unmount. if mount_point is not None: try: - self.ex.runv(['umount', mount_point], as_root=True) + self.ex.runv(['umount', mount_point]) except Exception: pass # Undo device mapping. try: - self.ex.runv(['kpartx', '-d', image_name], as_root=True) + self.ex.runv(['kpartx', '-d', image_name]) except Exception: pass raise # Undo device mapping. - self.build_watch.start('undo-device-mapper') - self.ex.runv(['kpartx', '-d', image_name], as_root=True) - self.build_watch.stop('undo-device-mapper') + with self.build_watch('undo-device-mapper'): + self.ex.runv(['kpartx', '-d', image_name]) # Move image file to cache. - self.build_watch.start('cache-image') - filename = self.filename(self.morph.name) - self.ex.runv(['mv', image_name, filename]) - self.build_watch.stop('cache-image') + with self.build_watch('cache-image'): + filename = self.filename(self.morph.name) + self.ex.runv(['mv', image_name, filename]) return { self.morph.name: filename } @@ -451,6 +462,8 @@ class Builder(object): self.dump_memory_profile = app.dump_memory_profile self.cachedir = morphlib.cachedir.CacheDir(self.settings['cachedir']) self.indent = 0 + self.morph_loader = \ + morphlib.morphologyloader.MorphologyLoader(self.settings) def msg(self, text): spaces = ' ' * self.indent @@ -468,11 +481,8 @@ class Builder(object): self.dump_memory_profile('at start of build method') self.indent_more() self.msg('build %s|%s|%s' % (repo, ref, filename)) - base_url = self.settings['git-base-url'] - if not base_url.endswith('/'): - base_url += '/' - repo = urlparse.urljoin(base_url, repo) - morph = self.get_morph_from_git(repo, ref, filename) + morph = self.morph_loader.load(repo, ref, filename) + repo = morph.repo self.dump_memory_profile('after getting morph from git') if morph.kind == 'chunk': @@ -510,24 +520,22 @@ class Builder(object): self.dump_memory_profile('after installing chunk') built = builds else: - blob.build_watch.start('overall-build') - - blob.build_watch.start('build-needed') - self.build_needed(blob) - blob.build_watch.stop('build-needed') - self.dump_memory_profile('after building needed') - - self.msg('Building %s %s' % (morph.kind, morph.name)) - self.indent_more() - built = blob.build() - self.dump_memory_profile('after building blob') - self.indent_less() - for x in built: - self.msg('%s %s cached at %s' % (morph.kind, x, built[x])) - self.install_chunk(morph, x, built[x], blob.staging) - self.dump_memory_profile('after installing chunks') - - blob.build_watch.stop('overall-build') + with blob.build_watch('overall-build'): + + with blob.build_watch('build-needed'): + self.build_needed(blob) + self.dump_memory_profile('after building needed') + + self.msg('Building %s %s' % (morph.kind, morph.name)) + self.indent_more() + built = blob.build() + self.dump_memory_profile('after building blob') + self.indent_less() + for x in built: + self.msg('%s %s cached at %s' % (morph.kind, x, built[x])) + self.install_chunk(morph, x, built[x], blob.staging) + self.dump_memory_profile('after installing chunks') + blob.save_build_times() self.indent_less() @@ -548,26 +556,18 @@ class Builder(object): if self.settings['bootstrap']: self.msg('Unpacking chunk %s onto system' % chunk_name) ex = morphlib.execute.Execute('/', self.msg) - morphlib.bins.unpack_binary(chunk_filename, '/', ex, as_root=True) + morphlib.bins.unpack_binary(chunk_filename, '/', ex) + ldconfig(ex, '/') else: self.msg('Unpacking chunk %s into staging' % chunk_name) ex = morphlib.execute.Execute(staging_dir, self.msg) - morphlib.bins.unpack_binary(chunk_filename, staging_dir, ex, - as_root=True) + morphlib.bins.unpack_binary(chunk_filename, staging_dir, ex) + ldconfig(ex, staging_dir) - def get_morph_from_git(self, repo, ref, filename): - morph_text = morphlib.git.get_morph_text(repo, ref, filename) - f = StringIO.StringIO(morph_text) - scheme, netlock, path, params, query, frag = urlparse.urlparse(repo) - f.name = os.path.join(path, filename) - morph = morphlib.morphology.Morphology(f, - self.settings['git-base-url']) - return morph - def get_cache_id(self, repo, ref, morph_filename): logging.debug('get_cache_id(%s, %s, %s)' % (repo, ref, morph_filename)) - morph = self.get_morph_from_git(repo, ref, morph_filename) + morph = self.morph_loader.load(repo, ref, morph_filename) if morph.kind == 'chunk': kids = [] elif morph.kind == 'stratum': diff --git a/morphlib/execute.py b/morphlib/execute.py index aedf8d4f..038a5cfd 100644 --- a/morphlib/execute.py +++ b/morphlib/execute.py @@ -39,35 +39,11 @@ class Execute(object): self._setup_env() self.dirname = dirname self.msg = msg - self._fakeroot_session = None - - def __del__(self): # pragma: no cover - try: - object.__del__(self) - except AttributeError: - pass - if self._fakeroot_session: - os.remove(self._fakeroot_session) def _setup_env(self): self.env = dict(os.environ) - def _prefix(self, argv, as_root, as_fakeroot): - if as_root: - if os.getuid() == 0: - prefix = ['env'] - else: - prefix = ['sudo'] - envs = ["%s=%s" % x for x in self.env.iteritems()] - argv = prefix + envs + argv - elif as_fakeroot and os.getuid() != 0: - if not self._fakeroot_session: - self._fakeroot_session = tempfile.mkstemp()[1] - argv = ['fakeroot', '-i', self._fakeroot_session, '-s', - self._fakeroot_session, '--'] + argv - return argv - - def run(self, commands, as_root=False, as_fakeroot=False, _log=True): + def run(self, commands, _log=True): '''Execute a list of commands. If a command fails (returns non-zero exit code), the rest are @@ -79,7 +55,6 @@ class Execute(object): for command in commands: self.msg('# %s' % command) argv = ['sh', '-c', command] - argv = self._prefix(argv, as_root, as_fakeroot) logging.debug('run: argv=%s' % repr(argv)) logging.debug('run: env=%s' % repr(self.env)) logging.debug('run: cwd=%s' % repr(self.dirname)) @@ -98,8 +73,7 @@ class Execute(object): stdouts.append(out) return stdouts - def runv(self, argv, feed_stdin=None, as_root=False, as_fakeroot=False, - _log=True, **kwargs): + def runv(self, argv, feed_stdin=None, _log=True, **kwargs): '''Run a command given as a list of argv elements. Return standard output. Raise ``CommandFailure`` if the command @@ -117,7 +91,6 @@ class Execute(object): if 'env' not in kwargs: kwargs['env'] = self.env - argv = self._prefix(argv, as_root, as_fakeroot) logging.debug('runv: argv=%s' % repr(argv)) logging.debug('runv: env=%s' % repr(self.env)) logging.debug('runv: cwd=%s' % repr(self.dirname)) diff --git a/morphlib/execute_tests.py b/morphlib/execute_tests.py index 86db9c25..da6f5d49 100644 --- a/morphlib/execute_tests.py +++ b/morphlib/execute_tests.py @@ -52,9 +52,3 @@ class ExecuteTests(unittest.TestCase): def test_runv_sets_working_directory(self): self.assertEqual(self.e.runv(['pwd']), '/\n') - def test_runs_as_fakeroot_when_requested(self): - self.assertEqual(self.e.run(['id -u'], as_fakeroot=True), ['0\n']) - - def test_runvs_as_fakeroot_when_requested(self): - self.assertEqual(self.e.runv(['id', '-u'], as_fakeroot=True), '0\n') - diff --git a/morphlib/morphology.py b/morphlib/morphology.py index f04e9f7c..8f787939 100644 --- a/morphlib/morphology.py +++ b/morphlib/morphology.py @@ -22,7 +22,10 @@ class Morphology(object): '''Represent a morphology: description of how to build binaries.''' - def __init__(self, fp, baseurl=None): + def __init__(self, repo, ref, fp, baseurl=None): + self.repo = repo + self.ref = ref + self._fp = fp self._baseurl = baseurl or '' self._load() @@ -62,7 +65,7 @@ class Morphology(object): @property def build_depends(self): - return self._dict.get('build-depends', []) + return self._dict.get('build-depends', None) @property def build_system(self): diff --git a/morphlib/morphology_tests.py b/morphlib/morphology_tests.py index 0119ebc6..cac80798 100644 --- a/morphlib/morphology_tests.py +++ b/morphlib/morphology_tests.py @@ -32,6 +32,7 @@ class MorphologyTests(unittest.TestCase): def test_accepts_valid_chunk_morphology(self): morph = morphlib.morphology.Morphology( + 'repo', 'ref', MockFile(''' { "name": "hello", @@ -57,6 +58,10 @@ class MorphologyTests(unittest.TestCase): ] } }''')) + + self.assertEqual(morph.repo, 'repo') + self.assertEqual(morph.ref, 'ref') + self.assertEqual(morph.filename, 'mockfile') self.assertEqual(morph.name, 'hello') self.assertEqual(morph.kind, 'chunk') self.assertEqual(morph.description, 'desc') @@ -77,6 +82,7 @@ class MorphologyTests(unittest.TestCase): def test_build_system_defaults_to_None(self): morph = morphlib.morphology.Morphology( + 'repo', 'ref', MockFile(''' { "name": "hello", @@ -86,6 +92,7 @@ class MorphologyTests(unittest.TestCase): def test_max_jobs_defaults_to_None(self): morph = morphlib.morphology.Morphology( + 'repo', 'ref', MockFile(''' { "name": "hello", @@ -95,6 +102,7 @@ class MorphologyTests(unittest.TestCase): def test_accepts_valid_stratum_morphology(self): morph = morphlib.morphology.Morphology( + 'repo', 'ref', MockFile(''' { "name": "hello", @@ -121,6 +129,7 @@ class MorphologyTests(unittest.TestCase): def test_accepts_valid_system_morphology(self): morph = morphlib.morphology.Morphology( + 'repo', 'ref', MockFile(''' { "name": "hello", @@ -145,6 +154,7 @@ class StratumRepoTests(unittest.TestCase): def stratum(self, repo): return morphlib.morphology.Morphology( + 'repo', 'ref', MockFile(''' { "name": "hello", diff --git a/morphlib/morphologyloader.py b/morphlib/morphologyloader.py new file mode 100644 index 00000000..a227214f --- /dev/null +++ b/morphlib/morphologyloader.py @@ -0,0 +1,57 @@ +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import os +import StringIO +import urlparse + +import morphlib + + +class MorphologyLoader(object): + + '''Load morphologies from git and parse them into Morphology objects.''' + + def __init__(self, settings): + self.settings = settings + self.morphologies = {} + + def load(self, repo, ref, filename): + base_url = self.settings['git-base-url'] + if not base_url.endswith('/'): + base_url += '/' + repo = urlparse.urljoin(base_url, repo) + + key = (repo, ref, filename) + + if key in self.morphologies: + return self.morphologies[key] + else: + morph = self._get_morph_from_git(repo, ref, filename) + self.morphologies[key] = morph + return morph + + def _get_morph_text(self, repo, ref, filename): # pragma: no cover + return morphlib.git.get_morph_text(repo, ref, filename) + + def _get_morph_from_git(self, repo, ref, filename): + morph_text = self._get_morph_text(repo, ref, filename) + scheme, netlock, path, params, query, frag = urlparse.urlparse(repo) + f = StringIO.StringIO(morph_text) + f.name = os.path.join(path, filename) + morph = morphlib.morphology.Morphology(repo, ref, f, + self.settings['git-base-url']) + return morph diff --git a/morphlib/morphologyloader_tests.py b/morphlib/morphologyloader_tests.py new file mode 100644 index 00000000..f9a03915 --- /dev/null +++ b/morphlib/morphologyloader_tests.py @@ -0,0 +1,47 @@ +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import os +import StringIO +import unittest + +import morphlib + + +class MorphologyLoaderTests(unittest.TestCase): + + def test_load_twice_verify_same_morph_object(self): + settings = { 'git-base-url': '' } + loader = morphlib.morphologyloader.MorphologyLoader(settings) + loader._get_morph_text = self.get_morph_text + + morph1 = loader.load('repo', 'ref', 'hello.morph') + morph2 = loader.load('repo', 'ref', 'hello.morph') + self.assertEqual(morph1, morph2) + + def get_morph_text(self, repo, ref, filename): + return (''' + { + "name": "foo", + "kind": "stratum", + "sources": [ + { + "name": "bar", + "repo": "bar", + "ref": "master" + } + ] + }''') diff --git a/morphlib/stopwatch.py b/morphlib/stopwatch.py index 492738d5..7446f897 100644 --- a/morphlib/stopwatch.py +++ b/morphlib/stopwatch.py @@ -23,6 +23,7 @@ class Stopwatch(object): def __init__(self): self.ticks = {} + self.context_stack = [] def tick(self, reference_object, name): if not reference_object in self.ticks: @@ -56,3 +57,17 @@ class Stopwatch(object): return (delta.days * 24 * 3600 + delta.seconds + operator.truediv(delta.microseconds, 10**6)) + + def __call__(self, reference_object): + self.context_stack.append(reference_object) + return self + + def __enter__(self): + self.start(self.context_stack[-1]) + return self + + def __exit__(self, *args): + self.stop(self.context_stack[-1]) + self.context_stack.pop() + return False # cause any exception to be re-raised + diff --git a/morphlib/stopwatch_tests.py b/morphlib/stopwatch_tests.py index 1a899f41..d4f1e3dd 100644 --- a/morphlib/stopwatch_tests.py +++ b/morphlib/stopwatch_tests.py @@ -56,3 +56,20 @@ class StopwatchTests(unittest.TestCase): self.assertEqual(our_delta, watch_delta) assert self.stopwatch.start_stop_seconds('start-stop') > 0 + + def test_with(self): + with self.stopwatch('foo'): + pass + self.assert_(self.stopwatch.start_stop_seconds('foo') < 1.0) + + def test_with_within_with(self): + with self.stopwatch('foo'): + with self.stopwatch('bar'): + pass + self.assert_(self.stopwatch.start_time('foo')) + self.assert_(self.stopwatch.stop_time('foo')) + self.assert_(self.stopwatch.start_time('bar')) + self.assert_(self.stopwatch.stop_time('bar')) + self.assert_(self.stopwatch.start_stop_seconds('foo') < 1.0) + self.assert_(self.stopwatch.start_stop_seconds('bar') < 1.0) + diff --git a/run-bootstrap-in-chroot b/run-bootstrap-in-chroot new file mode 100755 index 00000000..cdcf0612 --- /dev/null +++ b/run-bootstrap-in-chroot @@ -0,0 +1,36 @@ +#!/bin/sh + +set -e + +export LC_ALL=C + +dir="squeeze-chroot" +mirror="http://192.168.1.185/debian" + +mkdir "$dir" + +debootstrap \ +--include=build-essential,\ +gawk,bison,python,autoconf,autopoint,automake,gettext,libtool,\ +help2man,texinfo,sudo \ +squeeze "$dir" "$mirror" + +hostname > "$dir/etc/hostname" +cat <<EOF > "$dir/etc/hosts" +127.0.0.1 localhost +127.0.1.1 $(hostname) +EOF + +cp baserock-bootstrap "$dir/." +sed 's,^.*/,http://192.168.1.185/lfs/,' wget-list > "$dir/wget-list" +mount -t proc proc "$dir/proc" +mount -t sysfs sysfs "$dir/sys" +if chroot "$dir" bash -x baserock-bootstrap yes +then + exit=0 +else + exit=$? +fi +umount "$dir/sys" +umount "$dir/proc" +exit $exit diff --git a/tests/build-system.stdout b/tests/build-system.stdout deleted file mode 100644 index d00491fd..00000000 --- a/tests/build-system.stdout +++ /dev/null @@ -1 +0,0 @@ -1 diff --git a/tests/build-system.script b/tests/show-dependencies.script index 1553b85a..f38764b3 100755 --- a/tests/build-system.script +++ b/tests/show-dependencies.script @@ -1,8 +1,9 @@ -#!/bin/sh +#!/bin/bash # -# Test build a simple system image. +# Test a basic distributed build of two depending strata +# with a couple of chunks in them. # -# Copyright (C) 2011 Codethink Limited +# Copyright (C) 2012 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,10 +20,14 @@ set -e -cache="$DATADIR/build-system-cache" -log="$DATADIR/build-system-morph.log" -./morph --no-default-configs build morphs-repo master hello-system.morph \ - --git-base-url="file://$DATADIR" \ - --cachedir="$cache" --keep-path --no-distcc \ - --log="$log" || cat "$log" 1>&2 -find "$cache" -name '*.system.*' -type f | wc -l +cache="$DATADIR/show-dependencies-cache" +log="$DATADIR/show-dependencies-morph.log" + +./morph show-dependencies \ + test-repo master xfce-core.morph \ + --no-default-configs \ + --git-base-url="file://$DATADIR" \ + --cachedir="$cache" \ + --keep-path \ + --no-distcc \ + --log="$log" diff --git a/tests/show-dependencies.setup b/tests/show-dependencies.setup new file mode 100755 index 00000000..cbbbf369 --- /dev/null +++ b/tests/show-dependencies.setup @@ -0,0 +1,314 @@ +#!/bin/bash +# +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + +set -e + +# Create a repository +repo="$DATADIR/test-repo" +mkdir "$repo" +cd "$repo" +git init --quiet + +# Add a single source file to simulate compiling +cat <<EOF > hello.c +#include <stdio.h> +int main(void) +{ + puts("hello, world"); + return 0; +} +EOF +git add hello.c + +# Define a couple of chunk morphologies for the GTK stack +gtkcomponents=(freetype fontconfig cairo pango glib gdk-pixbuf gtk dbus-glib dbus) +for component in "${gtkcomponents[@]}" +do + cat <<EOF > $component.morph +{ + "name": "$component", + "kind": "chunk", + "build-commands": [ + "gcc -o hello hello.c" + ], + "install-commands": [ + "install -d \\"\$DESTDIR\\"/etc", + "install -d \\"\$DESTDIR\\"/bin", + "install hello \\"\$DESTDIR\\"/bin/$component" + ] +} +EOF + git add $component.morph +done +git commit --quiet -m "add .c source file and GTK chunk morphologies" + +# Define a stratum for the GTK stack +cat <<EOF > gtk-stack.morph +{ + "name": "gtk-stack", + "kind": "stratum", + "build-depends": [ + ], + "sources": [ + { + "name": "freetype", + "repo": "test-repo", + "ref": "master", + "build-depends": [ + ] + }, + { + "name": "fontconfig", + "repo": "test-repo", + "ref": "master", + "build-depends": [ + ] + }, + { + "name": "cairo", + "repo": "test-repo", + "ref": "master", + "build-depends": [ + ] + }, + { + "name": "pango", + "repo": "test-repo", + "ref": "master", + "build-depends": [ + "freetype", + "fontconfig" + ] + }, + { + "name": "glib", + "repo": "test-repo", + "ref": "master", + "build-depends": [ + ] + }, + { + "name": "gdk-pixbuf", + "repo": "test-repo", + "ref": "master", + "build-depends": [ + "glib" + ] + }, + { + "name": "gtk", + "repo": "test-repo", + "ref": "master", + "build-depends": [ + "cairo", + "gdk-pixbuf", + "glib", + "pango" + ] + }, + { + "name": "dbus", + "repo": "test-repo", + "ref": "master", + "build-depends": [ + ] + }, + { + "name": "dbus-glib", + "repo": "test-repo", + "ref": "master", + "build-depends": [ + "dbus", + "glib" + ] + } + ] +} +EOF +git add gtk-stack.morph +git commit --quiet -m "add gtk-stack.morph stratum" + +# Add a single source file to simulate compiling +cat <<EOF > hello.c +#include <stdio.h> +int main(void) +{ + puts("hello, world"); + return 0; +} +EOF +git add hello.c + +# Define a couple of chunk morphologies for the GTK stack +xfcecomponents=(xfce4-dev-tools libxfce4util libxfce4ui exo xfconf garcon thunar tumbler xfce4-panel xfce4-settings xfce4-session xfwm4 xfdesktop xfce4-appfinder gtk-xfce-engine) +for component in "${xfcecomponents[@]}" +do + cat <<EOF > $component.morph +{ + "name": "$component", + "kind": "chunk", + "build-commands": [ + "gcc -o hello hello.c" + ], + "install-commands": [ + "install -d \\"\$DESTDIR\\"/etc", + "install -d \\"\$DESTDIR\\"/bin", + "install hello \\"\$DESTDIR\\"/bin/$component" + ] +} +EOF + git add $component.morph +done +git commit --quiet -m "add .c source file and GTK chunk morphologies" + +# Define a stratum for the Xfce core +cat <<EOF > xfce-core.morph +{ + "name": "xfce-core", + "kind": "stratum", + "build-depends": [ + "gtk-stack" + ], + "sources": [ + { + "name": "libxfce4util", + "repo": "test-repo", + "ref": "master", + "build-depends": [ + ] + }, + { + "name": "xfconf", + "repo": "test-repo", + "ref": "master", + "build-depends": [ + "libxfce4util" + ] + }, + { + "name": "libxfce4ui", + "repo": "test-repo", + "ref": "master", + "build-depends": [ + "xfconf" + ] + }, + { + "name": "exo", + "repo": "test-repo", + "ref": "master", + "build-depends": [ + "libxfce4util" + ] + }, + { + "name": "garcon", + "repo": "test-repo", + "ref": "master", + "build-depends": [ + "libxfce4util" + ] + }, + { + "name": "thunar", + "repo": "test-repo", + "ref": "master", + "build-depends": [ + "libxfce4ui", + "exo" + ] + }, + { + "name": "tumbler", + "repo": "test-repo", + "ref": "master", + "build-depends": [ + ] + }, + { + "name": "xfce4-panel", + "repo": "test-repo", + "ref": "master", + "build-depends": [ + "libxfce4ui", + "exo", + "garcon" + ] + }, + { + "name": "xfce4-settings", + "repo": "test-repo", + "ref": "master", + "build-depends": [ + "libxfce4ui", + "exo", + "xfconf" + ] + }, + { + "name": "xfce4-session", + "repo": "test-repo", + "ref": "master", + "build-depends": [ + "libxfce4ui", + "exo", + "xfconf" + ] + }, + { + "name": "xfwm4", + "repo": "test-repo", + "ref": "master", + "build-depends": [ + "libxfce4ui", + "xfconf" + ] + }, + { + "name": "xfdesktop", + "repo": "test-repo", + "ref": "master", + "build-depends": [ + "libxfce4ui", + "xfconf" + ] + }, + { + "name": "xfce4-appfinder", + "repo": "test-repo", + "ref": "master", + "build-depends": [ + "libxfce4ui", + "garcon", + "xfconf" + ] + }, + { + "name": "gtk-xfce-engine", + "repo": "test-repo", + "ref": "master", + "build-depends": [ + "libxfce4ui", + "garcon", + "xfconf" + ] + } + ] +} +EOF +git add xfce-core.morph +git commit --quiet -m "add xfce-core.morph stratum" diff --git a/tests/show-dependencies.stdout b/tests/show-dependencies.stdout new file mode 100644 index 00000000..6f0096e2 --- /dev/null +++ b/tests/show-dependencies.stdout @@ -0,0 +1,132 @@ +dependency tree: + xfce-core (stratum) + -> libxfce4util (chunk) + -> xfconf (chunk) + -> libxfce4ui (chunk) + -> exo (chunk) + -> garcon (chunk) + -> thunar (chunk) + -> tumbler (chunk) + -> xfce4-panel (chunk) + -> xfce4-settings (chunk) + -> xfce4-session (chunk) + -> xfwm4 (chunk) + -> xfdesktop (chunk) + -> xfce4-appfinder (chunk) + -> gtk-xfce-engine (chunk) + gtk-stack (stratum) + -> freetype (chunk) + -> fontconfig (chunk) + -> cairo (chunk) + -> pango (chunk) + -> glib (chunk) + -> gdk-pixbuf (chunk) + -> gtk (chunk) + -> dbus (chunk) + -> dbus-glib (chunk) + libxfce4util (chunk) + -> gtk-stack (stratum) + xfconf (chunk) + -> libxfce4util (chunk) + -> gtk-stack (stratum) + libxfce4ui (chunk) + -> xfconf (chunk) + -> gtk-stack (stratum) + exo (chunk) + -> libxfce4util (chunk) + -> gtk-stack (stratum) + garcon (chunk) + -> libxfce4util (chunk) + -> gtk-stack (stratum) + thunar (chunk) + -> libxfce4ui (chunk) + -> exo (chunk) + -> gtk-stack (stratum) + tumbler (chunk) + -> gtk-stack (stratum) + xfce4-panel (chunk) + -> libxfce4ui (chunk) + -> exo (chunk) + -> garcon (chunk) + -> gtk-stack (stratum) + xfce4-settings (chunk) + -> libxfce4ui (chunk) + -> exo (chunk) + -> xfconf (chunk) + -> gtk-stack (stratum) + xfce4-session (chunk) + -> libxfce4ui (chunk) + -> exo (chunk) + -> xfconf (chunk) + -> gtk-stack (stratum) + xfwm4 (chunk) + -> libxfce4ui (chunk) + -> xfconf (chunk) + -> gtk-stack (stratum) + xfdesktop (chunk) + -> libxfce4ui (chunk) + -> xfconf (chunk) + -> gtk-stack (stratum) + xfce4-appfinder (chunk) + -> libxfce4ui (chunk) + -> garcon (chunk) + -> xfconf (chunk) + -> gtk-stack (stratum) + gtk-xfce-engine (chunk) + -> libxfce4ui (chunk) + -> garcon (chunk) + -> xfconf (chunk) + -> gtk-stack (stratum) + freetype (chunk) + fontconfig (chunk) + cairo (chunk) + pango (chunk) + -> freetype (chunk) + -> fontconfig (chunk) + glib (chunk) + gdk-pixbuf (chunk) + -> glib (chunk) + gtk (chunk) + -> cairo (chunk) + -> gdk-pixbuf (chunk) + -> glib (chunk) + -> pango (chunk) + dbus (chunk) + dbus-glib (chunk) + -> dbus (chunk) + -> glib (chunk) +build order: + group: + freetype (chunk) + fontconfig (chunk) + cairo (chunk) + glib (chunk) + dbus (chunk) + group: + pango (chunk) + gdk-pixbuf (chunk) + dbus-glib (chunk) + group: + gtk (chunk) + group: + gtk-stack (stratum) + group: + libxfce4util (chunk) + tumbler (chunk) + group: + xfconf (chunk) + exo (chunk) + garcon (chunk) + group: + libxfce4ui (chunk) + group: + thunar (chunk) + xfce4-panel (chunk) + xfce4-settings (chunk) + xfce4-session (chunk) + xfwm4 (chunk) + xfdesktop (chunk) + xfce4-appfinder (chunk) + gtk-xfce-engine (chunk) + group: + xfce-core (stratum) @@ -77,6 +77,6 @@ http://www.linuxfromscratch.org/patches/lfs/7.0/procps-3.2.8-fix_HZ_errors-1.pat http://www.linuxfromscratch.org/patches/lfs/7.0/procps-3.2.8-watch_unicode-1.patch http://www.linuxfromscratch.org/patches/lfs/7.0/readline-6.2-fixes-1.patch http://python.org/ftp/python/2.7.2/Python-2.7.2.tar.bz2 -http://code.liw.fi/debian/pool/main/p/python-cliapp/python-cliapp_0.22.orig.tar.gz +http://code.liw.fi/debian/pool/main/p/python-cliapp/python-cliapp_0.23.orig.tar.gz http://git-core.googlecode.com/files/git-1.7.7.3.tar.gz http://busybox.net/downloads/busybox-1.19.3.tar.bz2 |