diff options
Diffstat (limited to 'morphlib')
-rw-r--r-- | morphlib/__init__.py | 4 | ||||
-rw-r--r-- | morphlib/bins.py | 10 | ||||
-rw-r--r-- | morphlib/bins_tests.py | 6 | ||||
-rw-r--r-- | morphlib/builddependencygraph.py | 291 | ||||
-rw-r--r-- | morphlib/builddependencygraph_tests.py | 71 | ||||
-rw-r--r-- | morphlib/builder.py | 304 | ||||
-rw-r--r-- | morphlib/execute.py | 31 | ||||
-rw-r--r-- | morphlib/execute_tests.py | 6 | ||||
-rw-r--r-- | morphlib/morphology.py | 7 | ||||
-rw-r--r-- | morphlib/morphology_tests.py | 10 | ||||
-rw-r--r-- | morphlib/morphologyloader.py | 57 | ||||
-rw-r--r-- | morphlib/morphologyloader_tests.py | 47 | ||||
-rw-r--r-- | morphlib/stopwatch.py | 15 | ||||
-rw-r--r-- | morphlib/stopwatch_tests.py | 17 |
14 files changed, 675 insertions, 201 deletions
diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 5d8568b4..35939c45 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011 Codethink Limited +# Copyright (C) 2011-2012 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -21,11 +21,13 @@ __version__ = '0.0' import bins +import builddependencygraph import builder import cachedir import execute import git import morphology +import morphologyloader import stopwatch import tempdir import tester diff --git a/morphlib/bins.py b/morphlib/bins.py index 90d96d82..cd77a037 100644 --- a/morphlib/bins.py +++ b/morphlib/bins.py @@ -101,20 +101,16 @@ def create_stratum(rootdir, stratum_filename, ex): '''Create a stratum from the contents of a directory.''' logging.debug('Creating stratum file %s from %s' % (stratum_filename, rootdir)) - ex.runv(['tar', '-C', rootdir, '-caf', stratum_filename, '.'], - as_fakeroot=True) + ex.runv(['tar', '-C', rootdir, '-caf', stratum_filename, '.']) -def unpack_binary(filename, dirname, ex, as_fakeroot=False, as_root=False): +def unpack_binary(filename, dirname, ex): '''Unpack a binary into a directory. The directory must exist already. - If the binary will be packed up again by tar with the same Execute - object then as_fakeroot will suffice - If it will be creating a system image as_root will be needed ''' logging.debug('Unpacking %s into %s' % (filename, dirname)) - ex.runv(['tar', '-C', dirname, '-xvf', filename], as_fakeroot=as_fakeroot, as_root=as_root) + ex.runv(['tar', '-C', dirname, '-xvf', filename]) diff --git a/morphlib/bins_tests.py b/morphlib/bins_tests.py index 6fe48aa7..8a54033b 100644 --- a/morphlib/bins_tests.py +++ b/morphlib/bins_tests.py @@ -92,8 +92,7 @@ class ChunkTests(unittest.TestCase): morphlib.bins.create_chunk(self.instdir, self.chunk_file, ['.'], self.ex) os.mkdir(self.unpacked) - morphlib.bins.unpack_binary(self.chunk_file, self.unpacked, self.ex, - as_fakeroot=True) + morphlib.bins.unpack_binary(self.chunk_file, self.unpacked, self.ex) self.assertEqual(orig_files, recursive_lstat(self.unpacked)) def test_uses_only_matching_names(self): @@ -127,8 +126,7 @@ class StratumTests(unittest.TestCase): self.populate_instdir() morphlib.bins.create_stratum(self.instdir, self.stratum_file, self.ex) os.mkdir(self.unpacked) - morphlib.bins.unpack_binary(self.stratum_file, self.unpacked, self.ex, - as_fakeroot=True) + morphlib.bins.unpack_binary(self.stratum_file, self.unpacked, self.ex) self.assertEqual(recursive_lstat(self.instdir), recursive_lstat(self.unpacked)) diff --git a/morphlib/builddependencygraph.py b/morphlib/builddependencygraph.py new file mode 100644 index 00000000..7a432005 --- /dev/null +++ b/morphlib/builddependencygraph.py @@ -0,0 +1,291 @@ +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import collections +import copy +import os + +import morphlib + + +class Node(object): + + '''Node in the dependency graph.''' + + def __init__(self, morphology): + self.morphology = morphology + self.dependents = [] + self.dependencies = [] + + def add_dependency(self, other): + if other not in self.dependencies: + self.dependencies.append(other) + if self not in other.dependents: + other.dependents.append(self) + + def remove_dependency(self, other): + if other in self.dependencies: + self.dependencies.remove(other) + if self in other.dependents: + other.dependents.remove(self) + + def depends_on(self, other): + return other in self.dependencies + + def __str__(self): # pragma: no cover + return '%s (%s)' % (self.morphology.name, self.morphology.kind) + + def __deepcopy__(self, memo): # pragma: no cover + return Node(self.morphology) + + +class NodeList(list): + + def __deepcopy__(self, memo): # pragma: no cover + nodes = NodeList() + + old_to_new = {} + + for node in self: + node_copy = copy.deepcopy(node) + old_to_new[node] = node_copy + nodes.append(node_copy) + + for node in self: + node_copy = old_to_new[node] + for dep in node.dependencies: + dep_copy = old_to_new[dep] + node_copy.add_dependency(dep_copy) + + return nodes + + +class BuildDependencyGraph(object): # pragma: no cover + + def __init__(self, loader, morphology): + self.loader = loader + self.morphology = morphology + self.nodes = None + + def add(self, morphology): + node = Node(morphology) + if node not in self.nodes: + self.nodes.append(node) + + def resolve(self): + self.cached_morphologies = {} + self._resolve_strata() + self._resolve_chunks() + + def build_order(self): + sorting = self._compute_topological_sorting() + + #print + #print 'sorting: %s' % [str(x) for x in sorting] + #print + + groups = [] + group_nodes = {} + + group = [] + groups.append(group) + + for node in sorting: + create_group = False + for dep in node.dependencies: + if dep in group: + create_group = True + + if create_group: + group = [] + groups.append(group) + + group.append(node) + + morphology_groups = [] + for group in groups: + morphology_group = [] + for node in group: + morphology_group.append(node.morphology) + morphology_groups.append(morphology_group) + + return morphology_groups + + def _resolve_strata(self): + self.nodes = NodeList() + + if self.morphology.kind == 'stratum': + root = Node(self.morphology) + self.nodes.append(root) + + queue = collections.deque() + queue.append(root) + while len(queue) > 0: + node = queue.popleft() + + if not node.morphology.build_depends: + continue + + if isinstance(node.morphology.build_depends, list): + for other_name in node.morphology.build_depends: + # prepare a tuple for loading the morphology + repo = node.morphology.repo + ref = node.morphology.ref + filename = '%s.morph' % other_name + info = (repo, ref, filename) + + # look up or create a node for the morphology + other_node = self.cached_morphologies.get(info, None) + if not other_node: + #print other_name + morphology = self.loader.load(repo, ref, filename) + other_node = Node(morphology) + + # cache the node for this morphology + self.cached_morphologies[info] = other_node + + # add the morphology node to the graph + node.add_dependency(other_node) + self.nodes.append(other_node) + queue.append(other_node) + else: + raise Exception('%s uses an invalid "build-depends" format' % + os.path.basename(stratum_node.morphology.filename)) + + #print 'strata: %s' % [str(x) for x in self.nodes] + + def _resolve_chunks(self): + strata_nodes = list(self.nodes) + for stratum_node in strata_nodes: + self._resolve_stratum_chunks(stratum_node) + + def _morphology_node(self, repo, ref, filename): + info = (repo, ref, filename) + + if info in self.cached_morphologies: + return self.cached_morphologies[info] + else: + morphology = self.loader.load(repo, ref, filename) + node = Node(morphology) + self.nodes.append(node) + self.cached_morphologies[info] = node + return node + + def _resolve_stratum_chunks(self, stratum_node): + stratum_chunk_nodes = [] + chunk_lookup = {} + + # second, create nodes for all chunks in the stratum + for i in xrange(0, len(stratum_node.morphology.sources)): + source = stratum_node.morphology.sources[i] + + # construct the build tuple + repo = source['repo'] + ref = source['ref'] + filename = '%s.morph' % (source['morph'] + if 'morph' in source + else source['name']) + + chunk_node = self._morphology_node(repo, ref, filename) + + chunk_lookup[source['name']] = chunk_node + + stratum_chunk_nodes.append(chunk_node) + + # read the build-depends information + build_depends = (source['build-depends'] + if 'build-depends' in source + else None) + + # turn build-depends into proper dependencies in the graph + if build_depends is None: + for j in xrange(0, i): + chunk_node.add_dependency(stratum_chunk_nodes[j]) + elif isinstance(build_depends, list): + for depname in build_depends: + if depname in chunk_lookup: + depnode = chunk_lookup[depname] + chunk_node.add_dependency(depnode) + else: + raise Exception('%s: source "%s" references "%s" ' + 'before it is defined' % + (os.path.basename(stratum_node.morphology.filename), + source['name'], + depname)) + else: + raise Exception('%s: source "%s" uses an invalid ' + '"build-depends" format' % + (os.path.basename(stratum_node.morphology.filename), + source['name'])) + + # make the chunk nodes in this stratum depend on all strata + # that need to be built first + for chunk_node in stratum_chunk_nodes: + for stratum_dep in stratum_node.dependencies: + chunk_node.add_dependency(stratum_dep) + + # clear the dependencies of the stratum + stratum_node.dependencies = [] + + # make the stratum node depend on all its chunk nodes + for chunk_node in stratum_chunk_nodes: + stratum_node.add_dependency(chunk_node) + + def _compute_topological_sorting(self): + nodes = copy.deepcopy(self.nodes) + + original_node = {} + for node in self.nodes: + for node_copy in nodes: + if node.morphology == node_copy.morphology: + original_node[node_copy] = node + + #print 'compute topological sorting:' + #print ' nodes: %s' % [str(x) for x in nodes] + + sorting = [] + leafs = collections.deque() + + for node in nodes: + if len(node.dependencies) is 0: + leafs.append(node) + + #print ' leafs: %s' % [str(x) for x in leafs] + + while len(leafs) > 0: + leaf = leafs.popleft() + sorting.append(leaf) + + #print ' visit %s' % leaf + + #print ' visit %s' % leaf + + for parent in list(leaf.dependents): + #print ' parent %s' % parent + #print ' parent %s dependencies: %s' % (parent, [str(x) for x in parent.dependencies]) + parent.remove_dependency(leaf) + #print ' parent %s dependencies: %s' % (parent, [str(x) for x in parent.dependencies]) + if len(parent.dependencies) == 0: + #print ' add %s' % parent + leafs.append(parent) + + #print [str(node) for node in sorting] + if len(sorting) < len(nodes): + raise Exception('Cyclic dependencies found in the dependency ' + 'graph of "%s"' % self.morphology) + + #return sorting + return [original_node[node] for node in sorting] diff --git a/morphlib/builddependencygraph_tests.py b/morphlib/builddependencygraph_tests.py new file mode 100644 index 00000000..dec4e49c --- /dev/null +++ b/morphlib/builddependencygraph_tests.py @@ -0,0 +1,71 @@ +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import os +import StringIO +import unittest + +import morphlib + + +class BuildDependencyGraphTests(unittest.TestCase): + + def test_create_node_with_morphology(self): + fake_morphology = "fake morphology" + + node = morphlib.builddependencygraph.Node(fake_morphology) + self.assertEqual(node.morphology, fake_morphology) + + def test_node_add_remove_dependency(self): + node1 = morphlib.builddependencygraph.Node(None) + node2 = morphlib.builddependencygraph.Node(None) + + node1.add_dependency(node2) + + assert(node2 in node1.dependencies) + assert(node1 in node2.dependents) + + assert(node1.depends_on(node2)) + + node2.add_dependency(node1) + + assert(node2 in node1.dependencies) + assert(node1 in node2.dependents) + assert(node1 in node2.dependencies) + assert(node2 in node1.dependents) + + assert(node1.depends_on(node2)) + assert(node2.depends_on(node1)) + + node1.remove_dependency(node2) + + assert(node2 not in node1.dependencies) + assert(node1 not in node2.dependents) + assert(node1 in node2.dependencies) + assert(node2 in node1.dependents) + + assert(not node1.depends_on(node2)) + assert(node2.depends_on(node1)) + + node2.remove_dependency(node1) + + assert(node2 not in node1.dependencies) + assert(node1 not in node2.dependents) + assert(node1 not in node2.dependencies) + assert(node2 not in node1.dependents) + + assert(not node1.depends_on(node2)) + assert(not node2.depends_on(node1)) diff --git a/morphlib/builder.py b/morphlib/builder.py index 1add2835..88a46633 100644 --- a/morphlib/builder.py +++ b/morphlib/builder.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011 Codethink Limited +# Copyright (C) 2011-2012 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -25,6 +25,35 @@ import urlparse import morphlib +def ldconfig(ex, rootdir): + '''Run ldconfig for the filesystem below ``rootdir``. + + Essentially, ``rootdir`` specifies the root of a new system. + Only directories below it are considered. + + ``etc/ld.so.conf`` below ``rootdir`` is assumed to exist and + be populated by the right directories, and should assume + the root directory is ``rootdir``. Example: if ``rootdir`` + is ``/tmp/foo``, then ``/tmp/foo/etc/ld.so.conf`` should + contain ``/lib``, not ``/tmp/foo/lib``. + + The ldconfig found via ``$PATH`` is used, not the one in ``rootdir``, + since in bootstrap mode that might not yet exist, the various + implementations should be compatible enough. + + ''' + + conf = os.path.join(rootdir, 'etc', 'ld.so.conf') + if os.path.exists(conf): + logging.debug('Running ldconfig for %s' % rootdir) + cache = os.path.join(rootdir, 'etc', 'ld.so.cache') + old_path = ex.env['PATH'] + ex.env['PATH'] = '%s:/sbin:/usr/sbin:/usr/local/sbin' % old_path + ex.runv(['ldconfig', '-f', conf, '-C', cache, '-r', rootdir]) + ex.env['PATH'] = old_path + else: + logging.debug('No %s, not running ldconfig' % conf) + class BinaryBlob(object): def __init__(self, morph, repo, ref): @@ -210,15 +239,16 @@ class Chunk(BinaryBlob): def create_source_and_tarball(self): self.msg('Creating source tree and tarball') - self.build_watch.start('create-source-tarball') - self.dump_memory_profile('before creating source and tarball for chunk') - tarball = self.cache_prefix + '.src.tar' - morphlib.git.export_sources(self.repo, self.ref, tarball) - self.dump_memory_profile('after exporting sources') - os.mkdir(self.builddir) - self.ex.runv(['tar', '-C', self.builddir, '-xf', tarball]) - self.dump_memory_profile('after creating source and tarball for chunk') - self.build_watch.stop('create-source-tarball') + with self.build_watch('create-source-tarball'): + self.dump_memory_profile('before creating source and tarball ' + 'for chunk') + tarball = self.cache_prefix + '.src.tar' + morphlib.git.export_sources(self.repo, self.ref, tarball) + self.dump_memory_profile('after exporting sources') + os.mkdir(self.builddir) + self.ex.runv(['tar', '-C', self.builddir, '-xf', tarball]) + self.dump_memory_profile('after creating source and tarball ' + 'for chunk') def build_using_buildsystem(self): bs_name = self.morph.build_system @@ -227,48 +257,44 @@ class Chunk(BinaryBlob): self.run_sequentially('configure', bs['configure-commands']) self.run_in_parallel('build', bs['build-commands']) self.run_sequentially('test', bs['test-commands']) - self.run_sequentially('install', bs['install-commands'], - as_fakeroot=True) + self.run_sequentially('install', bs['install-commands']) def build_using_commands(self): self.msg('Building using explicit commands') self.run_sequentially('configure', self.morph.configure_commands) self.run_in_parallel('build', self.morph.build_commands) self.run_sequentially('test', self.morph.test_commands) - self.run_sequentially('install', self.morph.install_commands, - as_fakeroot=True) + self.run_sequentially('install', self.morph.install_commands) def run_in_parallel(self, what, commands): self.msg('commands: %s' % what) - self.build_watch.start(what) - self.ex.run(commands) - self.build_watch.stop(what) + with self.build_watch(what): + self.ex.run(commands) - def run_sequentially(self, what, commands, as_fakeroot=False, as_root=False): + def run_sequentially(self, what, commands): self.msg ('commands: %s' % what) - self.build_watch.start(what) - flags = self.ex.env['MAKEFLAGS'] - self.ex.env['MAKEFLAGS'] = '-j1' - logging.debug('Setting MAKEFLAGS=%s' % self.ex.env['MAKEFLAGS']) - self.ex.run(commands, as_fakeroot=as_fakeroot, as_root=as_root) - self.ex.env['MAKEFLAGS'] = flags - logging.debug('Restore MAKEFLAGS=%s' % self.ex.env['MAKEFLAGS']) - self.build_watch.stop(what) + with self.build_watch(what): + flags = self.ex.env['MAKEFLAGS'] + self.ex.env['MAKEFLAGS'] = '-j1' + logging.debug('Setting MAKEFLAGS=%s' % self.ex.env['MAKEFLAGS']) + self.ex.run(commands) + self.ex.env['MAKEFLAGS'] = flags + logging.debug('Restore MAKEFLAGS=%s' % self.ex.env['MAKEFLAGS']) def create_chunks(self, chunks): ret = {} - self.build_watch.start('create-chunks') - for chunk_name in chunks: - self.msg('Creating chunk %s' % chunk_name) - self.prepare_binary_metadata(chunk_name) - patterns = chunks[chunk_name] - patterns += [r'baserock/%s\.' % chunk_name] - filename = self.filename(chunk_name) - self.msg('Creating binary for %s' % chunk_name) - morphlib.bins.create_chunk(self.destdir, filename, patterns, - self.ex, self.dump_memory_profile) - ret[chunk_name] = filename - self.build_watch.stop('create-chunks') + with self.build_watch('create-chunks'): + for chunk_name in chunks: + self.msg('Creating chunk %s' % chunk_name) + self.prepare_binary_metadata(chunk_name) + patterns = chunks[chunk_name] + patterns += [r'baserock/%s\.' % chunk_name] + filename = self.filename(chunk_name) + self.msg('Creating binary for %s' % chunk_name) + morphlib.bins.create_chunk(self.destdir, filename, patterns, + self.ex, self.dump_memory_profile) + ret[chunk_name] = filename + files = os.listdir(self.destdir) if files: raise Exception('DESTDIR %s is not empty: %s' % @@ -294,18 +320,15 @@ class Stratum(BinaryBlob): def build(self): os.mkdir(self.destdir) ex = morphlib.execute.Execute(self.destdir, self.msg) - self.build_watch.start('unpack-chunks') - for chunk_name, filename in self.built: - self.msg('Unpacking chunk %s' % chunk_name) - morphlib.bins.unpack_binary(filename, self.destdir, ex, - as_fakeroot=True) - self.build_watch.stop('unpack-chunks') - self.prepare_binary_metadata(self.morph.name) - self.build_watch.start('create-binary') - self.msg('Creating binary for %s' % self.morph.name) - filename = self.filename(self.morph.name) - morphlib.bins.create_stratum(self.destdir, filename, ex) - self.build_watch.stop('create-binary') + with self.build_watch('unpack-chunks'): + for chunk_name, filename in self.built: + self.msg('Unpacking chunk %s' % chunk_name) + morphlib.bins.unpack_binary(filename, self.destdir, ex) + with self.build_watch('create-binary'): + self.prepare_binary_metadata(self.morph.name) + self.msg('Creating binary for %s' % self.morph.name) + filename = self.filename(self.morph.name) + morphlib.bins.create_stratum(self.destdir, filename, ex) return { self.morph.name: filename } @@ -323,118 +346,106 @@ class System(BinaryBlob): self.ex = morphlib.execute.Execute(self.tempdir.dirname, self.msg) # Create image. - self.build_watch.start('create-image') - image_name = self.tempdir.join('%s.img' % self.morph.name) - self.ex.runv(['qemu-img', 'create', '-f', 'raw', image_name, - self.morph.disk_size]) - self.build_watch.stop('create-image') + with self.build_watch('create-image'): + image_name = self.tempdir.join('%s.img' % self.morph.name) + self.ex.runv(['qemu-img', 'create', '-f', 'raw', image_name, + self.morph.disk_size]) # Partition it. - self.build_watch.start('partition-image') - self.ex.runv(['parted', '-s', image_name, 'mklabel', 'msdos'], - as_root=True) - self.ex.runv(['parted', '-s', image_name, 'mkpart', 'primary', - '0%', '100%'], as_root=True) - self.ex.runv(['parted', '-s', image_name, 'set', '1', 'boot', 'on'], - as_root=True) - self.build_watch.stop('partition-image') + with self.build_watch('partition-image'): + self.ex.runv(['parted', '-s', image_name, 'mklabel', 'msdos']) + self.ex.runv(['parted', '-s', image_name, 'mkpart', 'primary', + '0%', '100%']) + self.ex.runv(['parted', '-s', image_name, 'set', '1', 'boot', 'on']) # Install first stage boot loader into MBR. - self.build_watch.start('install-mbr') - self.ex.runv(['install-mbr', image_name], as_root=True) - self.build_watch.stop('install-mbr') + with self.build_watch('install-mbr'): + self.ex.runv(['install-mbr', image_name]) # Setup device mapper to access the partition. - self.build_watch.start('setup-device-mapper') - out = self.ex.runv(['kpartx', '-av', image_name], as_root=True) - devices = [line.split()[2] - for line in out.splitlines() - if line.startswith('add map ')] - partition = '/dev/mapper/%s' % devices[0] - self.build_watch.stop('setup-device-mapper') + with self.build_watch('setup-device-mapper'): + out = self.ex.runv(['kpartx', '-av', image_name]) + devices = [line.split()[2] + for line in out.splitlines() + if line.startswith('add map ')] + partition = '/dev/mapper/%s' % devices[0] mount_point = None try: # Create filesystem. - self.build_watch.start('create-filesystem') - self.ex.runv(['mkfs', '-t', 'ext3', partition], as_root=True) - self.build_watch.stop('create-filesystem') + with self.build_watch('create-filesystem'): + self.ex.runv(['mkfs', '-t', 'ext3', partition]) # Mount it. - self.build_watch.start('mount-filesystem') - mount_point = self.tempdir.join('mnt') - os.mkdir(mount_point) - self.ex.runv(['mount', partition, mount_point], as_root=True) - self.build_watch.stop('mount-filesystem') + with self.build_watch('mount-filesystem'): + mount_point = self.tempdir.join('mnt') + os.mkdir(mount_point) + self.ex.runv(['mount', partition, mount_point]) # Unpack all strata into filesystem. - self.build_watch.start('unpack-strata') - for name, filename in self.built: - self.msg('unpack %s from %s' % (name, filename)) - self.ex.runv(['tar', '-C', mount_point, '-xf', filename], - as_root=True) - self.build_watch.stop('unpack-strata') + # Also, run ldconfig. + with self.build_watch('unpack-strata'): + for name, filename in self.built: + self.msg('unpack %s from %s' % (name, filename)) + self.ex.runv(['tar', '-C', mount_point, '-xf', filename]) + ldconfig(ex, mount_point) # Create fstab. - self.build_watch.start('create-fstab') - fstab = self.tempdir.join('mnt/etc/fstab') - # sorry about the hack, I wish I knew a better way - self.ex.runv(['tee', fstab], feed_stdin=''' + with self.build_watch('create-fstab'): + fstab = self.tempdir.join('mnt/etc/fstab') + # sorry about the hack, I wish I knew a better way + self.ex.runv(['tee', fstab], feed_stdin=''' proc /proc proc defaults 0 0 sysfs /sys sysfs defaults 0 0 /dev/sda1 / ext4 errors=remount-ro 0 1 -''', as_root=True, stdout=open(os.devnull,'w')) - self.build_watch.stop('create-fstab') +''', stdout=open(os.devnull,'w')) # Install extlinux bootloader. - self.build_watch.start('install-bootloader') - conf = os.path.join(mount_point, 'extlinux.conf') - logging.debug('configure extlinux %s' % conf) - self.ex.runv(['tee', conf], feed_stdin=''' + with self.build_watch('install-bootloader'): + conf = os.path.join(mount_point, 'extlinux.conf') + logging.debug('configure extlinux %s' % conf) + self.ex.runv(['tee', conf], feed_stdin=''' default linux timeout 1 label linux kernel /vmlinuz append root=/dev/sda1 init=/sbin/init quiet rw -''', as_root=True, stdout=open(os.devnull, 'w')) +''', stdout=open(os.devnull, 'w')) - self.ex.runv(['extlinux', '--install', mount_point], as_root=True) - - # Weird hack that makes extlinux work. There is a bug somewhere. - self.ex.runv(['sync']) - import time; time.sleep(2) - self.build_watch.stop('install-bootloader') + self.ex.runv(['extlinux', '--install', mount_point]) + + # Weird hack that makes extlinux work. + # FIXME: There is a bug somewhere. + self.ex.runv(['sync']) + import time; time.sleep(2) # Unmount. - self.build_watch.start('unmount-filesystem') - self.ex.runv(['umount', mount_point], as_root=True) - self.build_watch.stop('unmount-filesystem') + with self.build_watch('unmount-filesystem'): + self.ex.runv(['umount', mount_point]) except BaseException, e: # Unmount. if mount_point is not None: try: - self.ex.runv(['umount', mount_point], as_root=True) + self.ex.runv(['umount', mount_point]) except Exception: pass # Undo device mapping. try: - self.ex.runv(['kpartx', '-d', image_name], as_root=True) + self.ex.runv(['kpartx', '-d', image_name]) except Exception: pass raise # Undo device mapping. - self.build_watch.start('undo-device-mapper') - self.ex.runv(['kpartx', '-d', image_name], as_root=True) - self.build_watch.stop('undo-device-mapper') + with self.build_watch('undo-device-mapper'): + self.ex.runv(['kpartx', '-d', image_name]) # Move image file to cache. - self.build_watch.start('cache-image') - filename = self.filename(self.morph.name) - self.ex.runv(['mv', image_name, filename]) - self.build_watch.stop('cache-image') + with self.build_watch('cache-image'): + filename = self.filename(self.morph.name) + self.ex.runv(['mv', image_name, filename]) return { self.morph.name: filename } @@ -451,6 +462,8 @@ class Builder(object): self.dump_memory_profile = app.dump_memory_profile self.cachedir = morphlib.cachedir.CacheDir(self.settings['cachedir']) self.indent = 0 + self.morph_loader = \ + morphlib.morphologyloader.MorphologyLoader(self.settings) def msg(self, text): spaces = ' ' * self.indent @@ -468,11 +481,8 @@ class Builder(object): self.dump_memory_profile('at start of build method') self.indent_more() self.msg('build %s|%s|%s' % (repo, ref, filename)) - base_url = self.settings['git-base-url'] - if not base_url.endswith('/'): - base_url += '/' - repo = urlparse.urljoin(base_url, repo) - morph = self.get_morph_from_git(repo, ref, filename) + morph = self.morph_loader.load(repo, ref, filename) + repo = morph.repo self.dump_memory_profile('after getting morph from git') if morph.kind == 'chunk': @@ -510,24 +520,22 @@ class Builder(object): self.dump_memory_profile('after installing chunk') built = builds else: - blob.build_watch.start('overall-build') - - blob.build_watch.start('build-needed') - self.build_needed(blob) - blob.build_watch.stop('build-needed') - self.dump_memory_profile('after building needed') - - self.msg('Building %s %s' % (morph.kind, morph.name)) - self.indent_more() - built = blob.build() - self.dump_memory_profile('after building blob') - self.indent_less() - for x in built: - self.msg('%s %s cached at %s' % (morph.kind, x, built[x])) - self.install_chunk(morph, x, built[x], blob.staging) - self.dump_memory_profile('after installing chunks') - - blob.build_watch.stop('overall-build') + with blob.build_watch('overall-build'): + + with blob.build_watch('build-needed'): + self.build_needed(blob) + self.dump_memory_profile('after building needed') + + self.msg('Building %s %s' % (morph.kind, morph.name)) + self.indent_more() + built = blob.build() + self.dump_memory_profile('after building blob') + self.indent_less() + for x in built: + self.msg('%s %s cached at %s' % (morph.kind, x, built[x])) + self.install_chunk(morph, x, built[x], blob.staging) + self.dump_memory_profile('after installing chunks') + blob.save_build_times() self.indent_less() @@ -548,26 +556,18 @@ class Builder(object): if self.settings['bootstrap']: self.msg('Unpacking chunk %s onto system' % chunk_name) ex = morphlib.execute.Execute('/', self.msg) - morphlib.bins.unpack_binary(chunk_filename, '/', ex, as_root=True) + morphlib.bins.unpack_binary(chunk_filename, '/', ex) + ldconfig(ex, '/') else: self.msg('Unpacking chunk %s into staging' % chunk_name) ex = morphlib.execute.Execute(staging_dir, self.msg) - morphlib.bins.unpack_binary(chunk_filename, staging_dir, ex, - as_root=True) + morphlib.bins.unpack_binary(chunk_filename, staging_dir, ex) + ldconfig(ex, staging_dir) - def get_morph_from_git(self, repo, ref, filename): - morph_text = morphlib.git.get_morph_text(repo, ref, filename) - f = StringIO.StringIO(morph_text) - scheme, netlock, path, params, query, frag = urlparse.urlparse(repo) - f.name = os.path.join(path, filename) - morph = morphlib.morphology.Morphology(f, - self.settings['git-base-url']) - return morph - def get_cache_id(self, repo, ref, morph_filename): logging.debug('get_cache_id(%s, %s, %s)' % (repo, ref, morph_filename)) - morph = self.get_morph_from_git(repo, ref, morph_filename) + morph = self.morph_loader.load(repo, ref, morph_filename) if morph.kind == 'chunk': kids = [] elif morph.kind == 'stratum': diff --git a/morphlib/execute.py b/morphlib/execute.py index aedf8d4f..038a5cfd 100644 --- a/morphlib/execute.py +++ b/morphlib/execute.py @@ -39,35 +39,11 @@ class Execute(object): self._setup_env() self.dirname = dirname self.msg = msg - self._fakeroot_session = None - - def __del__(self): # pragma: no cover - try: - object.__del__(self) - except AttributeError: - pass - if self._fakeroot_session: - os.remove(self._fakeroot_session) def _setup_env(self): self.env = dict(os.environ) - def _prefix(self, argv, as_root, as_fakeroot): - if as_root: - if os.getuid() == 0: - prefix = ['env'] - else: - prefix = ['sudo'] - envs = ["%s=%s" % x for x in self.env.iteritems()] - argv = prefix + envs + argv - elif as_fakeroot and os.getuid() != 0: - if not self._fakeroot_session: - self._fakeroot_session = tempfile.mkstemp()[1] - argv = ['fakeroot', '-i', self._fakeroot_session, '-s', - self._fakeroot_session, '--'] + argv - return argv - - def run(self, commands, as_root=False, as_fakeroot=False, _log=True): + def run(self, commands, _log=True): '''Execute a list of commands. If a command fails (returns non-zero exit code), the rest are @@ -79,7 +55,6 @@ class Execute(object): for command in commands: self.msg('# %s' % command) argv = ['sh', '-c', command] - argv = self._prefix(argv, as_root, as_fakeroot) logging.debug('run: argv=%s' % repr(argv)) logging.debug('run: env=%s' % repr(self.env)) logging.debug('run: cwd=%s' % repr(self.dirname)) @@ -98,8 +73,7 @@ class Execute(object): stdouts.append(out) return stdouts - def runv(self, argv, feed_stdin=None, as_root=False, as_fakeroot=False, - _log=True, **kwargs): + def runv(self, argv, feed_stdin=None, _log=True, **kwargs): '''Run a command given as a list of argv elements. Return standard output. Raise ``CommandFailure`` if the command @@ -117,7 +91,6 @@ class Execute(object): if 'env' not in kwargs: kwargs['env'] = self.env - argv = self._prefix(argv, as_root, as_fakeroot) logging.debug('runv: argv=%s' % repr(argv)) logging.debug('runv: env=%s' % repr(self.env)) logging.debug('runv: cwd=%s' % repr(self.dirname)) diff --git a/morphlib/execute_tests.py b/morphlib/execute_tests.py index 86db9c25..da6f5d49 100644 --- a/morphlib/execute_tests.py +++ b/morphlib/execute_tests.py @@ -52,9 +52,3 @@ class ExecuteTests(unittest.TestCase): def test_runv_sets_working_directory(self): self.assertEqual(self.e.runv(['pwd']), '/\n') - def test_runs_as_fakeroot_when_requested(self): - self.assertEqual(self.e.run(['id -u'], as_fakeroot=True), ['0\n']) - - def test_runvs_as_fakeroot_when_requested(self): - self.assertEqual(self.e.runv(['id', '-u'], as_fakeroot=True), '0\n') - diff --git a/morphlib/morphology.py b/morphlib/morphology.py index f04e9f7c..8f787939 100644 --- a/morphlib/morphology.py +++ b/morphlib/morphology.py @@ -22,7 +22,10 @@ class Morphology(object): '''Represent a morphology: description of how to build binaries.''' - def __init__(self, fp, baseurl=None): + def __init__(self, repo, ref, fp, baseurl=None): + self.repo = repo + self.ref = ref + self._fp = fp self._baseurl = baseurl or '' self._load() @@ -62,7 +65,7 @@ class Morphology(object): @property def build_depends(self): - return self._dict.get('build-depends', []) + return self._dict.get('build-depends', None) @property def build_system(self): diff --git a/morphlib/morphology_tests.py b/morphlib/morphology_tests.py index 0119ebc6..cac80798 100644 --- a/morphlib/morphology_tests.py +++ b/morphlib/morphology_tests.py @@ -32,6 +32,7 @@ class MorphologyTests(unittest.TestCase): def test_accepts_valid_chunk_morphology(self): morph = morphlib.morphology.Morphology( + 'repo', 'ref', MockFile(''' { "name": "hello", @@ -57,6 +58,10 @@ class MorphologyTests(unittest.TestCase): ] } }''')) + + self.assertEqual(morph.repo, 'repo') + self.assertEqual(morph.ref, 'ref') + self.assertEqual(morph.filename, 'mockfile') self.assertEqual(morph.name, 'hello') self.assertEqual(morph.kind, 'chunk') self.assertEqual(morph.description, 'desc') @@ -77,6 +82,7 @@ class MorphologyTests(unittest.TestCase): def test_build_system_defaults_to_None(self): morph = morphlib.morphology.Morphology( + 'repo', 'ref', MockFile(''' { "name": "hello", @@ -86,6 +92,7 @@ class MorphologyTests(unittest.TestCase): def test_max_jobs_defaults_to_None(self): morph = morphlib.morphology.Morphology( + 'repo', 'ref', MockFile(''' { "name": "hello", @@ -95,6 +102,7 @@ class MorphologyTests(unittest.TestCase): def test_accepts_valid_stratum_morphology(self): morph = morphlib.morphology.Morphology( + 'repo', 'ref', MockFile(''' { "name": "hello", @@ -121,6 +129,7 @@ class MorphologyTests(unittest.TestCase): def test_accepts_valid_system_morphology(self): morph = morphlib.morphology.Morphology( + 'repo', 'ref', MockFile(''' { "name": "hello", @@ -145,6 +154,7 @@ class StratumRepoTests(unittest.TestCase): def stratum(self, repo): return morphlib.morphology.Morphology( + 'repo', 'ref', MockFile(''' { "name": "hello", diff --git a/morphlib/morphologyloader.py b/morphlib/morphologyloader.py new file mode 100644 index 00000000..a227214f --- /dev/null +++ b/morphlib/morphologyloader.py @@ -0,0 +1,57 @@ +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import os +import StringIO +import urlparse + +import morphlib + + +class MorphologyLoader(object): + + '''Load morphologies from git and parse them into Morphology objects.''' + + def __init__(self, settings): + self.settings = settings + self.morphologies = {} + + def load(self, repo, ref, filename): + base_url = self.settings['git-base-url'] + if not base_url.endswith('/'): + base_url += '/' + repo = urlparse.urljoin(base_url, repo) + + key = (repo, ref, filename) + + if key in self.morphologies: + return self.morphologies[key] + else: + morph = self._get_morph_from_git(repo, ref, filename) + self.morphologies[key] = morph + return morph + + def _get_morph_text(self, repo, ref, filename): # pragma: no cover + return morphlib.git.get_morph_text(repo, ref, filename) + + def _get_morph_from_git(self, repo, ref, filename): + morph_text = self._get_morph_text(repo, ref, filename) + scheme, netlock, path, params, query, frag = urlparse.urlparse(repo) + f = StringIO.StringIO(morph_text) + f.name = os.path.join(path, filename) + morph = morphlib.morphology.Morphology(repo, ref, f, + self.settings['git-base-url']) + return morph diff --git a/morphlib/morphologyloader_tests.py b/morphlib/morphologyloader_tests.py new file mode 100644 index 00000000..f9a03915 --- /dev/null +++ b/morphlib/morphologyloader_tests.py @@ -0,0 +1,47 @@ +# Copyright (C) 2012 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import os +import StringIO +import unittest + +import morphlib + + +class MorphologyLoaderTests(unittest.TestCase): + + def test_load_twice_verify_same_morph_object(self): + settings = { 'git-base-url': '' } + loader = morphlib.morphologyloader.MorphologyLoader(settings) + loader._get_morph_text = self.get_morph_text + + morph1 = loader.load('repo', 'ref', 'hello.morph') + morph2 = loader.load('repo', 'ref', 'hello.morph') + self.assertEqual(morph1, morph2) + + def get_morph_text(self, repo, ref, filename): + return (''' + { + "name": "foo", + "kind": "stratum", + "sources": [ + { + "name": "bar", + "repo": "bar", + "ref": "master" + } + ] + }''') diff --git a/morphlib/stopwatch.py b/morphlib/stopwatch.py index 492738d5..7446f897 100644 --- a/morphlib/stopwatch.py +++ b/morphlib/stopwatch.py @@ -23,6 +23,7 @@ class Stopwatch(object): def __init__(self): self.ticks = {} + self.context_stack = [] def tick(self, reference_object, name): if not reference_object in self.ticks: @@ -56,3 +57,17 @@ class Stopwatch(object): return (delta.days * 24 * 3600 + delta.seconds + operator.truediv(delta.microseconds, 10**6)) + + def __call__(self, reference_object): + self.context_stack.append(reference_object) + return self + + def __enter__(self): + self.start(self.context_stack[-1]) + return self + + def __exit__(self, *args): + self.stop(self.context_stack[-1]) + self.context_stack.pop() + return False # cause any exception to be re-raised + diff --git a/morphlib/stopwatch_tests.py b/morphlib/stopwatch_tests.py index 1a899f41..d4f1e3dd 100644 --- a/morphlib/stopwatch_tests.py +++ b/morphlib/stopwatch_tests.py @@ -56,3 +56,20 @@ class StopwatchTests(unittest.TestCase): self.assertEqual(our_delta, watch_delta) assert self.stopwatch.start_stop_seconds('start-stop') > 0 + + def test_with(self): + with self.stopwatch('foo'): + pass + self.assert_(self.stopwatch.start_stop_seconds('foo') < 1.0) + + def test_with_within_with(self): + with self.stopwatch('foo'): + with self.stopwatch('bar'): + pass + self.assert_(self.stopwatch.start_time('foo')) + self.assert_(self.stopwatch.stop_time('foo')) + self.assert_(self.stopwatch.start_time('bar')) + self.assert_(self.stopwatch.stop_time('bar')) + self.assert_(self.stopwatch.start_stop_seconds('foo') < 1.0) + self.assert_(self.stopwatch.start_stop_seconds('bar') < 1.0) + |