diff options
author | Lars Wirzenius <lars.wirzenius@codethink.co.uk> | 2012-05-02 18:03:44 +0100 |
---|---|---|
committer | Lars Wirzenius <lars.wirzenius@codethink.co.uk> | 2012-05-02 18:03:44 +0100 |
commit | dc7f4153ae6f74fe747117154e02a2198be6e8c2 (patch) | |
tree | 5ce9a2af402565749ecb8dd32e1c115d254be8d1 | |
parent | c70d96187f13d63a06f36ec96be2a6033e507e41 (diff) | |
download | morph-dc7f4153ae6f74fe747117154e02a2198be6e8c2.tar.gz |
Get rid of the old internal morph APIs
-rwxr-xr-x | morph | 2 | ||||
-rw-r--r-- | morphlib/blobs.py | 87 | ||||
-rw-r--r-- | morphlib/blobs_tests.py | 203 | ||||
-rw-r--r-- | morphlib/builddependencygraph.py | 314 | ||||
-rw-r--r-- | morphlib/builder.py | 699 | ||||
-rw-r--r-- | morphlib/builder2.py | 40 | ||||
-rw-r--r-- | morphlib/builder_tests.py | 165 | ||||
-rw-r--r-- | morphlib/localrepocache.py | 23 | ||||
-rw-r--r-- | morphlib/morphology.py | 131 | ||||
-rw-r--r-- | morphlib/morphology_tests.py | 206 | ||||
-rw-r--r-- | morphlib/morphologyloader.py | 72 | ||||
-rw-r--r-- | morphlib/morphologyloader_tests.py | 50 | ||||
-rw-r--r-- | morphlib/sourcemanager.py | 286 | ||||
-rw-r--r-- | morphlib/sourcemanager_tests.py | 193 | ||||
-rwxr-xr-x | tests/build-chunk-distributed-local.script | 29 | ||||
-rw-r--r-- | tests/build-chunk-distributed-local.stdout | 8 | ||||
-rwxr-xr-x | tests/build-stratum-distributed-local.script | 24 | ||||
-rw-r--r-- | tests/build-stratum-distributed-local.stdout | 7 |
18 files changed, 57 insertions, 2482 deletions
@@ -27,8 +27,6 @@ import tempfile import morphlib from morphlib import buildworker from morphlib import buildcontroller -from morphlib.morphologyloader import MorphologyLoader -from morphlib.builddependencygraph import BuildDependencyGraph defaults = { diff --git a/morphlib/blobs.py b/morphlib/blobs.py deleted file mode 100644 index 1c8686b0..00000000 --- a/morphlib/blobs.py +++ /dev/null @@ -1,87 +0,0 @@ -# Copyright (C) 2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -class Blob(object): - - @staticmethod - def create_blob(morph): - if morph.kind == 'stratum': - return Stratum(morph) - elif morph.kind == 'chunk': - return Chunk(morph) - elif morph.kind == 'system': - return System(morph) - else: - raise TypeError('Morphology %s has an unknown type: %s' % - (morph.filename, morph.kind)) - - def __init__(self, morph): - self.parents = [] - self.morph = morph - self.dependencies = [] - self.dependents = [] - - def add_parent(self, parent): - if not parent in self.parents: - self.parents.append(parent) - - def remove_parent(self, parent): - if parent in self.parents: - self.parents.remove(parent) - - def add_dependency(self, other): - if not other in self.dependencies: - self.dependencies.append(other) - if not self in other.dependents: - other.dependents.append(self) - - def remove_dependency(self, other): - self.dependencies.remove(other) - other.dependents.remove(self) - - def depends_on(self, other): - return other in self.dependencies - - @property - def chunks(self): - if self.morph.chunks: - return self.morph.chunks - else: - return { self.morph.name: ['.'] } - - def __eq__(self, other): - return self.morph == other.morph - - def __hash__(self): - return hash(self.morph) - - def __str__(self): # pragma: no cover - return str(self.morph) - - -class Chunk(Blob): - - pass - - -class Stratum(Blob): - - pass - - -class System(Blob): - - pass diff --git a/morphlib/blobs_tests.py b/morphlib/blobs_tests.py deleted file mode 100644 index b1062459..00000000 --- a/morphlib/blobs_tests.py +++ /dev/null @@ -1,203 +0,0 @@ -# Copyright (C) 2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import unittest - -import morphlib - - -class FakeChunkMorph(object): - @property - def kind(self): - return 'chunk' - - -class FakeStratumMorph(object): - @property - def kind(self): - return 'stratum' - - -class BlobsTests(unittest.TestCase): - - def test_create_a_chunk_blob(self): - morph = FakeChunkMorph() - chunk = morphlib.blobs.Blob.create_blob(morph) - self.assertTrue(isinstance(chunk, morphlib.blobs.Chunk)) - self.assertEqual(morph, chunk.morph) - - def test_create_a_stratum_blob(self): - morph = FakeStratumMorph() - stratum = morphlib.blobs.Blob.create_blob(morph) - self.assertTrue(isinstance(stratum, morphlib.blobs.Stratum)) - self.assertEqual(morph, stratum.morph) - - def test_create_a_system_blob(self): - class FakeSystemMorph(object): - @property - def kind(self): - return 'system' - - morph = FakeSystemMorph() - system = morphlib.blobs.Blob.create_blob(morph) - self.assertTrue(isinstance(system, morphlib.blobs.System)) - self.assertEqual(morph, system.morph) - - def test_create_an_invalid_blob(self): - class FakeInvalidMorph(object): - @property - def kind(self): - return 'invalid' - - @property - def filename(self): - return '/foo/bar/baz.morph' - - morph = FakeInvalidMorph() - self.assertRaises(TypeError, morphlib.blobs.Blob.create_blob, morph) - - def test_blob_with_parents(self): - blob1 = morphlib.blobs.Blob(FakeChunkMorph()) - blob2 = morphlib.blobs.Blob(FakeStratumMorph()) - blob3 = morphlib.blobs.Blob(FakeStratumMorph()) - - self.assertEqual(len(blob1.parents), 0) - - blob1.add_parent(blob2) - self.assertTrue(blob2 in blob1.parents) - self.assertTrue(blob3 not in blob1.parents) - self.assertEqual(len(blob1.parents), 1) - - blob1.add_parent(blob3) - self.assertTrue(blob2 in blob1.parents) - self.assertTrue(blob3 in blob1.parents) - self.assertEqual(len(blob1.parents), 2) - - blob1.remove_parent(blob2) - self.assertTrue(blob2 not in blob1.parents) - self.assertTrue(blob3 in blob1.parents) - self.assertEqual(len(blob1.parents), 1) - - blob1.remove_parent(blob3) - self.assertTrue(blob2 not in blob1.parents) - self.assertTrue(blob3 not in blob1.parents) - self.assertEqual(len(blob1.parents), 0) - - def test_blob_add_remove_dependency(self): - blob1 = morphlib.blobs.Blob(None) - blob2 = morphlib.blobs.Blob(None) - - self.assertEqual(len(blob1.dependencies), 0) - self.assertEqual(len(blob2.dependencies), 0) - - blob1.add_dependency(blob2) - - self.assertTrue(blob2 in blob1.dependencies) - self.assertTrue(blob1 in blob2.dependents) - - self.assertTrue(blob1.depends_on(blob2)) - - blob2.add_dependency(blob1) - - self.assertTrue(blob2 in blob1.dependencies) - self.assertTrue(blob1 in blob2.dependents) - self.assertTrue(blob1 in blob2.dependencies) - self.assertTrue(blob2 in blob1.dependents) - - self.assertTrue(blob1.depends_on(blob2)) - self.assertTrue(blob2.depends_on(blob1)) - - blob1.remove_dependency(blob2) - - self.assertTrue(blob2 not in blob1.dependencies) - self.assertTrue(blob1 not in blob2.dependents) - self.assertTrue(blob1 in blob2.dependencies) - self.assertTrue(blob2 in blob1.dependents) - - self.assertFalse(blob1.depends_on(blob2)) - self.assertTrue(blob2.depends_on(blob1)) - - blob2.remove_dependency(blob1) - - self.assertTrue(blob2 not in blob1.dependencies) - self.assertTrue(blob1 not in blob2.dependents) - self.assertTrue(blob1 not in blob2.dependencies) - self.assertTrue(blob2 not in blob1.dependents) - - self.assertFalse(blob1.depends_on(blob2)) - self.assertFalse(blob2.depends_on(blob1)) - - def test_hashing_and_equality_checks(self): - morph = FakeChunkMorph() - blob1 = morphlib.blobs.Blob.create_blob(morph) - blob2 = morphlib.blobs.Blob.create_blob(morph) - blob3 = morphlib.blobs.Blob.create_blob(FakeChunkMorph()) - - self.assertEqual(hash(blob1), hash(blob2)) - self.assertEqual(blob1, blob2) - - self.assertNotEqual(hash(blob1), hash(blob3)) - self.assertNotEqual(blob1, blob3) - - def test_chunks(self): - settings = { 'git-base-url': [] } - loader = morphlib.morphologyloader.MorphologyLoader(settings) - loader._get_morph_text = self.get_morph_text - - class FakeTreeish(object): - def __init__(self): - self.original_repo = 'test-repo' - faketreeish = FakeTreeish() - - faketreeish.original_repo = 'hello' - stratum_morph = loader.load(faketreeish, 'foo.morph') - stratum = morphlib.blobs.Stratum(stratum_morph) - self.assertEquals(len(stratum.chunks), 1) - self.assertTrue('foo' in stratum.chunks) - self.assertEqual(['.'], stratum.chunks['foo']) - - chunk_morph = loader.load(faketreeish, 'bar.morph') - chunk = morphlib.blobs.Chunk(chunk_morph) - self.assertEqual(len(chunk.chunks), 2) - self.assertTrue('include' in chunk.chunks) - self.assertEqual(chunk.chunks['include'], ['include/']) - self.assertTrue('src' in chunk.chunks) - self.assertEqual(chunk.chunks['src'], ['src/']) - - def get_morph_text(self, treeish, filename): - if filename == 'foo.morph': - return (''' - { - "name": "foo", - "kind": "stratum", - "sources": [ - { - "name": "bar", - "repo": "bar", - "ref": "master" - } - ] - }''') - else: - return (''' - { - "name": "bar", - "kind": "chunk", - "chunks": { - "include": [ "include/" ], - "src": [ "src/" ] - } - }''') diff --git a/morphlib/builddependencygraph.py b/morphlib/builddependencygraph.py deleted file mode 100644 index 7fd93b66..00000000 --- a/morphlib/builddependencygraph.py +++ /dev/null @@ -1,314 +0,0 @@ -# Copyright (C) 2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import collections -import os - -import morphlib - - -class BuildDependencyGraph(object): # pragma: no cover - - '''This class constructs a build dependency graph from an input morphology. - - Given a chunk, stratum or system morphology, this class constructs a build - dependency graph and provides ways to traverse it. It also provides a - method to transform the dependency graph into groups of items that are - independent and can be built in parallel. - - ''' - - def __init__(self, source_manager, morph_loader, repo, ref, filename): - self.source_manager = source_manager - self.morph_loader = morph_loader - self.root_repo = repo - self.root_ref = ref - self.root_filename = filename - self.blobs = set() - - def create_blob(self, treeish, filename, chunk_name=None): - '''Creates a blob from a morphology.''' - - morph = self.morph_loader.load(treeish, filename, chunk_name) - - if morph.kind == 'stratum': - return morphlib.blobs.Stratum(morph) - elif morph.kind == 'chunk': - return morphlib.blobs.Chunk(morph) - else: - return morphlib.blobs.System(morph) - - def get_blob(self, treeish, filename, chunk_name=None): - '''Takes a repo, ref, filename and looks up the blob for them. - - Loads the corresponding morphology and chunk/stratum/system object - on-demand if it is not cached yet. - - ''' - - key = (treeish, filename) - blob = self.cached_blobs.get(key, None) - if not blob: - blob = self.create_blob(treeish, filename, chunk_name) - self.cached_blobs[key] = blob - return blob - - def resolve(self): - '''Constructs the graph by resolving dependencies recursively.''' - - self.cached_blobs = {} - self.blobs = set() - - root = self.resolve_root() - self.resolve_strata() - self.resolve_chunks() - return root - - def resolve_root(self): - # prepare the repo, load the morphology and blob information - treeish = self.source_manager.get_treeish(self.root_repo, - self.root_ref) - root = self.get_blob(treeish, self.root_filename) - self.blobs.add(root) - - # load all strata the morph depends on (only if it's a system image) - if root.morph.kind == 'system': - for stratum_name in root.morph.strata: - filename = '%s.morph' % stratum_name - stratum = self.get_blob(treeish, filename) - root.add_dependency(stratum) - stratum.add_parent(root) - self.blobs.add(stratum) - - return root - - def resolve_strata(self): - '''Generates a dependency graph of strata recursively. - - It starts with the input morphology and attempts to resolve all its - build dependencies recursively using breadth-first search. Morphologies - and blobs are loaded from disk on demand. - - ''' - - # start the BFS at all input strata - queue = collections.deque() - strata = [x for x in self.blobs if x.morph.kind == 'stratum'] - queue.extend(strata) - - while len(queue) > 0: - stratum = queue.popleft() - - # the DFS recursion ends whenever we have a stratum - # that depends on nothing else - if not stratum.morph.build_depends: - continue - - # verify that the build-depends format is valid - if isinstance(stratum.morph.build_depends, list): - for depname in stratum.morph.build_depends: - # load the dependency stratum on demand - depstratum = self.get_blob(stratum.morph.treeish, - '%s.morph' % depname) - self.blobs.add(depstratum) - - # add the dependency stratum to the graph - stratum.add_dependency(depstratum) - queue.append(depstratum) - else: - raise Exception('%s uses an invalid "build-depends" format' - % stratum) - - def resolve_chunks(self): - '''Resolves all chunks of the strata and inserts them into the graph. - - Starting with a dependency graph of strata, this method fills the - graph with all contained chunks and creates dependencies where - appropriate. Chunk morphologies and blobs are loaded on demand. - - ''' - - blobs = list(self.blobs) - for blob in blobs: - if isinstance(blob, morphlib.blobs.Stratum): - self.resolve_stratum_chunks(blob) - - def resolve_stratum_chunks(self, stratum): - # the set of chunks contained in the stratum - stratum_chunks = set() - - # dictionary that maps chunk names to chunks - name_to_chunk = {} - - # create objects for all chunks in the stratum - for i in xrange(0, len(stratum.morph.sources)): - source = stratum.morph.sources[i] - - # construct a tuple for loading the chunk - repo = source['repo'] - ref = source['ref'] - filename = '%s.morph' % (source['morph'] - if 'morph' in source - else source['name']) - - # load the chunk on demand - treeish = self.source_manager.get_treeish(repo, ref) - chunk = self.get_blob(treeish, filename, source['name']) - chunk.add_parent(stratum) - - # store (name -> chunk) association to avoid loading the chunk - # twice - name_to_chunk[source['name']] = chunk - - # read the build-depends information - build_depends = (source['build-depends'] - if 'build-depends' in source - else None) - - # turn build-depends into proper dependencies in the graph - if build_depends is None: - # chunks with no build-depends implicitly depend on all - # chunks listed earlier in the same stratum - for dependency in stratum_chunks: - if not dependency is chunk: - chunk.add_dependency(dependency) - elif isinstance(build_depends, list): - for depname in build_depends: - if depname in name_to_chunk: - dependency = name_to_chunk[depname] - if not dependency is chunk: - chunk.add_dependency(dependency) - else: - raise Exception('%s: source %s references %s before ' - 'it is defined' % - (stratum.morph.filename, - source['name'], - depname)) - else: - raise Exception('%s: source %s uses an invalid build-depends ' - 'format' % - (stratum.morph.filename, source['name'])) - - # add the chunk to stratum and graph - stratum_chunks.add(chunk) - self.blobs.add(chunk) - - # make the chunks in this stratum depend on all - # strata that need to be built first - for chunk in stratum_chunks: - for dependency in stratum.dependencies: - chunk.add_dependency(dependency) - - ## clear the dependencies of the stratum - #for dependency in set(stratum.dependencies): - # stratum.remove_dependency(dependency) - - # make the stratum depend on all its chunks - for chunk in stratum_chunks: - stratum.add_dependency(chunk) - - def build_order(self): - '''Returns a queue of build groups in a valid order. - - This function computes a topological sorting of the dependency graph - and generates a deque of groups, each of which contains a set of items - that are independent and can be built in parallel. The items in each - group are guaranteed to depend only on items in previous groups. - - ''' - - sorting = self.compute_topological_sorting() - groups = collections.deque() - - # create the first group - group = set() - groups.append(group) - - # traverse the graph in topological order - for blob in sorting: - # add the current item to the current group, or a new group - # if one of its dependencies is in the current one - create_group = False - for dependency in blob.dependencies: - if dependency in group: - create_group = True - if create_group: - group = set() - groups.append(group) - group.add(blob) - - # return the set of blobs and the build groups - return set(self.blobs), groups - - def compute_topological_sorting(self): - '''Computes a topological sorting of the dependency graph. - - A topological sorting basically is the result of a series of - breadth-first searches starting at each leaf node (blobs with no - dependencies). Blobs are added to the sorting as soon as all their - dependencies have been added (which means that by then, all - dependencies are satisfied). - - For more information, see - http://en.wikipedia.org/wiki/Topological_sorting. - - ''' - - # map blobs to sets of satisfied dependencies. this is to detect when - # we can actually add blobs to the BFS queue. rather than dropping - # links between nodes, like most topological sorting algorithms do, - # we simply remember all satisfied dependencies and check if all - # of them are met repeatedly - satisfied_dependencies = {} - - # create an empty sorting - sorting = collections.deque() - - # create a set of leafs to start the DFS from - leafs = collections.deque() - for blob in self.blobs: - satisfied_dependencies[blob] = set() - if len(blob.dependencies) == 0: - leafs.append(blob) - - while len(leafs) > 0: - # fetch a leaf blob from the DFS queue - blob = leafs.popleft() - - # add it to the sorting - sorting.append(blob) - - # mark this dependency as resolved - for dependent in blob.dependents: - satisfied_dependencies[dependent].add(blob) - - # add the dependent blob as a leaf if all - # its dependencies have been resolved - has = len(satisfied_dependencies[dependent]) - needs = len(dependent.dependencies) - if has == needs: - leafs.append(dependent) - - # if not all dependencies were resolved on the way, we - # have found at least one cyclic dependency - if len(sorting) < len(self.blobs): - raise Exception('Cyclic dependencies found in the dependency ' - 'graph of %s|%s|%s' % - (self.root_repo, - self.root_ref, - self.root_filename)) - - return sorting diff --git a/morphlib/builder.py b/morphlib/builder.py deleted file mode 100644 index 336a25b3..00000000 --- a/morphlib/builder.py +++ /dev/null @@ -1,699 +0,0 @@ -# Copyright (C) 2011-2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import collections -import json -import logging -import os -import shutil -import time - -import morphlib - - -def ldconfig(ex, rootdir): # pragma: no cover - '''Run ldconfig for the filesystem below ``rootdir``. - - Essentially, ``rootdir`` specifies the root of a new system. - Only directories below it are considered. - - ``etc/ld.so.conf`` below ``rootdir`` is assumed to exist and - be populated by the right directories, and should assume - the root directory is ``rootdir``. Example: if ``rootdir`` - is ``/tmp/foo``, then ``/tmp/foo/etc/ld.so.conf`` should - contain ``/lib``, not ``/tmp/foo/lib``. - - The ldconfig found via ``$PATH`` is used, not the one in ``rootdir``, - since in bootstrap mode that might not yet exist, the various - implementations should be compatible enough. - - ''' - - conf = os.path.join(rootdir, 'etc', 'ld.so.conf') - if os.path.exists(conf): - logging.debug('Running ldconfig for %s' % rootdir) - cache = os.path.join(rootdir, 'etc', 'ld.so.cache') - - # The following trickery with $PATH is necessary during the Baserock - # bootstrap build: we are not guaranteed that PATH contains the - # directory (/sbin conventionally) that ldconfig is in. Then again, - # it might, and if so, we don't want to hardware a particular - # location. So we add the possible locations to the end of $PATH - # and restore that aftewards. - old_path = ex.env['PATH'] - ex.env['PATH'] = '%s:/sbin:/usr/sbin:/usr/local/sbin' % old_path - ex.runv(['ldconfig', '-r', rootdir]) - ex.env['PATH'] = old_path - else: - logging.debug('No %s, not running ldconfig' % conf) - - -class Factory(object): - - '''Build Baserock binaries.''' - - def __init__(self, tempdir): - self._tempdir = tempdir - self.staging = None - - def create_staging(self): - '''Create the staging area.''' - self.staging = self._tempdir.join('staging') - os.mkdir(self.staging) - - def remove_staging(self): - '''Remove the staging area.''' - shutil.rmtree(self.staging) - self.staging = None - - def _unpack_binary(self, binary, dirname): - '''Unpack binary into a given directory.''' - ex = morphlib.execute.Execute('/', logging.debug) - morphlib.bins.unpack_binary(binary, dirname, ex) - - def unpack_binary_from_file(self, filename): - '''Unpack a binary package from a file, given its name.''' - self._unpack_binary(filename, self.staging) - - def unpack_binary_from_file_onto_system(self, filename): - '''Unpack contents of a binary package onto the running system. - - DANGER, WILL ROBINSON! This WILL modify your running system. - It should only be used during bootstrapping. - - ''' - - self._unpack_binary(filename, '/') - - def unpack_sources(self, treeish, srcdir): - '''Get sources from to a source directory, for building. - - The git repository and revision are given via a Treeish object. - The source directory must not exist. - - ''' - - def msg(s): - pass - - def extract_treeish(treeish, destdir): - logging.debug('Extracting %s into %s' % (treeish.repo, destdir)) - if not os.path.exists(destdir): - os.mkdir(destdir) - morphlib.git.copy_repository(treeish.repo, destdir, msg) - morphlib.git.checkout_ref(destdir, treeish.ref, msg) - morphlib.git.reset_workdir(destdir, msg) - return [(sub.treeish, os.path.join(destdir, sub.path)) - for sub in treeish.submodules] - - todo = [(treeish, srcdir)] - while todo: - treeish, srcdir = todo.pop() - todo += extract_treeish(treeish, srcdir) - self.set_mtime_recursively(srcdir) - - def set_mtime_recursively(self, root): - '''Set the mtime for every file in a directory tree to the same. - - We do this because git checkout does not set the mtime to anything, - and some projects (binutils, gperf for example) include formatted - documentation and try to randomly build things or not because of - the timestamps. This should help us get more reliable builds. - - ''' - - now = time.time() - for dirname, subdirs, basenames in os.walk(root, topdown=False): - for basename in basenames: - pathname = os.path.join(dirname, basename) - # we need the following check to ignore broken symlinks - if os.path.exists(pathname): - os.utime(pathname, (now, now)) - os.utime(dirname, (now, now)) - - -class BlobBuilder(object): # pragma: no cover - - def __init__(self, blob, factory, settings, cachedir, cache_key, tempdir, - env): - self.blob = blob - self.factory = factory - self.settings = settings - self.cachedir = cachedir - self.cache_key = cache_key - self.tempdir = tempdir - self.env = env - - self.logfile = None - self.stage_items = [] - self.real_msg = lambda s: None - self.dump_memory_profile = lambda msg: None - - self.destdir = os.path.join(self.factory.staging, - '%s.inst' % blob.morph.name) - - self.build_watch = morphlib.stopwatch.Stopwatch() - - def msg(self, text): - self.real_msg(text) - if self.logfile and not self.logfile.closed: - self.logfile.write('%s\n' % text) - - def builds(self): - raise NotImplemented() - - def build(self): - self.prepare_logfile() - with self.build_watch('overall-build'): - self.do_build() - self.save_build_times() - self.save_logfile() - - def filename(self, name): - return '%s.%s.%s' % (self.cachedir.name(self.cache_key), - self.blob.morph.kind, - name) - - def prepare_binary_metadata(self, blob_name): - '''Add metadata to a binary about to be built.''' - - self.msg('Adding metadata to %s' % blob_name) - meta = { - 'name': blob_name, - 'kind': self.blob.morph.kind, - 'description': self.blob.morph.description, - } - - dirname = os.path.join(self.destdir, 'baserock') - if not os.path.exists(dirname): - os.mkdir(dirname) - - filename = os.path.join(dirname, '%s.meta' % blob_name) - with open(filename, 'w') as f: - json.dump(meta, f, indent=4) - f.write('\n') - - def save_build_times(self): - meta = { - 'build-times': {} - } - for stage in self.build_watch.ticks.iterkeys(): - meta['build-times'][stage] = { - 'start': '%s' % self.build_watch.start_time(stage), - 'stop': '%s' % self.build_watch.stop_time(stage), - 'delta': '%.4f' % self.build_watch.start_stop_seconds(stage) - } - - self.msg('Writing metadata to the cache') - with self.cachedir.open(self.cache_key, suffix='.meta') as f: - json.dump(meta, f, indent=4) - f.write('\n') - - def prepare_logfile(self): - self.logfile = self.cachedir.open(self.cache_key, suffix='.log', - mode='w+', buffering=0) - - def save_logfile(self): - self.logfile.close() - - -class ChunkBuilder(BlobBuilder): # pragma: no cover - - def builds(self): - ret = {} - for chunk_name in self.blob.chunks: - ret[chunk_name] = self.filename(chunk_name) - return ret - - def do_build(self): - self.builddir = os.path.join(self.factory.staging, - '%s.build' % self.blob.morph.name) - self.msg('Creating build tree at %s' % self.builddir) - - self.ex = morphlib.execute.Execute(self.builddir, self.msg) - self.ex.env = self.env - - self.factory.unpack_sources(self.blob.morph.treeish, self.builddir) - - os.mkdir(self.destdir) - self.build_with_system_or_commands() - self.dump_memory_profile('after building chunk') - - chunks = self.create_chunks() - self.dump_memory_profile('after creating build chunks') - - # install all built items to the staging area - for name, filename in chunks: - self.msg('Fetching cached %s %s from %s' % - (self.blob.morph.kind, name, filename)) - self.install_chunk(name, filename) - self.dump_memory_profile('after installing chunk') - - def install_chunk(self, chunk_name, chunk_filename): - ex = morphlib.execute.Execute('/', self.msg) - if self.settings['bootstrap']: - self.msg('Unpacking item %s onto system' % chunk_name) - self.factory.unpack_binary_from_file_onto_system(chunk_filename) - ldconfig(ex, '/') - else: - self.msg('Unpacking chunk %s into staging' % chunk_name) - self.factory.unpack_binary_from_file(chunk_filename) - ldconfig(ex, self.factory.staging) - - def build_with_system_or_commands(self): - '''Run explicit commands or commands from build system. - - Use explicit commands, if given, and build system commands if one - has been specified. - - ''' - - bs_name = self.blob.morph.build_system - bs = morphlib.buildsystem.lookup_build_system(bs_name) - - def run_them(runner, what): - attr = '%s_commands' % what - cmds = getattr(self.blob.morph, attr, []) or bs[attr] - runner(what, cmds) - - run_them(self.run_sequentially, 'configure') - run_them(self.run_in_parallel, 'build') - run_them(self.run_sequentially, 'test') - run_them(self.run_sequentially, 'install') - - def run_in_parallel(self, what, commands): - self.msg('commands: %s' % what) - with self.build_watch(what): - max_jobs = self.blob.morph.max_jobs - if max_jobs is None: - max_jobs = self.settings['max-jobs'] - self.ex.env['MAKEFLAGS'] = '-j%s' % max_jobs - self.run_commands(commands) - - def run_sequentially(self, what, commands): - self.msg ('commands: %s' % what) - with self.build_watch(what): - self.ex.env['MAKEFLAGS'] = '-j1' - self.run_commands(commands) - - def run_commands(self, commands): - if self.settings['staging-chroot']: - ex = morphlib.execute.Execute(self.factory.staging, self.msg) - ex.env = self.ex.env.copy() - - assert self.builddir.startswith(self.factory.staging + '/') - assert self.destdir.startswith(self.factory.staging + '/') - builddir = self.builddir[len(self.factory.staging):] - destdir = self.destdir[len(self.factory.staging):] - ex.env['DESTDIR'] = destdir - - for cmd in commands: - ex.runv(['/usr/sbin/chroot', self.factory.staging, 'sh', '-c', - 'cd "$1" && shift && eval "$@"', '--', builddir, cmd]) - else: - self.ex.env['DESTDIR'] = self.destdir - self.ex.run(commands) - - def create_chunks(self): - chunks = [] - with self.build_watch('create-chunks'): - for chunk_name in self.blob.chunks: - self.msg('Creating chunk %s' % chunk_name) - self.prepare_binary_metadata(chunk_name) - patterns = self.blob.chunks[chunk_name] - patterns += [r'baserock/%s\.' % chunk_name] - filename = self.filename(chunk_name) - self.msg('Creating binary for %s' % chunk_name) - with self.cachedir.open(filename) as f: - morphlib.bins.create_chunk(self.destdir, f, patterns, - self.ex, - self.dump_memory_profile) - chunks.append((chunk_name, filename)) - - files = os.listdir(self.destdir) - if files: - raise Exception('DESTDIR %s is not empty: %s' % - (self.destdir, files)) - return chunks - - -class StratumBuilder(BlobBuilder): # pragma: no cover - - def builds(self): - filename = self.filename(self.blob.morph.name) - return { self.blob.morph.name: filename } - - def do_build(self): - os.mkdir(self.destdir) - ex = morphlib.execute.Execute(self.destdir, self.msg) - with self.build_watch('unpack-chunks'): - for chunk_name, filename in self.stage_items: - self.msg('Unpacking chunk %s' % chunk_name) - morphlib.bins.unpack_binary(filename, self.destdir, ex) - with self.build_watch('create-binary'): - self.prepare_binary_metadata(self.blob.morph.name) - self.msg('Creating binary for %s' % self.blob.morph.name) - filename = self.filename(self.blob.morph.name) - basename = os.path.basename(filename) - with self.cachedir.open(basename) as f: - morphlib.bins.create_stratum(self.destdir, f, ex) - - -class SystemBuilder(BlobBuilder): # pragma: no cover - - def builds(self): - filename = self.filename(self.blob.morph.name) - return { self.blob.morph.name: filename } - - def do_build(self): - logging.debug('SystemBuilder.do_build called') - self.ex = morphlib.execute.Execute(self.tempdir.dirname, self.msg) - - image_name = self.tempdir.join('%s.img' % self.blob.morph.name) - self._create_image(image_name) - self._partition_image(image_name) - self._install_mbr(image_name) - partition = self._setup_device_mapping(image_name) - - mount_point = None - try: - self._create_fs(partition) - mount_point = self.tempdir.join('mnt') - self._mount(partition, mount_point) - factory_path = os.path.join(mount_point, 'factory') - self._create_subvolume(factory_path) - self._unpack_strata(factory_path) - self._create_fstab(factory_path) - self._create_extlinux_config(factory_path) - self._create_subvolume_snapshot( - mount_point, 'factory', 'factory-run') - factory_run_path = os.path.join(mount_point, 'factory-run') - self._install_boot_files(factory_run_path, mount_point) - self._install_extlinux(mount_point) - self._unmount(mount_point) - except BaseException: - self._unmount(mount_point) - self._undo_device_mapping(image_name) - raise - - self._undo_device_mapping(image_name) - self._move_image_to_cache(image_name) - - def _create_image(self, image_name): - with self.build_watch('create-image'): - morphlib.fsutils.create_image(self.ex, image_name, - self.blob.morph.disk_size) - - def _partition_image(self, image_name): - with self.build_watch('partition-image'): - morphlib.fsutils.partition_image(self.ex, image_name) - - def _install_mbr(self, image_name): - with self.build_watch('install-mbr'): - morphlib.fsutils.install_mbr(self.ex, image_name) - - def _setup_device_mapping(self, image_name): - with self.build_watch('setup-device-mapper'): - return morphlib.fsutils.setup_device_mapping(self.ex, image_name) - - def _create_fs(self, partition): - with self.build_watch('create-filesystem'): - morphlib.fsutils.create_fs(self.ex, partition) - - def _mount(self, partition, mount_point): - with self.build_watch('mount-filesystem'): - morphlib.fsutils.mount(self.ex, partition, mount_point) - - def _create_subvolume(self, path): - with self.build_watch('create-factory-subvolume'): - self.ex.runv(['btrfs', 'subvolume', 'create', path]) - - def _unpack_strata(self, path): - with self.build_watch('unpack-strata'): - for name, filename in self.stage_items: - self.msg('unpack %s from %s' % (name, filename)) - self.ex.runv(['tar', '-C', path, '-xhf', filename]) - ldconfig(self.ex, path) - - def _create_fstab(self, path): - with self.build_watch('create-fstab'): - fstab = os.path.join(path, 'etc', 'fstab') - if not os.path.exists(os.path.dirname(fstab)): - os.makedirs(os.path.dirname(fstab)) - with open(fstab, 'w') as f: - f.write('proc /proc proc defaults 0 0\n') - f.write('sysfs /sys sysfs defaults 0 0\n') - f.write('/dev/sda1 / btrfs errors=remount-ro 0 1\n') - - def _create_extlinux_config(self, path): - config = os.path.join(path, 'extlinux.conf') - with open(config, 'w') as f: - f.write('default linux\n') - f.write('timeout 1\n') - f.write('label linux\n') - f.write('kernel /boot/vmlinuz\n') - f.write('append root=/dev/sda1 rootflags=subvol=factory-run ' - 'init=/sbin/init quiet rw\n') - - def _create_subvolume_snapshot(self, path, source, target): - with self.build_watch('create-subvolume-snapshot'): - self.ex.runv(['btrfs', 'subvolume', 'snapshot', source, target], - cwd=path) - - def _install_boot_files(self, sourcefs, targetfs): - with self.build_watch('install-boot-files'): - logging.debug('installing boot files into root volume') - shutil.copy2(os.path.join(sourcefs, 'extlinux.conf'), - os.path.join(targetfs, 'extlinux.conf')) - os.mkdir(os.path.join(targetfs, 'boot')) - shutil.copy2(os.path.join(sourcefs, 'boot', 'vmlinuz'), - os.path.join(targetfs, 'boot', 'vmlinuz')) - shutil.copy2(os.path.join(sourcefs, 'boot', 'System.map'), - os.path.join(targetfs, 'boot', 'System.map')) - - def _install_extlinux(self, path): - with self.build_watch('install-bootloader'): - self.ex.runv(['extlinux', '--install', path]) - - # FIXME this hack seems to be necessary to let extlinux finish - self.ex.runv(['sync']) - time.sleep(2) - - def _unmount(self, mount_point): - if mount_point is not None: - with self.build_watch('unmount-filesystem'): - morphlib.fsutils.unmount(self.ex, mount_point) - - def _undo_device_mapping(self, image_name): - with self.build_watch('undo-device-mapper'): - morphlib.fsutils.undo_device_mapping(self.ex, image_name) - - def _move_image_to_cache(self, image_name): - with self.build_watch('cache-image'): - filename = self.filename(self.blob.morph.name) - self.ex.runv(['mv', image_name, filename]) - - -class Builder(object): # pragma: no cover - - '''Build binary objects for Baserock. - - The objects may be chunks or strata.''' - - def __init__(self, tempdir, app, morph_loader, source_manager, factory): - self.tempdir = tempdir - self.app = app - self.real_msg = app.msg - self.dump_memory_profile = app.dump_memory_profile - self.cachedir = morphlib.cachedir.CacheDir( - self.app.settings['cachedir']) - self.morph_loader = morph_loader - self.source_manager = source_manager - self.factory = factory - self.indent = 0 - # create build environment string in advance - env_names = ("USER", "USERNAME", "LOGNAME", - "TOOLCHAIN_TARGET", "PREFIX", "BOOTSTRAP", "CFLAGS") - env = app.clean_env() - self.build_env = ''.join(k + env[k] for k in env_names) - - def msg(self, text): - spaces = ' ' * self.indent - self.real_msg('%s%s' % (spaces, text)) - - def indent_more(self): - self.indent += 1 - - def indent_less(self): - self.indent -= 1 - - def build(self, build_order): - '''Build a list of groups of morphologies. Items in a group - can be built in parallel.''' - - logging.debug('Builder.build called') - self.indent_more() - - # first pass: create builders for all blobs - builders = {} - for group in build_order: - for blob in group: - builder = self.create_blob_builder(blob) - builders[blob] = builder - - # second pass: build group by group, item after item - for group in build_order: - for blob in group: - logging.debug('Building blob %s' % repr(blob)) - self.msg('Building %s' % blob) - self.indent_more() - - ## TODO this needs to be done recursively - ## make sure all dependencies are in the staging area - #for dependency in blob.dependencies: - # depbuilder = builders[dependency] - # depbuilder.stage() - - ## TODO this needs the set of recursively collected - ## dependencies - ## make sure all non-dependencies are not staged - #for nondependency in (blobs - blob.dependencies): - # depbuilder = builders[nondependency] - # depbuilder.unstage() - - builder = builders[blob] - logging.debug('builder = %s' % repr(builder)) - - # get a list of all the items we have to build for this blob - builds = builder.builds() - logging.debug('builds = %s' % repr(builds)) - - # if not all build items are in the cache, rebuild the blob - if not self.all_built(builds): - logging.debug('calling builders build method') - builders[blob].build() - - # check again, fail if not all build items were actually built - if not self.all_built(builds): - raise Exception('Not all builds results expected from %s ' - 'were actually built' % blob) - - for parent in blob.parents: - for item, filename in builds.iteritems(): - self.msg('Marking %s to be staged for %s' % - (item, parent)) - - parent_builder = builders[parent] - parent_builder.stage_items += builds.items() - - self.indent_less() - - self.indent_less() - - def all_built(self, builds): - return all(os.path.isfile(builds[name]) for name in builds) - - def build_single(self, blob, build_order): - self.indent_more() - - # first pass: create builders for all blobs - builders = {} - for group in build_order: - for b in group: - builder = self.create_blob_builder(b) - builders[b] = builder - - # second pass: mark all dependencies for staging - queue = collections.deque() - visited = set() - for dependency in blob.dependencies: - queue.append(dependency) - visited.add(dependency) - while len(queue) > 0: - dependency = queue.popleft() - built_items = builders[dependency].builds() - for name, filename in built_items.iteritems(): - self.msg('Marking %s to be staged for %s' % (name, blob)) - builders[blob].stage_items.append((name, filename)) - for dep in dependency.dependencies: - if (dependency.morph.kind == 'stratum' and - not dep.morph.kind == 'chunk'): - if dep not in visited: - queue.append(dep) - visited.add(dep) - - # build the single blob now - builders[blob].build() - self.indent_less() - - def create_blob_builder(self, blob): - if isinstance(blob, morphlib.blobs.Stratum): - klass = StratumBuilder - elif isinstance(blob, morphlib.blobs.Chunk): - klass = ChunkBuilder - elif isinstance(blob, morphlib.blobs.System): - klass = SystemBuilder - else: - raise TypeError('Blob %s has unknown type %s' % - (str(blob), type(blob))) - - builder = klass(blob, self.factory, self.app.settings, self.cachedir, - self.get_cache_id(blob.morph), self.tempdir, - self.app.clean_env()) - builder.real_msg = self.msg - builder.dump_memory_profile = self.dump_memory_profile - - return builder - - def get_cache_id(self, morph): - logging.debug('get_cache_id(%s)' % morph) - - if morph.kind == 'chunk': - kids = [] - elif morph.kind == 'stratum': - kids = [] - for source in morph.sources: - repo = source['repo'] - ref = source['ref'] - treeish = self.source_manager.get_treeish(repo, ref) - filename = (source['morph'] - if 'morph' in source - else source['name']) - filename = '%s.morph' % filename - chunk = self.morph_loader.load(treeish, filename, - chunk_name=source['name']) - cache_id = self.get_cache_id(chunk) - kids.append(cache_id) - elif morph.kind == 'system': - kids = [] - for stratum_name in morph.strata: - filename = '%s.morph' % stratum_name - stratum = self.morph_loader.load(morph.treeish, filename) - cache_id = self.get_cache_id(stratum) - kids.append(cache_id) - else: - raise NotImplementedError('unknown morph kind %s' % - morph.kind) - - dict_key = { - 'filename': morph.filename, - 'arch': morphlib.util.arch(), - 'ref': morph.treeish.sha1, - 'kids': ''.join(self.cachedir.key(k) for k in kids), - 'env': self.build_env, - } - return dict_key - diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 774df675..1d0cbd98 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -22,6 +22,44 @@ import time import morphlib + +def ldconfig(ex, rootdir): # pragma: no cover + '''Run ldconfig for the filesystem below ``rootdir``. + + Essentially, ``rootdir`` specifies the root of a new system. + Only directories below it are considered. + + ``etc/ld.so.conf`` below ``rootdir`` is assumed to exist and + be populated by the right directories, and should assume + the root directory is ``rootdir``. Example: if ``rootdir`` + is ``/tmp/foo``, then ``/tmp/foo/etc/ld.so.conf`` should + contain ``/lib``, not ``/tmp/foo/lib``. + + The ldconfig found via ``$PATH`` is used, not the one in ``rootdir``, + since in bootstrap mode that might not yet exist, the various + implementations should be compatible enough. + + ''' + + conf = os.path.join(rootdir, 'etc', 'ld.so.conf') + if os.path.exists(conf): + logging.debug('Running ldconfig for %s' % rootdir) + cache = os.path.join(rootdir, 'etc', 'ld.so.cache') + + # The following trickery with $PATH is necessary during the Baserock + # bootstrap build: we are not guaranteed that PATH contains the + # directory (/sbin conventionally) that ldconfig is in. Then again, + # it might, and if so, we don't want to hardware a particular + # location. So we add the possible locations to the end of $PATH + # and restore that aftewards. + old_path = ex.env['PATH'] + ex.env['PATH'] = '%s:/sbin:/usr/sbin:/usr/local/sbin' % old_path + ex.runv(['ldconfig', '-r', rootdir]) + ex.env['PATH'] = old_path + else: + logging.debug('No %s, not running ldconfig' % conf) + + class BuilderBase(object): '''Base class for building artifacts.''' @@ -380,7 +418,7 @@ class SystemBuilder(BuilderBase): # pragma: no cover for stratum_artifact in self.artifact.dependencies: with self.artifact_cache.get(stratum_artifact) as f: morphlib.bins.unpack_binary_from_file(f, path) - morphlib.builder.ldconfig(self.ex, path) + ldconfig(self.ex, path) def _create_fstab(self, path): logging.debug('Creating fstab in %s' % path) diff --git a/morphlib/builder_tests.py b/morphlib/builder_tests.py deleted file mode 100644 index a8f06462..00000000 --- a/morphlib/builder_tests.py +++ /dev/null @@ -1,165 +0,0 @@ -# Copyright (C) 2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import os -import shutil -import unittest - -import morphlib - - -class FakeSubmodule(object): - - def __init__(self, **kwargs): - for name in kwargs: - setattr(self, name, kwargs[name]) - - -class FakeTreeish(object): - - def __init__(self, tempdir, repo, subtreeish=None): - self.repo = tempdir.join(repo) - self.ref = 'master' - self.submodules = [] - - temp_repo = tempdir.join('temp_repo') - - os.mkdir(temp_repo) - ex = morphlib.execute.Execute(temp_repo, lambda s: None) - ex.runv(['git', 'init', '--quiet']) - with open(os.path.join(temp_repo, 'file.txt'), 'w') as f: - f.write('foobar\n') - ex.runv(['git', 'add', 'file.txt']) - ex.runv(['git', 'commit', '--quiet', '-m', 'foo']) - - if subtreeish is not None: - ex.runv(['git', 'submodule', 'add', subtreeish.repo]) - path = os.path.basename(subtreeish.repo) - self.submodules = [FakeSubmodule(repo=subtreeish.repo, - ref='master', - path=path, - treeish=subtreeish)] - - ex = morphlib.execute.Execute(tempdir.dirname, lambda s: None) - ex.runv(['git', 'clone', '-n', temp_repo, self.repo]) - - shutil.rmtree(temp_repo) - - -class FactoryTests(unittest.TestCase): - - def setUp(self): - self.tempdir = morphlib.tempdir.Tempdir() - self.factory = morphlib.builder.Factory(self.tempdir) - - def tearDown(self): - self.tempdir.remove() - - def create_chunk(self): - '''Create a simple binary chunk.''' - - inst = self.tempdir.join('dummy-inst') - os.mkdir(inst) - for x in ['bin', 'etc', 'lib']: - os.mkdir(os.path.join(inst, x)) - - binary = self.tempdir.join('dummy-chunk') - ex = None # this is not actually used by the function! - with open(binary, 'wb') as f: - morphlib.bins.create_chunk(inst, f, ['.'], ex) - return binary - - def test_has_no_staging_area_initially(self): - self.assertEqual(self.factory.staging, None) - - def test_creates_staging_area(self): - self.factory.create_staging() - self.assertEqual(os.listdir(self.factory.staging), []) - - def test_removes_staging_area(self): - self.factory.create_staging() - staging = self.factory.staging - self.factory.remove_staging() - self.assertEqual(self.factory.staging, None) - self.assertFalse(os.path.exists(staging)) - - def test_unpacks_binary_from_file(self): - binary = self.create_chunk() - self.factory.create_staging() - self.factory.unpack_binary_from_file(binary) - self.assertEqual(sorted(os.listdir(self.factory.staging)), - sorted(['bin', 'etc', 'lib'])) - - def test_removes_staging_area_with_contents(self): - binary = self.create_chunk() - self.factory.create_staging() - self.factory.unpack_binary_from_file(binary) - staging = self.factory.staging - self.factory.remove_staging() - self.assertEqual(self.factory.staging, None) - self.assertFalse(os.path.exists(staging)) - - def test_unpacks_onto_system(self): - - # We can't test this by really unpacking onto the system. - # Instead, we rely on the fact that if the normal unpacking - # works, the actual worker function for unpacking works, and - # we can just verify that it gets called with the right - # parameters. - - def fake_unpack(binary, dirname): - self.dirname = dirname - - binary = self.create_chunk() - self.factory._unpack_binary = fake_unpack - self.factory.unpack_binary_from_file_onto_system(binary) - self.assertEqual(self.dirname, '/') - - def test_unpacks_simple_sources(self): - self.factory.create_staging() - srcdir = self.tempdir.join('src') - treeish = FakeTreeish(self.tempdir, 'repo') - self.factory.unpack_sources(treeish, srcdir) - self.assertTrue(os.path.exists(os.path.join(srcdir, 'file.txt'))) - - def test_unpacks_submodules(self): - self.factory.create_staging() - srcdir = self.tempdir.join('src') - subtreeish = FakeTreeish(self.tempdir, 'subrepo') - supertreeish = FakeTreeish(self.tempdir, 'repo', subtreeish=subtreeish) - self.factory.unpack_sources(supertreeish, srcdir) - self.assertEqual(sorted(os.listdir(srcdir)), - sorted(['.git', 'file.txt', 'subrepo'])) - self.assertEqual(sorted(os.listdir(os.path.join(srcdir, 'subrepo'))), - sorted(['.git', 'file.txt'])) - - def test_sets_timestamp_for_unpacked_files(self): - self.factory.create_staging() - srcdir = self.tempdir.join('src') - treeish = FakeTreeish(self.tempdir, 'repo') - self.factory.unpack_sources(treeish, srcdir) - - mtime = None - for dirname, subdirs, basenames in os.walk(srcdir): - pathnames = [os.path.join(dirname, x) for x in basenames] - for pathname in pathnames + [dirname]: - st = os.lstat(pathname) - if mtime is None: - mtime = st.st_mtime - else: - self.assertEqual((pathname, mtime), - (pathname, st.st_mtime)) - diff --git a/morphlib/localrepocache.py b/morphlib/localrepocache.py index b8dfea88..a3e28d57 100644 --- a/morphlib/localrepocache.py +++ b/morphlib/localrepocache.py @@ -19,6 +19,7 @@ import os import urllib2 import urlparse import shutil +import string import morphlib @@ -34,6 +35,19 @@ urlparse.uses_fragment.extend(gitscheme) +def quote_url(url): + ''' Convert URIs to strings that only contain digits, letters, % and _. + + NOTE: When changing the code of this function, make sure to also apply + the same to the quote_url() function of lorry. Otherwise the git bundles + generated by lorry may no longer be found by morph. + + ''' + valid_chars = string.digits + string.letters + '%_' + transl = lambda x: x if x in valid_chars else '_' + return ''.join([transl(x) for x in url]) + + class NoRemote(morphlib.Error): def __init__(self, reponame, errors): @@ -151,13 +165,12 @@ class LocalRepoCache(object): def _escape(self, url): '''Escape a URL so it can be used as a basename in a file.''' - # FIXME: The following is a nicer way than what source manager does. - # However, for compatibility, we need to use the same as the source - # manager uses, since that's what the bundle server (set up by - # Lorry) uses. + # FIXME: The following is a nicer way than to do this. + # However, for compatibility, we need to use the same as the + # bundle server (set up by Lorry) uses. # return urllib.quote(url, safe='') - return morphlib.sourcemanager.quote_url(url) + return quote_url(url) def _cache_name(self, url): basename = self._escape(url) diff --git a/morphlib/morphology.py b/morphlib/morphology.py deleted file mode 100644 index aec63c90..00000000 --- a/morphlib/morphology.py +++ /dev/null @@ -1,131 +0,0 @@ -# Copyright (C) 2011-2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import json -import logging -import os - - -class Morphology(object): - - '''Represent a morphology: description of how to build binaries.''' - - def __init__(self, treeish, fp): - self.treeish = treeish - self.filename = fp.name - - self._fp = fp - self._load() - - def _load(self): - logging.debug('Loading morphology %s from %s' % - (self._fp.name, self.treeish)) - try: - self._dict = json.load(self._fp) - except ValueError: - logging.error('Failed to load morphology %s from %s' % - (self._fp.name, self.treeish)) - raise - - if self.kind == 'stratum': - for source in self.sources: - if 'repo' not in source: - source[u'repo'] = source['name'] - source[u'repo'] = unicode(source['repo']) - - @property - def name(self): - return self._dict['name'] - - @property - def kind(self): - return self._dict['kind'] - - @property - def description(self): - return self._dict.get('description', '') - - @property - def sources(self): - return self._dict['sources'] - - @property - def build_depends(self): - return self._dict.get('build-depends', None) - - @property - def build_system(self): - return self._dict.get('build-system', 'manual') - - @property - def max_jobs(self): - if 'max-jobs' in self._dict: - return int(self._dict['max-jobs']) - return None - - @property - def configure_commands(self): - return self._dict.get('configure-commands', []) - - @property - def build_commands(self): - return self._dict.get('build-commands', []) - - @property - def test_commands(self): - return self._dict.get('test-commands', []) - - @property - def install_commands(self): - return self._dict.get('install-commands', []) - - @property - def chunks(self): - return self._dict.get('chunks', {}) - - @property - def strata(self): - return self._dict.get('strata', []) - - @property - def disk_size(self): - size = self._dict['disk-size'] - size = size.lower() - if size.endswith('g'): - size = int(size[:-1]) * 1024**3 - elif size.endswith('m'): # pragma: no cover - size = int(size[:-1]) * 1024**2 - elif size.endswith('k'): # pragma: no cover - size = int(size[:-1]) * 1024 - else: # pragma: no cover - size = int(size) - return size - - @property - def test_stories(self): - return self._dict.get('test-stories', []) - - def __eq__(self, other): - return (self.filename == other.filename and - self.treeish == other.treeish) - - def __hash__(self): - return hash((self.filename, self.treeish)) - - def __str__(self): # pragma: no cover - return '%s|%s|%s' % (self.treeish.original_repo, - self.treeish.ref, - os.path.basename(self.filename)) diff --git a/morphlib/morphology_tests.py b/morphlib/morphology_tests.py deleted file mode 100644 index c314f63c..00000000 --- a/morphlib/morphology_tests.py +++ /dev/null @@ -1,206 +0,0 @@ -# Copyright (C) 2011-2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import StringIO -import unittest - -import morphlib - - -class FakeTreeish(object): - - pass - - -class MockFile(StringIO.StringIO): - - def __init__(self, *args, **kwargs): - StringIO.StringIO.__init__(self, *args, **kwargs) - self.name = 'mockfile' - - -class MorphologyTests(unittest.TestCase): - - def test_constructor_with_treeish(self): - faketreeish = FakeTreeish() - morph = morphlib.morphology.Morphology( - faketreeish, - MockFile(''' - { - "name": "hello", - "kind": "chunk" - }''')) - self.assertEqual(morph.treeish, faketreeish) - - def test_fails_invalid_chunk_morphology(self): - def failtest(): - morphlib.morphology.Morphology( - FakeTreeish(), - MockFile(''' - { - "name": "hello", - }''')) - self.assertRaises(ValueError, failtest) - - def test_accepts_valid_chunk_morphology(self): - faketreeish = FakeTreeish() - morph = morphlib.morphology.Morphology( - faketreeish, - MockFile(''' - { - "name": "hello", - "kind": "chunk", - "description": "desc", - "build-depends": [ - "devel" - ], - "build-system": "autotools", - "max-jobs": "42", - "configure-commands": ["./configure"], - "build-commands": ["make"], - "test-commands": ["make check"], - "install-commands": ["make install"], - "chunks": { - "hello": [ - "usr/bin/hello", - "usr/lib/libhello.so*" - ], - "hello-dev": [ - "usr/include/*", - "usr/lib/*" - ] - } - }''')) - - self.assertEqual(morph.treeish, faketreeish) - self.assertEqual(morph.filename, 'mockfile') - self.assertEqual(morph.name, 'hello') - self.assertEqual(morph.kind, 'chunk') - self.assertEqual(morph.description, 'desc') - self.assertEqual(morph.filename, 'mockfile') - self.assertEqual(morph.build_depends, ['devel']) - self.assertEqual(morph.build_system, 'autotools') - self.assertEqual(morph.max_jobs, 42) - self.assertEqual(morph.configure_commands, ['./configure']) - self.assertEqual(morph.build_commands, ['make']) - self.assertEqual(morph.test_commands, ['make check']) - self.assertEqual(morph.install_commands, ['make install']) - self.assertEqual(morph.chunks, - { - u'hello': [u'usr/bin/hello', - u'usr/lib/libhello.so*'], - u'hello-dev': [u'usr/include/*', u'usr/lib/*'], - }) - - def test_build_system_defaults_to_manual(self): - morph = morphlib.morphology.Morphology( - FakeTreeish(), - MockFile(''' - { - "name": "hello", - "kind": "chunk" - }''')) - self.assertEqual(morph.build_system, 'manual') - - def test_max_jobs_defaults_to_None(self): - morph = morphlib.morphology.Morphology( - FakeTreeish(), - MockFile(''' - { - "name": "hello", - "kind": "chunk" - }''')) - self.assertEqual(morph.max_jobs, None) - - def test_accepts_valid_stratum_morphology(self): - morph = morphlib.morphology.Morphology( - FakeTreeish(), - MockFile(''' - { - "name": "hello", - "kind": "stratum", - "sources": - [ - { - "name": "foo", - "ref": "ref" - } - ] - }''')) - self.assertEqual(morph.kind, 'stratum') - self.assertEqual(morph.filename, 'mockfile') - self.assertEqual(morph.sources, - [ - { - u'name': u'foo', - u'repo': u'foo', - u'ref': u'ref', - }, - ]) - - def test_accepts_valid_system_morphology(self): - morph = morphlib.morphology.Morphology( - FakeTreeish(), - MockFile(''' - { - "name": "hello", - "kind": "system", - "disk-size": "1G", - "strata": [ - "foo", - "bar" - ], - "test-stories": [ - "test-1", - "test-2" - ] - }''')) - self.assertEqual(morph.kind, 'system') - self.assertEqual(morph.disk_size, 1024**3) - self.assertEqual(morph.strata, ['foo', 'bar']) - self.assertEqual(morph.test_stories, ['test-1', 'test-2']) - - def test_hashing_and_equality_checks(self): - mockfile1 = MockFile(''' - { - "name": "foo", - "kind": "chunk" - }''') - mockfile1.name = 'mockfile1' - mockfile2 = MockFile(''' - { - "name": "foo", - "kind": "chunk" - }''') - mockfile2.name = 'mockfile1' - mockfile3 = MockFile(''' - { - "name": "bar", - "kind": "chunk" - }''') - mockfile3.name = 'mockfile2' - - treeish = FakeTreeish() - - morph1 = morphlib.morphology.Morphology(treeish, mockfile1) - morph2 = morphlib.morphology.Morphology(treeish, mockfile2) - morph3 = morphlib.morphology.Morphology(treeish, mockfile3) - - self.assertEqual(hash(morph1), hash(morph2)) - self.assertEqual(morph1, morph2) - - self.assertNotEqual(hash(morph1), hash(morph3)) - self.assertNotEqual(morph1, morph3) diff --git a/morphlib/morphologyloader.py b/morphlib/morphologyloader.py deleted file mode 100644 index b0da97dc..00000000 --- a/morphlib/morphologyloader.py +++ /dev/null @@ -1,72 +0,0 @@ -# Copyright (C) 2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import StringIO - -import morphlib - - -class MorphologyLoader(object): - - '''Load morphologies from git and parse them into Morphology objects.''' - - def __init__(self, settings): - self.settings = settings - self.morphologies = {} - - def load(self, treeish, filename, chunk_name=None): - key = (treeish, filename) - if key in self.morphologies: - return self.morphologies[key] - else: - try: - morph = self._get_morph_from_git(treeish, filename) - except morphlib.execute.CommandFailure: # pragma: no cover - morph = None - if morph is None and chunk_name is not None: # pragma: no cover - morph = self._autodetect_morph(treeish, filename, chunk_name) - if morph is None: # pragma: no cover - raise Exception("Can't find morphology %s" % filename) - self.morphologies[key] = morph - return morph - - def _autodetect_morph(self, treeish, filename, chunk): # pragma: no cover - def exists(x): - try: - self._get_morph_text(treeish, x) - except morphlib.execute.CommandFailure: - return False - else: - return True - bs = morphlib.buildsystem.detect_build_system(exists) - if bs is not None: - morph_text = bs.get_morphology_text(chunk) - return self._get_morph(treeish, filename, morph_text) - else: - return None - - def _get_morph_text(self, treeish, filename): # pragma: no cover - return morphlib.git.get_morph_text(treeish, filename) - - def _get_morph_from_git(self, treeish, filename): - morph_text = self._get_morph_text(treeish, filename) - return self._get_morph(treeish, filename, morph_text) - - def _get_morph(self, treeish, filename, morph_text): - fp = StringIO.StringIO(morph_text) - fp.name = filename - return morphlib.morphology.Morphology(treeish, fp) - diff --git a/morphlib/morphologyloader_tests.py b/morphlib/morphologyloader_tests.py deleted file mode 100644 index 33528086..00000000 --- a/morphlib/morphologyloader_tests.py +++ /dev/null @@ -1,50 +0,0 @@ -# Copyright (C) 2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import unittest - -import morphlib - - -class MorphologyLoaderTests(unittest.TestCase): - - def test_load_twice_verify_same_morph_object(self): - settings = { 'git-base-url': '' } - loader = morphlib.morphologyloader.MorphologyLoader(settings) - loader._get_morph_text = self.get_morph_text - - class FakeTreeish(object): - def __init__(self): - self.original_repo = 'test-repo' - faketreeish = FakeTreeish() - - morph1 = loader.load(faketreeish, 'foo.morph') - morph2 = loader.load(faketreeish, 'foo.morph') - self.assertEqual(morph1, morph2) - - def get_morph_text(self, treeish, filename): - return ''' - { - "name": "foo", - "kind": "stratum", - "sources": [ - { - "name": "bar", - "repo": "bar", - "ref": "master" - } - ] - }''' diff --git a/morphlib/sourcemanager.py b/morphlib/sourcemanager.py deleted file mode 100644 index a1a350ca..00000000 --- a/morphlib/sourcemanager.py +++ /dev/null @@ -1,286 +0,0 @@ -# Copyright (C) 2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import os -import urlparse -import urllib2 -import shutil -import string - -import morphlib - - -gitscheme=["git",] -urlparse.uses_relative.extend(gitscheme) -urlparse.uses_netloc.extend(gitscheme) -urlparse.uses_params.extend(gitscheme) -urlparse.uses_query.extend(gitscheme) -urlparse.uses_fragment.extend(gitscheme) - - -def quote_url(url): - ''' Convert URIs to strings that only contain digits, letters, % and _. - - NOTE: When changing the code of this function, make sure to also apply - the same to the quote_url() function of lorry. Otherwise the git bundles - generated by lorry may no longer be found by morph. - - ''' - valid_chars = string.digits + string.letters + '%_' - transl = lambda x: x if x in valid_chars else '_' - return ''.join([transl(x) for x in url]) - - -class RepositoryUpdateError(Exception): # pragma: no cover - - def __init__(self, repo, ref, error): - Exception.__init__(self, 'Failed to update %s:%s: %s' % - (repo, ref, error)) - - -class RepositoryFetchError(Exception): - - def __init__(self, repo): - Exception.__init__(self, 'Failed to fetch %s' % repo) - - -class SourceManager(object): - - def __init__(self, app, cachedir=None, update=True): - self.real_msg = app.msg - self.settings = app.settings - self.cached_treeishes = {} - self.cache_dir = cachedir - self.update = update - if not self.cache_dir: - self.cache_dir = os.path.join(app.settings['cachedir'], 'gits') - self.indent = 0 - - def indent_more(self): - self.indent += 1 - - def indent_less(self): - self.indent -= 1 - - def msg(self, text): - spaces = ' ' * self.indent - self.real_msg('%s%s' % (spaces, text)) - - def _wget(self, url, filename): # pragma: no cover - # the following doesn't work during bootstrapping - # ex = morphlib.execute.Execute(self.cache_dir, msg=self.msg) - # ex.runv(['wget', '-c', url]) - # so we do it poorly in pure Python instead - source_handle = urllib2.urlopen(url) - target_handle = open(filename, 'wb') - - data = source_handle.read(4096) - while data: - target_handle.write(data) - data = source_handle.read(4096) - - source_handle.close() - target_handle.close() - - return filename - - def _cache_repo_from_bundle(self, server, repo_url): - quoted_url = quote_url(repo_url) - cached_repo = os.path.join(self.cache_dir, quoted_url) - bundle_name = '%s.bndl' % quoted_url - bundle_url = server + bundle_name - bundle = os.path.join(self.cache_dir, bundle_name) - self.msg('Trying to fetch bundle %s' % bundle_url) - request = urllib2.Request(bundle_url) - try: - urllib2.urlopen(request) - try: - self._wget(bundle_url, bundle) - self.msg('Extracting bundle %s into %s' % - (bundle, cached_repo)) - try: - os.mkdir(cached_repo) - morphlib.git.extract_bundle(cached_repo, bundle, - self.msg) - self.msg('Setting origin to %s' % repo_url) - morphlib.git.set_remote(cached_repo, 'origin', - repo_url, self.msg) - return cached_repo, None - except morphlib.execute.CommandFailure, e: # pragma: no cover - if os.path.exists(cached_repo): - shutil.rmtree(cached_repo) - self.msg('Unable to extract bundle %s' % bundle) - return None, 'Unable to extract bundle %s: %s' % (bundle, - e) - finally: - if os.path.exists(bundle): - os.remove(bundle) - except morphlib.execute.CommandFailure, e: # pragma: no cover - return None, 'Unable to fetch bundle %s: %s' % (bundle, e) - except urllib2.URLError, e: - return None, 'Unable to fetch bundle %s: %s' % (bundle_url, e) - - def _cache_repo_from_url(self, repo_url): - # quote the URL and calculate the location for the cached repo - quoted_url = quote_url(repo_url) - cached_repo = os.path.join(self.cache_dir, quoted_url) - - if os.path.exists(cached_repo): # pragma: no cover - # the cache location exists, assume this is what we want - self.msg('Using cached clone %s of %s' % (cached_repo, repo_url)) - return cached_repo, None - else: - # bundle server did not have a bundle for the repo - self.msg('Trying to clone %s into %s' % (repo_url, cached_repo)) - try: - morphlib.git.clone(cached_repo, repo_url, self.msg) - return cached_repo, None - except morphlib.execute.CommandFailure, e: - if os.path.exists(cached_repo): # pragma: no cover - shutil.rmtree(cached_repo) - return None, 'Unable to clone from %s: %s' % (repo_url, e) - - def _cache_repo_from_base_urls(self, repo, ref): - self.msg('Checking repository %s' % repo) - self.indent_more() - - def fixup_url(url): - return (url if url.endswith('/') else url + '/') - - # create absolute repo URLs - repo_urls = [urlparse.urljoin(fixup_url(x), repo) - for x in self.settings['git-base-url']] - - orig_url = None - cached_repo = None - errors = [] - - # check if we have a cached version of the repo - for repo_url in repo_urls: - quoted_url = quote_url(repo_url) - cached_repo_dirname = os.path.join(self.cache_dir, quoted_url) - if os.path.exists(cached_repo_dirname): - orig_url = repo_url - cached_repo = cached_repo_dirname - break - - # first pass, try all base URLs with the bundle server - if not cached_repo and self.settings['bundle-server']: - server = fixup_url(self.settings['bundle-server']) - - for repo_url in repo_urls: - cached_repo, error = self._cache_repo_from_bundle(server, - repo_url) - if cached_repo: - orig_url = repo_url - break - else: - errors.append(error) - - # second pass, try cloning from base URLs directly - if not cached_repo: - # try all URLs to find or obtain a cached clone of the repo - for repo_url in repo_urls: - cached_repo, error = self._cache_repo_from_url(repo_url) - if cached_repo: - orig_url = repo_url - break - else: - errors.append(error) - - if cached_repo: - # we have a cached version of the repo now - if self.update: - # we are supposed to update 'origin', so do that now - try: - self.msg('Updating %s' % cached_repo) - morphlib.git.update_remote(cached_repo, 'origin', - self.msg) - except morphlib.execute.CommandFailure, e: # pragma: no cover - self.msg('Failed to update origin: %s' % e) - self.indent_less() - # ignore remote update failures during bootstrap - if not self.settings['bootstrap']: - raise RepositoryUpdateError(repo, ref, e) - else: # pragma: no cover - self.msg('Assuming cached repository %s is up to date' % - cached_repo) - else: # pragma: no cover - # cloning using all individual base URLs failed - - # print all the errors at once to give the user an overview - # over what went wrong in which order - for error in errors: - self.msg(error) - - self.indent_less() - raise RepositoryFetchError(repo) - - # we should have a cached version of the repo now, return a treeish - # for the repo and ref tuple - treeish = morphlib.git.Treeish(cached_repo, orig_url, ref, self.msg) - self.indent_less() - return treeish - - def _resolve_submodules(self, treeish): # pragma: no cover - self.indent_more() - - # resolve submodules - treeish.submodules = morphlib.git.Submodules(treeish.repo, - treeish.ref, - self.msg) - try: - # load submodules from .gitmodules - treeish.submodules.load() - - # resolve the tree-ishes for all submodules recursively - for submodule in treeish.submodules: # pragma: no cover - submodule.treeish = self.get_treeish(submodule.url, - submodule.commit) - except morphlib.git.NoModulesFileError: - # this is not really an error, the repository simply - # does not specify any git submodules - pass - - self.indent_less() - - def get_treeish(self, repo, ref): - '''Returns a Treeish for a URL or repo name with a given reference. - - If the source hasn't been cloned yet, this will fetch it, either using - clone or by fetching a bundle. - - Raises morphlib.git.InvalidReferenceError if the reference cannot be - found. Raises morphlib.sourcemanager.RepositoryUpdateError if the - repository cannot be cloned or updated. - - ''' - - if (repo, ref) not in self.cached_treeishes: # pragma: no cover - # load the corresponding treeish on demand - treeish = self._cache_repo_from_base_urls(repo, ref) - - # have a treeish now, cache it to avoid loading it twice - self.cached_treeishes[(repo, ref)] = treeish - - # load tree-ishes for submodules, if necessary and desired - if self.settings['ignore-submodules']: - treeish.submodules = [] - else: - self._resolve_submodules(treeish) - - # we should now have a cached treeish to use now - return self.cached_treeishes[(repo, ref)] # pragma: no cover diff --git a/morphlib/sourcemanager_tests.py b/morphlib/sourcemanager_tests.py deleted file mode 100644 index eb18dcc3..00000000 --- a/morphlib/sourcemanager_tests.py +++ /dev/null @@ -1,193 +0,0 @@ -# Copyright (C) 2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import glob -import unittest -import tempfile -import shutil -import os -import subprocess -import urlparse - -import morphlib - - -class DummyApp(object): - - def __init__(self): - self.settings = { - 'git-base-url': ['.',], - 'bundle-server': None, - 'cachedir': '/foo/bar/baz', - 'ignore-submodules': False - } - self.msg = lambda msg: None - - -class SourceManagerTests(unittest.TestCase): - - def setUp(self): - self.temprepodir = tempfile.mkdtemp() - env = os.environ - env["DATADIR"]=self.temprepodir - subprocess.call("./tests/show-dependencies.setup", shell=True, env=env) - self.temprepo = self.temprepodir + '/test-repo/' - bundle_name = morphlib.sourcemanager.quote_url(self.temprepo) + '.bndl' - with open('/dev/null', 'w') as f: - subprocess.check_call("git bundle create %s/%s master" % - (self.temprepodir, bundle_name), - stderr=f, shell=True, cwd=self.temprepo) - - def tearDown(self): - shutil.rmtree(self.temprepodir) - - def test_uses_provided_cache_dir(self): - app = DummyApp() - - tempdir = '/bla/bla/bla' - s = morphlib.sourcemanager.SourceManager(app, tempdir) - self.assertEqual(s.cache_dir, tempdir) - - def test_uses_cachedir_gits_if_no_cache_dir_provided(self): - app = DummyApp() - - s = morphlib.sourcemanager.SourceManager(app) - self.assertEqual(s.cache_dir, - os.path.join(app.settings['cachedir'], 'gits')) - - def test_resolves_sha1_treeish_for_test_repo_correctly(self): - tempdir = tempfile.mkdtemp() - - s = morphlib.sourcemanager.SourceManager(DummyApp(), tempdir) - t = s.get_treeish(self.temprepo, - 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') - self.assertEquals(t.sha1, 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') - - shutil.rmtree(tempdir) - - def test_resolves_sha1_treeish_for_test_repo_correctly_twice(self): - tempdir = tempfile.mkdtemp() - - # try two times with different source manager objects - s = morphlib.sourcemanager.SourceManager(DummyApp(), tempdir) - t = s.get_treeish(self.temprepo, - 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') - self.assertEquals(t.sha1, 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') - s = morphlib.sourcemanager.SourceManager(DummyApp(), tempdir) - t = s.get_treeish(self.temprepo, - 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') - self.assertEquals(t.sha1, 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') - - # try two times with the same source manager object - s = morphlib.sourcemanager.SourceManager(DummyApp(), tempdir) - t = s.get_treeish(self.temprepo, - 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') - self.assertEquals(t.sha1, 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') - t = s.get_treeish(self.temprepo, - 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') - self.assertEquals(t.sha1, 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') - - shutil.rmtree(tempdir) - - def test_resolves_ref_treeish_for_test_repo_correctly(self): - tempdir = tempfile.mkdtemp() - - s = morphlib.sourcemanager.SourceManager(DummyApp(), tempdir) - t = s.get_treeish(self.temprepo, 'master') - self.assertEquals(t.ref, 'refs/remotes/origin/master') - - shutil.rmtree(tempdir) - - def test_resolves_ref_treeish_for_test_repo_without_submodules(self): - tempdir = tempfile.mkdtemp() - - s = morphlib.sourcemanager.SourceManager(DummyApp(), tempdir) - t = s.get_treeish(self.temprepo, 'master') - self.assertEquals(len(t.submodules), 0) - - shutil.rmtree(tempdir) - - def test_resolves_sha1_treeish_for_test_repo_correctly_from_bundle(self): - tempdir = tempfile.mkdtemp() - bundle_server_loc = self.temprepodir - - app = DummyApp() - app.settings['bundle-server'] = 'file://' + bundle_server_loc - - s = morphlib.sourcemanager.SourceManager(app, tempdir) - - def wget(url, filename): - bundle_file = os.path.join(self.temprepodir, - os.path.basename(filename)) - shutil.copy(bundle_file, s.cache_dir) - - s._wget = wget - - t = s.get_treeish(self.temprepo, - 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') - self.assertEquals(t.sha1, 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') - - shutil.rmtree(tempdir) - - def test_fails_to_resolves_sha1_treeish_for_non_existent_repo(self): - tempdir = tempfile.mkdtemp() - app = DummyApp() - - s = morphlib.sourcemanager.SourceManager(app, tempdir) - - def wget(url, filename): - bundle_file = os.path.join(self.temprepodir, - os.path.basename(filename)) - shutil.copy(bundle_file, s.cache_dir) - - s._wget = wget - self.assertRaises(morphlib.sourcemanager.RepositoryFetchError, - s.get_treeish, 'asdf', - 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') - - shutil.rmtree(tempdir) - - def test_fails_to_resolves_sha1_treeish_for_non_existent_repo_bundle(self): - tempdir = tempfile.mkdtemp() - app = DummyApp() - app.settings['bundle-server'] = 'file://' + self.temprepodir - - s = morphlib.sourcemanager.SourceManager(app, tempdir) - - def wget(url, filename): - bundle_file = os.path.join(self.temprepodir, - os.path.basename(filename)) - shutil.copy(bundle_file, s.cache_dir) - - s._wget = wget - self.assertRaises(morphlib.sourcemanager.RepositoryFetchError, - s.get_treeish, 'asdf', - 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') - - shutil.rmtree(tempdir) - - def test_get_sha1_treeish_for_self_multiple_base(self): - - tempdir = tempfile.mkdtemp() - app = DummyApp() - app.settings['git-base-url'] = ['.', '/somewhere/else'] - - s = morphlib.sourcemanager.SourceManager(app, tempdir) - t = s.get_treeish(self.temprepo, - 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') - self.assertEquals(t.sha1, 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') - - shutil.rmtree(tempdir) diff --git a/tests/build-chunk-distributed-local.script b/tests/build-chunk-distributed-local.script deleted file mode 100755 index f38a0673..00000000 --- a/tests/build-chunk-distributed-local.script +++ /dev/null @@ -1,29 +0,0 @@ -#!/bin/sh -# -# Test building a chunk using as many local workers as make sense. -# -# Copyright (C) 2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -set -eu - -"$SRCDIR/scripts/test-morph" build-distributed chunk-repo farrokh hello.morph - -for chunk in "$DATADIR/cache/"*.chunk.* -do - echo "$chunk:" | sed 's/[^.]*//' - tar -tf "$chunk" | LC_ALL=C sort | sed '/^\.\/./s:^\./::' - echo -done diff --git a/tests/build-chunk-distributed-local.stdout b/tests/build-chunk-distributed-local.stdout deleted file mode 100644 index 8077cac2..00000000 --- a/tests/build-chunk-distributed-local.stdout +++ /dev/null @@ -1,8 +0,0 @@ -.chunk.hello: -./ -baserock/ -baserock/hello.meta -bin/ -bin/hello -etc/ - diff --git a/tests/build-stratum-distributed-local.script b/tests/build-stratum-distributed-local.script deleted file mode 100755 index d69946eb..00000000 --- a/tests/build-stratum-distributed-local.script +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/sh -# -# Test build a simple stratum using as many local workers as makes sense. -# -# Copyright (C) 2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - -set -eu - -"$SRCDIR/scripts/test-morph" \ - build-distributed morphs-repo master hello-stratum.morph -tar -tf "$DATADIR/cache/"*.stratum.* | LC_ALL=C sort | sed '/^\.\/./s:^\./::' diff --git a/tests/build-stratum-distributed-local.stdout b/tests/build-stratum-distributed-local.stdout deleted file mode 100644 index 4f422ea4..00000000 --- a/tests/build-stratum-distributed-local.stdout +++ /dev/null @@ -1,7 +0,0 @@ -./ -baserock/ -baserock/hello-stratum.meta -baserock/hello.meta -bin/ -bin/hello -etc/ |