diff options
author | Richard Maw <richard.maw@codethink.co.uk> | 2012-05-29 17:18:46 +0000 |
---|---|---|
committer | Richard Maw <richard.maw@codethink.co.uk> | 2012-05-30 13:11:13 +0100 |
commit | b088e31b740c62b90ddf9c3342b876736790c1e8 (patch) | |
tree | b73ea9c49d130c584c50c3b8c55f755f5cfa6d4a | |
parent | 34817d54a193c6c0937ef3868f31d822e90aa740 (diff) | |
download | morph-b088e31b740c62b90ddf9c3342b876736790c1e8.tar.gz |
morph: remove dead code and replace Execute with app.runcmd
-rwxr-xr-x | morph | 128 | ||||
-rw-r--r-- | morphlib/__init__.py | 3 | ||||
-rw-r--r-- | morphlib/bins.py | 4 | ||||
-rw-r--r-- | morphlib/bins_tests.py | 7 | ||||
-rw-r--r-- | morphlib/buildcontroller.py | 127 | ||||
-rw-r--r-- | morphlib/buildcontroller_tests.py | 86 | ||||
-rw-r--r-- | morphlib/builder2.py | 68 | ||||
-rw-r--r-- | morphlib/builder2_tests.py | 16 | ||||
-rw-r--r-- | morphlib/buildworker.py | 277 | ||||
-rw-r--r-- | morphlib/buildworker_tests.py | 75 | ||||
-rw-r--r-- | morphlib/cachedrepo.py | 37 | ||||
-rw-r--r-- | morphlib/cachedrepo_tests.py | 14 | ||||
-rw-r--r-- | morphlib/execute.py | 112 | ||||
-rw-r--r-- | morphlib/execute_tests.py | 54 | ||||
-rw-r--r-- | morphlib/fsutils.py | 44 | ||||
-rw-r--r-- | morphlib/git.py | 153 | ||||
-rw-r--r-- | morphlib/localrepocache.py | 15 | ||||
-rw-r--r-- | morphlib/localrepocache_tests.py | 6 | ||||
-rw-r--r-- | morphlib/stagingarea.py | 6 | ||||
-rw-r--r-- | morphlib/stagingarea_tests.py | 5 | ||||
-rwxr-xr-x | tests/uses-tempdir.script | 3 |
21 files changed, 165 insertions, 1075 deletions
@@ -26,8 +26,6 @@ import shutil import tempfile import morphlib -from morphlib import buildworker -from morphlib import buildcontroller defaults = { @@ -144,11 +142,6 @@ class Morph(cliapp.Application): 'build things in a staging chroot ' '(require real root to use)') - self.settings.string_list(['worker'], - 'IP or host name of a machine to distribute ' - 'build work to', - metavar='HOSTNAME') - def _itertriplets(self, args): '''Generate repo, ref, filename triples from args.''' @@ -208,7 +201,7 @@ class Morph(cliapp.Application): rac = None repo_resolver = morphlib.repoaliasresolver.RepoAliasResolver( self.settings['repo-alias']) - lrc = morphlib.localrepocache.LocalRepoCache( + lrc = morphlib.localrepocache.LocalRepoCache(self, os.path.join(cachedir, 'gits'), repo_resolver, bundle_base_url=self.settings['bundle-server']) if self.settings['cache-server']: @@ -274,12 +267,13 @@ class Morph(cliapp.Application): install_chunks = False setup_proc = False - staging_area = morphlib.stagingarea.StagingArea(staging_root, + staging_area = morphlib.stagingarea.StagingArea(self, + staging_root, staging_temp) if self.settings['staging-chroot']: self._install_initial_staging(staging_area) - builder = morphlib.builder2.Builder( + builder = morphlib.builder2.Builder(self, staging_area, lac, rac, lrc, build_env, self.settings['max-jobs']) if setup_proc: @@ -341,7 +335,7 @@ class Morph(cliapp.Application): bundle_base_url = self.settings['bundle-server'] repo_resolver = morphlib.repoaliasresolver.RepoAliasResolver( self.settings['repo-alias']) - lrc = morphlib.localrepocache.LocalRepoCache( + lrc = morphlib.localrepocache.LocalRepoCache(self, cachedir, repo_resolver, bundle_base_url) if self.settings['cache-server']: rrc = morphlib.remoterepocache.RemoteRepoCache( @@ -434,7 +428,7 @@ class Morph(cliapp.Application): repo_resolver = morphlib.repoaliasresolver.RepoAliasResolver( self.settings['repo-alias']) bundle_base_url = self.settings['bundle-server'] - cache = morphlib.localrepocache.LocalRepoCache( + cache = morphlib.localrepocache.LocalRepoCache(self, cachedir, repo_resolver, bundle_base_url) subs_to_process = set() @@ -444,7 +438,8 @@ class Morph(cliapp.Application): assert cache.has_repo(reponame) cached_repo = cache.get_repo(reponame) try: - submodules = morphlib.git.Submodules(cached_repo.path, absref) + submodules = morphlib.git.Submodules(self, cached_repo.path, + absref) submodules.load() except morphlib.git.NoModulesFileError: pass @@ -469,7 +464,8 @@ class Morph(cliapp.Application): cached_repo.update() try: - submodules = morphlib.git.Submodules(cached_repo.path, ref) + submodules = morphlib.git.Submodules(self, cached_repo.path, + ref) submodules.load() except morphlib.git.NoModulesFileError: pass @@ -499,7 +495,7 @@ class Morph(cliapp.Application): os.path.join(cachedir, 'artifacts')) repo_resolver = morphlib.repoaliasresolver.RepoAliasResolver( self.settings['repo-alias']) - lrc = morphlib.localrepocache.LocalRepoCache( + lrc = morphlib.localrepocache.LocalRepoCache(self, os.path.join(cachedir, 'gits'), repo_resolver, bundle_base_url=self.settings['bundle-server']) if self.settings['cache-server']: @@ -530,18 +526,18 @@ class Morph(cliapp.Application): image_path_2 = lac.get(artifact2).name def setup(path): - part = morphlib.fsutils.setup_device_mapping(ex, path) + part = morphlib.fsutils.setup_device_mapping(self.runcmd, path) mount_point = tempfile.mkdtemp(dir=self.settings['tempdir']) - morphlib.fsutils.mount(ex, part, mount_point) + morphlib.fsutils.mount(self.runcmd, part, mount_point) return mount_point def cleanup(path, mount_point): try: - morphlib.fsutils.unmount(ex, mount_point) + morphlib.fsutils.unmount(self.runcmd, mount_point) except: pass try: - morphlib.fsutils.undo_device_mapping(ex, path) + morphlib.fsutils.undo_device_mapping(self.runcmd, path) except: pass try: @@ -549,14 +545,13 @@ class Morph(cliapp.Application): except: pass - ex = morphlib.execute.Execute('.', logging.debug) try: mount_point_1 = setup(image_path_1) mount_point_2 = setup(image_path_2) - ex.runv(['tbdiff-create', output, - os.path.join(mount_point_1, 'factory'), - os.path.join(mount_point_2, 'factory')]) + self.runcmd(['tbdiff-create', output, + os.path.join(mount_point_1, 'factory'), + os.path.join(mount_point_2, 'factory')]) except BaseException: raise finally: @@ -621,7 +616,7 @@ class Morph(cliapp.Application): repo_resolver = morphlib.repoaliasresolver.RepoAliasResolver( self.settings['repo-alias']) bundle_base_url = self.settings['bundle-server'] - cache = morphlib.localrepocache.LocalRepoCache( + cache = morphlib.localrepocache.LocalRepoCache(self, cachedir, repo_resolver, bundle_base_url) # Get the repository into the cache; make sure it is up to date. @@ -633,14 +628,14 @@ class Morph(cliapp.Application): repo.checkout(ref, os.path.abspath(dirname)) # Set the origin to point at the original repository. - morphlib.git.set_remote(dirname, 'origin', repo.url) + morphlib.git.set_remote(self.runcmd, dirname, 'origin', repo.url) # Add push url rewrite rule to .git/config. filename = os.path.join(dirname, '.git', 'config') with open(filename, 'a') as f: f.write('\n') f.write('[url "%s"]\n' % repo_resolver.push_url(reponame)) - f.write('\tpushInsteadOf = %s\n' % repo_resolver.pull_url(reponame)) + f.write('\tpushInsteadOf = %s\n' %repo_resolver.pull_url(reponame)) # Update remotes. self.runcmd(['git', 'remote', 'update'], cwd=dirname) @@ -823,7 +818,7 @@ class Morph(cliapp.Application): repo_resolver = morphlib.repoaliasresolver.RepoAliasResolver( self.settings['repo-alias']) bundle_base_url = self.settings['bundle-server'] - cache = morphlib.localrepocache.LocalRepoCache( + cache = morphlib.localrepocache.LocalRepoCache(self, cachedir, repo_resolver, bundle_base_url) for filename in args: @@ -860,9 +855,10 @@ class Morph(cliapp.Application): msg = kwargs['msg'] del kwargs['msg'] - if 'env' not in kwargs: + if 'env' not in kwargs: kwargs['env'] = dict(os.environ) - kwargs['env'].update(TMPDIR=self.settings['tempdir']) + if self.settings['tempdir'] is not None: + kwargs['env'].update(TMPDIR=self.settings['tempdir']) # convert the command line arguments into a string commands = [argv] + list(args) @@ -878,79 +874,5 @@ class Morph(cliapp.Application): # run the command line return cliapp.Application.runcmd(self, argv, *args, **kwargs) - # This is in morph so that policy is easily visible, and not embedded - # deep down in the call stack. - def clean_env(self): - '''Create a fresh set of environment variables for a clean build. - - Return a dict with the new environment. - - ''' - - path = os.environ['PATH'] - tools = os.environ.get('BOOTSTRAP_TOOLS') - distcc_hosts = os.environ.get('DISTCC_HOSTS') - - # copy a set of white-listed variables from the original env - copied_vars = dict.fromkeys([ - 'TMPDIR', - 'LD_PRELOAD', - 'LD_LIBRARY_PATH', - 'FAKEROOTKEY', - 'FAKED_MODE', - 'FAKEROOT_FD_BASE', - ]) - for name in copied_vars: - copied_vars[name] = os.environ.get(name, None) - - env = {} - - # apply the copied variables to the clean env - for name in copied_vars: - if copied_vars[name] is not None: - env[name] = copied_vars[name] - - env['TERM'] = 'dumb' - env['SHELL'] = '/bin/sh' - env['USER'] = \ - env['USERNAME'] = \ - env['LOGNAME'] = 'tomjon' - env['LC_ALL'] = 'C' - env['HOME'] = '/tmp' - - if self.settings['keep-path'] or self.settings['bootstrap']: - env['PATH'] = path - else: - env['PATH'] = '/sbin:/usr/sbin:/bin:/usr/bin' - - env['TOOLCHAIN_TARGET'] = self.settings['toolchain-target'] - env['CFLAGS'] = self.settings['target-cflags'] - env['PREFIX'] = self.settings['prefix'] - env['BOOTSTRAP'] = 'true' if self.settings['bootstrap'] else 'false' - if tools is not None: - env['BOOTSTRAP_TOOLS'] = tools - if distcc_hosts is not None: - env['DISTCC_HOSTS'] = distcc_hosts - - if not self.settings['no-ccache']: - env['PATH'] = ('/usr/lib/ccache:%s' % env['PATH']) -# FIXME: This needs to be made working, but it doesn't really, right now: -# the tempdir is not available inside the staging chroot. -# env['CCACHE_BASEDIR'] = self.tempdir.dirname - env['CCACHE_EXTRAFILES'] = ':'.join( - f for f in ('/baserock/binutils.meta', - '/baserock/eglibc.meta', - '/baserock/gcc.meta') if os.path.exists(f) - ) - if self.settings['ccache-remotedir'] != '': - env['CCACHE_REMOTEDIR'] = self.settings['ccache-remotedir'] - env['CCACHE_REMOTENLEVELS'] = \ - str(self.settings['ccache-remotenlevels']) - if not self.settings['no-distcc']: - env['CCACHE_PREFIX'] = 'distcc' - - return env - - if __name__ == '__main__': Morph().run() diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 23d9f8b6..56917402 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -28,16 +28,13 @@ class Error(cliapp.AppException): import artifact import artifactresolver import bins -import buildcontroller import buildenvironment import buildorder import buildsystem -import buildworker import builder2 import cachedir import cachedrepo import cachekeycomputer -import execute import fsutils import git import localartifactcache diff --git a/morphlib/bins.py b/morphlib/bins.py index f42951b7..93aa7b15 100644 --- a/morphlib/bins.py +++ b/morphlib/bins.py @@ -29,7 +29,7 @@ import stat import tarfile -def create_chunk(rootdir, f, regexps, ex, dump_memory_profile=None): +def create_chunk(rootdir, f, regexps, dump_memory_profile=None): '''Create a chunk from the contents of a directory. Only files and directories that match at least one of the regular @@ -95,7 +95,7 @@ def create_chunk(rootdir, f, regexps, ex, dump_memory_profile=None): dump_memory_profile('after removing in create_chunks') -def create_stratum(rootdir, f, ex): +def create_stratum(rootdir, f): '''Create a stratum from the contents of a directory.''' logging.debug('Creating stratum file %s from %s' % (f.name, rootdir)) tar = tarfile.open(fileobj=f, mode='w:gz') diff --git a/morphlib/bins_tests.py b/morphlib/bins_tests.py index dd2fb886..2da5d047 100644 --- a/morphlib/bins_tests.py +++ b/morphlib/bins_tests.py @@ -73,7 +73,6 @@ class BinsTest(unittest.TestCase): class ChunkTests(BinsTest): def setUp(self): - self.ex = morphlib.execute.Execute('.', lambda s: None) self.tempdir = tempfile.mkdtemp() self.instdir = os.path.join(self.tempdir, 'inst') self.chunk_file = os.path.join(self.tempdir, 'chunk') @@ -107,8 +106,7 @@ class ChunkTests(BinsTest): def create_chunk(self, regexps): self.populate_instdir() - morphlib.bins.create_chunk(self.instdir, self.chunk_f, regexps, - self.ex) + morphlib.bins.create_chunk(self.instdir, self.chunk_f, regexps) self.chunk_f.flush() def unpack_chunk(self): @@ -138,7 +136,6 @@ class ChunkTests(BinsTest): class StratumTests(BinsTest): def setUp(self): - self.ex = morphlib.execute.Execute('.', lambda s: None) self.tempdir = tempfile.mkdtemp() self.instdir = os.path.join(self.tempdir, 'inst') self.stratum_file = os.path.join(self.tempdir, 'stratum') @@ -155,7 +152,7 @@ class StratumTests(BinsTest): def test_creates_and_unpacks_stratum_exactly(self): self.populate_instdir() - morphlib.bins.create_stratum(self.instdir, self.stratum_f, self.ex) + morphlib.bins.create_stratum(self.instdir, self.stratum_f) self.stratum_f.flush() os.mkdir(self.unpacked) morphlib.bins.unpack_binary(self.stratum_file, self.unpacked) diff --git a/morphlib/buildcontroller.py b/morphlib/buildcontroller.py deleted file mode 100644 index fdc6c5e0..00000000 --- a/morphlib/buildcontroller.py +++ /dev/null @@ -1,127 +0,0 @@ -# Copyright (C) 2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import collections -import time - - -class BuildController(object): - - def __init__(self, app, tempdir): - self.settings = app.settings - self.real_msg = app.msg - self.tempdir = tempdir - self.indent = 1 - - self.workers = set() - self.busy_workers = set() - self.idle_workers = set() - - self.blobs = set() - self.build_order = collections.deque() - - def indent_more(self): # pragma: no cover - self.indent += 1 - - def indent_less(self): # pragma: no cover - self.indent -= 1 - - def msg(self, text): # pragma: no cover - spaces = ' ' * self.indent - self.real_msg('%s%s' % (spaces, text)) - - def generate_worker_name(self, ident): - similar_workers = [x for x in self.workers if x.ident == ident] - return '%s-%s' % (ident, len(similar_workers) + 1) - - def add_worker(self, worker): - self.workers.add(worker) - self.mark_idle(worker) - - def wait_for_workers(self, need_idle=False,timeout=0.1): # pragma: no cover - # first, check if any of the busy workers are finished - while all([not x.check_complete(timeout) for x in self.busy_workers]): - # wait and repeat if they are all busy and we have no idle workers - if need_idle and len(self.idle_workers) == 0: - time.sleep(0.250) - else: - break - - # get a list of all finished busy workers - finished = [x for x in self.busy_workers if x.check_complete(0)] - - # log the result of all workers that we are moving from busy to idle - for worker in finished: - self.msg('Built %s using worker %s' % (worker.blob, worker)) - if worker.output: - for line in worker.output.split('\n'): - self.msg('> %s' % line) - if worker.error: - import morphlib - raise morphlib.execute.CommandFailure(worker.error['command'], - worker.error['error']) - - # mark all finished workers as being idle - for worker in finished: - self.mark_idle(worker) - - def wait_for_worker(self): # pragma: no cover - # wait for at least one worker to be idle - self.wait_for_workers(need_idle = True) - - # sort idle workers by their idle timestamps (ascending) - idle_workers = sorted(self.idle_workers, key=lambda x: x.idle_since) - - # return the worker that has been idling for the longest period of time - return idle_workers[0] - - def build(self, blobs, build_order): # pragma: no cover - self.blobs = blobs - self.build_order = build_order - - result = [] - - while len(build_order) > 0: - group = build_order.popleft() - group_str = ', '.join([x.morph.filename for x in group]) - self.msg('Building parallel group %s' % group_str) - self.indent_more() - - while len(group) > 0: - blob = group.pop() - - worker = self.wait_for_worker() - self.msg('Distributing %s to worker %s' % (blob, worker)) - self.mark_busy(worker) - worker.build(blob) - - self.wait_for_workers(need_idle = False, timeout = None) - - self.indent_less() - - return result - - def mark_idle(self, worker): # pragma: no cover - if worker not in self.idle_workers: - self.idle_workers.add(worker) - if worker in self.busy_workers: - self.busy_workers.remove(worker) - - def mark_busy(self, worker): # pragma: no cover - if worker not in self.busy_workers: - self.busy_workers.add(worker) - if worker in self.idle_workers: - self.idle_workers.remove(worker) diff --git a/morphlib/buildcontroller_tests.py b/morphlib/buildcontroller_tests.py deleted file mode 100644 index 40b00213..00000000 --- a/morphlib/buildcontroller_tests.py +++ /dev/null @@ -1,86 +0,0 @@ -# Copyright (C) 2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import unittest - -import morphlib - - -class DummyApp(object): - - def __init__(self): - self.settings = {} - self.msg = lambda x: '%s' % x - - -class DummyWorker(object): - - def __init__(self, name, ident): - self.name = name - self.ident = ident - - -class BuildControllerTests(unittest.TestCase): - - def test_construction_with_app_and_tempdir(self): - app = DummyApp() - tempdir = '/foo/bar' - controller = morphlib.buildcontroller.BuildController(app, tempdir) - self.assertEqual(app.settings, controller.settings) - self.assertEqual(tempdir, controller.tempdir) - - def test_adding_workers(self): - app = DummyApp() - tempdir = '/foo/bar' - controller = morphlib.buildcontroller.BuildController(app, tempdir) - - worker1 = object() - worker2 = object() - worker3 = object() - - controller.add_worker(worker1) - self.assertTrue(worker1 in controller.workers) - self.assertTrue(worker2 not in controller.workers) - self.assertTrue(worker3 not in controller.workers) - - controller.add_worker(worker2) - self.assertTrue(worker1 in controller.workers) - self.assertTrue(worker2 in controller.workers) - self.assertTrue(worker3 not in controller.workers) - - controller.add_worker(worker3) - self.assertTrue(worker1 in controller.workers) - self.assertTrue(worker2 in controller.workers) - self.assertTrue(worker3 in controller.workers) - - def test_generation_of_worker_names(self): - app = DummyApp() - tempdir = '/foo/bar' - controller = morphlib.buildcontroller.BuildController(app, tempdir) - - localname1 = controller.generate_worker_name('local') - worker1 = DummyWorker(localname1, 'local') - controller.add_worker(worker1) - localname2 = controller.generate_worker_name('local') - worker2 = DummyWorker(localname1, 'local') - controller.add_worker(worker2) - localname3 = controller.generate_worker_name('local') - worker3 = DummyWorker(localname1, 'local') - controller.add_worker(worker3) - - self.assertEqual(localname1, 'local-1') - self.assertEqual(localname2, 'local-2') - self.assertEqual(localname3, 'local-3') diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 19a4b1b3..e42e7b0f 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -25,7 +25,7 @@ import tarfile import morphlib -def ldconfig(ex, rootdir): # pragma: no cover +def ldconfig(runcmd, rootdir): # pragma: no cover '''Run ldconfig for the filesystem below ``rootdir``. Essentially, ``rootdir`` specifies the root of a new system. @@ -53,11 +53,10 @@ def ldconfig(ex, rootdir): # pragma: no cover # directory (/sbin conventionally) that ldconfig is in. Then again, # it might, and if so, we don't want to hardware a particular # location. So we add the possible locations to the end of $PATH - # and restore that aftewards. - old_path = ex.env['PATH'] - ex.env['PATH'] = '%s:/sbin:/usr/sbin:/usr/local/sbin' % old_path - ex.runv(['ldconfig', '-r', rootdir]) - ex.env['PATH'] = old_path + env = dict(os.environ) + old_path = env['PATH'] + env['PATH'] = '%s:/sbin:/usr/sbin:/usr/local/sbin' % old_path + runcmd(['ldconfig', '-r', rootdir], env=env) else: logging.debug('No %s, not running ldconfig' % conf) @@ -91,9 +90,10 @@ class BuilderBase(object): '''Base class for building artifacts.''' - def __init__(self, staging_area, local_artifact_cache, + def __init__(self, app, staging_area, local_artifact_cache, remote_artifact_cache, artifact, repo_cache, build_env, max_jobs, setup_proc): + self.app = app self.staging_area = staging_area self.local_artifact_cache = local_artifact_cache self.remote_artifact_cache = remote_artifact_cache @@ -218,8 +218,7 @@ class ChunkBuilder(BuilderBase): logging.debug('Mounting /proc in staging area') path = os.path.join(self.staging_area.dirname, 'proc') if os.path.exists(path) and self.setup_proc: - ex = morphlib.execute.Execute('.', logging.debug) - ex.runv(['mount', '-t', 'proc', 'none', path]) + self.app.runcmd(['mount', '-t', 'proc', 'none', path]) return path else: logging.debug('Not mounting /proc after all, %s does not exist' % @@ -230,8 +229,7 @@ class ChunkBuilder(BuilderBase): if (mounted and self.setup_proc and mounted and os.path.exists(os.path.join(mounted, 'self'))): logging.error('Unmounting /proc in staging area: %s' % mounted) - ex = morphlib.execute.Execute('.', logging.debug) - ex.runv(['umount', mounted]) + morphlib.fsutils.unmount(self.app.runcmd, mounted) def get_sources(self, srcdir): # pragma: no cover '''Get sources from git to a source directory, for building.''' @@ -242,10 +240,10 @@ class ChunkBuilder(BuilderBase): logging.debug('Extracting %s into %s' % (path, destdir)) if not os.path.exists(destdir): os.mkdir(destdir) - morphlib.git.copy_repository(path, destdir, logging.debug) - morphlib.git.checkout_ref(destdir, sha1, logging.debug) - morphlib.git.reset_workdir(destdir, logging.debug) - submodules = morphlib.git.Submodules(path, sha1) + morphlib.git.copy_repository(self.app.runcmd, path, destdir) + morphlib.git.checkout_ref(self.app.runcmd, destdir, sha1) + morphlib.git.reset_workdir(self.app.runcmd, destdir) + submodules = morphlib.git.Submodules(self.app, path, sha1) try: submodules.load() except morphlib.git.NoModulesFileError: @@ -311,7 +309,6 @@ class ChunkBuilder(BuilderBase): def assemble_chunk_artifacts(self, destdir): # pragma: no cover with self.build_watch('create-chunks'): - ex = None # create_chunk doesn't actually use this specs = self.artifact.source.morphology['chunks'] if len(specs) == 0: specs = { @@ -328,7 +325,7 @@ class ChunkBuilder(BuilderBase): with self.local_artifact_cache.put(artifact) as f: logging.debug('assembling chunk %s' % artifact_name) logging.debug('assembling into %s' % f.name) - morphlib.bins.create_chunk(destdir, f, patterns, ex) + morphlib.bins.create_chunk(destdir, f, patterns) files = os.listdir(destdir) if files: @@ -375,7 +372,7 @@ class StratumBuilder(BuilderBase): self.write_metadata(destdir, artifact_name) artifact = self.new_artifact(artifact_name) with self.local_artifact_cache.put(artifact) as f: - morphlib.bins.create_stratum(destdir, f, None) + morphlib.bins.create_stratum(destdir, f) self.save_build_times() @@ -386,8 +383,6 @@ class SystemBuilder(BuilderBase): # pragma: no cover def build_and_cache(self): with self.build_watch('overall-build'): logging.debug('SystemBuilder.do_build called') - self.ex = morphlib.execute.Execute(self.staging_area.tempdir, - logging.debug) handle = self.local_artifact_cache.put(self.artifact) image_name = handle.name @@ -429,38 +424,39 @@ class SystemBuilder(BuilderBase): # pragma: no cover logging.debug('Creating disk image %s' % image_name) with self.build_watch('create-image'): morphlib.fsutils.create_image( - self.ex, image_name, + self.app.runcmd, image_name, self.artifact.source.morphology['disk-size']) def _partition_image(self, image_name): logging.debug('Partitioning disk image %s' % image_name) with self.build_watch('partition-image'): - morphlib.fsutils.partition_image(self.ex, image_name) + morphlib.fsutils.partition_image(self.app.runcmd, image_name) def _install_mbr(self, image_name): logging.debug('Installing mbr on disk image %s' % image_name) with self.build_watch('install-mbr'): - morphlib.fsutils.install_mbr(self.ex, image_name) + morphlib.fsutils.install_mbr(self.app.runcmd, image_name) def _setup_device_mapping(self, image_name): logging.debug('Device mapping partitions in %s' % image_name) with self.build_watch('setup-device-mapper'): - return morphlib.fsutils.setup_device_mapping(self.ex, image_name) + return morphlib.fsutils.setup_device_mapping(self.app.runcmd, + image_name) def _create_fs(self, partition): logging.debug('Creating filesystem on %s' % partition) with self.build_watch('create-filesystem'): - morphlib.fsutils.create_fs(self.ex, partition) + morphlib.fsutils.create_fs(self.app.runcmd, partition) def _mount(self, partition, mount_point): logging.debug('Mounting %s on %s' % (partition, mount_point)) with self.build_watch('mount-filesystem'): - morphlib.fsutils.mount(self.ex, partition, mount_point) + morphlib.fsutils.mount(self.app.runcmd, partition, mount_point) def _create_subvolume(self, path): logging.debug('Creating subvolume %s' % path) with self.build_watch('create-factory-subvolume'): - self.ex.runv(['btrfs', 'subvolume', 'create', path]) + self.app.runcmd(['btrfs', 'subvolume', 'create', path]) def _unpack_strata(self, path): logging.debug('Unpacking strata to %s' % path) @@ -483,7 +479,7 @@ class SystemBuilder(BuilderBase): # pragma: no cover f = self.local_artifact_cache.get(stratum_artifact) morphlib.bins.unpack_binary_from_file(f, path) f.close() - ldconfig(self.ex, path) + ldconfig(self.app.runcmd, path) def _create_fstab(self, path): logging.debug('Creating fstab in %s' % path) @@ -512,7 +508,7 @@ class SystemBuilder(BuilderBase): # pragma: no cover logging.debug('Creating subvolume snapshot %s to %s' % (source, target)) with self.build_watch('create-runtime-snapshot'): - self.ex.runv(['btrfs', 'subvolume', 'snapshot', source, target], + self.app.runcmd(['btrfs', 'subvolume', 'snapshot', source, target], cwd=path) def _install_boot_files(self, sourcefs, targetfs): @@ -529,22 +525,22 @@ class SystemBuilder(BuilderBase): # pragma: no cover def _install_extlinux(self, path): logging.debug('Installing extlinux to %s' % path) with self.build_watch('install-bootloader'): - self.ex.runv(['extlinux', '--install', path]) + self.app.runcmd(['extlinux', '--install', path]) # FIXME this hack seems to be necessary to let extlinux finish - self.ex.runv(['sync']) + self.app.runcmd(['sync']) time.sleep(2) def _unmount(self, mount_point): logging.debug('Unmounting %s' % mount_point) with self.build_watch('unmount-filesystem'): if mount_point is not None: - morphlib.fsutils.unmount(self.ex, mount_point) + morphlib.fsutils.unmount(self.app.runcmd, mount_point) def _undo_device_mapping(self, image_name): logging.debug('Undoing device mappings for %s' % image_name) with self.build_watch('undo-device-mapper'): - morphlib.fsutils.undo_device_mapping(self.ex, image_name) + morphlib.fsutils.undo_device_mapping(self.app.runcmd, image_name) class Builder(object): # pragma: no cover @@ -557,8 +553,9 @@ class Builder(object): # pragma: no cover 'system': SystemBuilder, } - def __init__(self, staging_area, local_artifact_cache, + def __init__(self, app, staging_area, local_artifact_cache, remote_artifact_cache, repo_cache, build_env, max_jobs): + self.app = app self.staging_area = staging_area self.local_artifact_cache = local_artifact_cache self.remote_artifact_cache = remote_artifact_cache @@ -569,7 +566,8 @@ class Builder(object): # pragma: no cover def build_and_cache(self, artifact): kind = artifact.source.morphology['kind'] - o = self.classes[kind](self.staging_area, self.local_artifact_cache, + o = self.classes[kind](self.app, self.staging_area, + self.local_artifact_cache, self.remote_artifact_cache, artifact, self.repo_cache, self.build_env, self.max_jobs, self.setup_proc) diff --git a/morphlib/builder2_tests.py b/morphlib/builder2_tests.py index 93a2f8b7..9730b59e 100644 --- a/morphlib/builder2_tests.py +++ b/morphlib/builder2_tests.py @@ -27,6 +27,9 @@ class FakeBuildSystem(object): def __init__(self): self.build_commands = ['buildsys-it'] +class FakeApp(object): + def __init__(self, runcmd=None): + self.runcmd = runcmd class FakeStagingArea(object): @@ -43,7 +46,8 @@ class FakeSource(object): 'description': 'c', } - self.repo = morphlib.cachedrepo.CachedRepo('repo', 'url', 'path') + self.repo = morphlib.cachedrepo.CachedRepo(FakeApp(), 'repo', + 'url', 'path') self.original_ref = 'e' self.sha1 = 'f' self.filename = 'g' @@ -131,13 +135,15 @@ class BuilderBaseTests(unittest.TestCase): def setUp(self): self.commands_run = [] + self.app = FakeApp(self.fake_runcmd) self.staging_area = FakeStagingArea(self.fake_runcmd) self.artifact_cache = FakeArtifactCache() self.artifact = FakeArtifact('le-artifact') self.repo_cache = None self.build_env = FakeBuildEnv() self.max_jobs = 1 - self.builder = morphlib.builder2.BuilderBase(self.staging_area, + self.builder = morphlib.builder2.BuilderBase(self.app, + self.staging_area, self.artifact_cache, None, self.artifact, @@ -206,8 +212,10 @@ class BuilderBaseTests(unittest.TestCase): class ChunkBuilderTests(unittest.TestCase): def setUp(self): - self.build = morphlib.builder2.ChunkBuilder(None, None, None, None, - None, None, 1, False) + self.app = FakeApp() + self.build = morphlib.builder2.ChunkBuilder(self.app, None, None, + None, None, None, None, 1, + False) def test_uses_morphology_commands_when_given(self): m = { 'build-commands': ['build-it'] } diff --git a/morphlib/buildworker.py b/morphlib/buildworker.py deleted file mode 100644 index d839b09a..00000000 --- a/morphlib/buildworker.py +++ /dev/null @@ -1,277 +0,0 @@ -# Copyright (C) 2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import datetime -from multiprocessing import Manager, Process - -import morphlib - - -class BuildWorker(object): - - def __init__(self, name, ident, app): - self.name = name - self.ident = ident - self.settings = app.settings - self.real_msg = app.msg - self.indent = 2 - self.idle_since = datetime.datetime.now() - self.manager = Manager() - self.reset() - - def indent_more(self): # pragma: no cover - self.indent += 1 - - def indent_less(self): # pragma: no cover - self.indent -= 1 - - def msg(self, text): # pragma: no cover - spaces = ' ' * self.indent - self.real_msg('%s%s' % (spaces, text)) - - def reset(self): - self.process = None - self.blob = None - self._output = self.manager.list() - self._error = self.manager.dict() - - def build(self, blob): - raise NotImplementedError - - def check_complete(self, timeout): # pragma: no cover - if self.process: - self.process.join(timeout) - if self.process.is_alive(): - return False - else: - self.idle_since = datetime.datetime.now() - return True - else: - return True - - @property - def output(self): - try: - return self._output[0] - except IndexError: - return None - - @property - def error(self): - return self._error - - def options(self): # pragma: no cover - '''Returns an array of command line options for the settings. - - NOTE: This is just a hack that uses internals of the cliapp.Settings - class. We need to merge this kind of functionality into cliapp and - remove this hack! - - ''' - - # use the canonical names of the settings to generate cmdline options - names = list(self.settings._canonical_names) - - # internal function to combine an option name and a value into string - def combine(name, value): - if name.startswith('--'): - if isinstance(value, bool): - if value: - return '%s' % name - else: - return '%s=%s' % (name, value) - else: - if isinstance(value, bool): - if value: - return '%s' % name - else: - return '%s %s' % (name, value) - - # generate a list of (setting name, option name) pairs - options_list = [] - options = self.settings._option_names(names) - name_pairs = [(names[i], options[i]) for i in xrange(0, len(names))] - - # convert all settings into command line arguments; drop absent - # settings and make sure to convert all values correctly - for name, option in name_pairs: - value = self.settings[name] - if not value: - continue - if isinstance(value, list): - for listval in value: - if isinstance(listval, str): - if len(listval) > 0: - string = combine(option, listval) - if string: - options_list.append(string) - else: - string = combine(option, listval) - if string: - options_list.append(string) - else: - if isinstance(value, str): - if len(value) > 0: - string = combine(option, value) - if string: - options_list.append(string) - else: - string = combine(option, value) - if string: - options_list.append(string) - return options_list - - def __str__(self): - return self.name - - -class LocalBuildWorker(BuildWorker): - - def __init__(self, name, ident, app): - BuildWorker.__init__(self, name, ident, app) - - def run(self, first_tuple, second_tuple, sudo, output, - error): # pragma: no cover - ex = morphlib.execute.Execute('.', self.msg) - - # generate command line options - args = self.options() - cmdline = [] - if sudo: - cmdline.extend(['sudo']) - cmdline.extend(['morph', 'build-single']) - cmdline.extend([first_tuple['repo'], - first_tuple['ref'], - first_tuple['filename']]) - if second_tuple: - cmdline.extend([second_tuple['repo'], - second_tuple['ref'], - second_tuple['filename']]) - cmdline.extend(args) - - # run morph locally in a child process - try: - stdout = ex.runv(cmdline) - output.append(stdout) - except OSError, e: - error['error'] = str(e) - error['command'] = ' '.join(cmdline) - except morphlib.execute.CommandFailure, e: - error['error'] = str(e) - error['command'] = ' '.join(cmdline) - - def build(self, blob): # pragma: no cover - self.reset() - self.blob = blob - - first_tuple = None - if len(blob.parents) > 0: - first_tuple = { - 'repo': blob.parents[0].morph.treeish.original_repo, - 'ref': blob.parents[0].morph.treeish.ref, - 'filename': blob.parents[0].morph.filename, - } - - blob_tuple = { - 'repo': blob.morph.treeish.original_repo, - 'ref': blob.morph.treeish.ref, - 'filename': blob.morph.filename, - } - - args = (first_tuple if first_tuple else blob_tuple, - blob_tuple if first_tuple else None, - blob.morph.kind == 'system', - self._output, - self._error) - self.process = Process(group=None, target=self.run, args=args) - self.process.start() - - -class RemoteBuildWorker(BuildWorker): - - def __init__(self, name, ident, app): - BuildWorker.__init__(self, name, ident, app) - self.hostname = ident - - def run(self, first_tuple, second_tuple, sudo, output, - error): # pragma: no cover - ex = morphlib.execute.Execute('.', self.msg) - - # generate command line options - args = self.options() - cmdline = ['ssh', '-q', self.hostname] - if sudo: - cmdline.extend(['-t', '-t', '-t', 'sudo', '-S', - 'bash', '--login', '-c']) - cmdline.extend(['"']) - cmdline.extend(['morph', 'build-single', repo, ref, filename]) - cmdline.extend([first_tuple['repo'], - first_tuple['ref'], - first_tuple['filename']]) - if second_tuple: - cmdline.extend([second_tuple['repo'], - second_tuple['ref'], - second_tuple['filename']]) - cmdline.extend(args) - cmdline.extend(['"']) - else: - cmdline.extend(['fakeroot']) - cmdline.extend(['morph', 'build-single', repo, ref, filename]) - cmdline.extend([first_tuple['repo'], - first_tuple['ref'], - first_tuple['filename']]) - if second_tuple: - cmdline.extend([second_tuple['repo'], - second_tuple['ref'], - second_tuple['filename']]) - cmdline.extend(args) - - # run morph on the other machine - try: - stdout = ex.runv(cmdline) - output.append(stdout) - except OSError, e: - error['error'] = str(e) - error['command'] = ' '.join(cmdline) - except morphlib.execute.CommandFailure, e: - error['error'] = str(e) - error['command'] = ' '.join(cmdline) - - def build(self, blob): # pragma: no cover - self.reset() - self.blob = blob - - first_tuple = None - if len(blob.parents) > 0: - first_tuple = { - 'repo': blob.parents[0].morph.treeish.original_repo, - 'ref': blob.parents[0].morph.treeish.ref, - 'filename': blob.parents[0].morph.filename, - } - - blob_tuple = { - 'repo': blob.morph.treeish.original_repo, - 'ref': blob.morph.treeish.ref, - 'filename': blob.morph.filename, - } - - args = (first_tuple if first_tuple else blob_tuple, - blob_tuple if first_tuple else None, - blob.morph.kind == 'system', - self._output, - self._error) - self.process = Process(group=None, target=self.run, args=args) - self.process.start() diff --git a/morphlib/buildworker_tests.py b/morphlib/buildworker_tests.py deleted file mode 100644 index 6cf4b38b..00000000 --- a/morphlib/buildworker_tests.py +++ /dev/null @@ -1,75 +0,0 @@ -# Copyright (C) 2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import unittest - -import morphlib -from morphlib import buildworker - - -class DummyApp(object): - - def __init__(self): - self.settings = {} - self.msg = lambda x: '%s' % x - - -class BuildWorkerTests(unittest.TestCase): - - def test_construction_with_name_ident_and_app(self): - app = DummyApp() - worker = buildworker.BuildWorker('local-1', 'local', app) - - self.assertEqual(worker.name, 'local-1') - self.assertEqual(worker.ident, 'local') - self.assertEqual(worker.settings, app.settings) - - def test_methods_that_need_to_be_overloaded(self): - app = DummyApp() - worker = buildworker.BuildWorker('local-1', 'local', app) - self.assertRaises(NotImplementedError, worker.build, []) - - def test_conversion_to_a_string(self): - app = DummyApp() - worker = buildworker.BuildWorker('local-1', 'local', app) - self.assertEquals(str(worker), 'local-1') - - def test_local_builder_construction_with_name_ident_and_app(self): - app = DummyApp() - worker = buildworker.LocalBuildWorker('local-1', 'local', app) - - self.assertEqual(worker.name, 'local-1') - self.assertEqual(worker.ident, 'local') - self.assertEqual(worker.settings, app.settings) - - def test_remote_builder_construction_with_name_ident_and_app(self): - app = DummyApp() - worker = buildworker.RemoteBuildWorker('user@host-1', 'user@host', app) - - self.assertEqual(worker.name, 'user@host-1') - self.assertEqual(worker.ident, 'user@host') - self.assertEqual(worker.hostname, 'user@host') - self.assertEqual(worker.settings, app.settings) - - def test_output_property(self): - app = DummyApp() - worker = buildworker.BuildWorker('local-1', 'local', app) - self.assertEqual(worker.output, None) - - def test_error_property(self): - app = DummyApp() - worker = buildworker.BuildWorker('local-1', 'local', app) - self.assertTrue(len(worker.error) == 0) diff --git a/morphlib/cachedrepo.py b/morphlib/cachedrepo.py index 4c651130..575eedd6 100644 --- a/morphlib/cachedrepo.py +++ b/morphlib/cachedrepo.py @@ -18,8 +18,7 @@ import cliapp import logging import os -import morphlib.execute - +import morphlib class InvalidReferenceError(cliapp.AppException): @@ -76,13 +75,13 @@ class CachedRepo(object): ''' - def __init__(self, original_name, url, path): + def __init__(self, app, original_name, url, path): '''Creates a new CachedRepo for a repo name, URL and local path.''' + self.app = app self.original_name = original_name self.url = url self.path = path - self.ex = morphlib.execute.Execute(self.path, logging.debug) def is_valid_sha1(self, ref): '''Checks whether a string is a valid SHA1.''' @@ -103,14 +102,14 @@ class CachedRepo(object): # split each ref line into an array, drop non-origin branches refs = [x.split() for x in refs if 'origin' in x] return refs[0][0] - except morphlib.execute.CommandFailure: + except cliapp.AppException: pass if not self.is_valid_sha1(ref): raise InvalidReferenceError(self, ref) try: return self._rev_list(ref).strip() - except morphlib.execute.CommandFailure: + except cliapp.AppException: raise InvalidReferenceError(self, ref) def cat(self, ref, filename): @@ -127,12 +126,12 @@ class CachedRepo(object): raise UnresolvedNamedReferenceError(self, ref) try: sha1 = self._rev_list(ref).strip() - except morphlib.execute.CommandFailure: + except cliapp.AppException: raise InvalidReferenceError(self, ref) try: return self._cat_file(sha1, filename) - except morphlib.execute.CommandFailure: + except cliapp.AppException: raise IOError('File %s does not exist in ref %s of repo %s' % (filename, ref, self)) @@ -155,7 +154,7 @@ class CachedRepo(object): try: self._copy_repository(self.path, target_dir) self._checkout_ref(ref, target_dir) - except morphlib.execute.CommandFailure: + except cliapp.AppException: raise CheckoutError(self, ref, target_dir) def update(self): @@ -168,28 +167,32 @@ class CachedRepo(object): try: self._update() - except morphlib.execute.CommandFailure: + except cliapp.AppException, e: raise UpdateError(self) + def _runcmd(self, *args, **kwargs): # pragma: no cover + if not 'cwd' in kwargs: + kwargs['cwd'] = self.path + return self.app.runcmd(*args, **kwargs) + def _show_ref(self, ref): # pragma: no cover - return self.ex.runv(['git', 'show-ref', ref]) + return self._runcmd(['git', 'show-ref', ref]) def _rev_list(self, ref): # pragma: no cover - return self.ex.runv(['git', 'rev-list', '--no-walk', ref]) + return self._runcmd(['git', 'rev-list', '--no-walk', ref]) def _cat_file(self, ref, filename): # pragma: no cover - return self.ex.runv(['git', 'cat-file', 'blob', + return self._runcmd(['git', 'cat-file', 'blob', '%s:%s' % (ref, filename)]) def _copy_repository(self, source_dir, target_dir): # pragma: no cover - self.ex.runv(['cp', '-a', os.path.join(source_dir, '.git'), - target_dir]) + morphlib.git.copy_repository(self._runcmd, source_dir, target_dir) def _checkout_ref(self, ref, target_dir): # pragma: no cover - self.ex.runv(['git', 'checkout', ref], cwd=target_dir) + morphlib.git.checkout_ref(self._runcmd, target_dir, ref) def _update(self): # pragma: no cover - self.ex.runv(['git', 'remote', 'update', 'origin']) + self._runcmd(['git', 'remote', 'update', 'origin']) def __str__(self): # pragma: no cover return self.url diff --git a/morphlib/cachedrepo_tests.py b/morphlib/cachedrepo_tests.py index f3d4d1fc..0baeba26 100644 --- a/morphlib/cachedrepo_tests.py +++ b/morphlib/cachedrepo_tests.py @@ -17,6 +17,8 @@ import os import unittest +import cliapp + import morphlib from morphlib import cachedrepo @@ -36,7 +38,7 @@ class CachedRepoTests(unittest.TestCase): try: return output[ref] except: - raise morphlib.execute.CommandFailure('git show-ref %s' % ref, '') + raise cliapp.AppException('git show-ref %s' % ref, '') def rev_list(self, ref): output = { @@ -48,7 +50,7 @@ class CachedRepoTests(unittest.TestCase): try: return output[ref] except: - raise morphlib.execute.CommandFailure('git rev-list %s' % ref, '') + raise cliapp.AppException('git rev-list %s' % ref, '') def cat_file(self, ref, filename): output = { @@ -58,7 +60,7 @@ class CachedRepoTests(unittest.TestCase): try: return output['%s:%s' % (ref, filename)] except: - raise morphlib.execute.CommandFailure( + raise cliapp.AppException( 'git cat-file blob %s:%s' % (ref, filename), '') def copy_repository(self, source_dir, target_dir): @@ -72,7 +74,7 @@ class CachedRepoTests(unittest.TestCase): if ref in bad_refs: # simulate a git failure or something similar to # trigger a CheckoutError - raise morphlib.execute.CommandFailure('git checkout %s' % ref, '') + raise cliapp.AppException('git checkout %s' % ref, '') else: with open(os.path.join(target_dir, 'foo.morph'), 'w') as f: f.write('contents of foo.morph') @@ -81,13 +83,13 @@ class CachedRepoTests(unittest.TestCase): pass def update_with_failure(self): - raise morphlib.execute.CommandFailure('git remote update origin', '') + raise cliapp.AppException('git remote update origin', '') def setUp(self): self.repo_name = 'foo' self.repo_url = 'git://foo.bar/foo.git' self.repo_path = '/tmp/foo' - self.repo = cachedrepo.CachedRepo( + self.repo = cachedrepo.CachedRepo(object(), self.repo_name, self.repo_url, self.repo_path) self.repo._show_ref = self.show_ref self.repo._rev_list = self.rev_list diff --git a/morphlib/execute.py b/morphlib/execute.py deleted file mode 100644 index 71053faa..00000000 --- a/morphlib/execute.py +++ /dev/null @@ -1,112 +0,0 @@ -# Copyright (C) 2011-2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import cliapp -import logging -import os -import subprocess - -import morphlib - - -class CommandFailure(cliapp.AppException): - - def __init__(self, command, stderr): - cliapp.AppException.__init__(self, - 'Command failed: %s\nOutput from command:\n%s' % - (command, stderr)) - - -class Execute(object): - - '''Execute commands for morph.''' - - def __init__(self, dirname, msg): - self._setup_env() - self.dirname = dirname - self.msg = msg - - def _setup_env(self): - self.env = dict(os.environ) - - def run(self, commands, _log=True): - '''Execute a list of commands. - - If a command fails (returns non-zero exit code), the rest are - not run, and CommandFailure is returned. - - ''' - - stdouts = [] - for command in commands: - self.msg('# %s' % command) - argv = ['sh', '-c', command] - logging.debug('run: argv=%s' % repr(argv)) - logging.debug('run: env=%s' % repr(self.env)) - logging.debug('run: cwd=%s' % repr(self.dirname)) - p = subprocess.Popen(argv, shell=False, - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, - env=self.env, - cwd=self.dirname) - out, err = p.communicate() - if p.returncode != 0: - if _log: # pragma: no cover - logging.error('Exit code: %d' % p.returncode) - logging.error('Standard output and error:\n%s' % - morphlib.util.indent(out)) - raise CommandFailure(command, out) - stdouts.append(out) - return stdouts - - def runv(self, argv, feed_stdin=None, _log=True, **kwargs): - '''Run a command given as a list of argv elements. - - Return standard output. Raise ``CommandFailure`` if the command - fails. Log standard output and error in any case. - - ''' - if 'stdout' not in kwargs: - kwargs['stdout'] = subprocess.PIPE - if feed_stdin is not None and 'stdin' not in kwargs: - kwargs['stdin'] = subprocess.PIPE # pragma: no cover - if 'stderr' not in kwargs: - kwargs['stderr'] = subprocess.STDOUT - if 'cwd' not in kwargs: - kwargs['cwd'] = self.dirname - if 'env' not in kwargs: - kwargs['env'] = self.env - - logging.debug('runv: argv=%s' % repr(argv)) - logging.debug('runv: env=%s' % repr(kwargs['env'])) - logging.debug('runv: cwd=%s' % repr(self.dirname)) - self.msg('# %s' % ' '.join(argv)) - p = subprocess.Popen(argv, **kwargs) - out, err = p.communicate(feed_stdin) - - if _log: # pragma: no cover - if p.returncode == 0: - logger = logging.debug - else: - logger = logging.error - logger('Exit code: %d' % p.returncode) - logger('Standard output:\n%s' % morphlib.util.indent(out or '')) - logger('Standard error:\n%s' % morphlib.util.indent(err or '')) - if p.returncode != 0: - raise CommandFailure(' '.join(argv), out) - else: - return out - diff --git a/morphlib/execute_tests.py b/morphlib/execute_tests.py deleted file mode 100644 index 927ab07f..00000000 --- a/morphlib/execute_tests.py +++ /dev/null @@ -1,54 +0,0 @@ -# Copyright (C) 2011-2012 Codethink Limited -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along -# with this program; if not, write to the Free Software Foundation, Inc., -# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. - - -import os -import unittest - -import morphlib - - -class ExecuteTests(unittest.TestCase): - - def setUp(self): - self.e = morphlib.execute.Execute('/', lambda msg: None) - - def test_has_same_path_as_environment(self): - self.assertEqual(self.e.env['PATH'], os.environ['PATH']) - - def test_executes_true_ok(self): - self.assertEqual(self.e.run(['true']), ['']) - - def test_raises_commandfailure_for_false(self): - self.assertRaises(morphlib.execute.CommandFailure, - self.e.run, ['false'], _log=False) - - def test_returns_stdout_from_all_commands(self): - self.assertEqual(self.e.run(['echo foo', 'echo bar']), - ['foo\n', 'bar\n']) - - def test_sets_working_directory(self): - self.assertEqual(self.e.run(['pwd']), ['/\n']) - - def test_executes_argv(self): - self.assertEqual(self.e.runv(['echo', 'foo']), 'foo\n') - - def test_raises_error_when_argv_fails(self): - self.assertRaises(morphlib.execute.CommandFailure, - self.e.runv, ['false'], _log=False) - - def test_runv_sets_working_directory(self): - self.assertEqual(self.e.runv(['pwd']), '/\n') - diff --git a/morphlib/fsutils.py b/morphlib/fsutils.py index 15ad7ebf..d4f9f474 100644 --- a/morphlib/fsutils.py +++ b/morphlib/fsutils.py @@ -17,26 +17,26 @@ import os import re -def create_image(ex, image_name, size): +def create_image(runcmd, image_name, size): # FIXME a pure python implementation may be better - ex.runv(['dd', 'if=/dev/zero', 'of=' + image_name, 'bs=1', - 'seek=%d' % size, 'count=0']) + runcmd(['dd', 'if=/dev/zero', 'of=' + image_name, 'bs=1', + 'seek=%d' % size, 'count=0']) -def partition_image(ex, image_name): +def partition_image(runcmd, image_name): # FIXME make this more flexible with partitioning options - ex.runv(['sfdisk', image_name], feed_stdin='1,,83,*\n') + runcmd(['sfdisk', image_name], feed_stdin='1,,83,*\n') -def install_mbr(ex, image_name): +def install_mbr(runcmd, image_name): for path in ['/usr/lib/extlinux/mbr.bin', '/usr/share/syslinux/mbr.bin']: if os.path.exists(path): - ex.runv(['dd', 'if=' + path, 'of=' + image_name, - 'conv=notrunc']) + runcmd(['dd', 'if=' + path, 'of=' + image_name, + 'conv=notrunc']) break -def setup_device_mapping(ex, image_name): +def setup_device_mapping(runcmd, image_name): findstart = re.compile(r"start=\s+(\d+),") - out = ex.runv(['sfdisk', '-d', image_name]) + out = runcmd(['sfdisk', '-d', image_name]) for line in out.splitlines(): match = findstart.search(line) if match is None: @@ -45,30 +45,30 @@ def setup_device_mapping(ex, image_name): if start != 0: break - ex.runv(['losetup', '-o', str(start), '-f', image_name]) + runcmd(['losetup', '-o', str(start), '-f', image_name]) - out = ex.runv(['losetup', '-j', image_name]) + out = runcmd(['losetup', '-j', image_name]) line = out.strip() i = line.find(':') return line[:i] -def create_fs(ex, partition): +def create_fs(runcmd, partition): # FIXME: the hardcoded size of 4GB is icky but the default broke # when we used mkfs -t ext4 - ex.runv(['mkfs.btrfs', '-L', 'baserock', - '-b', '4294967296', partition]) + runcmd(['mkfs.btrfs', '-L', 'baserock', + '-b', '4294967296', partition]) -def mount(ex, partition, mount_point): +def mount(runcmd, partition, mount_point): if not os.path.exists(mount_point): os.mkdir(mount_point) - ex.runv(['mount', partition, mount_point]) + runcmd(['mount', partition, mount_point]) -def unmount(ex, mount_point): - ex.runv(['umount', mount_point]) +def unmount(runcmd, mount_point): + runcmd(['umount', mount_point]) -def undo_device_mapping(ex, image_name): - out = ex.runv(['losetup', '-j', image_name]) +def undo_device_mapping(runcmd, image_name): + out = runcmd(['losetup', '-j', image_name]) for line in out.splitlines(): i = line.find(':') device = line[:i] - ex.runv(['losetup', '-d', device]) + runcmd(['losetup', '-d', device]) diff --git a/morphlib/git.py b/morphlib/git.py index 1e9190df..5a3f8503 100644 --- a/morphlib/git.py +++ b/morphlib/git.py @@ -22,78 +22,9 @@ import os import re import StringIO -import morphlib - - -class NoMorphs(Exception): - - def __init__(self, repo, ref): - Exception.__init__(self, 'Cannot find any morpologies at %s:%s' % - (repo, ref)) - - -class TooManyMorphs(Exception): - - def __init__(self, repo, ref, morphs): - Exception.__init__(self, 'Too many morphologies at %s:%s: %s' % - (repo, ref, ', '.join(morphs))) - - -class InvalidReferenceError(cliapp.AppException): - - def __init__(self, repo, ref): - Exception.__init__(self, '%s is an invalid reference for repo %s' % - (ref, repo)) - - -class Treeish(object): - - def __init__(self, repo, original_repo, ref, msg=logging.debug): - self.repo = repo - self.msg = msg - self.sha1 = None - self.ref = None - self.original_repo = original_repo - self._resolve_ref(ref) - - def __hash__(self): - return hash((self.repo, self.ref)) - - def __eq__(self, other): - return other.repo == self.repo and other.ref == self.ref - - def __str__(self): - return '%s:%s' % (self.repo, self.ref) - - def _resolve_ref(self, ref): - ex = morphlib.execute.Execute(self.repo, self.msg) - try: - refs = ex.runv(['git', 'show-ref', ref]).split('\n') - - # split each ref line into an array, drop non-origin branches - refs = [x.split() for x in refs if 'origin' in x] - - binascii.unhexlify(refs[0][0]) #Valid hex? - self.sha1 = refs[0][0] - self.ref = refs[0][1] - except morphlib.execute.CommandFailure: - self._is_sha(ref) - - def _is_sha(self, ref): - if len(ref) != 40: - raise InvalidReferenceError(self.original_repo, ref) +import cliapp - try: - binascii.unhexlify(ref) - ex = morphlib.execute.Execute(self.repo, self.msg) - ex.runv(['git', 'rev-list', '--no-walk', ref]) - self.sha1 = ref - # NOTE this is a hack, but we really don't want to add - # conditionals all over the place in order to handle - # situations where we only have a .sha1, do we? - self.ref = ref - except (TypeError, morphlib.execute.CommandFailure): - raise InvalidReferenceError(self.original_repo, ref) +import morphlib class NoModulesFileError(cliapp.AppException): @@ -111,13 +42,6 @@ class Submodule(object): self.path = path -class ModulesFileParseError(cliapp.AppException): - - def __init__(self, repo, ref, message): - Exception.__init__(self, 'Failed to parse %s:%s:.gitmodules: %s' % - (repo, ref, message)) - - class InvalidSectionError(cliapp.AppException): def __init__(self, repo, ref, section): @@ -136,7 +60,8 @@ class MissingSubmoduleCommitError(cliapp.AppException): class Submodules(object): - def __init__(self, repo, ref, msg=logging.debug): + def __init__(self, app, repo, ref, msg=logging.debug): + self.app = app self.repo = repo self.ref = ref self.msg = msg @@ -154,17 +79,16 @@ class Submodules(object): def _read_gitmodules_file(self): try: # try to read the .gitmodules file from the repo/ref - ex = morphlib.execute.Execute(self.repo, self.msg) - content = ex.runv(['git', 'cat-file', 'blob', '%s:.gitmodules' % - self.ref]) + content = self.app.runcmd( + ['git', 'cat-file', 'blob', '%s:.gitmodules' % self.ref], + cwd=self.repo) # drop indentation in sections, as RawConfigParser cannot handle it return '\n'.join([line.strip() for line in content.splitlines()]) - except morphlib.execute.CommandFailure: + except cliapp.AppException: raise NoModulesFileError(self.repo, self.ref) def _validate_and_read_entries(self, parser): - ex = morphlib.execute.Execute(self.repo, self.msg) for section in parser.sections(): # validate section name against the 'section "foo"' pattern section_pattern = r'submodule "(.*)"' @@ -179,8 +103,8 @@ class Submodules(object): try: # list objects in the parent repo tree to find the commit # object that corresponds to the submodule - commit = ex.runv(['git', 'ls-tree', self.ref, - submodule.name]) + commit = self.app.runcmd(['git', 'ls-tree', self.ref, + submodule.name], cwd=self.repo) # read the commit hash from the output fields = commit.split() @@ -199,7 +123,7 @@ class Submodules(object): self.msg('Skipping submodule "%s" as %s:%s has ' 'a non-commit object for it' % (submodule.name, self.repo, self.ref)) - except morphlib.execute.CommandFailure: + except cliapp.AppException: raise MissingSubmoduleCommitError(self.repo, self.ref, submodule.name) else: @@ -213,57 +137,20 @@ class Submodules(object): return len(self.submodules) -def get_morph_text(treeish, filename, msg=logging.debug): - '''Return a morphology from a git repository.''' - ex = morphlib.execute.Execute(treeish.repo, msg=msg) - return ex.runv(['git', 'cat-file', 'blob', '%s:%s' - % (treeish.sha1, filename)]) - -def extract_bundle(location, bundle, msg=logging.debug): - '''Extract a bundle into git at location''' - ex = morphlib.execute.Execute(location, msg=msg) - return ex.runv(['git', 'clone', '-n', bundle, '.']) - -def clone(location, repo, msg=logging.debug): - '''clone at git repo into location''' - ex = morphlib.execute.Execute('.', msg=msg) - return ex.runv(['git', 'clone', '-n', '-l', repo, location]) - -def init(location, msg=logging.debug): - '''initialise git repo at location''' - os.mkdir(location) - ex = morphlib.execute.Execute(location, msg=msg) - return ex.runv(['git', 'init']) - -def set_remote(gitdir, name, url, msg=logging.debug): +def set_remote(runcmd, gitdir, name, url): '''Set remote with name 'name' use a given url at gitdir''' - ex = morphlib.execute.Execute(gitdir, msg=msg) - return ex.runv(['git', 'remote', 'set-url', name, url]) - -def update_remote(gitdir, name, msg=logging.debug): - ex = morphlib.execute.Execute(gitdir, msg=msg) - return ex.runv(['git', 'remote', 'update', name]) + return runcmd(['git', 'remote', 'set-url', name, url], cwd=gitdir) -def copy_repository(repo, destdir, msg=logging.debug): +def copy_repository(runcmd, repo, destdir): '''Copies a cached repository into a directory using cp.''' - ex = morphlib.execute.Execute('.', msg=msg) - return ex.runv(['cp', '-a', os.path.join(repo, '.git'), destdir]) + return runcmd(['cp', '-a', os.path.join(repo, '.git'), destdir]) -def checkout_ref(gitdir, ref, msg=logging.debug): +def checkout_ref(runcmd, gitdir, ref): '''Checks out a specific ref/SHA1 in a git working tree.''' - ex = morphlib.execute.Execute(gitdir, msg=msg) - ex.runv(['git', 'checkout', ref]) + runcmd(['git', 'checkout', ref], cwd=gitdir) -def reset_workdir(gitdir, msg=logging.debug): +def reset_workdir(runcmd, gitdir): '''Removes any differences between the current commit ''' '''and the status of the working directory''' - ex = morphlib.execute.Execute(gitdir, msg=msg) - ex.runv(['git', 'clean', '-fxd']) - ex.runv(['git', 'reset', '--hard', 'HEAD']) - -def set_submodule_url(gitdir, name, url, msg=logging.debug): - '''Changes the URL of a submodule to point to a specific location.''' - ex = morphlib.execute.Execute(gitdir, msg=msg) - ex.runv(['git', 'config', 'submodule.%s.url' % name, url]) - ex.runv(['git', 'config', '-f', '.gitmodules', - 'submodule.%s.url' % name, url]) + runcmd(['git', 'clean', '-fxd'], cwd=gitdir) + runcmd(['git', 'reset', '--hard', 'HEAD'], cwd=gitdir) diff --git a/morphlib/localrepocache.py b/morphlib/localrepocache.py index 3a79967c..84b75577 100644 --- a/morphlib/localrepocache.py +++ b/morphlib/localrepocache.py @@ -22,6 +22,8 @@ import urlparse import shutil import string +import cliapp + import morphlib @@ -89,13 +91,13 @@ class LocalRepoCache(object): ''' - def __init__(self, cachedir, resolver, bundle_base_url=None): + def __init__(self, app, cachedir, resolver, bundle_base_url=None): + self._app = app self._cachedir = cachedir self._resolver = resolver if bundle_base_url and not bundle_base_url.endswith('/'): bundle_base_url += '/' # pragma: no cover self._bundle_base_url = bundle_base_url - self._ex = morphlib.execute.Execute(cachedir, logging.debug) self._cached_repo_objects = {} def _exists(self, filename): # pragma: no cover @@ -116,7 +118,7 @@ class LocalRepoCache(object): ''' - self._ex.runv(['git'] + args, cwd=cwd) + self._app.runcmd(['git'] + args, cwd=cwd) def _fetch(self, url, filename): # pragma: no cover '''Fetch contents of url into a file. @@ -197,7 +199,7 @@ class LocalRepoCache(object): try: self._git(['clone', '-n', bundle_path, path]) self._git(['remote', 'set-url', 'origin', repourl], cwd=path) - except morphlib.execute.CommandFailure, e: # pragma: no cover + except cliapp.AppException, e: # pragma: no cover if self._exists(path): shutil.rmtree(path) return False, 'Unable to extract bundle %s: %s' % (bundle_path, e) @@ -235,7 +237,7 @@ class LocalRepoCache(object): path = self._cache_name(repourl) try: self._git(['clone', '-n', repourl, path]) - except morphlib.execute.CommandFailure, e: + except cliapp.AppException, e: errors.append('Unable to clone from %s to %s: %s' % (repourl, path, e)) raise NoRemote(reponame, errors) @@ -251,7 +253,8 @@ class LocalRepoCache(object): repourl = self._resolver.pull_url(reponame) path = self._cache_name(repourl) if self._exists(path): - repo = morphlib.cachedrepo.CachedRepo(reponame, repourl, path) + repo = morphlib.cachedrepo.CachedRepo(self._app, reponame, + repourl, path) self._cached_repo_objects[reponame] = repo return repo raise NotCached(reponame) diff --git a/morphlib/localrepocache_tests.py b/morphlib/localrepocache_tests.py index 47f92a1d..de894794 100644 --- a/morphlib/localrepocache_tests.py +++ b/morphlib/localrepocache_tests.py @@ -17,6 +17,8 @@ import unittest import urllib2 +import cliapp + import morphlib @@ -36,7 +38,7 @@ class LocalRepoCacheTests(unittest.TestCase): self.remotes = {} self.fetched = [] self.removed = [] - self.lrc = morphlib.localrepocache.LocalRepoCache( + self.lrc = morphlib.localrepocache.LocalRepoCache(object(), self.cachedir, repo_resolver, bundle_base_url) self.lrc._git = self.fake_git self.lrc._exists = self.fake_exists @@ -106,7 +108,7 @@ class LocalRepoCacheTests(unittest.TestCase): def test_fails_to_cache_when_remote_does_not_exist(self): def fail(args): - raise morphlib.execute.CommandFailure('', '') + raise cliapp.AppException('', '') self.lrc._git = fail self.assertRaises(morphlib.localrepocache.NoRemote, self.lrc.cache_repo, self.repourl) diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index e4d233f0..f3be5209 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -36,7 +36,8 @@ class StagingArea(object): ''' - def __init__(self, dirname, tempdir): + def __init__(self, app, dirname, tempdir): + self._app = app self.dirname = dirname self.tempdir = tempdir @@ -104,7 +105,6 @@ class StagingArea(object): def runcmd(self, argv, **kwargs): # pragma: no cover '''Run a command in a chroot in the staging area.''' - ex = morphlib.execute.Execute('/', logging.debug) cwd = kwargs.get('cwd') or '/' if 'cwd' in kwargs: cwd = kwargs['cwd'] @@ -113,5 +113,5 @@ class StagingArea(object): cwd = '/' real_argv = ['chroot', self.dirname, 'sh', '-c', 'cd "$1" && shift && exec "$@"', '--', cwd] + argv - return ex.runv(real_argv, **kwargs) + return self._app.runcmd(real_argv, **kwargs) diff --git a/morphlib/stagingarea_tests.py b/morphlib/stagingarea_tests.py index ea2de291..4fbea487 100644 --- a/morphlib/stagingarea_tests.py +++ b/morphlib/stagingarea_tests.py @@ -37,7 +37,8 @@ class StagingAreaTests(unittest.TestCase): self.tempdir = tempfile.mkdtemp() self.staging = os.path.join(self.tempdir, 'staging') self.created_dirs = [] - self.sa = morphlib.stagingarea.StagingArea(self.staging, self.staging) + self.sa = morphlib.stagingarea.StagingArea(object(), self.staging, + self.staging) def tearDown(self): shutil.rmtree(self.tempdir) @@ -69,7 +70,7 @@ class StagingAreaTests(unittest.TestCase): self.assertEqual(self.sa.dirname, self.staging) def test_accepts_root_directory(self): - sa = morphlib.stagingarea.StagingArea('/', '/tmp') + sa = morphlib.stagingarea.StagingArea(object(), '/', '/tmp') self.assertEqual(sa.dirname, '/') def test_creates_build_directory(self): diff --git a/tests/uses-tempdir.script b/tests/uses-tempdir.script index d00293a2..4143926f 100755 --- a/tests/uses-tempdir.script +++ b/tests/uses-tempdir.script @@ -23,4 +23,5 @@ export TMPDIR TMPDIR="$DATADIR"/unwritable-tmp install -m 000 -d "$TMPDIR" mkdir "$DATADIR"/tmp -"$SRCDIR/scripts/test-morph" build --tempdir "$DATADIR"/tmp test:morphs-repo master hello-stratum.morph +"$SRCDIR/scripts/test-morph" build --tempdir "$DATADIR"/tmp \ + test:morphs-repo master hello-stratum.morph |