diff options
| author | Richard Maw <richard.maw@codethink.co.uk> | 2012-07-31 12:52:12 +0000 |
|---|---|---|
| committer | Richard Maw <richard.maw@codethink.co.uk> | 2012-08-01 13:12:03 +0000 |
| commit | 0032ccbd60f202ec76d9b738cddc8bdbf9f54133 (patch) | |
| tree | cd398ab063be0bdab20b91cbd2420ad4981de813 | |
| parent | 93a3d28ee51cc301cd79ff8bfabb2010defda09b (diff) | |
| download | morph-0032ccbd60f202ec76d9b738cddc8bdbf9f54133.tar.gz | |
python scripts: pep8ize codebase
This was done with the aid of the pep8 script, available by running
`easy_install pep8`.
It may be worth making this part of ./check, but that will require
putting pep8 into the development tools stratum.
This should be easy, given pep8 has no external dependencies.
64 files changed, 1250 insertions, 1254 deletions
diff --git a/dump-build-times b/dump-build-times index 5135ca04..48fdca65 100755 --- a/dump-build-times +++ b/dump-build-times @@ -1,16 +1,16 @@ #!/usr/bin/env python # # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -1,16 +1,16 @@ #!/usr/bin/python # # Copyright (C) 2011-2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. diff --git a/morphlib/__init__.py b/morphlib/__init__.py index 4d9ad22e..60e289ea 100644 --- a/morphlib/__init__.py +++ b/morphlib/__init__.py @@ -1,14 +1,14 @@ # Copyright (C) 2011-2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -53,5 +53,4 @@ import stopwatch import tempdir import util -import app # this needs to be last - +import app # this needs to be last diff --git a/morphlib/app.py b/morphlib/app.py index 491adb16..3e4a0bf5 100755 --- a/morphlib/app.py +++ b/morphlib/app.py @@ -1,14 +1,14 @@ # Copyright (C) 2011-2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -29,21 +29,21 @@ import morphlib defaults = { 'repo-alias': [ - 'upstream=' + ('upstream=' 'git://roadtrain.codethink.co.uk/delta/#' - 'gitano@roadtrain.codethink.co.uk:delta/', - 'baserock=' + 'gitano@roadtrain.codethink.co.uk:delta/'), + ('baserock=' 'git://roadtrain.codethink.co.uk/baserock/#' - 'gitano@roadtrain.codethink.co.uk:baserock/', - 'freedesktop=' + 'gitano@roadtrain.codethink.co.uk:baserock/'), + ('freedesktop=' 'git://anongit.freedesktop.org/#' - 'ssh://git.freedesktop.org/', - 'gnome=' + 'ssh://git.freedesktop.org/'), + ('gnome=' 'git://git.gnome.org/%s#' - 'ssh://git.gnome.org/git/%s', - 'github=' + 'ssh://git.gnome.org/git/%s'), + ('github=' 'git://github.com/%s#' - 'git@github.com:%s', + 'git@github.com:%s'), ], 'cachedir': os.path.expanduser('~/.cache/morph'), 'max-jobs': morphlib.util.make_concurrency(), @@ -57,13 +57,13 @@ defaults = { class BuildCommand(object): '''High level logic for building. - + This controls how the whole build process goes. This is a separate class to enable easy experimentation of different approaches to the various parts of the process. - + ''' - + def __init__(self, app): self.app = app self.build_env = self.new_build_env() @@ -75,7 +75,7 @@ class BuildCommand(object): '''Build triplets specified on command line.''' self.app.status(msg='Build starts', chatty=True) - + for repo_name, ref, filename in self.app._itertriplets(args): self.app.status(msg='Building %(repo_name)s %(ref)s %(filename)s', repo_name=repo_name, ref=ref, filename=filename) @@ -111,7 +111,7 @@ class BuildCommand(object): '''Create a new directory for the local artifact cache.''' artifact_cachedir = os.path.join( - self.app.settings['cachedir'], 'artifacts') + self.app.settings['cachedir'], 'artifacts') if not os.path.exists(artifact_cachedir): os.mkdir(artifact_cachedir) return artifact_cachedir @@ -124,8 +124,8 @@ class BuildCommand(object): gits_dir = os.path.join(cachedir, 'gits') bundle_base_url = self.app.settings['bundle-server'] repo_resolver = morphlib.repoaliasresolver.RepoAliasResolver(aliases) - lrc = morphlib.localrepocache.LocalRepoCache(self.app, - gits_dir, repo_resolver, bundle_base_url=bundle_base_url) + lrc = morphlib.localrepocache.LocalRepoCache( + self.app, gits_dir, repo_resolver, bundle_base_url=bundle_base_url) url = self.app.settings['cache-server'] if url: @@ -149,7 +149,7 @@ class BuildCommand(object): self.app.status(msg='Creating source pool', chatty=True) srcpool = self.app._create_source_pool( - self.lrc, self.rrc, (repo_name, ref, filename)) + self.lrc, self.rrc, (repo_name, ref, filename)) self.app.status(msg='Creating artifact resolver', chatty=True) ar = morphlib.artifactresolver.ArtifactResolver() @@ -164,7 +164,7 @@ class BuildCommand(object): self.app.status(msg='Computing build order', chatty=True) order = morphlib.buildorder.BuildOrder(artifacts) - + return order def build_in_order(self, order): @@ -173,13 +173,13 @@ class BuildCommand(object): chatty=True) for group in order.groups: self.build_artifacts(group) - + def build_artifacts(self, artifacts): '''Build a set of artifact. - + Typically, this would be a build group, but might be anything. At this level of abstraction we don't care. - + ''' self.app.status(msg='Building a set of artifacts', chatty=True) @@ -188,10 +188,10 @@ class BuildCommand(object): def build_artifact(self, artifact): '''Build one artifact. - + All the dependencies are assumed to be built and available in either the local or remote cache already. - + ''' self.app.status(msg='Checking if %(kind)s %(name)s needs building', @@ -213,7 +213,7 @@ class BuildCommand(object): if self.app.settings['staging-chroot']: if artifact.source.morphology.needs_staging_area: self.install_fillers(staging_area) - self.install_chunk_artifacts(staging_area, + self.install_chunk_artifacts(staging_area, deps) self.build_and_cache(staging_area, artifact) if self.app.settings['bootstrap']: @@ -270,8 +270,8 @@ class BuildCommand(object): # Update submodules. done = set() self.app._cache_repo_and_submodules( - self.lrc, artifact.source.repo.url, - artifact.source.sha1, done) + self.lrc, artifact.source.repo.url, + artifact.source.sha1, done) def cache_artifacts_locally(self, artifacts): '''Get artifacts missing from local cache from remote cache.''' @@ -280,22 +280,22 @@ class BuildCommand(object): shutil.copyfileobj(remote, local) remote.close() local.close() - + for artifact in artifacts: if not self.lac.has(artifact): self.app.status(msg='Fetching to local cache: ' 'artifact %(name)s', name=artifact.name) copy(self.rac.get(artifact), self.lac.put(artifact)) - + if artifact.source.morphology.needs_artifact_metadata_cached: if not self.lac.has_artifact_metadata(artifact, 'meta'): self.app.status(msg='Fetching to local cache: ' 'artifact metadata %(name)s', name=artifact.name) - copy(self.rac.get_artifact_metadata(artifact, 'meta'), + copy(self.rac.get_artifact_metadata(artifact, 'meta'), self.lac.put_artifact_metadata(artifact, 'meta')) - + def create_staging_area(self, artifact): '''Create the staging area for building a single artifact.''' @@ -318,36 +318,36 @@ class BuildCommand(object): if staging_area.dirname != '/': self.app.status(msg='Removing staging area') staging_area.remove() - if (staging_area.tempdir != '/' and - os.path.exists(staging_area.tempdir)): + temp_path = staging_area.tempdir + if temp_path != '/' and os.path.exists(temp_path): self.app.status(msg='Removing temporary staging directory') - shutil.rmtree(staging_area.tempdir) + shutil.rmtree(temp_path) def install_fillers(self, staging_area): '''Install staging fillers into the staging area. - + This must not be called in bootstrap mode. - + ''' - + logging.debug('Pre-populating staging area %s' % staging_area.dirname) - logging.debug('Fillers: %s' % - repr(self.app.settings['staging-filler'])) + logging.debug('Fillers: %s' % + repr(self.app.settings['staging-filler'])) for filename in self.app.settings['staging-filler']: with open(filename, 'rb') as f: - self.app.status(msg='Installing %(filename)s', + self.app.status(msg='Installing %(filename)s', filename=filename) staging_area.install_artifact(f) def install_chunk_artifacts(self, staging_area, artifacts): '''Install chunk artifacts into staging area. - + We only ever care about chunk artifacts as build dependencies, so this is not a generic artifact installer into staging area. Any non-chunk artifacts are silently ignored. - + All artifacts MUST be in the local artifact cache already. - + ''' for artifact in artifacts: @@ -363,9 +363,9 @@ class BuildCommand(object): self.app.status(msg='Starting actual build') setup_mounts = self.app.settings['staging-chroot'] - builder = morphlib.builder2.Builder(self.app, - staging_area, self.lac, self.rac, self.lrc, self.build_env, - self.app.settings['max-jobs'], setup_mounts) + builder = morphlib.builder2.Builder( + self.app, staging_area, self.lac, self.rac, self.lrc, + self.build_env, self.app.settings['max-jobs'], setup_mounts) return builder.build_and_cache(artifact) @@ -375,18 +375,19 @@ class Morph(cliapp.Application): system_repo_name = 'baserock:%s' % system_repo_base def add_settings(self): - self.settings.boolean(['verbose', 'v'], + self.settings.boolean(['verbose', 'v'], 'show what is happening in much detail') self.settings.boolean(['quiet', 'q'], 'show no output unless there is an error') self.settings.string_list(['repo-alias'], - 'define URL prefix aliases to allow repository ' - 'addresses to be shortened; ' - 'use alias=pullpattern=pushpattern ' - 'to allow alias:shortname to be used instead ' - 'of the full URL; the patterns must contain ' - 'a %s where the shortname gets replaced', - default=defaults['repo-alias']) + 'define URL prefix aliases to allow ' + 'repository addresses to be shortened; ' + 'use alias=pullpattern=pushpattern ' + 'to allow alias:shortname to be used ' + 'instead of the full URL; the patterns must ' + 'contain a %s where the shortname gets ' + 'replaced', + default=defaults['repo-alias']) self.settings.string(['bundle-server'], 'base URL to download bundles', metavar='URL', @@ -395,9 +396,9 @@ class Morph(cliapp.Application): 'HTTP URL of the morph cache server to use', metavar='URL', default=None) - self.settings.string(['cachedir'], + self.settings.string(['cachedir'], 'put build results in DIR', - metavar='DIR', + metavar='DIR', default=defaults['cachedir']) self.settings.string(['prefix'], 'build chunks with prefix PREFIX', @@ -415,65 +416,65 @@ class Morph(cliapp.Application): default='') self.settings.string(['tempdir'], 'temporary directory to use for builds ' - '(this is separate from just setting $TMPDIR ' - 'or /tmp because those are used internally ' - 'by things that cannot be on NFS, but ' - 'this setting can point at a directory in ' - 'NFS)', - metavar='DIR', + '(this is separate from just setting $TMPDIR ' + 'or /tmp because those are used internally ' + 'by things that cannot be on NFS, but ' + 'this setting can point at a directory in ' + 'NFS)', + metavar='DIR', default=os.environ.get('TMPDIR')) self.settings.boolean(['no-ccache'], 'do not use ccache') self.settings.string(['ccache-remotedir'], 'allow ccache to download objects from REMOTEDIR ' - 'if they are not cached locally', + 'if they are not cached locally', metavar='REMOTEDIR', default=defaults['ccache-remotedir']) self.settings.integer(['ccache-remotenlevels'], 'assume ccache directory objects are split into ' - 'NLEVELS levels of subdirectories', + 'NLEVELS levels of subdirectories', metavar='NLEVELS', default=defaults['ccache-remotenlevels']) self.settings.boolean(['no-distcc'], 'do not use distcc') - self.settings.integer(['max-jobs'], + self.settings.integer(['max-jobs'], 'run at most N parallel jobs with make (default ' - 'is to a value based on the number of CPUs ' - 'in the machine running morph', + 'is to a value based on the number of CPUs ' + 'in the machine running morph', metavar='N', default=defaults['max-jobs']) - self.settings.boolean(['keep-path'], + self.settings.boolean(['keep-path'], 'do not touch the PATH environment variable') - self.settings.boolean(['bootstrap'], + self.settings.boolean(['bootstrap'], 'build stuff in bootstrap mode; this is ' - 'DANGEROUS and will install stuff on your ' - 'system') + 'DANGEROUS and will install stuff on your ' + 'system') self.settings.boolean(['ignore-submodules'], 'do not cache repositories of git submodules ' 'or unpack them into the build directory') self.settings.boolean(['no-git-update'], 'do not update the cached git repositories ' - 'during a build (user must have done that ' - 'already using the update-gits subcommand)') + 'during a build (user must have done that ' + 'already using the update-gits subcommand)') self.settings.string_list(['staging-filler'], 'unpack BLOB into staging area for ' - 'non-bootstrap builds (this will ' - 'eventually be replaced with proper ' - 'build dependencies)', - metavar='BLOB') + 'non-bootstrap builds (this will ' + 'eventually be replaced with proper ' + 'build dependencies)', + metavar='BLOB') self.settings.boolean(['staging-chroot'], 'build things in a staging chroot ' - '(require real root to use)') + '(require real root to use)') def setup_plugin_manager(self): cliapp.Application.setup_plugin_manager(self) self.pluginmgr.locations += os.path.join( - os.path.dirname(morphlib.__file__), 'plugins') + os.path.dirname(morphlib.__file__), 'plugins') s = os.environ.get('MORPH_PLUGIN_PATH', '') self.pluginmgr.locations += s.split(':') - + self.hookmgr = cliapp.HookManager() self.hookmgr.new('new-build-command', cliapp.FilterHook()) self.system_kind_builder_factory = \ @@ -481,10 +482,10 @@ class Morph(cliapp.Application): def _itertriplets(self, args): '''Generate repo, ref, filename triples from args.''' - + if (len(args) % 3) != 0: raise cliapp.AppException('Argument list must have full triplets') - + while args: assert len(args) >= 2, args yield args[0], args[1], args[2] @@ -505,17 +506,17 @@ class Morph(cliapp.Application): def cmd_build(self, args): '''Build a binary from a morphology. - + Command line arguments are the repository, git tree-ish reference, and morphology filename. Morph takes care of building all dependencies before building the morphology. All generated binaries are put into the cache. - + (The triplet of command line arguments may be repeated as many times as necessary.) - + ''' - + build_command = BuildCommand(self) build_command = self.hookmgr.call('new-build-command', build_command) build_command.build(args) @@ -528,12 +529,12 @@ class Morph(cliapp.Application): cachedir = os.path.join(self.settings['cachedir'], 'gits') bundle_base_url = self.settings['bundle-server'] repo_resolver = morphlib.repoaliasresolver.RepoAliasResolver( - self.settings['repo-alias']) - lrc = morphlib.localrepocache.LocalRepoCache(self, - cachedir, repo_resolver, bundle_base_url) + self.settings['repo-alias']) + lrc = morphlib.localrepocache.LocalRepoCache( + self, cachedir, repo_resolver, bundle_base_url) if self.settings['cache-server']: rrc = morphlib.remoterepocache.RemoteRepoCache( - self.settings['cache-server'], repo_resolver) + self.settings['cache-server'], repo_resolver) else: rrc = None @@ -544,7 +545,7 @@ class Morph(cliapp.Application): resolver = morphlib.artifactresolver.ArtifactResolver() artifacts = resolver.resolve_artifacts(pool) - self.output.write('dependency graph for %s|%s|%s:\n' % + self.output.write('dependency graph for %s|%s|%s:\n' % (repo, ref, filename)) for artifact in sorted(artifacts, key=str): self.output.write(' %s\n' % artifact) @@ -563,7 +564,7 @@ class Morph(cliapp.Application): '''Resolves the sha1 of the ref in reponame and returns it. If update is True then this has the side-effect of updating - or cloning the repository into the local repo cache. + or cloning the repository into the local repo cache. ''' absref = None @@ -574,12 +575,12 @@ class Morph(cliapp.Application): reponame=reponame) repo.update() absref = repo.resolve_ref(ref) - elif rrc != None: + elif rrc is not None: try: absref = rrc.resolve_ref(reponame, ref) except: pass - if absref == None: + if absref is None: if update: self.status(msg='Caching git repository %(reponame)s', reponame=reponame) @@ -599,7 +600,7 @@ class Morph(cliapp.Application): reponame, ref, filename = queue.popleft() absref = self._resolveref(lrc, rrc, reponame, ref, update) morphology = morph_factory.get_morphology( - reponame, absref, filename) + reponame, absref, filename) visit(reponame, ref, filename, absref, morphology) if morphology['kind'] == 'system': queue.extend((reponame, ref, '%s.morph' % s) @@ -608,7 +609,7 @@ class Morph(cliapp.Application): if morphology['build-depends']: queue.extend((reponame, ref, '%s.morph' % s) for s in morphology['build-depends']) - queue.extend((c['repo'], c['ref'], '%s.morph' % c['morph']) + queue.extend((c['repo'], c['ref'], '%s.morph' % c['morph']) for c in morphology['sources']) def cmd_update_gits(self, args): @@ -617,20 +618,20 @@ class Morph(cliapp.Application): Parse the given morphologies, and their dependencies, and update all the git repositories referred to by them in the morph cache directory. - + ''' if not os.path.exists(self.settings['cachedir']): os.mkdir(self.settings['cachedir']) cachedir = os.path.join(self.settings['cachedir'], 'gits') repo_resolver = morphlib.repoaliasresolver.RepoAliasResolver( - self.settings['repo-alias']) + self.settings['repo-alias']) bundle_base_url = self.settings['bundle-server'] - cache = morphlib.localrepocache.LocalRepoCache(self, - cachedir, repo_resolver, bundle_base_url) + cache = morphlib.localrepocache.LocalRepoCache( + self, cachedir, repo_resolver, bundle_base_url) subs_to_process = set() - + def visit(reponame, ref, filename, absref, morphology): self.status(msg='Updating %(repo_name)s %(ref)s %(filename)s', repo_name=reponame, ref=ref, filename=filename) @@ -645,7 +646,7 @@ class Morph(cliapp.Application): else: for submod in submodules: subs_to_process.add((submod.url, submod.commit)) - + self._traverse_morphs(self._itertriplets(args), cache, None, update=True, visit=visit) @@ -675,9 +676,9 @@ class Morph(cliapp.Application): def cmd_make_patch(self, args): '''Create a Trebuchet patch between two system images.''' - + logging.debug('cmd_make_patch starting') - + if len(args) != 7: raise cliapp.AppException('make-patch requires arguments: ' 'name of output file plus two triplest') @@ -691,15 +692,15 @@ class Morph(cliapp.Application): build_env = morphlib.buildenvironment.BuildEnvironment(self.settings) ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) lac = morphlib.localartifactcache.LocalArtifactCache( - os.path.join(cachedir, 'artifacts')) + os.path.join(cachedir, 'artifacts')) repo_resolver = morphlib.repoaliasresolver.RepoAliasResolver( - self.settings['repo-alias']) - lrc = morphlib.localrepocache.LocalRepoCache(self, - os.path.join(cachedir, 'gits'), repo_resolver, - bundle_base_url=self.settings['bundle-server']) + self.settings['repo-alias']) + lrc = morphlib.localrepocache.LocalRepoCache( + self, os.path.join(cachedir, 'gits'), repo_resolver, + bundle_base_url=self.settings['bundle-server']) if self.settings['cache-server']: rrc = morphlib.remoterepocache.RemoteRepoCache( - self.settings['cache-server'], repo_resolver) + self.settings['cache-server'], repo_resolver) else: rrc = None @@ -710,14 +711,15 @@ class Morph(cliapp.Application): def get_artifact(repo_name, ref, filename): srcpool = self._create_source_pool( - lrc, rrc, (repo_name, ref, filename)) + lrc, rrc, (repo_name, ref, filename)) ar = morphlib.artifactresolver.ArtifactResolver() artifacts = ar.resolve_artifacts(srcpool) for artifact in artifacts: artifact.cache_key = ckc.compute_key(artifact) if the_one(artifact.source, repo_name, ref, filename): - a = morphlib.artifact.Artifact(artifact.source, - artifact.source.morphology['name']+'-rootfs') + a = morphlib.artifact.Artifact( + artifact.source, + artifact.source.morphology['name'] + '-rootfs') a.cache_key = artifact.cache_key return a @@ -726,7 +728,7 @@ class Morph(cliapp.Application): image_path_1 = lac.get(artifact1).name image_path_2 = lac.get(artifact2).name - + def setup(path): part = morphlib.fsutils.setup_device_mapping(self.runcmd, path) mount_point = tempfile.mkdtemp(dir=self.settings['tempdir']) @@ -762,10 +764,10 @@ class Morph(cliapp.Application): finally: cleanup(image_path_1, mount_point_1) cleanup(image_path_2, mount_point_2) - + def cmd_init(self, args): '''Initialize a mine.''' - + if not args: args = ['.'] elif len(args) > 1: @@ -779,7 +781,7 @@ class Morph(cliapp.Application): 'directory: %s' % dirname) else: raise cliapp.AppException('can only initialize an existing ' - 'empty directory: %s' % dirname) + 'empty directory: %s' % dirname) os.mkdir(os.path.join(dirname, '.morph')) @@ -794,24 +796,24 @@ class Morph(cliapp.Application): def cmd_minedir(self, args): '''Find morph mine directory from current working directory.''' - + dirname = self._deduce_mine_directory() if dirname is None: raise cliapp.AppException("Can't find the mine directory") self.output.write('%s\n' % dirname) - + def _resolve_reponame(self, reponame): '''Return the full pull URL of a reponame.''' resolver = morphlib.repoaliasresolver.RepoAliasResolver( - self.settings['repo-alias']) + self.settings['repo-alias']) return resolver.pull_url(reponame) - + def _clone_to_directory(self, dirname, reponame, ref): '''Clone a repository below a directory. - + As a side effect, clone it into the morph repository. - + ''' # Setup. @@ -819,36 +821,37 @@ class Morph(cliapp.Application): os.mkdir(self.settings['cachedir']) cachedir = os.path.join(self.settings['cachedir'], 'gits') repo_resolver = morphlib.repoaliasresolver.RepoAliasResolver( - self.settings['repo-alias']) + self.settings['repo-alias']) bundle_base_url = self.settings['bundle-server'] - cache = morphlib.localrepocache.LocalRepoCache(self, - cachedir, repo_resolver, bundle_base_url) + cache = morphlib.localrepocache.LocalRepoCache( + self, cachedir, repo_resolver, bundle_base_url) # Get the repository into the cache; make sure it is up to date. repo = cache.cache_repo(reponame) if not self.settings['no-git-update']: repo.update() - + # Clone it from cache to target directory. repo.checkout(ref, os.path.abspath(dirname)) - + # Set the origin to point at the original repository. morphlib.git.set_remote(self.runcmd, dirname, 'origin', repo.url) - + # Add push url rewrite rule to .git/config. self.runcmd(['git', 'config', - 'url.%s.pushInsteadOf'% repo_resolver.push_url(reponame), + ('url.%s.pushInsteadOf' % + repo_resolver.push_url(reponame)), repo_resolver.pull_url(reponame)], cwd=dirname) - + # Update remotes. self.runcmd(['git', 'remote', 'update'], cwd=dirname) def cmd_branch(self, args): '''Branch the whole system.''' - + if len(args) not in [1, 2]: raise cliapp.AppException('morph branch needs name of branch ' - 'as parameter') + 'as parameter') new_branch = args[0] commit = 'master' if len(args) == 1 else args[1] @@ -859,7 +862,7 @@ class Morph(cliapp.Application): # Clone into system branch directory. new_repo = os.path.join(new_branch, self.system_repo_base) self._clone_to_directory(new_repo, self.system_repo_name, commit) - + # Create a new branch in the local morphs repository. self.runcmd(['git', 'checkout', '-b', new_branch, commit], cwd=new_repo) @@ -869,7 +872,7 @@ class Morph(cliapp.Application): if len(args) != 1: raise cliapp.AppException('morph checkout needs name of ' - 'branch as parameter') + 'branch as parameter') system_branch = args[0] @@ -888,31 +891,31 @@ class Morph(cliapp.Application): if not minedir.endswith('/'): minedir += '/' - + cwd = os.getcwd() if not cwd.startswith(minedir): return None - + return os.path.dirname(cwd[len(minedir):]) - + def cmd_show_system_branch(self, args): '''Print name of current system branch. - + This must be run in the system branch's ``morphs`` repository. - + ''' - + system_branch = self._deduce_system_branch() if system_branch is None: raise cliapp.AppException("Can't determine system branch") self.output.write('%s\n' % system_branch) - + def cmd_edit(self, args): '''Edit a component in a system branch.''' - - if len(args) not in (1,2): + + if len(args) not in (1, 2): raise cliapp.AppException('morph edit must get a repository name ' - 'and commit ref as argument') + 'and commit ref as argument') mine_directory = self._deduce_mine_directory() system_branch = self._deduce_system_branch() @@ -932,10 +935,10 @@ class Morph(cliapp.Application): if ref is None: raise morphlib.Error('Cannot deduce commit to start edit from') - new_repo = os.path.join(mine_directory, system_branch, + new_repo = os.path.join(mine_directory, system_branch, os.path.basename(repo)) self._clone_to_directory(new_repo, args[0], ref) - + if system_branch == ref: self.runcmd(['git', 'checkout', system_branch], cwd=new_repo) @@ -950,7 +953,7 @@ class Morph(cliapp.Application): if spec_repo == repo and spec['ref'] != system_branch: if self.settings['verbose']: print ('Replacing ref "%s" with "%s" in %s' % - (spec['ref'], system_branch, filename)) + (spec['ref'], system_branch, filename)) spec['ref'] = system_branch changed = True if changed: @@ -996,15 +999,15 @@ class Morph(cliapp.Application): def cmd_merge(self, args): '''Merge specified repositories from another system branch.''' - + if len(args) == 0: raise cliapp.AppException('morph merge must get a branch name ' - 'and some repo names as arguments') + 'and some repo names as arguments') other_branch = args[0] mine = self._deduce_mine_directory() this_branch = self._deduce_system_branch() - + for repo in args[1:]: repo = self._resolve_reponame(repo) basename = os.path.basename(repo) @@ -1019,15 +1022,15 @@ class Morph(cliapp.Application): os.mkdir(self.settings['cachedir']) cachedir = os.path.join(self.settings['cachedir'], 'gits') repo_resolver = morphlib.repoaliasresolver.RepoAliasResolver( - self.settings['repo-alias']) + self.settings['repo-alias']) bundle_base_url = self.settings['bundle-server'] - cache = morphlib.localrepocache.LocalRepoCache(self, - cachedir, repo_resolver, bundle_base_url) - + cache = morphlib.localrepocache.LocalRepoCache( + self, cachedir, repo_resolver, bundle_base_url) + for filename in args: with open(filename) as f: morph = json.load(f) - + if morph['kind'] != 'stratum': self.status(msg='Not a stratum: %(filename)s', filename=filename) @@ -1039,28 +1042,28 @@ class Morph(cliapp.Application): reponame = source.get('repo', source['name']) ref = source['ref'] self.status(msg='Looking up sha1 for %(repo_name)s %(ref)s', - repo_name=reponame, + repo_name=reponame, ref=ref) repo = cache.get_repo(reponame) source['ref'] = repo.resolve_ref(ref) - + with open(filename, 'w') as f: json.dump(morph, f, indent=2) def status(self, **kwargs): '''Show user a status update. - + The keyword arguments are formatted and presented to the user in a pleasing manner. Some keywords are special: - + * ``msg`` is the message text; it can use ``%(foo)s`` to embed the value of keyword argument ``foo`` * ``chatty`` should be true when the message is only informative, and only useful for users who want to know everything (--verbose) * ``error`` should be true when it is an error message - + All other keywords are ignored unless embedded in ``msg``. - + ''' assert 'msg' in kwargs @@ -1077,7 +1080,7 @@ class Morph(cliapp.Application): logging.debug(text) else: logging.info(text) - + ok = verbose or error or (not quiet and not chatty) if ok: timestamp = time.strftime('%Y-%m-%d %H:%M:%S', time.gmtime()) @@ -1100,11 +1103,10 @@ class Morph(cliapp.Application): self.status(msg='# %(cmdline)s', cmdline=' | '.join(commands), chatty=True) - + # Log the environment. for name in kwargs['env']: logging.debug('environment: %s=%s' % (name, kwargs['env'][name])) # run the command line return cliapp.Application.runcmd(self, argv, *args, **kwargs) - diff --git a/morphlib/artifact.py b/morphlib/artifact.py index 9be47d48..ccde11d4 100644 --- a/morphlib/artifact.py +++ b/morphlib/artifact.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -17,21 +17,21 @@ class Artifact(object): '''Represent a build result generated from a source. - + Has the following properties: - + * ``source`` -- the source from which the artifact is built * ``name`` -- the name of the artifact * ``cache_key`` -- a cache key to uniquely identify the artifact * ``cache_id`` -- a dict describing the components of the cache key * ``dependencies`` -- list of Artifacts that need to be built beforehand * ``dependents`` -- list of Artifacts that need this Artifact to be built - + The ``dependencies`` and ``dependents`` lists MUST be modified by the ``add_dependencies`` and ``add_dependent`` methods only. - + ''' - + def __init__(self, source, name): self.source = source self.name = name @@ -50,16 +50,16 @@ class Artifact(object): '''Do we depend on ``artifact``?''' return artifact in self.dependencies - def basename(self): # pragma: no cover + def basename(self): # pragma: no cover return '%s.%s.%s' % (self.cache_key, self.source.morphology['kind'], self.name) - def metadata_basename(self, metadata_name): # pragma: no cover + def metadata_basename(self, metadata_name): # pragma: no cover return '%s.%s.%s.%s' % (self.cache_key, self.source.morphology['kind'], self.name, metadata_name) - def __str__(self): # pragma: no cover + def __str__(self): # pragma: no cover return '%s|%s' % (self.source, self.name) diff --git a/morphlib/artifact_tests.py b/morphlib/artifact_tests.py index 5c912cd7..a8637fd0 100644 --- a/morphlib/artifact_tests.py +++ b/morphlib/artifact_tests.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -23,30 +23,30 @@ class ArtifactTests(unittest.TestCase): def setUp(self): morph = morphlib.morph2.Morphology( - ''' - { - "chunk": "chunk", - "kind": "chunk", - "chunks": { - "chunk-runtime": [ - "usr/bin", - "usr/sbin", - "usr/lib", - "usr/libexec" - ], - "chunk-devel": [ - "usr/include" - ] - } + ''' + { + "chunk": "chunk", + "kind": "chunk", + "chunks": { + "chunk-runtime": [ + "usr/bin", + "usr/sbin", + "usr/lib", + "usr/libexec" + ], + "chunk-devel": [ + "usr/include" + ] } - ''') + } + ''') self.source = morphlib.source.Source( - 'repo', 'ref', 'sha1', morph, 'chunk.morph') + 'repo', 'ref', 'sha1', morph, 'chunk.morph') self.artifact_name = 'chunk-runtime' self.artifact = morphlib.artifact.Artifact( - self.source, self.artifact_name) + self.source, self.artifact_name) self.other = morphlib.artifact.Artifact( - self.source, self.artifact_name) + self.source, self.artifact_name) def test_constructor_sets_source(self): self.assertEqual(self.artifact.source, self.source) @@ -59,7 +59,7 @@ class ArtifactTests(unittest.TestCase): def test_sets_dependencies_to_empty(self): self.assertEqual(self.artifact.dependencies, []) - + def test_sets_dependents_to_empty(self): self.assertEqual(self.artifact.dependents, []) @@ -78,4 +78,3 @@ class ArtifactTests(unittest.TestCase): self.assertEqual(self.artifact.dependencies, [self.other]) self.assertEqual(self.other.dependents, [self.artifact]) self.assertTrue(self.artifact.depends_on(self.other)) - diff --git a/morphlib/artifactcachereference.py b/morphlib/artifactcachereference.py index 169bf96d..8211f6b5 100644 --- a/morphlib/artifactcachereference.py +++ b/morphlib/artifactcachereference.py @@ -1,20 +1,20 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. -class ArtifactCacheReference(object): +class ArtifactCacheReference(object): '''Represent the information needed to retrieve an artifact The artifact cache doesn't need to know the dependencies or the @@ -27,6 +27,7 @@ class ArtifactCacheReference(object): Conversely if it generated the basename then old strata wouldn't be able to refer to new chunks, but strata change more often than the chunks. ''' + def __init__(self, basename): self._basename = basename diff --git a/morphlib/artifactresolver.py b/morphlib/artifactresolver.py index 1b47a722..929dae57 100644 --- a/morphlib/artifactresolver.py +++ b/morphlib/artifactresolver.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -24,50 +24,50 @@ class MutualDependencyError(cliapp.AppException): def __init__(self, a, b): cliapp.AppException.__init__( - self, 'Cyclic dependency between %s and %s detected' % (a, b)) + self, 'Cyclic dependency between %s and %s detected' % (a, b)) class DependencyOrderError(cliapp.AppException): def __init__(self, stratum, chunk, dependency_name): cliapp.AppException.__init__( - self, 'In stratum %s, chunk %s references its dependency %s ' - 'before it is defined' % - (stratum.source, chunk, dependency_name)) + self, 'In stratum %s, chunk %s references its dependency %s ' + 'before it is defined' % + (stratum.source, chunk, dependency_name)) class DependencyFormatError(cliapp.AppException): def __init__(self, stratum, chunk): cliapp.AppException.__init__( - self, 'In stratum %s, chunk %s uses an invalid ' - 'build-depends format' % (stratum.source, chunk)) + self, 'In stratum %s, chunk %s uses an invalid ' + 'build-depends format' % (stratum.source, chunk)) class UndefinedChunkArtifactError(cliapp.AppException): '''Exception raised when non-existent artifacts are referenced. - + Usually, this will only occur when a stratum refers to a chunk artifact that is not defined in a chunk. - + ''' def __init__(self, parent, reference): cliapp.AppException.__init__( - self, 'Undefined chunk artifact "%s" referenced in ' - 'stratum %s' % (reference, parent)) + self, 'Undefined chunk artifact "%s" referenced in ' + 'stratum %s' % (reference, parent)) class ArtifactResolver(object): '''Resolves sources into artifacts that would be build from the sources. - + This class takes a CacheKeyComputer and a SourcePool, analyses the sources and their dependencies and creates a list of artifacts (represented by Artifact objects) that are involved in building the sources in the pool. - + ''' def __init__(self): @@ -95,14 +95,13 @@ class ArtifactResolver(object): if source.morphology['kind'] == 'system': systems = [self._get_artifact(source, a) for a in source.morphology.builds_artifacts] - if any(a not in self._added_artifacts for a in systems): artifacts.extend(systems) self._added_artifacts.update(systems) resolved_artifacts = self._resolve_system_dependencies( - systems, source, queue) + systems, source, queue) for artifact in resolved_artifacts: if not artifact in self._added_artifacts: @@ -110,15 +109,15 @@ class ArtifactResolver(object): self._added_artifacts.add(artifact) elif source.morphology['kind'] == 'stratum': assert len(source.morphology.builds_artifacts) == 1 - artifact = self._get_artifact(source, - source.morphology.builds_artifacts[0]) + artifact = self._get_artifact( + source, source.morphology.builds_artifacts[0]) if not artifact in self._added_artifacts: artifacts.append(artifact) self._added_artifacts.add(artifact) resolved_artifacts = self._resolve_stratum_dependencies( - artifact, queue) + artifact, queue) for artifact in resolved_artifacts: if not artifact in self._added_artifacts: @@ -156,9 +155,9 @@ class ArtifactResolver(object): for stratum_name in source.morphology['strata']: stratum_source = self._source_pool.lookup( - source.repo_name, - source.original_ref, - '%s.morph' % stratum_name) + source.repo_name, + source.original_ref, + '%s.morph' % stratum_name) stratum = self._get_artifact(stratum_source, stratum_name) @@ -178,9 +177,9 @@ class ArtifactResolver(object): if stratum.source.morphology['build-depends']: for stratum_name in stratum.source.morphology['build-depends']: other_source = self._source_pool.lookup( - stratum.source.repo_name, - stratum.source.original_ref, - '%s.morph' % stratum_name) + stratum.source.repo_name, + stratum.source.original_ref, + '%s.morph' % stratum_name) other_stratum = self._get_artifact(other_source, stratum_name) @@ -200,9 +199,9 @@ class ArtifactResolver(object): for info in stratum.source.morphology['sources']: chunk_source = self._source_pool.lookup( - info['repo'], - info['ref'], - '%s.morph' % info['morph']) + info['repo'], + info['ref'], + '%s.morph' % info['morph']) possible_names = chunk_source.morphology.builds_artifacts if not info['name'] in possible_names: @@ -226,7 +225,7 @@ class ArtifactResolver(object): continue if earlier_artifact.depends_on(chunk_artifact): raise MutualDependencyError( - chunk_artifact, earlier_artifact) + chunk_artifact, earlier_artifact) chunk_artifact.add_dependency(earlier_artifact) elif isinstance(build_depends, list): for name in build_depends: @@ -237,11 +236,10 @@ class ArtifactResolver(object): chunk_artifact.add_dependency(other_artifact) else: raise DependencyOrderError( - stratum, info['name'], name) + stratum, info['name'], name) else: raise DependencyFormatError(stratum, info['name']) processed_artifacts.append(chunk_artifact) name_to_processed_artifact[info['name']] = chunk_artifact return artifacts - diff --git a/morphlib/artifactresolver_tests.py b/morphlib/artifactresolver_tests.py index 055343dc..5869cb27 100644 --- a/morphlib/artifactresolver_tests.py +++ b/morphlib/artifactresolver_tests.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -47,7 +47,7 @@ class FakeChunkMorphology(morphlib.morph2.Morphology): ''' % name) self.builds_artifacts = [name] morphlib.morph2.Morphology.__init__(self, text) - + class FakeStratumMorphology(morphlib.morph2.Morphology): @@ -99,10 +99,10 @@ class ArtifactResolverTests(unittest.TestCase): def test_resolve_single_chunk_with_no_subartifacts(self): pool = morphlib.sourcepool.SourcePool() - + morph = FakeChunkMorphology('chunk') source = morphlib.source.Source( - 'repo', 'ref', 'sha1', morph, 'chunk.morph') + 'repo', 'ref', 'sha1', morph, 'chunk.morph') pool.add(source) artifacts = self.resolver.resolve_artifacts(pool) @@ -116,10 +116,10 @@ class ArtifactResolverTests(unittest.TestCase): def test_resolve_single_chunk_with_one_artifact(self): pool = morphlib.sourcepool.SourcePool() - + morph = FakeChunkMorphology('chunk', ['chunk-runtime']) source = morphlib.source.Source( - 'repo', 'ref', 'sha1', morph, 'chunk.morph') + 'repo', 'ref', 'sha1', morph, 'chunk.morph') pool.add(source) artifacts = self.resolver.resolve_artifacts(pool) @@ -132,10 +132,10 @@ class ArtifactResolverTests(unittest.TestCase): def test_resolve_single_chunk_with_two_artifact(self): pool = morphlib.sourcepool.SourcePool() - + morph = FakeChunkMorphology('chunk', ['chunk-runtime', 'chunk-devel']) source = morphlib.source.Source( - 'repo', 'ref', 'sha1', morph, 'chunk.morph') + 'repo', 'ref', 'sha1', morph, 'chunk.morph') pool.add(source) artifacts = self.resolver.resolve_artifacts(pool) @@ -157,15 +157,15 @@ class ArtifactResolverTests(unittest.TestCase): pool = morphlib.sourcepool.SourcePool() morph = morphlib.morph2.Morphology( - ''' - { - "name": "foo", - "kind": "stratum" - } - ''') + ''' + { + "name": "foo", + "kind": "stratum" + } + ''') morph.builds_artifacts = ['foo'] stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'foo.morph') + 'repo', 'original/ref', 'sha1', morph, 'foo.morph') pool.add(stratum) artifacts = self.resolver.resolve_artifacts(pool) @@ -179,15 +179,15 @@ class ArtifactResolverTests(unittest.TestCase): pool = morphlib.sourcepool.SourcePool() morph = morphlib.morph2.Morphology( - ''' - { - "name": "foo", - "kind": "system" - } - ''') + ''' + { + "name": "foo", + "kind": "system" + } + ''') morph.builds_artifacts = ['foo-rootfs'] system = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'foo.morph') + 'repo', 'original/ref', 'sha1', morph, 'foo.morph') pool.add(system) artifacts = self.resolver.resolve_artifacts(pool) @@ -201,39 +201,39 @@ class ArtifactResolverTests(unittest.TestCase): pool = morphlib.sourcepool.SourcePool() morph = morphlib.morph2.Morphology( - ''' - { - "name": "foo", - "kind": "system", - "arch": "arm" - } - ''') + ''' + { + "name": "foo", + "kind": "system", + "arch": "arm" + } + ''') morph.builds_artifacts = ['foo-rootfs', 'foo-kernel'] system = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'foo.morph') + 'repo', 'original/ref', 'sha1', morph, 'foo.morph') pool.add(system) artifacts = self.resolver.resolve_artifacts(pool) self.assertTrue(any((a.source == system and a.name == 'foo-rootfs' and a.dependencies == [] and a.dependents == []) - for a in artifacts)) + for a in artifacts)) self.assertTrue(any((a.source == system and a.name == 'foo-kernel' and a.dependencies == [] and a.dependents == []) - for a in artifacts)) + for a in artifacts)) def test_resolve_stratum_and_chunk_with_no_subartifacts(self): pool = morphlib.sourcepool.SourcePool() - + morph = FakeChunkMorphology('chunk') chunk = morphlib.source.Source( - 'repo', 'ref', 'sha1', morph, 'chunk.morph') + 'repo', 'ref', 'sha1', morph, 'chunk.morph') pool.add(chunk) morph = FakeStratumMorphology( - 'stratum', [('chunk', 'chunk', 'repo', 'ref')]) + 'stratum', [('chunk', 'chunk', 'repo', 'ref')]) stratum = morphlib.source.Source( - 'repo', 'ref', 'sha1', morph, 'stratum.morph') + 'repo', 'ref', 'sha1', morph, 'stratum.morph') pool.add(stratum) artifacts = self.resolver.resolve_artifacts(pool) @@ -252,19 +252,19 @@ class ArtifactResolverTests(unittest.TestCase): def test_resolve_stratum_and_chunk_with_two_subartifacts(self): pool = morphlib.sourcepool.SourcePool() - + morph = FakeChunkMorphology('chunk', ['chunk-devel', 'chunk-runtime']) chunk = morphlib.source.Source( - 'repo', 'ref', 'sha1', morph, 'chunk.morph') + 'repo', 'ref', 'sha1', morph, 'chunk.morph') pool.add(chunk) morph = FakeStratumMorphology( - 'stratum', [ - ('chunk-devel', 'chunk', 'repo', 'ref'), - ('chunk-runtime', 'chunk', 'repo', 'ref') - ]) + 'stratum', [ + ('chunk-devel', 'chunk', 'repo', 'ref'), + ('chunk-runtime', 'chunk', 'repo', 'ref') + ]) stratum = morphlib.source.Source( - 'repo', 'ref', 'sha1', morph, 'stratum.morph') + 'repo', 'ref', 'sha1', morph, 'stratum.morph') pool.add(stratum) artifacts = self.resolver.resolve_artifacts(pool) @@ -289,18 +289,18 @@ class ArtifactResolverTests(unittest.TestCase): def test_resolve_stratum_and_chunk_with_one_used_subartifacts(self): pool = morphlib.sourcepool.SourcePool() - + morph = FakeChunkMorphology('chunk', ['chunk-devel', 'chunk-runtime']) chunk = morphlib.source.Source( - 'repo', 'ref', 'sha1', morph, 'chunk.morph') + 'repo', 'ref', 'sha1', morph, 'chunk.morph') pool.add(chunk) morph = FakeStratumMorphology( - 'stratum', [ - ('chunk-runtime', 'chunk', 'repo', 'ref') - ]) + 'stratum', [ + ('chunk-runtime', 'chunk', 'repo', 'ref') + ]) stratum = morphlib.source.Source( - 'repo', 'ref', 'sha1', morph, 'stratum.morph') + 'repo', 'ref', 'sha1', morph, 'stratum.morph') pool.add(stratum) artifacts = self.resolver.resolve_artifacts(pool) @@ -319,24 +319,24 @@ class ArtifactResolverTests(unittest.TestCase): def test_resolving_two_different_chunk_artifacts_in_a_stratum(self): pool = morphlib.sourcepool.SourcePool() - + morph = FakeChunkMorphology('foo') foo_chunk = morphlib.source.Source( - 'repo', 'ref', 'sha1', morph, 'foo.morph') + 'repo', 'ref', 'sha1', morph, 'foo.morph') pool.add(foo_chunk) morph = FakeChunkMorphology('bar') bar_chunk = morphlib.source.Source( - 'repo', 'ref', 'sha1', morph, 'bar.morph') + 'repo', 'ref', 'sha1', morph, 'bar.morph') pool.add(bar_chunk) morph = FakeStratumMorphology( - 'stratum', [ - ('foo', 'foo', 'repo', 'ref'), - ('bar', 'bar', 'repo', 'ref') - ]) + 'stratum', [ + ('foo', 'foo', 'repo', 'ref'), + ('bar', 'bar', 'repo', 'ref') + ]) stratum = morphlib.source.Source( - 'repo', 'ref', 'sha1', morph, 'stratum.morph') + 'repo', 'ref', 'sha1', morph, 'stratum.morph') pool.add(stratum) artifacts = self.resolver.resolve_artifacts(pool) @@ -361,15 +361,15 @@ class ArtifactResolverTests(unittest.TestCase): def test_resolving_artifacts_for_a_chain_of_two_strata(self): pool = morphlib.sourcepool.SourcePool() - + morph = FakeStratumMorphology('stratum1') stratum1 = morphlib.source.Source( - 'repo', 'ref', 'sha1', morph, 'stratum1.morph') + 'repo', 'ref', 'sha1', morph, 'stratum1.morph') pool.add(stratum1) morph = FakeStratumMorphology('stratum2', [], ['stratum1']) stratum2 = morphlib.source.Source( - 'repo', 'ref', 'sha1', morph, 'stratum2.morph') + 'repo', 'ref', 'sha1', morph, 'stratum2.morph') pool.add(stratum2) artifacts = self.resolver.resolve_artifacts(pool) @@ -391,26 +391,26 @@ class ArtifactResolverTests(unittest.TestCase): morph = FakeStratumMorphology('stratum1') stratum1 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'stratum1.morph') + 'repo', 'original/ref', 'sha1', morph, 'stratum1.morph') pool.add(stratum1) morph = FakeStratumMorphology( - 'stratum2', [ - ('chunk1', 'chunk1', 'repo', 'original/ref'), - ('chunk2', 'chunk2', 'repo', 'original/ref') - ], ['stratum1']) + 'stratum2', [ + ('chunk1', 'chunk1', 'repo', 'original/ref'), + ('chunk2', 'chunk2', 'repo', 'original/ref') + ], ['stratum1']) stratum2 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'stratum2.morph') + 'repo', 'original/ref', 'sha1', morph, 'stratum2.morph') pool.add(stratum2) morph = FakeChunkMorphology('chunk1') chunk1 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'chunk1.morph') + 'repo', 'original/ref', 'sha1', morph, 'chunk1.morph') pool.add(chunk1) morph = FakeChunkMorphology('chunk2') chunk2 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'chunk2.morph') + 'repo', 'original/ref', 'sha1', morph, 'chunk2.morph') pool.add(chunk2) artifacts = self.resolver.resolve_artifacts(pool) @@ -421,19 +421,19 @@ class ArtifactResolverTests(unittest.TestCase): self.assertEqual(artifacts[0].name, 'stratum1') self.assertEqual(artifacts[0].dependencies, []) self.assertEqual(artifacts[0].dependents, - [artifacts[1], artifacts[2], artifacts[3]]) + [artifacts[1], artifacts[2], artifacts[3]]) self.assertEqual(artifacts[1].source, stratum2) self.assertEqual(artifacts[1].name, 'stratum2') self.assertEqual(artifacts[1].dependencies, [artifacts[0], artifacts[2], artifacts[3]]) self.assertEqual(artifacts[1].dependents, []) - + self.assertEqual(artifacts[2].source, chunk1) self.assertEqual(artifacts[2].name, 'chunk1') self.assertEqual(artifacts[2].dependencies, [artifacts[0]]) self.assertEqual(artifacts[2].dependents, [artifacts[1], artifacts[3]]) - + self.assertEqual(artifacts[3].source, chunk2) self.assertEqual(artifacts[3].name, 'chunk2') self.assertEqual(artifacts[3].dependencies, @@ -442,31 +442,31 @@ class ArtifactResolverTests(unittest.TestCase): def test_resolving_artifacts_for_a_system_with_two_strata(self): pool = morphlib.sourcepool.SourcePool() - + morph = FakeStratumMorphology('stratum1') stratum1 = morphlib.source.Source( - 'repo', 'ref', 'sha1', morph, 'stratum1.morph') + 'repo', 'ref', 'sha1', morph, 'stratum1.morph') pool.add(stratum1) morph = morphlib.morph2.Morphology( - ''' - { - "name": "system", - "kind": "system", - "strata": [ - "stratum1", - "stratum2" - ] - } - ''') + ''' + { + "name": "system", + "kind": "system", + "strata": [ + "stratum1", + "stratum2" + ] + } + ''') morph.builds_artifacts = ['system-rootfs'] system = morphlib.source.Source( - 'repo', 'ref', 'sha1', morph, 'system.morph') + 'repo', 'ref', 'sha1', morph, 'system.morph') pool.add(system) morph = FakeStratumMorphology('stratum2', [], ['stratum1']) stratum2 = morphlib.source.Source( - 'repo', 'ref', 'sha1', morph, 'stratum2.morph') + 'repo', 'ref', 'sha1', morph, 'stratum2.morph') pool.add(stratum2) artifacts = self.resolver.resolve_artifacts(pool) @@ -493,53 +493,53 @@ class ArtifactResolverTests(unittest.TestCase): pool = morphlib.sourcepool.SourcePool() morph = morphlib.morph2.Morphology( - ''' - { - "name": "stratum", - "kind": "stratum", - "sources": [ - { - "name": "chunk1", - "repo": "repo", - "ref": "original/ref", - "build-depends": [] - }, - { - "name": "chunk2", - "repo": "repo", - "ref": "original/ref", - "build-depends": [] - }, - { - "name": "chunk3", - "repo": "repo", - "ref": "original/ref", - "build-depends": [ - "chunk1", - "chunk2" - ] - } - ] - } - ''') + ''' + { + "name": "stratum", + "kind": "stratum", + "sources": [ + { + "name": "chunk1", + "repo": "repo", + "ref": "original/ref", + "build-depends": [] + }, + { + "name": "chunk2", + "repo": "repo", + "ref": "original/ref", + "build-depends": [] + }, + { + "name": "chunk3", + "repo": "repo", + "ref": "original/ref", + "build-depends": [ + "chunk1", + "chunk2" + ] + } + ] + } + ''') morph.builds_artifacts = ['stratum'] stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'stratum.morph') + 'repo', 'original/ref', 'sha1', morph, 'stratum.morph') pool.add(stratum) morph = FakeChunkMorphology('chunk1') chunk1 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'chunk1.morph') + 'repo', 'original/ref', 'sha1', morph, 'chunk1.morph') pool.add(chunk1) morph = FakeChunkMorphology('chunk2') chunk2 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'chunk2.morph') + 'repo', 'original/ref', 'sha1', morph, 'chunk2.morph') pool.add(chunk2) morph = FakeChunkMorphology('chunk3') chunk3 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'chunk3.morph') + 'repo', 'original/ref', 'sha1', morph, 'chunk3.morph') pool.add(chunk3) artifacts = self.resolver.resolve_artifacts(pool) @@ -571,35 +571,35 @@ class ArtifactResolverTests(unittest.TestCase): def test_detection_of_invalid_chunk_artifact_references(self): pool = morphlib.sourcepool.SourcePool() - + morph = FakeChunkMorphology('chunk') chunk = morphlib.source.Source( - 'repo', 'ref', 'sha1', morph, 'chunk.morph') + 'repo', 'ref', 'sha1', morph, 'chunk.morph') pool.add(chunk) morph = FakeStratumMorphology( - 'stratum', [ - ('chunk-runtime', 'chunk', 'repo', 'ref') - ]) + 'stratum', [ + ('chunk-runtime', 'chunk', 'repo', 'ref') + ]) stratum = morphlib.source.Source( - 'repo', 'ref', 'sha1', morph, 'stratum.morph') + 'repo', 'ref', 'sha1', morph, 'stratum.morph') pool.add(stratum) self.assertRaises( - morphlib.artifactresolver.UndefinedChunkArtifactError, - self.resolver.resolve_artifacts, pool) + morphlib.artifactresolver.UndefinedChunkArtifactError, + self.resolver.resolve_artifacts, pool) def test_detection_of_mutual_dependency_between_two_strata(self): pool = morphlib.sourcepool.SourcePool() morph = FakeStratumMorphology('stratum1', [], ['stratum2']) stratum1 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'stratum1.morph') + 'repo', 'original/ref', 'sha1', morph, 'stratum1.morph') pool.add(stratum1) morph = FakeStratumMorphology('stratum2', [], ['stratum1']) stratum2 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'stratum2.morph') + 'repo', 'original/ref', 'sha1', morph, 'stratum2.morph') pool.add(stratum2) self.assertRaises(morphlib.artifactresolver.MutualDependencyError, @@ -609,31 +609,31 @@ class ArtifactResolverTests(unittest.TestCase): pool = morphlib.sourcepool.SourcePool() morph = FakeStratumMorphology( - 'stratum1', [ - ('chunk1', 'chunk1', 'repo', 'original/ref'), - ('chunk2', 'chunk2', 'repo', 'original/ref') - ], []) + 'stratum1', [ + ('chunk1', 'chunk1', 'repo', 'original/ref'), + ('chunk2', 'chunk2', 'repo', 'original/ref') + ], []) stratum1 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'stratum1.morph') + 'repo', 'original/ref', 'sha1', morph, 'stratum1.morph') pool.add(stratum1) morph = FakeStratumMorphology( - 'stratum2', [ - ('chunk2', 'chunk2', 'repo', 'original/ref'), - ('chunk1', 'chunk1', 'repo', 'original/ref') - ], ['stratum1']) + 'stratum2', [ + ('chunk2', 'chunk2', 'repo', 'original/ref'), + ('chunk1', 'chunk1', 'repo', 'original/ref') + ], ['stratum1']) stratum2 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'stratum2.morph') + 'repo', 'original/ref', 'sha1', morph, 'stratum2.morph') pool.add(stratum2) morph = FakeChunkMorphology('chunk1') chunk1 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'chunk1.morph') + 'repo', 'original/ref', 'sha1', morph, 'chunk1.morph') pool.add(chunk1) morph = FakeChunkMorphology('chunk2') chunk2 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'chunk2.morph') + 'repo', 'original/ref', 'sha1', morph, 'chunk2.morph') pool.add(chunk2) self.assertRaises(morphlib.artifactresolver.MutualDependencyError, @@ -643,40 +643,40 @@ class ArtifactResolverTests(unittest.TestCase): pool = morphlib.sourcepool.SourcePool() morph = morphlib.morph2.Morphology( - ''' - { - "name": "stratum", - "kind": "stratum", - "sources": [ - { - "name": "chunk", - "repo": "repo", - "ref": "original/ref" - }, - { - "name": "chunk", - "repo": "repo", - "ref": "original/ref" - }, - { - "name": "chunk", - "repo": "repo", - "ref": "original/ref", - "build-depends": [ - "chunk" - ] - } - ] - } - ''') + ''' + { + "name": "stratum", + "kind": "stratum", + "sources": [ + { + "name": "chunk", + "repo": "repo", + "ref": "original/ref" + }, + { + "name": "chunk", + "repo": "repo", + "ref": "original/ref" + }, + { + "name": "chunk", + "repo": "repo", + "ref": "original/ref", + "build-depends": [ + "chunk" + ] + } + ] + } + ''') morph.builds_artifacts = ['stratum'] stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'stratum.morph') + 'repo', 'original/ref', 'sha1', morph, 'stratum.morph') pool.add(stratum) morph = FakeChunkMorphology('chunk') chunk = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'chunk.morph') + 'repo', 'original/ref', 'sha1', morph, 'chunk.morph') pool.add(chunk) artifacts = self.resolver.resolve_artifacts(pool) @@ -697,40 +697,40 @@ class ArtifactResolverTests(unittest.TestCase): pool = morphlib.sourcepool.SourcePool() morph = morphlib.morph2.Morphology( - ''' - { - "name": "stratum", - "kind": "stratum", - "sources": [ - { - "name": "chunk1", - "repo": "repo", - "ref": "original/ref", - "build-depends": [ - "chunk2" - ] - }, - { - "name": "chunk2", - "repo": "repo", - "ref": "original/ref" - } - ] - } - ''') + ''' + { + "name": "stratum", + "kind": "stratum", + "sources": [ + { + "name": "chunk1", + "repo": "repo", + "ref": "original/ref", + "build-depends": [ + "chunk2" + ] + }, + { + "name": "chunk2", + "repo": "repo", + "ref": "original/ref" + } + ] + } + ''') morph.builds_artifacts = ['stratum'] stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'stratum.morph') + 'repo', 'original/ref', 'sha1', morph, 'stratum.morph') pool.add(stratum) morph = FakeChunkMorphology('chunk1') chunk1 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'chunk1.morph') + 'repo', 'original/ref', 'sha1', morph, 'chunk1.morph') pool.add(chunk1) morph = FakeChunkMorphology('chunk2') chunk2 = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'chunk2.morph') + 'repo', 'original/ref', 'sha1', morph, 'chunk2.morph') pool.add(chunk2) self.assertRaises(morphlib.artifactresolver.DependencyOrderError, @@ -740,28 +740,28 @@ class ArtifactResolverTests(unittest.TestCase): pool = morphlib.sourcepool.SourcePool() morph = morphlib.morph2.Morphology( - ''' - { - "name": "stratum", - "kind": "stratum", - "sources": [ - { - "name": "chunk", - "repo": "repo", - "ref": "original/ref", - "build-depends": "whatever" - } - ] - } - ''') + ''' + { + "name": "stratum", + "kind": "stratum", + "sources": [ + { + "name": "chunk", + "repo": "repo", + "ref": "original/ref", + "build-depends": "whatever" + } + ] + } + ''') morph.builds_artifacts = ['stratum'] stratum = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'stratum.morph') + 'repo', 'original/ref', 'sha1', morph, 'stratum.morph') pool.add(stratum) morph = FakeChunkMorphology('chunk') chunk = morphlib.source.Source( - 'repo', 'original/ref', 'sha1', morph, 'chunk.morph') + 'repo', 'original/ref', 'sha1', morph, 'chunk.morph') pool.add(chunk) self.assertRaises(morphlib.artifactresolver.DependencyFormatError, diff --git a/morphlib/bins.py b/morphlib/bins.py index 3c6c1bad..ec750db1 100644 --- a/morphlib/bins.py +++ b/morphlib/bins.py @@ -1,14 +1,14 @@ # Copyright (C) 2011-2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -31,24 +31,24 @@ import tarfile def create_chunk(rootdir, f, regexps, dump_memory_profile=None): '''Create a chunk from the contents of a directory. - + Only files and directories that match at least one of the regular expressions are accepted. The regular expressions are implicitly - anchored to the beginning of the string, but not the end. The + anchored to the beginning of the string, but not the end. The filenames are relative to rootdir. - + ``f`` is an open file handle, to which the tar file is written. - + ''' - dump_memory_profile = dump_memory_profile or (lambda msg: None ) - + dump_memory_profile = dump_memory_profile or (lambda msg: None) + # This timestamp is used to normalize the mtime for every file in # chunk artifact. This is useful to avoid problems from smallish # clock skew. It needs to be recent enough, however, that GNU tar # does not complain about an implausibly old timestamp. normalized_timestamp = 683074800 - + def mkrel(filename): assert filename.startswith(rootdir) if filename == rootdir: @@ -65,8 +65,8 @@ def create_chunk(rootdir, f, regexps, dump_memory_profile=None): filename = os.path.dirname(filename) yield filename - logging.debug('Creating chunk file %s from %s with regexps %s' % - (getattr(f, 'name', 'UNNAMED'), rootdir, regexps)) + logging.debug('Creating chunk file %s from %s with regexps %s' % + (getattr(f, 'name', 'UNNAMED'), rootdir, regexps)) dump_memory_profile('at beginning of create_chunk') compiled = [re.compile(x) for x in regexps] @@ -85,7 +85,7 @@ def create_chunk(rootdir, f, regexps, dump_memory_profile=None): logging.debug('regexp MISMATCH: %s' % filename) dump_memory_profile('after walking') - include = sorted(include) # get dirs before contents + include = sorted(include) # get dirs before contents tar = tarfile.open(fileobj=f, mode='w:gz') for filename in include: # Normalize mtime for everything. @@ -109,15 +109,15 @@ def create_chunk(rootdir, f, regexps, dump_memory_profile=None): dump_memory_profile('after removing in create_chunks') -def unpack_binary_from_file(f, dirname): # pragma: no cover +def unpack_binary_from_file(f, dirname): # pragma: no cover '''Unpack a binary into a directory. - + The directory must exist already. - + ''' tf = tarfile.open(fileobj=f) - + # This is evil, but necessary. For some reason Python's system # call wrappers (os.mknod and such) do not (always?) set the # filename attribute of the OSError exception they raise. We @@ -125,34 +125,34 @@ def unpack_binary_from_file(f, dirname): # pragma: no cover # for the relevant methods to add things. The wrapper further # ignores EEXIST errors, since we do not (currently!) care about # overwriting files. - - def follow_symlink(path): # pragma: no cover + + def follow_symlink(path): # pragma: no cover try: return os.stat(path) except OSError: return None - - def prepare_extract(tarinfo, targetpath): # pragma: no cover + + def prepare_extract(tarinfo, targetpath): # pragma: no cover '''Prepare to extract a tar file member onto targetpath? - + If the target already exist, and we can live with it or remove it, we do so. Otherwise, raise an error. - + It's OK to extract if: * the target does not exist - * the member is a directory a directory and the + * the member is a directory a directory and the target is a directory or a symlink to a directory (just extract, no need to remove) * the member is not a directory, and the target is not a directory or a symlink to a directory (remove target, then extract) - + ''' try: existing = os.lstat(targetpath) except OSError: - return True # target does not exist + return True # target does not exist if tarinfo.isdir(): if stat.S_ISDIR(existing.st_mode): @@ -174,7 +174,7 @@ def unpack_binary_from_file(f, dirname): # pragma: no cover return False def monkey_patcher(real): - def make_something(tarinfo, targetpath): # pragma: no cover + def make_something(tarinfo, targetpath): # pragma: no cover prepare_extract(tarinfo, targetpath) try: return real(tarinfo, targetpath) @@ -197,6 +197,7 @@ def unpack_binary_from_file(f, dirname): # pragma: no cover tf.extractall(path=dirname) tf.close + def unpack_binary(filename, dirname): f = open(filename, "rb") unpack_binary_from_file(f, dirname) diff --git a/morphlib/bins_tests.py b/morphlib/bins_tests.py index df020e0c..3288207b 100644 --- a/morphlib/bins_tests.py +++ b/morphlib/bins_tests.py @@ -1,14 +1,14 @@ # Copyright (C) 2011-2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -29,22 +29,22 @@ class BinsTest(unittest.TestCase): def recursive_lstat(self, root): '''Return a list of (pathname, stat) pairs for everything in root. - + Pathnames are relative to root. Directories are recursed into. The stat result is selective, not all fields are used. - + ''' - + def remove_root(pathname): self.assertTrue(pathname.startswith(root)) if pathname == root: return '.' else: - return pathname[len(root)+1:] - + return pathname[(len(root) + 1):] + def lstat(filename): st = os.lstat(filename) - + # For directories, the size is dependent on the contents, and # possibly on things that have been deleted already. An unpacked # directory can be identical even if the size field is different. @@ -59,16 +59,16 @@ class BinsTest(unittest.TestCase): return (st.st_mode, 0, 0) else: return (st.st_mode, st.st_size, 0) - + result = [] - + for dirname, subdirs, basenames in os.walk(root): result.append((remove_root(dirname), lstat(dirname))) for basename in sorted(basenames): filename = os.path.join(dirname, basename) result.append((remove_root(filename), lstat(filename))) subdirs.sort() - + return result @@ -80,7 +80,7 @@ class ChunkTests(BinsTest): self.chunk_file = os.path.join(self.tempdir, 'chunk') self.chunk_f = open(self.chunk_file, 'wb') self.unpacked = os.path.join(self.tempdir, 'unpacked') - + def tearDown(self): self.chunk_f.close() shutil.rmtree(self.tempdir) @@ -89,7 +89,7 @@ class ChunkTests(BinsTest): timestamp = 12765 os.mkdir(self.instdir) - + bindir = os.path.join(self.instdir, 'bin') os.mkdir(bindir) filename = os.path.join(bindir, 'foo') @@ -114,26 +114,27 @@ class ChunkTests(BinsTest): def unpack_chunk(self): os.mkdir(self.unpacked) morphlib.bins.unpack_binary(self.chunk_file, self.unpacked) - + def test_empties_everything(self): self.create_chunk(['.']) - self.assertEqual([x for x,y in self.recursive_lstat(self.instdir)], + self.assertEqual([x for x, y in self.recursive_lstat(self.instdir)], ['.']) def test_creates_and_unpacks_chunk_exactly(self): self.create_chunk(['.']) self.unpack_chunk() - self.assertEqual(self.instdir_orig_files, + self.assertEqual(self.instdir_orig_files, self.recursive_lstat(self.unpacked)) def test_uses_only_matching_names(self): self.create_chunk(['bin']) self.unpack_chunk() - self.assertEqual([x for x,y in self.recursive_lstat(self.unpacked)], + self.assertEqual([x for x, y in self.recursive_lstat(self.unpacked)], ['.', 'bin', 'bin/foo']) - self.assertEqual([x for x,y in self.recursive_lstat(self.instdir)], + self.assertEqual([x for x, y in self.recursive_lstat(self.instdir)], ['.', 'lib', 'lib/libfoo.so']) + class ExtractTests(unittest.TestCase): def setUp(self): @@ -156,13 +157,15 @@ class ExtractTests(unittest.TestCase): def test_extracted_files_replace_links(self): def make_linkfile(basedir): - with open(os.path.join(basedir, 'babar'), 'w') as f: pass + with open(os.path.join(basedir, 'babar'), 'w') as f: + pass os.symlink('babar', os.path.join(basedir, 'bar')) return ['.'] linktar = self.create_chunk(make_linkfile) def make_file(basedir): - with open(os.path.join(basedir, 'bar'), 'w') as f: pass + with open(os.path.join(basedir, 'bar'), 'w') as f: + pass return ['.'] filetar = self.create_chunk(make_file) @@ -177,7 +180,7 @@ class ExtractTests(unittest.TestCase): os.symlink('.', os.path.join(basedir, 'usr')) return ['.'] linktar = self.create_chunk(make_usrlink) - + def make_usrdir(basedir): os.mkdir(os.path.join(basedir, 'usr')) return ['.'] @@ -196,7 +199,8 @@ class ExtractTests(unittest.TestCase): def make_usrdir(basedir): os.mkdir(os.path.join(basedir, 'usr')) - with open(os.path.join(basedir, 'usr', 'foo'), 'w') as f: pass + with open(os.path.join(basedir, 'usr', 'foo'), 'w') as f: + pass return ['.'] dirtar = self.create_chunk(make_usrdir) @@ -204,4 +208,3 @@ class ExtractTests(unittest.TestCase): morphlib.bins.unpack_binary_from_file(dirtar, self.unpacked) mode = os.lstat(os.path.join(self.unpacked, 'foo')).st_mode self.assertTrue(stat.S_ISREG(mode)) - diff --git a/morphlib/buildenvironment.py b/morphlib/buildenvironment.py index 0b7cccb2..c9c12cc4 100644 --- a/morphlib/buildenvironment.py +++ b/morphlib/buildenvironment.py @@ -17,10 +17,11 @@ import os import morphlib + class BuildEnvironment(): def __init__(self, settings, arch=None): - self.arch = morphlib.util.arch() if arch == None else arch + self.arch = morphlib.util.arch() if arch is None else arch self.env = self._clean_env(settings) _osenv = os.environ diff --git a/morphlib/buildenvironment_tests.py b/morphlib/buildenvironment_tests.py index f0efdb60..7cb0f07b 100644 --- a/morphlib/buildenvironment_tests.py +++ b/morphlib/buildenvironment_tests.py @@ -19,6 +19,7 @@ import unittest import morphlib from morphlib import buildenvironment + class BuildEnvironmentTests(unittest.TestCase): def setUp(self): diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 5820dc44..6edb7c61 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -30,7 +30,7 @@ import morphlib from morphlib.artifactcachereference import ArtifactCacheReference -def ldconfig(runcmd, rootdir): # pragma: no cover +def ldconfig(runcmd, rootdir): # pragma: no cover '''Run ldconfig for the filesystem below ``rootdir``. Essentially, ``rootdir`` specifies the root of a new system. @@ -41,9 +41,9 @@ def ldconfig(runcmd, rootdir): # pragma: no cover the root directory is ``rootdir``. Example: if ``rootdir`` is ``/tmp/foo``, then ``/tmp/foo/etc/ld.so.conf`` should contain ``/lib``, not ``/tmp/foo/lib``. - + The ldconfig found via ``$PATH`` is used, not the one in ``rootdir``, - since in bootstrap mode that might not yet exist, the various + since in bootstrap mode that might not yet exist, the various implementations should be compatible enough. ''' @@ -52,7 +52,7 @@ def ldconfig(runcmd, rootdir): # pragma: no cover if os.path.exists(conf): logging.debug('Running ldconfig for %s' % rootdir) cache = os.path.join(rootdir, 'etc', 'ld.so.cache') - + # The following trickery with $PATH is necessary during the Baserock # bootstrap build: we are not guaranteed that PATH contains the # directory (/sbin conventionally) that ldconfig is in. Then again, @@ -65,6 +65,7 @@ def ldconfig(runcmd, rootdir): # pragma: no cover else: logging.debug('No %s, not running ldconfig' % conf) + def download_depends(constituents, lac, rac, metadatas=None): for constituent in constituents: if not lac.has(constituent): @@ -75,29 +76,32 @@ def download_depends(constituents, lac, rac, metadatas=None): source.close() if metadatas is not None: for metadata in metadatas: - if (not lac.has_artifact_metadata(constituent, metadata) - and rac.has_artifact_metadata(constituent, metadata)): + if not lac.has_artifact_metadata(constituent, metadata): + if rac.has_artifact_metadata(constituent, metadata): src = rac.get_artifact_metadata(constituent, metadata) dst = lac.put_artifact_metadata(constituent, metadata) shutil.copyfileobj(src, dst) dst.close() src.close() -def get_chunk_files(f): # pragma: no cover + +def get_chunk_files(f): # pragma: no cover tar = tarfile.open(fileobj=f) for member in tar.getmembers(): if member.type is not tarfile.DIRTYPE: yield member.name tar.close() -def get_stratum_files(f, lac): # pragma: no cover + +def get_stratum_files(f, lac): # pragma: no cover for ca in (ArtifactCacheReference(a) for a in json.load(f)): cf = lac.get(ca) for filename in get_chunk_files(cf): yield filename cf.close() -def get_overlaps(artifact, constituents, lac): #pragma: no cover + +def get_overlaps(artifact, constituents, lac): # pragma: no cover # check whether strata overlap installed = defaultdict(set) for dep in constituents: @@ -107,7 +111,7 @@ def get_overlaps(artifact, constituents, lac): #pragma: no cover installed[filename].add(dep) elif artifact.source.morphology['kind'] == 'system': for filename in get_stratum_files(handle, lac): - installed[filename].add(dep) + installed[filename].add(dep) handle.close() overlaps = defaultdict(set) for filename, artifacts in installed.iteritems(): @@ -115,25 +119,28 @@ def get_overlaps(artifact, constituents, lac): #pragma: no cover overlaps[frozenset(artifacts)].add(filename) return overlaps -def log_overlaps(overlaps): #pragma: no cover + +def log_overlaps(overlaps): # pragma: no cover for overlapping, files in sorted(overlaps.iteritems()): logging.warning(' Artifacts %s overlap with files:' % - ', '.join(sorted(a.name for a in overlapping)) - ) + ', '.join(sorted(a.name for a in overlapping))) for filename in sorted(files): logging.warning(' %s' % filename) -def write_overlap_metadata(artifact, overlaps, lac): #pragma: no cover + +def write_overlap_metadata(artifact, overlaps, lac): # pragma: no cover f = lac.put_artifact_metadata(artifact, 'overlaps') # the big list comprehension is because json can't serialize # artifacts, sets or dicts with non-string keys - json.dump([ - [ - [a.name for a in afs], list(files) - ] for afs, files in overlaps.iteritems() - ], f, indent=4) + json.dump( + [ + [ + [a.name for a in afs], list(files) + ] for afs, files in overlaps.iteritems() + ], f, indent=4) f.close() + class BuilderBase(object): '''Base class for building artifacts.''' @@ -170,16 +177,16 @@ class BuilderBase(object): 'meta') as f: json.dump(meta, f, indent=4, sort_keys=True) f.write('\n') - + def create_metadata(self, artifact_name): '''Create metadata to artifact to allow it to be reproduced later. - + The metadata is represented as a dict, which later on will be written out as a JSON file. - + ''' - - assert isinstance(self.artifact.source.repo, + + assert isinstance(self.artifact.source.repo, morphlib.cachedrepo.CachedRepo) meta = { 'artifact-name': artifact_name, @@ -193,11 +200,11 @@ class BuilderBase(object): 'cache-key': self.artifact.cache_key, 'cache-id': self.artifact.cache_id, } - + return meta # Wrapper around open() to allow it to be overridden by unit tests. - def _open(self, filename, mode): # pragma: no cover + def _open(self, filename, mode): # pragma: no cover dirname = os.path.dirname(filename) if not os.path.exists(dirname): os.makedirs(dirname) @@ -205,13 +212,13 @@ class BuilderBase(object): def write_metadata(self, instdir, artifact_name): '''Write the metadata for an artifact. - + The file will be located under the ``baserock`` directory under instdir, named after ``cache_key`` with ``.meta`` as the suffix. It will be in JSON format. - + ''' - + meta = self.create_metadata(artifact_name) basename = '%s.meta' % artifact_name @@ -222,13 +229,13 @@ class BuilderBase(object): f = self._open(filename, 'w') f.write(json.dumps(meta, indent=4, sort_keys=True)) f.close() - + def new_artifact(self, artifact_name): '''Return an Artifact object for something built from our source.''' a = morphlib.artifact.Artifact(self.artifact.source, artifact_name) a.cache_key = self.artifact.cache_key return a - + def runcmd(self, *args, **kwargs): kwargs['env'] = self.build_env.env return self.staging_area.runcmd(*args, **kwargs) @@ -237,7 +244,7 @@ class BuilderBase(object): class ChunkBuilder(BuilderBase): '''Build chunk artifacts.''' - + def get_commands(self, which, morphology, build_system): '''Return the commands to run from a morphology or the build system.''' if morphology[which] is None: @@ -246,7 +253,7 @@ class ChunkBuilder(BuilderBase): else: return morphology[which] - def build_and_cache(self): # pragma: no cover + def build_and_cache(self): # pragma: no cover with self.build_watch('overall-build'): mounted = self.do_mounts() log_name = None @@ -264,8 +271,8 @@ class ChunkBuilder(BuilderBase): if log_name: with open(log_name) as f: for line in f: - logging.error('OUTPUT FROM FAILED BUILD: %s' % - line.rstrip('\n')) + logging.error('OUTPUT FROM FAILED BUILD: %s' % + line.rstrip('\n')) raise self.do_unmounts(mounted) built_artifacts = self.assemble_chunk_artifacts(destdir) @@ -277,7 +284,8 @@ class ChunkBuilder(BuilderBase): ('proc', 'proc', 'none'), ('dev/shm', 'tmpfs', 'none'), ) - def do_mounts(self): # pragma: no cover + + def do_mounts(self): # pragma: no cover mounted = [] if not self.setup_mounts: return mounted @@ -290,12 +298,12 @@ class ChunkBuilder(BuilderBase): mounted.append(path) return mounted - def do_unmounts(self, mounted): # pragma: no cover + def do_unmounts(self, mounted): # pragma: no cover for path in mounted: logging.debug('Unmounting %s in staging area' % path) morphlib.fsutils.unmount(self.app.runcmd, path) - def get_sources(self, srcdir): # pragma: no cover + def get_sources(self, srcdir): # pragma: no cover '''Get sources from git to a source directory, for building.''' cache_dir = os.path.dirname(self.artifact.source.repo.path) @@ -329,16 +337,16 @@ class ChunkBuilder(BuilderBase): todo += extract_repo(path, sha1, srcdir) self.set_mtime_recursively(srcdir) - def set_mtime_recursively(self, root): # pragma: no cover + def set_mtime_recursively(self, root): # pragma: no cover '''Set the mtime for every file in a directory tree to the same. - + We do this because git checkout does not set the mtime to anything, and some projects (binutils, gperf for example) include formatted documentation and try to randomly build things or not because of the timestamps. This should help us get more reliable builds. - + ''' - + now = time.time() for dirname, subdirs, basenames in os.walk(root, topdown=False): for basename in basenames: @@ -348,7 +356,7 @@ class ChunkBuilder(BuilderBase): os.utime(pathname, (now, now)) os.utime(dirname, (now, now)) - def run_commands(self, builddir, destdir, logfile): # pragma: no cover + def run_commands(self, builddir, destdir, logfile): # pragma: no cover m = self.artifact.source.morphology bs = morphlib.buildsystem.lookup_build_system(m['build-system']) @@ -356,7 +364,7 @@ class ChunkBuilder(BuilderBase): relative_destdir = self.staging_area.relative(destdir) self.build_env.env['DESTDIR'] = relative_destdir - steps = [('configure', False), + steps = [('configure', False), ('build', True), ('test', False), ('install', False)] @@ -393,7 +401,7 @@ class ChunkBuilder(BuilderBase): shutil.copyfileobj(readlog, self.app.output) raise e - def assemble_chunk_artifacts(self, destdir): # pragma: no cover + def assemble_chunk_artifacts(self, destdir): # pragma: no cover built_artifacts = [] with self.build_watch('create-chunks'): specs = self.artifact.source.morphology['chunks'] @@ -407,7 +415,7 @@ class ChunkBuilder(BuilderBase): self.write_metadata(destdir, artifact_name) patterns = specs[artifact_name] patterns += [r'baserock/%s\.' % artifact_name] - + artifact = self.new_artifact(artifact_name) with self.local_artifact_cache.put(artifact) as f: logging.debug('assembling chunk %s' % artifact_name) @@ -416,7 +424,7 @@ class ChunkBuilder(BuilderBase): name=artifact.name) morphlib.bins.create_chunk(destdir, f, patterns) built_artifacts.append(artifact) - + files = os.listdir(destdir) if files: raise Exception('DESTDIR %s is not empty: %s' % @@ -428,12 +436,12 @@ class StratumBuilder(BuilderBase): '''Build stratum artifacts.''' - def build_and_cache(self): # pragma: no cover + def build_and_cache(self): # pragma: no cover with self.build_watch('overall-build'): constituents = [dependency for dependency in self.artifact.dependencies if dependency.source.morphology['kind'] == 'chunk'] - # the only reason the StratumBuilder has to download chunks is to + # the only reason the StratumBuilder has to download chunks is to # check for overlap now that strata are lists of chunks with self.build_watch('check-chunks'): # download the chunk artifact if necessary @@ -467,13 +475,13 @@ class StratumBuilder(BuilderBase): return [artifact] -class SystemKindBuilder(BuilderBase): # pragma: no cover +class SystemKindBuilder(BuilderBase): # pragma: no cover '''Build a specific kind of a system. - + Subclasses should set the ``system_kind`` attribute to the kind of system they build. - + ''' def unpack_strata(self, path): @@ -520,10 +528,10 @@ class SystemKindBuilder(BuilderBase): # pragma: no cover chunk_handle.close() f.close() meta = self.local_artifact_cache.get_artifact_metadata( - stratum_artifact, 'meta') + stratum_artifact, 'meta') dst = morphlib.savefile.SaveFile( - os.path.join(path, 'baserock', - '%s.meta' % stratum_artifact.name), 'w') + os.path.join(path, 'baserock', + '%s.meta' % stratum_artifact.name), 'w') shutil.copyfileobj(meta, dst) dst.close() meta.close() @@ -532,18 +540,19 @@ class SystemKindBuilder(BuilderBase): # pragma: no cover def create_fstab(self, path): '''Create an /etc/fstab inside a system tree. - + The fstab is created using assumptions of the disk layout. If the assumptions are wrong, extend this code so it can deal with other cases. - + ''' self.app.status(msg='Creating fstab in %(path)s', path=path, chatty=True) with self.build_watch('create-fstab'): fstab = os.path.join(path, 'etc', 'fstab') - if not os.path.exists(os.path.dirname(fstab)):# FIXME: should exist + # FIXME: should exist + if not os.path.exists(os.path.dirname(fstab)): os.makedirs(os.path.dirname(fstab)) with open(fstab, 'w') as f: f.write('proc /proc proc defaults 0 0\n') @@ -555,11 +564,11 @@ class SystemKindBuilder(BuilderBase): # pragma: no cover The kernel image will be a separate artifact from the root filesystem/disk image/whatever. This is sometimes useful with - funky bootloaders or virtualisation. - + funky bootloaders or virtualisation. + ''' - name = self.artifact.source.morphology['name']+'-kernel' + name = self.artifact.source.morphology['name'] + '-kernel' a = self.new_artifact(name) with self.local_artifact_cache.put(a) as dest: for basename in ['zImage', 'vmlinuz']: @@ -570,7 +579,7 @@ class SystemKindBuilder(BuilderBase): # pragma: no cover break -class SystemKindBuilderFactory(object): # pragma: no cover +class SystemKindBuilderFactory(object): # pragma: no cover '''A factory class for SystemKindBuilder objects.''' @@ -579,16 +588,16 @@ class SystemKindBuilderFactory(object): # pragma: no cover def register(self, klass): self.system_kinds.append(klass) - + def new(self, system_kind, args, kwargs): for klass in self.system_kinds: if klass.system_kind == system_kind: return klass(*args, **kwargs) raise morphlib.Error("Don't know how to build system kind %s" % - system_kind) + system_kind) -class SystemBuilder(BuilderBase): # pragma: no cover +class SystemBuilder(BuilderBase): # pragma: no cover '''Build system image artifacts.''' @@ -596,21 +605,21 @@ class SystemBuilder(BuilderBase): # pragma: no cover BuilderBase.__init__(self, *args, **kwargs) self.args = args self.kwargs = kwargs - + def build_and_cache(self): system_kind = self.artifact.source.morphology['system-kind'] builder = self.app.system_kind_builder_factory.new( - system_kind, self.args, self.kwargs) + system_kind, self.args, self.kwargs) logging.debug('Building system with %s' % repr(builder)) self.app.status(msg='Building system %(system_name)s', system_name=self.artifact.name) return builder.build_and_cache() -class Builder(object): # pragma: no cover +class Builder(object): # pragma: no cover '''Helper class to build with the right BuilderBase subclass.''' - + classes = { 'chunk': ChunkBuilder, 'stratum': StratumBuilder, @@ -628,17 +637,16 @@ class Builder(object): # pragma: no cover self.build_env = build_env self.max_jobs = max_jobs self.setup_mounts = setup_mounts - + def build_and_cache(self, artifact): kind = artifact.source.morphology['kind'] o = self.classes[kind](self.app, self.staging_area, self.local_artifact_cache, self.remote_artifact_cache, artifact, - self.repo_cache, self.build_env, + self.repo_cache, self.build_env, self.max_jobs, self.setup_mounts) logging.debug('Builder.build: artifact %s with %s' % (artifact.name, repr(o))) built_artifacts = o.build_and_cache() logging.debug('Builder.build: done') return built_artifacts - diff --git a/morphlib/builder2_tests.py b/morphlib/builder2_tests.py index 6214891d..ccf3bc81 100644 --- a/morphlib/builder2_tests.py +++ b/morphlib/builder2_tests.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -27,10 +27,12 @@ class FakeBuildSystem(object): def __init__(self): self.build_commands = ['buildsys-it'] + class FakeApp(object): def __init__(self, runcmd=None): self.runcmd = runcmd + class FakeStagingArea(object): def __init__(self, runcmd): @@ -38,14 +40,14 @@ class FakeStagingArea(object): class FakeSource(object): - + def __init__(self): self.morphology = { 'name': 'a', 'kind': 'b', 'description': 'c', } - + self.repo = morphlib.cachedrepo.CachedRepo(FakeApp(), 'repo', 'url', 'path') self.original_ref = 'e' @@ -70,6 +72,7 @@ class FakeBuildEnv(object): 'PATH': '/le-bin:/le-bon:/le-bin-bon', } + class FakeFileHandle(object): def __init__(self, cache, key): @@ -92,6 +95,7 @@ class FakeFileHandle(object): def write(self, string): self._string += string + class FakeArtifactCache(object): def __init__(self): @@ -108,15 +112,14 @@ class FakeArtifactCache(object): def get(self, artifact): return StringIO.StringIO( - self._cached[(artifact.cache_key, artifact.name)]) + self._cached[(artifact.cache_key, artifact.name)]) def get_artifact_metadata(self, artifact, name): return StringIO.StringIO( - self._cached[(artifact.cache_key, artifact.name, name)]) + self._cached[(artifact.cache_key, artifact.name, name)]) def get_source_metadata(self, source, cachekey, name): - return StringIO.StringIO( - self._cached[(cachekey, name)]) + return StringIO.StringIO(self._cached[(cachekey, name)]) def has(self, artifact): return (artifact.cache_key, artifact.name) in self._cached @@ -183,17 +186,17 @@ class BuilderBaseTests(unittest.TestCase): def test_writes_metadata(self): artifact_name = 'le-artifact' orig_meta = self.builder.create_metadata(artifact_name) - + instdir = '/inst/dir' self.builder._open = self.fake_open self.builder.write_metadata(instdir, artifact_name) self.assertTrue(self.open_filename.startswith( - os.path.join(instdir, 'baserock', - artifact_name + '.'))) + os.path.join(instdir, 'baserock', + artifact_name + '.'))) self.assertTrue(self.open_filename.endswith('.meta')) - + meta = json.loads(self.open_handle.getvalue()) self.assertEqual(meta, orig_meta) @@ -202,7 +205,7 @@ class BuilderBaseTests(unittest.TestCase): pass self.builder.save_build_times() self.assertTrue(self.artifact_cache.has_source_metadata( - self.artifact.source, self.artifact.cache_key, 'meta')) + self.artifact.source, self.artifact.cache_key, 'meta')) def test_watched_events_in_cache(self): events = ["configure", "build", "install"] @@ -211,7 +214,7 @@ class BuilderBaseTests(unittest.TestCase): pass self.builder.save_build_times() meta = json.load(self.artifact_cache.get_source_metadata( - self.artifact.source, self.artifact.cache_key, 'meta')) + self.artifact.source, self.artifact.cache_key, 'meta')) self.assertEqual(sorted(events), sorted(meta['build-times'].keys())) @@ -252,20 +255,19 @@ class ChunkBuilderTests(unittest.TestCase): False) def test_uses_morphology_commands_when_given(self): - m = { 'build-commands': ['build-it'] } + m = {'build-commands': ['build-it']} bs = FakeBuildSystem() cmds = self.build.get_commands('build-commands', m, bs) self.assertEqual(cmds, ['build-it']) def test_uses_build_system_commands_when_morphology_doesnt(self): - m = { 'build-commands': None } + m = {'build-commands': None} bs = FakeBuildSystem() cmds = self.build.get_commands('build-commands', m, bs) self.assertEqual(cmds, ['buildsys-it']) def test_uses_morphology_commands_when_morphology_has_empty_list(self): - m = { 'build-commands': [] } + m = {'build-commands': []} bs = FakeBuildSystem() cmds = self.build.get_commands('build-commands', m, bs) self.assertEqual(cmds, []) - diff --git a/morphlib/buildorder.py b/morphlib/buildorder.py index 7e15814f..39b53f05 100644 --- a/morphlib/buildorder.py +++ b/morphlib/buildorder.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -22,7 +22,7 @@ class CyclicDependencyChainError(cliapp.AppException): def __init__(self): cliapp.AppException.__init__( - self, 'Cyclic dependency chain detected') + self, 'Cyclic dependency chain detected') class BuildOrder: @@ -37,8 +37,8 @@ class BuildOrder: self.groups = [] def _compute_reverse_topological_sorting(self, artifacts): - '''Computes a reverse topological sorting of the build graph. - + '''Computes a reverse topological sorting of the build graph. + A reverse topological sorting basically is the result of a series of breadth-first searches starting at each leaf node (artifacts with no dependents). Artifacts are added to the sorting as soon as all their @@ -46,7 +46,7 @@ class BuildOrder: For more information, see http://en.wikipedia.org/wiki/Topological_sorting. - + ''' # map artifacts to sets of satisfied dependents. this is to detect diff --git a/morphlib/buildorder_tests.py b/morphlib/buildorder_tests.py index 32a14120..ab1a7e1c 100644 --- a/morphlib/buildorder_tests.py +++ b/morphlib/buildorder_tests.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -77,7 +77,7 @@ class BuildOrderTests(unittest.TestCase): artifact3.add_dependency(artifact2) order = morphlib.buildorder.BuildOrder( - [artifact1, artifact2, artifact3]) + [artifact1, artifact2, artifact3]) self.assertEqual(len(order.groups), 3) self.assertEqual(order.groups[0], [artifact1]) @@ -97,7 +97,7 @@ class BuildOrderTests(unittest.TestCase): artifact3.add_dependency(artifact1) order = morphlib.buildorder.BuildOrder( - [artifact1, artifact2, artifact3]) + [artifact1, artifact2, artifact3]) self.assertEqual(len(order.groups), 2) self.assertEqual(order.groups[0], [artifact1]) @@ -116,7 +116,7 @@ class BuildOrderTests(unittest.TestCase): artifact3.add_dependency(artifact2) order = morphlib.buildorder.BuildOrder( - [artifact1, artifact2, artifact3]) + [artifact1, artifact2, artifact3]) self.assertEqual(len(order.groups), 2) self.assertEqual(order.groups[0], [artifact1, artifact2]) diff --git a/morphlib/buildsystem.py b/morphlib/buildsystem.py index c9f04054..0efd8e73 100644 --- a/morphlib/buildsystem.py +++ b/morphlib/buildsystem.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -20,30 +20,30 @@ import os class BuildSystem(object): '''An abstraction of an upstream build system. - + Some build systems are well known: autotools, for example. Others are purely manual: there's a set of commands to run that are specific for that project, and (almost) no other project uses them. The Linux kernel would be an example of that. - + This class provides an abstraction for these, including a method to autodetect well known build systems. - + ''' - + def __init__(self): self.configure_commands = [] self.build_commands = [] self.test_commands = [] self.install_commands = [] - + def __getitem__(self, key): key = '_'.join(key.split('-')) return getattr(self, key) - + def get_morphology_text(self, name): '''Return the text of an autodetected chunk morphology.''' - + return ''' { "name": "%(name)s", @@ -54,23 +54,23 @@ class BuildSystem(object): 'name': name, 'bs': self.name, } - + def used_by_project(self, exists): '''Does a project use this build system? - + ``exists`` is a function that returns a boolean telling if a filename, relative to the project source directory, exists or not. - + ''' - raise NotImplementedError() # pragma: no cover - + raise NotImplementedError() # pragma: no cover + class ManualBuildSystem(BuildSystem): '''A manual build system where the morphology must specify all commands.''' name = 'manual' - + def used_by_project(self, exists): return False @@ -80,7 +80,7 @@ class DummyBuildSystem(BuildSystem): '''A dummy build system, useful for debugging morphologies.''' name = 'dummy' - + def __init__(self): self.configure_commands = ['echo dummy configure'] self.build_commands = ['echo dummy build'] @@ -96,7 +96,7 @@ class AutotoolsBuildSystem(BuildSystem): '''The automake/autoconf/libtool holy trinity.''' name = 'autotools' - + def __init__(self): self.configure_commands = [ 'export NOCONFIGURE=1; ' + @@ -121,7 +121,7 @@ class AutotoolsBuildSystem(BuildSystem): 'configure.in', 'configure.in.in', ] - + return any(exists(x) for x in indicators) @@ -130,7 +130,7 @@ class PythonDistutilsBuildSystem(BuildSystem): '''The Python distutils build systems.''' name = 'python-distutils' - + def __init__(self): self.configure_commands = [ ] @@ -147,7 +147,7 @@ class PythonDistutilsBuildSystem(BuildSystem): indicators = [ 'setup.py', ] - + return any(exists(x) for x in indicators) @@ -159,12 +159,12 @@ class CPANBuildSystem(BuildSystem): def __init__(self): self.configure_commands = [ - 'perl Makefile.PL INSTALLDIRS=perl '\ - 'INSTALLARCHLIB="$PREFIX/lib/perl" '\ - 'INSTALLPRIVLIB="$PREFIX/lib/perl" '\ - 'INSTALLBIN="$PREFIX/bin" '\ - 'INSTALLSCRIPT="$PREFIX/bin" '\ - 'INSTALLMAN1DIR="$PREFIX/share/man/man1" '\ + 'perl Makefile.PL INSTALLDIRS=perl ' + 'INSTALLARCHLIB="$PREFIX/lib/perl" ' + 'INSTALLPRIVLIB="$PREFIX/lib/perl" ' + 'INSTALLBIN="$PREFIX/bin" ' + 'INSTALLSCRIPT="$PREFIX/bin" ' + 'INSTALLMAN1DIR="$PREFIX/share/man/man1" ' 'INSTALLMAN3DIR="$PREFIX/share/man/man3"', ] self.build_commands = [ @@ -195,12 +195,12 @@ build_systems = [ def detect_build_system(exists): '''Automatically detect the build system, if possible. - + If the build system cannot be detected automatically, return None. For ``exists`` see the ``BuildSystem.exists`` method. - + ''' - + for bs in build_systems: if bs.used_by_project(exists): return bs @@ -209,13 +209,12 @@ def detect_build_system(exists): def lookup_build_system(name): '''Return build system that corresponds to the name. - + If the name does not match any build system, raise ``KeyError``. - + ''' - + for bs in build_systems: if bs.name == name: return bs raise KeyError('Unknown build system: %s' % name) - diff --git a/morphlib/buildsystem_tests.py b/morphlib/buildsystem_tests.py index 8c790219..07ae3b98 100644 --- a/morphlib/buildsystem_tests.py +++ b/morphlib/buildsystem_tests.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -33,8 +33,8 @@ def create_manual_project(srcdir): def create_autotools_project(srcdir): touch(os.path.join(srcdir, 'configure.in')) - - + + class BuildSystemTests(unittest.TestCase): def setUp(self): @@ -51,7 +51,7 @@ class BuildSystemTests(unittest.TestCase): def test_has_install_commands(self): self.assertEqual(self.bs['install-commands'], []) - + def test_returns_morphology_text(self): self.bs.name = 'fake' text = self.bs.get_morphology_text('foobar') @@ -63,17 +63,17 @@ class ManualBuildSystemTests(unittest.TestCase): def setUp(self): self.bs = morphlib.buildsystem.ManualBuildSystem() self.tempdir = tempfile.mkdtemp() - + def tearDown(self): shutil.rmtree(self.tempdir) - + def exists(self, filename): return os.path.exists(os.path.join(self.tempdir, filename)) - + def test_does_not_autodetect_empty(self): create_manual_project(self.tempdir) self.assertFalse(self.bs.used_by_project(self.exists)) - + def test_does_not_autodetect_autotools(self): create_autotools_project(self.tempdir) self.assertFalse(self.bs.used_by_project(self.exists)) @@ -84,17 +84,17 @@ class DummyBuildSystemTests(unittest.TestCase): def setUp(self): self.bs = morphlib.buildsystem.DummyBuildSystem() self.tempdir = tempfile.mkdtemp() - + def tearDown(self): shutil.rmtree(self.tempdir) - + def exists(self, filename): return os.path.exists(os.path.join(self.tempdir, filename)) - + def test_does_not_autodetect_empty(self): create_manual_project(self.tempdir) self.assertFalse(self.bs.used_by_project(self.exists)) - + def test_does_not_autodetect_autotools(self): create_autotools_project(self.tempdir) self.assertFalse(self.bs.used_by_project(self.exists)) @@ -105,17 +105,17 @@ class AutotoolsBuildSystemTests(unittest.TestCase): def setUp(self): self.bs = morphlib.buildsystem.ManualBuildSystem() self.tempdir = tempfile.mkdtemp() - + def tearDown(self): shutil.rmtree(self.tempdir) - + def exists(self, filename): return os.path.exists(os.path.join(self.tempdir, filename)) - + def test_does_not_autodetect_empty(self): create_manual_project(self.tempdir) self.assertFalse(self.bs.used_by_project(self.exists)) - + def test_autodetects_autotools(self): create_autotools_project(self.tempdir) self.assertFalse(self.bs.used_by_project(self.exists)) @@ -126,10 +126,10 @@ class DetectBuildSystemTests(unittest.TestCase): def setUp(self): self.bs = morphlib.buildsystem.ManualBuildSystem() self.tempdir = tempfile.mkdtemp() - + def tearDown(self): shutil.rmtree(self.tempdir) - + def exists(self, filename): return os.path.exists(os.path.join(self.tempdir, filename)) @@ -163,4 +163,3 @@ class LookupBuildSystemTests(unittest.TestCase): def test_looks_up_dummy(self): self.assertEqual(type(self.lookup('dummy')), morphlib.buildsystem.DummyBuildSystem) - diff --git a/morphlib/cachedir.py b/morphlib/cachedir.py index a7daaf3b..de1b87ff 100644 --- a/morphlib/cachedir.py +++ b/morphlib/cachedir.py @@ -1,14 +1,14 @@ # Copyright (C) 2011-2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -23,37 +23,37 @@ import morphlib class CacheDir(object): '''Manage Baserock cached binaries.''' - + def __init__(self, dirname): self.dirname = os.path.abspath(dirname) - + def key(self, dict_key): '''Create a string key from a dictionary key. - + The string key can be used as a filename, or as part of one. The dictionary key is a dict that maps any set of strings to another set of strings. - + The same string key is guaranteed to be returned for a given dictionary key. It is highly unlikely that two different dictionary keys result in the same string key. - + ''' - + data = ''.join(key + value for key, value in dict_key.iteritems()) return hashlib.sha256(data).hexdigest() def name(self, dict_key): '''Return a filename for an object described by dictionary key. - + It is the caller's responsibility to set the fields in the dictionary key suitably. For example, if there is a field specifying a commit id, it should be the full git SHA-1 identifier, not something ephemeral like HEAD. - + If the field 'kind' has a value, it is used as a suffix for the filename. - + ''' key = self.key(dict_key) @@ -66,14 +66,14 @@ class CacheDir(object): def open(self, relative_name_or_cache_key, suffix='', **kwargs): '''Open a file for writing in the cache. - + The file will be written with a temporary name, and renamed to the final name when the file is closed. Additionally, if the caller decides, mid-writing, that they don't want to write the file after all (e.g., a log file), then the ``abort`` method in the returned file handle can be called to remove the temporary file. - + ''' if type(relative_name_or_cache_key) is dict: @@ -84,4 +84,3 @@ class CacheDir(object): if 'mode' not in kwargs: kwargs['mode'] = 'w' return morphlib.savefile.SaveFile(path, **kwargs) - diff --git a/morphlib/cachedir_tests.py b/morphlib/cachedir_tests.py index e7ab9354..43686427 100644 --- a/morphlib/cachedir_tests.py +++ b/morphlib/cachedir_tests.py @@ -1,14 +1,14 @@ # Copyright (C) 2011-2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -51,7 +51,7 @@ class CacheDirTests(unittest.TestCase): 'foo': 'bar', 'xyzzy': 'plugh', } - self.assertEqual(self.cachedir.key(dict_key), + self.assertEqual(self.cachedir.key(dict_key), self.cachedir.key(dict_key)) def test_generates_different_string_keys(self): @@ -63,7 +63,7 @@ class CacheDirTests(unittest.TestCase): 'foo': 'foobar', 'xyzzy': 'stevenage', } - self.assertNotEqual(self.cachedir.key(dict_key_1), + self.assertNotEqual(self.cachedir.key(dict_key_1), self.cachedir.key(dict_key_2)) def test_returns_a_chunk_pathname_in_cache_directory(self): @@ -126,4 +126,3 @@ class CacheDirTests(unittest.TestCase): f.abort() pathname = os.path.join(self.cachedir.dirname, 'foo') self.assertFalse(os.path.exists(pathname)) - diff --git a/morphlib/cachedrepo.py b/morphlib/cachedrepo.py index 575eedd6..d7f22400 100644 --- a/morphlib/cachedrepo.py +++ b/morphlib/cachedrepo.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -20,6 +20,7 @@ import os import morphlib + class InvalidReferenceError(cliapp.AppException): def __init__(self, repo, ref): @@ -38,16 +39,16 @@ class CheckoutDirectoryExistsError(cliapp.AppException): def __init__(self, repo, target_dir): cliapp.AppException.__init__( - self, - 'Checkout directory %s for repo %s already exists' % - (target_dir, repo)) + self, + 'Checkout directory %s for repo %s already exists' % + (target_dir, repo)) class CheckoutError(cliapp.AppException): def __init__(self, repo, ref, target_dir): cliapp.AppException.__init__( - self, + self, 'Failed to check out %s:%s into %s' % (repo, ref, target_dir)) @@ -61,7 +62,7 @@ class UpdateError(cliapp.AppException): class CachedRepo(object): '''A locally cached Git repository with an origin remote set up. - + On instance of this class represents a locally cached version of a remote Git repository. This remote repository is set up as the 'origin' remote. @@ -148,9 +149,9 @@ class CachedRepo(object): if os.path.exists(target_dir): raise CheckoutDirectoryExistsError(self, target_dir) - + os.mkdir(target_dir) - + try: self._copy_repository(self.path, target_dir) self._checkout_ref(ref, target_dir) @@ -159,10 +160,10 @@ class CachedRepo(object): def update(self): '''Updates the cached repository using its origin remote. - + Raises an UpdateError if anything goes wrong while performing the update. - + ''' try: @@ -170,29 +171,29 @@ class CachedRepo(object): except cliapp.AppException, e: raise UpdateError(self) - def _runcmd(self, *args, **kwargs): # pragma: no cover + def _runcmd(self, *args, **kwargs): # pragma: no cover if not 'cwd' in kwargs: kwargs['cwd'] = self.path return self.app.runcmd(*args, **kwargs) - def _show_ref(self, ref): # pragma: no cover + def _show_ref(self, ref): # pragma: no cover return self._runcmd(['git', 'show-ref', ref]) - def _rev_list(self, ref): # pragma: no cover + def _rev_list(self, ref): # pragma: no cover return self._runcmd(['git', 'rev-list', '--no-walk', ref]) - def _cat_file(self, ref, filename): # pragma: no cover + def _cat_file(self, ref, filename): # pragma: no cover return self._runcmd(['git', 'cat-file', 'blob', '%s:%s' % (ref, filename)]) - def _copy_repository(self, source_dir, target_dir): # pragma: no cover + def _copy_repository(self, source_dir, target_dir): # pragma: no cover morphlib.git.copy_repository(self._runcmd, source_dir, target_dir) - def _checkout_ref(self, ref, target_dir): # pragma: no cover + def _checkout_ref(self, ref, target_dir): # pragma: no cover morphlib.git.checkout_ref(self._runcmd, target_dir, ref) - def _update(self): # pragma: no cover + def _update(self): # pragma: no cover self._runcmd(['git', 'remote', 'update', 'origin']) - def __str__(self): # pragma: no cover + def __str__(self): # pragma: no cover return self.url diff --git a/morphlib/cachedrepo_tests.py b/morphlib/cachedrepo_tests.py index b60038e4..2a0c2c4b 100644 --- a/morphlib/cachedrepo_tests.py +++ b/morphlib/cachedrepo_tests.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -28,12 +28,12 @@ class CachedRepoTests(unittest.TestCase): def show_ref(self, ref): output = { - 'master': - 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9' - ' refs/remotes/origin/master', + 'master': + 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9' + ' refs/remotes/origin/master', 'baserock/morph': - '8b780e2e6f102fcf400ff973396566d36d730501' - ' refs/remotes/origin/baserock/morph', + '8b780e2e6f102fcf400ff973396566d36d730501' + ' refs/remotes/origin/baserock/morph', } try: return output[ref] @@ -42,10 +42,10 @@ class CachedRepoTests(unittest.TestCase): def rev_list(self, ref): output = { - 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9': - 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9', + 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9': + 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9', 'a4da32f5a81c8bc6d660404724cedc3bc0914a75': - 'a4da32f5a81c8bc6d660404724cedc3bc0914a75' + 'a4da32f5a81c8bc6d660404724cedc3bc0914a75', } try: return output[ref] @@ -55,13 +55,13 @@ class CachedRepoTests(unittest.TestCase): def cat_file(self, ref, filename): output = { 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9:foo.morph': - 'contents of foo.morph' + 'contents of foo.morph' } try: return output['%s:%s' % (ref, filename)] except: raise cliapp.AppException( - 'git cat-file blob %s:%s' % (ref, filename)) + 'git cat-file blob %s:%s' % (ref, filename)) def copy_repository(self, source_dir, target_dir): pass @@ -89,8 +89,8 @@ class CachedRepoTests(unittest.TestCase): self.repo_name = 'foo' self.repo_url = 'git://foo.bar/foo.git' self.repo_path = '/tmp/foo' - self.repo = cachedrepo.CachedRepo(object(), - self.repo_name, self.repo_url, self.repo_path) + self.repo = cachedrepo.CachedRepo( + object(), self.repo_name, self.repo_url, self.repo_path) self.repo._show_ref = self.show_ref self.repo._rev_list = self.rev_list self.repo._cat_file = self.cat_file @@ -120,7 +120,7 @@ class CachedRepoTests(unittest.TestCase): def test_resolve_sha1_ref(self): sha1 = self.repo.resolve_ref( - 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') + 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') self.assertEqual(sha1, 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9') def test_fail_resolving_an_invalid_sha1_ref(self): diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index 85d13934..ca6776b7 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -34,8 +34,8 @@ class CacheKeyComputer(object): def compute_key(self, artifact): logging.debug('computing cache key for artifact %s from source ' 'repo %s, sha1 %s, filename %s' % - (artifact.name, artifact.source.repo_name, - artifact.source.sha1, artifact.source.filename)) + (artifact.name, artifact.source.repo_name, + artifact.source.sha1, artifact.source.filename)) return self._hash_id(self.get_cache_id(artifact)) def _hash_id(self, cache_id): @@ -68,8 +68,8 @@ class CacheKeyComputer(object): def get_cache_id(self, artifact): logging.debug('computing cache id for artifact %s from source ' 'repo %s, sha1 %s, filename %s' % - (artifact.name, artifact.source.repo_name, - artifact.source.sha1, artifact.source.filename)) + (artifact.name, artifact.source.repo_name, + artifact.source.sha1, artifact.source.filename)) try: return self._calculated[artifact] except KeyError: @@ -84,13 +84,13 @@ class CacheKeyComputer(object): 'filename': artifact.source.filename, 'kids': [self.compute_key(x) for x in artifact.dependencies] } - + kind = artifact.source.morphology['kind'] if kind == 'chunk': keys['ref'] = artifact.source.sha1 elif kind in ('system', 'stratum'): morphology = artifact.source.morphology - le_dict = dict((k,morphology[k]) for k in morphology.keys()) + le_dict = dict((k, morphology[k]) for k in morphology.keys()) checksum = hashlib.sha1() self._hash_thing(checksum, le_dict) keys['morphology-sha1'] = checksum.hexdigest() @@ -100,4 +100,3 @@ class CacheKeyComputer(object): keys['system-compatibility-version'] = "1~ (temporary, root rw)" return keys - diff --git a/morphlib/cachekeycomputer_tests.py b/morphlib/cachekeycomputer_tests.py index 4847218f..3aba2175 100644 --- a/morphlib/cachekeycomputer_tests.py +++ b/morphlib/cachekeycomputer_tests.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -25,7 +25,7 @@ class DummyBuildEnvironment: a dict representing it ''' def __init__(self, env, arch=None): - self.arch = morphlib.util.arch() if arch == None else arch + self.arch = morphlib.util.arch() if arch is None else arch self.env = env @@ -82,8 +82,9 @@ class CacheKeyComputerTests(unittest.TestCase): ] }''', }.iteritems(): - source = morphlib.source.Source('repo', 'original/ref', 'sha', - morphlib.morph2.Morphology(text), name) + source = morphlib.source.Source( + 'repo', 'original/ref', 'sha', + morphlib.morph2.Morphology(text), name) self.source_pool.add(source) # FIXME: This should use MorphologyFactory m = source.morphology @@ -94,16 +95,16 @@ class CacheKeyComputerTests(unittest.TestCase): elif m['kind'] == 'chunk': m.builds_artifacts = [m['name']] self.build_env = DummyBuildEnvironment({ - "USER": "foouser", - "USERNAME": "foouser", - "LOGNAME": "foouser", - "TOOLCHAIN_TARGET": "dummy-baserock-linux-gnu", - "PREFIX": "/baserock", - "BOOTSTRAP": "false", - "CFLAGS": "-O4"}) + "USER": "foouser", + "USERNAME": "foouser", + "LOGNAME": "foouser", + "TOOLCHAIN_TARGET": "dummy-baserock-linux-gnu", + "PREFIX": "/baserock", + "BOOTSTRAP": "false", + "CFLAGS": "-O4"}) self.artifact_resolver = morphlib.artifactresolver.ArtifactResolver() self.artifacts = self.artifact_resolver.resolve_artifacts( - self.source_pool) + self.source_pool) self.ckc = morphlib.cachekeycomputer.CacheKeyComputer(self.build_env) def _find_artifact(self, name): @@ -114,6 +115,7 @@ class CacheKeyComputerTests(unittest.TestCase): def test_compute_key_hashes_all_types(self): runcount = {'thing': 0, 'dict': 0, 'list': 0, 'tuple': 0} + def inccount(func, name): def f(sha, item): runcount[name] = runcount[name] + 1 @@ -146,22 +148,22 @@ class CacheKeyComputerTests(unittest.TestCase): artifact = self._find_artifact('system-rootfs') oldsha = self.ckc.compute_key(artifact) build_env = DummyBuildEnvironment({ - "USER": "foouser", - "USERNAME": "foouser", - "LOGNAME": "foouser", - "TOOLCHAIN_TARGET": "dummy-baserock-linux-gnu", - "PREFIX": "/baserock", - "BOOTSTRAP": "false", - "CFLAGS": "-Os"}) + "USER": "foouser", + "USERNAME": "foouser", + "LOGNAME": "foouser", + "TOOLCHAIN_TARGET": "dummy-baserock-linux-gnu", + "PREFIX": "/baserock", + "BOOTSTRAP": "false", + "CFLAGS": "-Os"}) ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) self.assertNotEqual(oldsha, ckc.compute_key(artifact)) - + def test_same_morphology_text_but_changed_sha1_gives_same_cache_key(self): old_artifact = self._find_artifact('system-rootfs') morphology = old_artifact.source.morphology - new_source = morphlib.source.Source('repo', 'original/ref', 'newsha', - morphology, + new_source = morphlib.source.Source('repo', 'original/ref', 'newsha', + morphology, old_artifact.source.filename) self.source_pool.add(new_source) artifacts = self.artifact_resolver.resolve_artifacts(self.source_pool) @@ -174,4 +176,3 @@ class CacheKeyComputerTests(unittest.TestCase): old_sha = self.ckc.compute_key(old_artifact) new_sha = self.ckc.compute_key(new_artifact) self.assertEqual(old_sha, new_sha) - diff --git a/morphlib/fsutils.py b/morphlib/fsutils.py index c74e97db..45fb9e2f 100644 --- a/morphlib/fsutils.py +++ b/morphlib/fsutils.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -22,10 +22,12 @@ def create_image(runcmd, image_name, size): runcmd(['dd', 'if=/dev/zero', 'of=' + image_name, 'bs=1', 'seek=%d' % size, 'count=0']) + def partition_image(runcmd, image_name): # FIXME make this more flexible with partitioning options runcmd(['sfdisk', image_name], feed_stdin='1,,83,*\n') + def install_syslinux_mbr(runcmd, image_name): for path in ['/usr/lib/extlinux/mbr.bin', '/usr/share/syslinux/mbr.bin']: @@ -34,6 +36,7 @@ def install_syslinux_mbr(runcmd, image_name): 'conv=notrunc']) break + def setup_device_mapping(runcmd, image_name): findstart = re.compile(r"start=\s+(\d+),") out = runcmd(['sfdisk', '-d', image_name]) @@ -48,17 +51,21 @@ def setup_device_mapping(runcmd, image_name): device = runcmd(['losetup', '--show', '-o', str(start), '-f', image_name]) return device.strip() + def create_fs(runcmd, partition): runcmd(['mkfs.btrfs', '-L', 'baserock', partition]) + def mount(runcmd, partition, mount_point): if not os.path.exists(mount_point): os.mkdir(mount_point) runcmd(['mount', partition, mount_point]) + def unmount(runcmd, mount_point): runcmd(['umount', mount_point]) + def undo_device_mapping(runcmd, image_name): out = runcmd(['losetup', '-j', image_name]) for line in out.splitlines(): diff --git a/morphlib/git.py b/morphlib/git.py index 81e7213b..70915770 100644 --- a/morphlib/git.py +++ b/morphlib/git.py @@ -1,14 +1,14 @@ # Copyright (C) 2011-2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -30,7 +30,7 @@ import morphlib class NoModulesFileError(cliapp.AppException): def __init__(self, repo, ref): - Exception.__init__(self, + Exception.__init__(self, '%s:%s has no .gitmodules file.' % (repo, ref)) @@ -79,8 +79,8 @@ class Submodules(object): try: # try to read the .gitmodules file from the repo/ref content = self.app.runcmd( - ['git', 'cat-file', 'blob', '%s:.gitmodules' % self.ref], - cwd=self.repo) + ['git', 'cat-file', 'blob', '%s:.gitmodules' % self.ref], + cwd=self.repo) # drop indentation in sections, as RawConfigParser cannot handle it return '\n'.join([line.strip() for line in content.splitlines()]) @@ -120,8 +120,8 @@ class Submodules(object): self.submodules.append(submodule) else: logging.warning('Skipping submodule "%s" as %s:%s has ' - 'a non-commit object for it' % - (submodule.name, self.repo, self.ref)) + 'a non-commit object for it' % + (submodule.name, self.repo, self.ref)) except cliapp.AppException: raise MissingSubmoduleCommitError(self.repo, self.ref, submodule.name) @@ -140,14 +140,17 @@ def set_remote(runcmd, gitdir, name, url): '''Set remote with name 'name' use a given url at gitdir''' return runcmd(['git', 'remote', 'set-url', name, url], cwd=gitdir) + def copy_repository(runcmd, repo, destdir): '''Copies a cached repository into a directory using cp.''' return runcmd(['cp', '-a', os.path.join(repo, '.git'), destdir]) + def checkout_ref(runcmd, gitdir, ref): '''Checks out a specific ref/SHA1 in a git working tree.''' runcmd(['git', 'checkout', ref], cwd=gitdir) + def reset_workdir(runcmd, gitdir): '''Removes any differences between the current commit ''' '''and the status of the working directory''' diff --git a/morphlib/localartifactcache.py b/morphlib/localartifactcache.py index 83a668e4..51ccb4f6 100644 --- a/morphlib/localartifactcache.py +++ b/morphlib/localartifactcache.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. diff --git a/morphlib/localartifactcache_tests.py b/morphlib/localartifactcache_tests.py index a2fb0810..aacba8fb 100644 --- a/morphlib/localartifactcache_tests.py +++ b/morphlib/localartifactcache_tests.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -25,36 +25,36 @@ class LocalArtifactCacheTests(unittest.TestCase): self.tempdir = morphlib.tempdir.Tempdir() morph = morphlib.morph2.Morphology( - ''' - { - "chunk": "chunk", - "kind": "chunk", - "artifacts": { - "chunk-runtime": [ - "usr/bin", - "usr/sbin", - "usr/lib", - "usr/libexec" - ], - "chunk-devel": [ - "usr/include" - ] - } + ''' + { + "chunk": "chunk", + "kind": "chunk", + "artifacts": { + "chunk-runtime": [ + "usr/bin", + "usr/sbin", + "usr/lib", + "usr/libexec" + ], + "chunk-devel": [ + "usr/include" + ] } - ''') + } + ''') self.source = morphlib.source.Source( - 'repo', 'ref', 'sha1', morph, 'chunk.morph') + 'repo', 'ref', 'sha1', morph, 'chunk.morph') self.runtime_artifact = morphlib.artifact.Artifact( - self.source, 'chunk-runtime') + self.source, 'chunk-runtime') self.devel_artifact = morphlib.artifact.Artifact( - self.source, 'chunk-devel') + self.source, 'chunk-devel') def tearDown(self): self.tempdir.remove() def test_put_artifacts_and_check_whether_the_cache_has_them(self): cache = morphlib.localartifactcache.LocalArtifactCache( - self.tempdir.dirname) + self.tempdir.dirname) handle = cache.put(self.runtime_artifact) handle.write('runtime') @@ -71,7 +71,7 @@ class LocalArtifactCacheTests(unittest.TestCase): def test_put_artifacts_and_get_them_afterwards(self): cache = morphlib.localartifactcache.LocalArtifactCache( - self.tempdir.dirname) + self.tempdir.dirname) handle = cache.put(self.runtime_artifact) handle.write('runtime') @@ -101,7 +101,7 @@ class LocalArtifactCacheTests(unittest.TestCase): def test_put_check_and_get_artifact_metadata(self): cache = morphlib.localartifactcache.LocalArtifactCache( - self.tempdir.dirname) + self.tempdir.dirname) handle = cache.put_artifact_metadata(self.runtime_artifact, 'log') handle.write('log line 1\nlog line 2\n') @@ -118,7 +118,7 @@ class LocalArtifactCacheTests(unittest.TestCase): def test_put_check_and_get_source_metadata(self): cache = morphlib.localartifactcache.LocalArtifactCache( - self.tempdir.dirname) + self.tempdir.dirname) handle = cache.put_source_metadata(self.source, 'mycachekey', 'log') handle.write('source log line 1\nsource log line 2\n') diff --git a/morphlib/localrepocache.py b/morphlib/localrepocache.py index 84b75577..c5a95ebc 100644 --- a/morphlib/localrepocache.py +++ b/morphlib/localrepocache.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -29,7 +29,7 @@ import morphlib # urlparse.urljoin needs to know details of the URL scheme being used. # It does not know about git:// by default, so we teach it here. -gitscheme=['git'] +gitscheme = ['git'] urlparse.uses_relative.extend(gitscheme) urlparse.uses_netloc.extend(gitscheme) urlparse.uses_params.extend(gitscheme) @@ -37,7 +37,6 @@ urlparse.uses_query.extend(gitscheme) urlparse.uses_fragment.extend(gitscheme) - def quote_url(url): ''' Convert URIs to strings that only contain digits, letters, % and _. @@ -45,7 +44,7 @@ def quote_url(url): the same to the quote_url() function of lorry. Otherwise the git bundles generated by lorry may no longer be found by morph. - ''' + ''' valid_chars = string.digits + string.letters + '%_' transl = lambda x: x if x in valid_chars else '_' return ''.join([transl(x) for x in url]) @@ -56,77 +55,78 @@ class NoRemote(morphlib.Error): def __init__(self, reponame, errors): self.reponame = reponame self.errors = errors - + def __str__(self): return '\n\t'.join(['Cannot find remote git repository: %s' % self.reponame] + self.errors) + class NotCached(morphlib.Error): def __init__(self, reponame): self.reponame = reponame - def __str__(self): # pragma: no cover + def __str__(self): # pragma: no cover return 'Repository %s is not cached yet' % self.reponame class LocalRepoCache(object): '''Manage locally cached git repositories. - + When we build stuff, we need a local copy of the git repository. To avoid having to clone the repositories for every build, we maintain a local cache of the repositories: we first clone the remote repository to the cache, and then make a local clone from the cache to the build environment. This class manages the local cached repositories. - + Repositories may be specified either using a full URL, in a form understood by git(1), or as a repository name to which a base url is prepended. The base urls are given to the class when it is created. - + Instead of cloning via a normal 'git clone' directly from the git server, we first try to download a bundle from a url, and if that works, we clone from the bundle. - + ''' - + def __init__(self, app, cachedir, resolver, bundle_base_url=None): self._app = app self._cachedir = cachedir self._resolver = resolver if bundle_base_url and not bundle_base_url.endswith('/'): - bundle_base_url += '/' # pragma: no cover + bundle_base_url += '/' # pragma: no cover self._bundle_base_url = bundle_base_url self._cached_repo_objects = {} - def _exists(self, filename): # pragma: no cover + def _exists(self, filename): # pragma: no cover '''Does a file exist? - + This is a wrapper around os.path.exists, so that unit tests may override it. - + ''' - + return os.path.exists(filename) - - def _git(self, args, cwd=None): # pragma: no cover + + def _git(self, args, cwd=None): # pragma: no cover '''Execute git command. - + This is a method of its own so that unit tests can easily override all use of the external git command. - + ''' - + self._app.runcmd(['git'] + args, cwd=cwd) - def _fetch(self, url, filename): # pragma: no cover + def _fetch(self, url, filename): # pragma: no cover '''Fetch contents of url into a file. - + This method is meant to be overridden by unit tests. - + ''' - + source_handle = urllib2.urlopen(url) target_handle = open(filename, 'wb') @@ -138,48 +138,48 @@ class LocalRepoCache(object): source_handle.close() target_handle.close() - def _mkdir(self, dirname): # pragma: no cover + def _mkdir(self, dirname): # pragma: no cover '''Create a directory. - + This method is meant to be overridden by unit tests. - + ''' - + os.mkdir(dirname) - def _remove(self, filename): # pragma: no cover + def _remove(self, filename): # pragma: no cover '''Remove given file. - + This method is meant to be overridden by unit tests. - + ''' - + os.remove(filename) - def _rmtree(self, dirname): # pragma: no cover + def _rmtree(self, dirname): # pragma: no cover '''Remove given directory tree. This method is meant to be overridden by unit tests. ''' - + shutil.rmtree(dirname) def _escape(self, url): '''Escape a URL so it can be used as a basename in a file.''' - + # FIXME: The following is a nicer way than to do this. - # However, for compatibility, we need to use the same as the + # However, for compatibility, we need to use the same as the # bundle server (set up by Lorry) uses. # return urllib.quote(url, safe='') - + return quote_url(url) def _cache_name(self, url): basename = self._escape(url) path = os.path.join(self._cachedir, basename) return path - + def has_repo(self, reponame): '''Have we already got a cache of a given repo?''' url = self._resolver.pull_url(reponame) @@ -199,7 +199,7 @@ class LocalRepoCache(object): try: self._git(['clone', '-n', bundle_path, path]) self._git(['remote', 'set-url', 'origin', repourl], cwd=path) - except cliapp.AppException, e: # pragma: no cover + except cliapp.AppException, e: # pragma: no cover if self._exists(path): shutil.rmtree(path) return False, 'Unable to extract bundle %s: %s' % (bundle_path, e) @@ -211,9 +211,9 @@ class LocalRepoCache(object): def cache_repo(self, reponame): '''Clone the given repo into the cache. - + If the repo is already cloned, do nothing. - + ''' errors = [] if not self._exists(self._cachedir): @@ -231,7 +231,7 @@ class LocalRepoCache(object): if ok: return self.get_repo(reponame) else: - errors.append(error) + errors.append(error) repourl = self._resolver.pull_url(reponame) path = self._cache_name(repourl) @@ -239,7 +239,7 @@ class LocalRepoCache(object): self._git(['clone', '-n', repourl, path]) except cliapp.AppException, e: errors.append('Unable to clone from %s to %s: %s' % - (repourl, path, e)) + (repourl, path, e)) raise NoRemote(reponame, errors) return self.get_repo(reponame) @@ -258,4 +258,3 @@ class LocalRepoCache(object): self._cached_repo_objects[reponame] = repo return repo raise NotCached(reponame) - diff --git a/morphlib/localrepocache_tests.py b/morphlib/localrepocache_tests.py index 29feba07..dcc27728 100644 --- a/morphlib/localrepocache_tests.py +++ b/morphlib/localrepocache_tests.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -38,14 +38,14 @@ class LocalRepoCacheTests(unittest.TestCase): self.remotes = {} self.fetched = [] self.removed = [] - self.lrc = morphlib.localrepocache.LocalRepoCache(object(), - self.cachedir, repo_resolver, bundle_base_url) + self.lrc = morphlib.localrepocache.LocalRepoCache( + object(), self.cachedir, repo_resolver, bundle_base_url) self.lrc._git = self.fake_git self.lrc._exists = self.fake_exists self.lrc._fetch = self.not_found self.lrc._mkdir = self.fake_mkdir self.lrc._remove = self.fake_remove - + def fake_git(self, args, cwd=None): if args[0] == 'clone': self.assertEqual(len(args), 4) @@ -61,7 +61,7 @@ class LocalRepoCacheTests(unittest.TestCase): self.remotes[remote]['url'] = url else: raise NotImplementedError() - + def fake_exists(self, filename): return filename in self.cache @@ -110,7 +110,7 @@ class LocalRepoCacheTests(unittest.TestCase): def fail(args): raise cliapp.AppException('') self.lrc._git = fail - self.assertRaises(morphlib.localrepocache.NoRemote, + self.assertRaises(morphlib.localrepocache.NoRemote, self.lrc.cache_repo, self.repourl) def test_does_not_mind_a_missing_bundle(self): @@ -144,4 +144,3 @@ class LocalRepoCacheTests(unittest.TestCase): def test_noremote_error_message_contains_repo_name(self): e = morphlib.localrepocache.NoRemote(self.repourl, []) self.assertTrue(self.repourl in str(e)) - diff --git a/morphlib/morph2.py b/morphlib/morph2.py index b1c027b9..ad9e1d89 100644 --- a/morphlib/morph2.py +++ b/morphlib/morph2.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -20,12 +20,12 @@ import json class Morphology(object): '''An in-memory representation of a morphology. - + This is a parsed version of the morphology, with rules for default values applied. No other processing. - + ''' - + static_defaults = [ ('configure-commands', None), ('build-commands', None), @@ -41,20 +41,20 @@ class Morphology(object): ('arch', None), ('system-kind', None), ] - + def __init__(self, text): self._dict = json.loads(text) self._set_defaults() - + def __getitem__(self, key): return self._dict[key] - + def __contains__(self, key): return key in self._dict def keys(self): return self._dict.keys() - + def _set_defaults(self): if 'max-jobs' in self: self._dict['max-jobs'] = int(self['max-jobs']) @@ -63,22 +63,22 @@ class Morphology(object): size = self['disk-size'] size = size.lower() if size.endswith('g'): - size = int(size[:-1]) * 1024**3 - elif size.endswith('m'): # pragma: no cover - size = int(size[:-1]) * 1024**2 - elif size.endswith('k'): # pragma: no cover + size = int(size[:-1]) * 1024 ** 3 + elif size.endswith('m'): # pragma: no cover + size = int(size[:-1]) * 1024 ** 2 + elif size.endswith('k'): # pragma: no cover size = int(size[:-1]) * 1024 - else: # pragma: no cover + else: # pragma: no cover size = int(size) self._dict['disk-size'] = size - + for name, value in self.static_defaults: if name not in self._dict: self._dict[name] = value if self['kind'] == 'stratum': self._set_stratum_defaults() - + def _set_stratum_defaults(self): for source in self['sources']: if 'repo' not in source: @@ -87,4 +87,3 @@ class Morphology(object): source['morph'] = source['name'] if 'build-depends' not in source: source['build-depends'] = None - diff --git a/morphlib/morph2_tests.py b/morphlib/morph2_tests.py index 0d3e7168..bed6b918 100644 --- a/morphlib/morph2_tests.py +++ b/morphlib/morph2_tests.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -29,7 +29,7 @@ class MorphologyTests(unittest.TestCase): "build-system": "manual" } ''') - + self.assertEqual(m['name'], 'foo') self.assertEqual(m['kind'], 'chunk') self.assertEqual(m['build-system'], 'manual') @@ -48,7 +48,7 @@ class MorphologyTests(unittest.TestCase): "max-jobs": "42" } ''') - + self.assertEqual(m['max-jobs'], 42) def test_sets_stratum_sources_repo_and_morph_from_name(self): @@ -77,7 +77,7 @@ class MorphologyTests(unittest.TestCase): } ''') - self.assertEqual(m['disk-size'], 1024**3) + self.assertEqual(m['disk-size'], 1024 ** 3) def test_returns_dict_keys(self): m = Morphology(''' @@ -91,4 +91,3 @@ class MorphologyTests(unittest.TestCase): self.assertTrue('name' in m.keys()) self.assertTrue('kind' in m.keys()) self.assertTrue('disk-size' in m.keys()) - diff --git a/morphlib/morphologyfactory.py b/morphlib/morphologyfactory.py index 925829e9..78a49125 100644 --- a/morphlib/morphologyfactory.py +++ b/morphlib/morphologyfactory.py @@ -8,7 +8,7 @@ # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -23,16 +23,16 @@ class MorphologyFactoryError(cliapp.AppException): class AutodetectError(MorphologyFactoryError): def __init__(self, repo_name, ref): - MorphologyFactoryError.__init__(self, - "Failed to determine the build system of repo %s at " - "ref %s" % (repo_name, ref)) + MorphologyFactoryError.__init__( + self, "Failed to determine the build system of repo %s at " + "ref %s" % (repo_name, ref)) class NotcachedError(MorphologyFactoryError): def __init__(self, repo_name): - MorphologyFactoryError.__init__(self, - "Repository %s is not cached locally and there is no " - "remote cache specified" % repo_name) + MorphologyFactoryError.__init__( + self, "Repository %s is not cached locally and there is no " + "remote cache specified" % repo_name) class MorphologyFactory(object): @@ -50,7 +50,7 @@ class MorphologyFactory(object): text = self._autodetect_text(reponame, sha1, filename) morphology = morphlib.morph2.Morphology(text) - + method_name = '_check_and_tweak_%s' % morphology['kind'] if hasattr(self, method_name): method = getattr(self, method_name) @@ -71,6 +71,7 @@ class MorphologyFactory(object): # TODO get lists of files from the cache to reduce round trips if self._lrc.has_repo(reponame): repo = self._lrc.get_repo(reponame) + def has_file(filename): try: repo.cat(sha1, filename) @@ -98,14 +99,14 @@ class MorphologyFactory(object): morph_name = filename[:-len('.morph')] morph_text = bs.get_morphology_text(morph_name) return morph_text - + def _check_and_tweak_system(self, morphology, reponame, sha1, filename): '''Check and tweak a system morphology.''' - if morphology['arch'] is None: #pragma: no cover + if morphology['arch'] is None: # pragma: no cover raise morphlib.Error('No arch specified in system %s ' '(arch is a mandatory field)' % - filename) + filename) if not morphology['system-kind']: raise morphlib.Error('No system-kind defined in system %s ' @@ -121,23 +122,23 @@ class MorphologyFactory(object): morphology.needs_staging_area = False morphology.needs_artifact_metadata_cached = False - + def _check_and_tweak_stratum(self, morphology, reponame, sha1, filename): '''Check and tweak a stratum morphology.''' - for source in morphology['sources']: # pragma: no cover + for source in morphology['sources']: # pragma: no cover if source.get('build-depends', None) is None: name = source.get('name', source.get('repo', 'unknown')) raise morphlib.Error('No build dependencies ' 'stratum %s for chunk %s ' '(build-depends is a mandatory ' 'field)' % - (filename, name)) - + (filename, name)) + morphology.builds_artifacts = [morphology['name']] morphology.needs_staging_area = False morphology.needs_artifact_metadata_cached = True - + def _check_and_tweak_chunk(self, morphology, reponame, sha1, filename): '''Check and tweak a chunk morphology.''' @@ -148,4 +149,3 @@ class MorphologyFactory(object): morphology.needs_staging_area = True morphology.needs_artifact_metadata_cached = False - diff --git a/morphlib/morphologyfactory_tests.py b/morphlib/morphologyfactory_tests.py index cc45bb21..b00a496a 100644 --- a/morphlib/morphologyfactory_tests.py +++ b/morphlib/morphologyfactory_tests.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -28,11 +28,11 @@ class FakeRemoteRepoCache(object): def cat_file(self, reponame, sha1, filename): if filename.endswith('.morph'): - return '''{ - "name": "remote-foo", - "kind": "chunk", - "build-system": "bar" - }''' + return '''{ + "name": "remote-foo", + "kind": "chunk", + "build-system": "bar" + }''' return 'text' @@ -138,7 +138,7 @@ class MorphologyFactoryTests(unittest.TestCase): def test_autodetects_local_morphology(self): self.lr.cat = self.nolocalmorph - morph = self.mf.get_morphology('reponame', 'sha1', + morph = self.mf.get_morphology('reponame', 'sha1', 'assumed-local.morph') self.assertEqual('assumed-local', morph['name']) @@ -162,7 +162,7 @@ class MorphologyFactoryTests(unittest.TestCase): 'reponame', 'sha1', 'unreached.morph') def test_looks_locally_with_no_remote(self): - morph = self.lmf.get_morphology('reponame', 'sha1', + morph = self.lmf.get_morphology('reponame', 'sha1', 'foo.morph') self.assertEqual('local-foo', morph['name']) @@ -183,7 +183,7 @@ class MorphologyFactoryTests(unittest.TestCase): def test_sets_builds_artifacts_for_split_chunk(self): morph = self.mf.get_morphology('reponame', 'sha1', 'chunk-split.morph') - self.assertEqual(morph.builds_artifacts, + self.assertEqual(morph.builds_artifacts, ['local-foo-runtime', 'local-foo-devel']) def test_sets_builds_artifacts_for_stratum(self): @@ -229,4 +229,3 @@ class MorphologyFactoryTests(unittest.TestCase): self.lr.system_kind = '' self.assertRaises(morphlib.Error, self.mf.get_morphology, 'reponame', 'sha1', 'system.morph') - diff --git a/morphlib/plugins/graphing_plugin.py b/morphlib/plugins/graphing_plugin.py index 3ca39e33..ed949da1 100644 --- a/morphlib/plugins/graphing_plugin.py +++ b/morphlib/plugins/graphing_plugin.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -23,13 +23,13 @@ import morphlib class GraphingPlugin(cliapp.Plugin): def enable(self): - self.app.add_subcommand('graph-build-depends', - self.graph_build_depends, + self.app.add_subcommand('graph-build-depends', + self.graph_build_depends, arg_synopsis='REPO REF MORPHOLOGY') def disable(self): pass - + def graph_build_depends(self, args): for repo_name, ref, filename in self.app._itertriplets(args): self.app.status(msg='Creating build order for ' @@ -54,15 +54,15 @@ class GraphingPlugin(cliapp.Plugin): f.write('digraph "%s" {\n' % basename) for i, group in enumerate(order.groups): for artifact in group: - f.write(' "%s" [shape=%s];\n' % - (artifact.name, + f.write( + ' "%s" [shape=%s];\n' % + (artifact.name, shape_name[artifact.source.morphology['kind']])) for dep in artifact.dependencies: if artifact.source.morphology['kind'] == 'stratum': if dep.dependents == [artifact]: - f.write(dep_fmt % + f.write(dep_fmt % (artifact.name, dep.name)) else: f.write(dep_fmt % (artifact.name, dep.name)) f.write('}\n') - diff --git a/morphlib/plugins/hello_plugin.py b/morphlib/plugins/hello_plugin.py index 8d166d16..85fbdf28 100644 --- a/morphlib/plugins/hello_plugin.py +++ b/morphlib/plugins/hello_plugin.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -24,7 +24,6 @@ class Hello(cliapp.Plugin): def disable(self): pass - + def hello(self, args): self.app.output.write('hello, world\n') - diff --git a/morphlib/plugins/syslinux-disk-systembuilder_plugin.py b/morphlib/plugins/syslinux-disk-systembuilder_plugin.py index 4c6f25ad..f6ebf059 100644 --- a/morphlib/plugins/syslinux-disk-systembuilder_plugin.py +++ b/morphlib/plugins/syslinux-disk-systembuilder_plugin.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -33,16 +33,16 @@ from morphlib.builder2 import (SystemKindBuilder, download_depends, write_overlap_metadata) -class SyslinuxDiskBuilder(SystemKindBuilder): # pragma: no cover +class SyslinuxDiskBuilder(SystemKindBuilder): # pragma: no cover system_kind = 'syslinux-disk' def build_and_cache(self): with self.build_watch('overall-build'): arch = self.artifact.source.morphology['arch'] - + rootfs_artifact = self.new_artifact( - self.artifact.source.morphology['name'] + '-rootfs') + self.artifact.source.morphology['name'] + '-rootfs') handle = self.local_artifact_cache.put(rootfs_artifact) image_name = handle.name @@ -50,7 +50,7 @@ class SyslinuxDiskBuilder(SystemKindBuilder): # pragma: no cover self._partition_image(image_name) self._install_mbr(arch, image_name) partition = self._setup_device_mapping(image_name) - + mount_point = None try: self._create_fs(partition) @@ -62,7 +62,7 @@ class SyslinuxDiskBuilder(SystemKindBuilder): # pragma: no cover self.create_fstab(factory_path) self._create_extlinux_config(factory_path) self._create_subvolume_snapshot( - mount_point, 'factory', 'factory-run') + mount_point, 'factory', 'factory-run') factory_run_path = os.path.join(mount_point, 'factory-run') self._install_boot_files(arch, factory_run_path, mount_point) self._install_extlinux(mount_point) @@ -76,7 +76,7 @@ class SyslinuxDiskBuilder(SystemKindBuilder): # pragma: no cover self._undo_device_mapping(image_name) handle.abort() raise - + self._undo_device_mapping(image_name) handle.close() @@ -126,7 +126,7 @@ class SyslinuxDiskBuilder(SystemKindBuilder): # pragma: no cover morphlib.fsutils.mount(self.app.runcmd, partition, mount_point) def _create_subvolume(self, path): - self.app.status(msg='Creating subvolume %(path)s', + self.app.status(msg='Creating subvolume %(path)s', path=path, chatty=True) with self.build_watch('create-factory-subvolume'): self.app.runcmd(['btrfs', 'subvolume', 'create', path]) @@ -143,14 +143,14 @@ class SyslinuxDiskBuilder(SystemKindBuilder): # pragma: no cover f.write('kernel /boot/vmlinuz\n') f.write('append root=/dev/sda1 rootflags=subvol=factory-run ' 'init=/sbin/init rw\n') - + def _create_subvolume_snapshot(self, path, source, target): self.app.status(msg='Creating subvolume snapshot ' '%(source)s to %(target)s', source=source, target=target, chatty=True) with self.build_watch('create-runtime-snapshot'): self.app.runcmd(['btrfs', 'subvolume', 'snapshot', source, target], - cwd=path) + cwd=path) def _install_boot_files(self, arch, sourcefs, targetfs): with self.build_watch('install-boot-files'): @@ -196,7 +196,6 @@ class SyslinuxDiskBuilderPlugin(cliapp.Plugin): # supported by syslinux. if morphlib.util.arch() in ['x86_64', 'i386', 'i486', 'i586', 'i686']: self.app.system_kind_builder_factory.register(SyslinuxDiskBuilder) - + def disable(self): pass - diff --git a/morphlib/plugins/tarball-systembuilder_plugin.py b/morphlib/plugins/tarball-systembuilder_plugin.py index 8afe6bce..fd70fcd9 100644 --- a/morphlib/plugins/tarball-systembuilder_plugin.py +++ b/morphlib/plugins/tarball-systembuilder_plugin.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -33,14 +33,14 @@ from morphlib.builder2 import (SystemKindBuilder, download_depends, write_overlap_metadata) -class RootfsTarballBuilder(SystemKindBuilder): # pragma: no cover +class RootfsTarballBuilder(SystemKindBuilder): # pragma: no cover system_kind = 'rootfs-tarball' def build_and_cache(self): with self.build_watch('overall-build'): arch = self.artifact.source.morphology['arch'] - + rootfs_name = self.artifact.source.morphology['name'] + '-rootfs' rootfs_artifact = self.new_artifact(rootfs_name) handle = self.local_artifact_cache.put(rootfs_artifact) @@ -56,18 +56,17 @@ class RootfsTarballBuilder(SystemKindBuilder): # pragma: no cover error=True) handle.abort() raise - + handle.close() self.save_build_times() return [self.artifact] - + class RootfsTarballBuilderPlugin(cliapp.Plugin): def enable(self): self.app.system_kind_builder_factory.register(RootfsTarballBuilder) - + def disable(self): pass - diff --git a/morphlib/remoteartifactcache.py b/morphlib/remoteartifactcache.py index aaab4a3c..c0c0f723 100644 --- a/morphlib/remoteartifactcache.py +++ b/morphlib/remoteartifactcache.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -20,7 +20,7 @@ import urllib2 import urlparse -class HeadRequest(urllib2.Request): # pragma: no cover +class HeadRequest(urllib2.Request): # pragma: no cover def get_method(self): return 'HEAD' @@ -30,26 +30,26 @@ class GetError(cliapp.AppException): def __init__(self, cache, artifact): cliapp.AppException.__init__( - self, 'Failed to get the artifact %s with cache key %s ' - 'from the artifact cache %s' % - (artifact, artifact.cache_key, cache)) + self, 'Failed to get the artifact %s with cache key %s ' + 'from the artifact cache %s' % + (artifact, artifact.cache_key, cache)) class GetArtifactMetadataError(cliapp.AppException): def __init__(self, cache, artifact, name): cliapp.AppException.__init__( - self, 'Failed to get metadata %s for the artifact %s ' - 'from the artifact cache %s' % (name, artifact, cache)) + self, 'Failed to get metadata %s for the artifact %s ' + 'from the artifact cache %s' % (name, artifact, cache)) class GetSourceMetadataError(cliapp.AppException): def __init__(self, cache, source, cache_key, name): cliapp.AppException.__init__( - self, 'Failed to get metadata %s for source %s ' - 'and cache key %s from the artifact cache %s' % - (name, source, cache_key, cache)) + self, 'Failed to get metadata %s for source %s ' + 'and cache key %s from the artifact cache %s' % + (name, source, cache_key, cache)) class RemoteArtifactCache(object): @@ -88,7 +88,7 @@ class RemoteArtifactCache(object): except urllib2.URLError: raise GetSourceMetadataError(self, source, cachekey, name) - def _has_file(self, filename): # pragma: no cover + def _has_file(self, filename): # pragma: no cover url = self._request_url(filename) logging.debug('RemoteArtifactCache._has_file: url=%s' % url) request = HeadRequest(url) @@ -98,17 +98,17 @@ class RemoteArtifactCache(object): except urllib2.HTTPError: return False - def _get_file(self, filename): # pragma: no cover + def _get_file(self, filename): # pragma: no cover url = self._request_url(filename) logging.debug('RemoteArtifactCache._get_file: url=%s' % url) return urllib2.urlopen(url) - def _request_url(self, filename): # pragma: no cover + def _request_url(self, filename): # pragma: no cover server_url = self.server_url if not server_url.endswith('/'): server_url += '/' return urlparse.urljoin( - server_url, '/1.0/artifacts?filename=%s' % filename) - - def __str__(self): # pragma: no cover + server_url, '/1.0/artifacts?filename=%s' % filename) + + def __str__(self): # pragma: no cover return self.server_url diff --git a/morphlib/remoteartifactcache_tests.py b/morphlib/remoteartifactcache_tests.py index 201cfc50..d77f8f53 100644 --- a/morphlib/remoteartifactcache_tests.py +++ b/morphlib/remoteartifactcache_tests.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -25,36 +25,36 @@ class RemoteArtifactCacheTests(unittest.TestCase): def setUp(self): morph = morphlib.morph2.Morphology( - ''' - { - "chunk": "chunk", - "kind": "chunk", - "artifacts": { - "chunk-runtime": [ - "usr/bin", - "usr/sbin", - "usr/lib", - "usr/libexec" - ], - "chunk-devel": [ - "usr/include" - ], - "chunk-doc": [ - "usr/share/doc" - ] - } + ''' + { + "chunk": "chunk", + "kind": "chunk", + "artifacts": { + "chunk-runtime": [ + "usr/bin", + "usr/sbin", + "usr/lib", + "usr/libexec" + ], + "chunk-devel": [ + "usr/include" + ], + "chunk-doc": [ + "usr/share/doc" + ] } - ''') + } + ''') self.source = morphlib.source.Source( - 'repo', 'ref', 'sha1', morph, 'chunk.morph') + 'repo', 'ref', 'sha1', morph, 'chunk.morph') self.runtime_artifact = morphlib.artifact.Artifact( - self.source, 'chunk-runtime') + self.source, 'chunk-runtime') self.runtime_artifact.cache_key = 'CHUNK-RUNTIME' self.devel_artifact = morphlib.artifact.Artifact( - self.source, 'chunk-devel') + self.source, 'chunk-devel') self.devel_artifact.cache_key = 'CHUNK-DEVEL' self.doc_artifact = morphlib.artifact.Artifact( - self.source, 'chunk-doc') + self.source, 'chunk-doc') self.doc_artifact.cache_key = 'CHUNK-DOC' self.existing_files = set([ @@ -66,7 +66,7 @@ class RemoteArtifactCacheTests(unittest.TestCase): self.server_url = 'http://foo.bar:8080' self.cache = morphlib.remoteartifactcache.RemoteArtifactCache( - self.server_url) + self.server_url) self.cache._has_file = self._has_file self.cache._get_file = self._get_file @@ -127,31 +127,31 @@ class RemoteArtifactCacheTests(unittest.TestCase): def test_get_existing_artifact_metadata(self): handle = self.cache.get_artifact_metadata( - self.runtime_artifact, 'meta') + self.runtime_artifact, 'meta') data = handle.read() self.assertEqual( - data, '%s.%s' % (self.runtime_artifact.basename(), 'meta')) + data, '%s.%s' % (self.runtime_artifact.basename(), 'meta')) def test_fails_to_get_non_existent_artifact_metadata(self): self.assertRaises( - morphlib.remoteartifactcache.GetArtifactMetadataError, - self.cache.get_artifact_metadata, - self.runtime_artifact, - 'non-existent-meta') + morphlib.remoteartifactcache.GetArtifactMetadataError, + self.cache.get_artifact_metadata, + self.runtime_artifact, + 'non-existent-meta') def test_get_existing_source_metadata(self): handle = self.cache.get_source_metadata( - self.runtime_artifact.source, - self.runtime_artifact.cache_key, - 'meta') + self.runtime_artifact.source, + self.runtime_artifact.cache_key, + 'meta') data = handle.read() self.assertEqual( - data, '%s.%s' % (self.runtime_artifact.cache_key, 'meta')) + data, '%s.%s' % (self.runtime_artifact.cache_key, 'meta')) def test_fails_to_get_non_existent_source_metadata(self): self.assertRaises( - morphlib.remoteartifactcache.GetSourceMetadataError, - self.cache.get_source_metadata, - self.runtime_artifact.source, - self.runtime_artifact.cache_key, - 'non-existent-meta') + morphlib.remoteartifactcache.GetSourceMetadataError, + self.cache.get_source_metadata, + self.runtime_artifact.source, + self.runtime_artifact.cache_key, + 'non-existent-meta') diff --git a/morphlib/remoterepocache.py b/morphlib/remoterepocache.py index 39e671eb..d8526562 100644 --- a/morphlib/remoterepocache.py +++ b/morphlib/remoterepocache.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -24,16 +24,16 @@ class ResolveRefError(cliapp.AppException): def __init__(self, repo_name, ref): cliapp.AppException.__init__( - self, 'Failed to resolve ref %s for repo %s' % - (ref, repo_name)) + self, 'Failed to resolve ref %s for repo %s' % + (ref, repo_name)) class CatFileError(cliapp.AppException): def __init__(self, repo_name, ref, filename): cliapp.AppException.__init__( - self, 'Failed to cat file %s in ref %s of repo %s' % - (filename, ref, repo_name)) + self, 'Failed to cat file %s in ref %s of repo %s' % + (filename, ref, repo_name)) class RemoteRepoCache(object): @@ -56,17 +56,17 @@ class RemoteRepoCache(object): except: raise CatFileError(repo_name, ref, filename) - def _resolve_ref_for_repo_url(self, repo_url, ref): # pragma: no cover + def _resolve_ref_for_repo_url(self, repo_url, ref): # pragma: no cover data = self._make_request('sha1s?repo=%s&ref=%s' % (repo_url, ref)) info = json.loads(data) return info['sha1'] def _cat_file_for_repo_url(self, repo_url, ref, - filename): # pragma: no cover + filename): # pragma: no cover return self._make_request( - 'files?repo=%s&ref=%s&filename=%s' % (repo_url, ref, filename)) + 'files?repo=%s&ref=%s&filename=%s' % (repo_url, ref, filename)) - def _make_request(self, path): # pragma: no cover + def _make_request(self, path): # pragma: no cover server_url = self.server_url if not server_url.endswith('/'): server_url += '/' diff --git a/morphlib/remoterepocache_tests.py b/morphlib/remoterepocache_tests.py index 6826daa3..6f162f7d 100644 --- a/morphlib/remoterepocache_tests.py +++ b/morphlib/remoterepocache_tests.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -26,7 +26,7 @@ class RemoteRepoCacheTests(unittest.TestCase): def _cat_file_for_repo_url(self, repo_url, sha1, filename): return self.files[repo_url][sha1][filename] - + def setUp(self): self.sha1s = { 'git://gitorious.org/baserock/morph': { @@ -47,7 +47,7 @@ class RemoteRepoCacheTests(unittest.TestCase): ] resolver = morphlib.repoaliasresolver.RepoAliasResolver(aliases) self.cache = morphlib.remoterepocache.RemoteRepoCache( - self.server_url, resolver) + self.server_url, resolver) self.cache._resolve_ref_for_repo_url = self._resolve_ref_for_repo_url self.cache._cat_file_for_repo_url = self._cat_file_for_repo_url @@ -57,8 +57,8 @@ class RemoteRepoCacheTests(unittest.TestCase): def test_resolve_existing_ref_for_existing_repo(self): sha1 = self.cache.resolve_ref('baserock:morph', 'master') self.assertEqual( - sha1, - self.sha1s['git://gitorious.org/baserock/morph']['master']) + sha1, + self.sha1s['git://gitorious.org/baserock/morph']['master']) def test_fail_resolving_existing_ref_for_non_existent_repo(self): self.assertRaises(morphlib.remoterepocache.ResolveRefError, @@ -77,10 +77,10 @@ class RemoteRepoCacheTests(unittest.TestCase): def test_cat_existing_file_in_existing_repo_and_ref(self): content = self.cache.cat_file( - 'upstream:linux', 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9', - 'linux.morph') + 'upstream:linux', 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9', + 'linux.morph') self.assertEqual(content, 'linux morphology') - + def test_fail_cat_file_using_invalid_sha1(self): self.assertRaises(morphlib.remoterepocache.CatFileError, self.cache.cat_file, 'upstream:linux', 'blablabla', @@ -103,4 +103,3 @@ class RemoteRepoCacheTests(unittest.TestCase): self.cache.cat_file, 'non-existent-repo', 'e28a23812eadf2fce6583b8819b9c5dbd36b9fb9', 'some-file') - diff --git a/morphlib/repoaliasresolver.py b/morphlib/repoaliasresolver.py index 3daf34dd..731c1a76 100644 --- a/morphlib/repoaliasresolver.py +++ b/morphlib/repoaliasresolver.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -57,15 +57,15 @@ class RepoAliasResolver(object): return self._apply_url_pattern(pullpat, suffix) # Unknown prefix. Which means it may be a real URL instead. - # Let the caller deal with it. + # Let the caller deal with it. logging.debug('expanding: unknown prefix') return reponame - + def _split_reponame(self, reponame): '''Split reponame into prefix and suffix. - + The prefix is returned as None if there was no prefix. - + ''' pat = r'^(?P<prefix>[a-z0-9]+):(?P<rest>.*)$' @@ -74,10 +74,9 @@ class RepoAliasResolver(object): return m.group('prefix'), m.group('rest') else: return None, reponame - + def _apply_url_pattern(self, pattern, shortname): if '%s' in pattern: return shortname.join(pattern.split('%s')) else: return pattern + shortname - diff --git a/morphlib/repoaliasresolver_tests.py b/morphlib/repoaliasresolver_tests.py index 9a287282..21421878 100644 --- a/morphlib/repoaliasresolver_tests.py +++ b/morphlib/repoaliasresolver_tests.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -22,18 +22,18 @@ class RepoAliasResolverTests(unittest.TestCase): def setUp(self): self.aliases = [ - 'upstream=' + ('upstream=' 'git://gitorious.org/baserock-morphs/%s#' - 'git@gitorious.org:baserock-morphs/%s.git', - 'baserock=' + 'git@gitorious.org:baserock-morphs/%s.git'), + ('baserock=' 'git://gitorious.org/baserock/%s#' - 'git@gitorious.org:baserock/%s.git', - 'append=' + 'git@gitorious.org:baserock/%s.git'), + ('append=' 'git://append/#' - 'git@append/', + 'git@append/'), ] self.resolver = morphlib.repoaliasresolver.RepoAliasResolver( - self.aliases) + self.aliases) def test_resolve_urls_without_alias_prefix(self): self.assertEqual(self.resolver.pull_url('bar'), 'bar') @@ -41,7 +41,7 @@ class RepoAliasResolverTests(unittest.TestCase): self.assertEqual(self.resolver.pull_url('foo'), 'foo') self.assertEqual(self.resolver.push_url('foo'), 'foo') - + def test_resolve_urls_for_repos_of_one_alias(self): url = self.resolver.pull_url('upstream:foo') self.assertEqual(url, 'git://gitorious.org/baserock-morphs/foo') diff --git a/morphlib/savefile.py b/morphlib/savefile.py index 9392f1a2..2d87a54f 100644 --- a/morphlib/savefile.py +++ b/morphlib/savefile.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -22,25 +22,25 @@ import tempfile class SaveFile(file): '''Save files with a temporary name and rename when they're ready. - + This class acts exactly like the normal ``file`` class, except that it is meant only for saving data to files. The data is written to a temporary file, which gets renamed to the target name when the open file is closed. This avoids readers of the file from getting an incomplete file. - + Example: - + f = SaveFile('foo', 'w') f.write(stuff) f.close() - + The file will be called something like ``tmpCAFEBEEF`` until ``close`` is called, at which point it gets renamed to ``foo``. - + If the writer decides the file is not worth saving, they can call the ``abort`` method, which deletes the temporary file. - + ''' def __init__(self, filename, *args, **kwargs): @@ -52,10 +52,10 @@ class SaveFile(file): def abort(self): '''Abort file saving. - + The temporary file will be removed, and the universe is almost exactly as if the file save had never started. - + ''' os.remove(self._savefile_tempname) @@ -64,7 +64,6 @@ class SaveFile(file): def close(self): ret = file.close(self) logging.debug('Rename temporary file %s to %s' % - (self._savefile_tempname, self.real_filename)) + (self._savefile_tempname, self.real_filename)) os.rename(self._savefile_tempname, self.real_filename) return ret - diff --git a/morphlib/savefile_tests.py b/morphlib/savefile_tests.py index 63c80c9d..7ae2360d 100644 --- a/morphlib/savefile_tests.py +++ b/morphlib/savefile_tests.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -36,10 +36,10 @@ class SaveFileTests(unittest.TestCase): self.tempdir = tempfile.mkdtemp() self.basename = 'filename' self.filename = os.path.join(self.tempdir, self.basename) - + def tearDown(self): shutil.rmtree(self.tempdir) - + def test_there_are_no_files_initially(self): self.assertEqual(os.listdir(self.tempdir), []) @@ -95,4 +95,3 @@ class SaveFileTests(unittest.TestCase): pass self.assertEqual(os.listdir(self.tempdir), [self.basename]) self.assertEqual(self.cat(self.filename), 'foo') - diff --git a/morphlib/source.py b/morphlib/source.py index 50b6e279..fceed9ef 100644 --- a/morphlib/source.py +++ b/morphlib/source.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -20,9 +20,9 @@ import morphlib class Source(object): '''Represent the source to be built. - + Has the following properties: - + * ``repo`` -- the git repository which contains the source * ``repo_name`` -- name of the git repository which contains the source * ``original_ref`` -- the git ref provided by the user or a morphology @@ -31,12 +31,12 @@ class Source(object): * ``filename`` -- basename of the morphology filename * ``dependencies`` -- list of Sources for build dependencies for us * ``dependents`` -- list of Source for whom we are a build dependency - + The ``dependencies`` and ``dependents`` lists MUST be modified by the ``add_dependencies`` and ``add_dependent`` methods only. - + ''' - + def __init__(self, repo_name, original_ref, sha1, morphology, filename): self.repo = None self.repo_name = repo_name @@ -45,7 +45,7 @@ class Source(object): self.morphology = morphology self.filename = filename - def __str__(self): # pragma: no cover + def __str__(self): # pragma: no cover return '%s|%s|%s' % (self.repo_name, self.original_ref, self.filename) diff --git a/morphlib/source_tests.py b/morphlib/source_tests.py index 39635b6b..7ad5d370 100644 --- a/morphlib/source_tests.py +++ b/morphlib/source_tests.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -35,27 +35,26 @@ class SourceTests(unittest.TestCase): self.morphology = morphlib.morph2.Morphology(self.morphology_text) self.filename = 'foo.morph' self.source = morphlib.source.Source( - self.repo_name,self.original_ref, self.sha1, self.morphology, - self.filename) + self.repo_name, self.original_ref, self.sha1, self.morphology, + self.filename) self.other = morphlib.source.Source( - self.repo_name, self.original_ref, self.sha1, self.morphology, - self.filename) - + self.repo_name, self.original_ref, self.sha1, self.morphology, + self.filename) + def test_sets_repo_name(self): self.assertEqual(self.source.repo_name, self.repo_name) - + def test_sets_repo_to_none_initially(self): self.assertEqual(self.source.repo, None) - + def test_sets_original_ref(self): self.assertEqual(self.source.original_ref, self.original_ref) - + def test_sets_sha1(self): self.assertEqual(self.source.sha1, self.sha1) - + def test_sets_morphology(self): self.assertEqual(self.source.morphology, self.morphology) - + def test_sets_filename(self): self.assertEqual(self.source.filename, self.filename) - diff --git a/morphlib/sourcepool.py b/morphlib/sourcepool.py index b91f0f25..ef21ba5a 100644 --- a/morphlib/sourcepool.py +++ b/morphlib/sourcepool.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -35,11 +35,11 @@ class SourcePool(object): def lookup(self, repo_name, original_ref, filename): '''Find a source in the pool. - + Raise KeyError if it is not found. - + ''' - + key = self._key(repo_name, original_ref, filename) return self._sources[key] @@ -50,4 +50,3 @@ class SourcePool(object): def __len__(self): return len(self._sources) - diff --git a/morphlib/sourcepool_tests.py b/morphlib/sourcepool_tests.py index 9243bb1d..95264140 100644 --- a/morphlib/sourcepool_tests.py +++ b/morphlib/sourcepool_tests.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -67,4 +67,3 @@ class SourcePoolTests(unittest.TestCase): self.pool.add(source) sources.append(source) self.assertEqual(list(self.pool), sources) - diff --git a/morphlib/stagingarea.py b/morphlib/stagingarea.py index f3be5209..a87b45c3 100644 --- a/morphlib/stagingarea.py +++ b/morphlib/stagingarea.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -24,48 +24,48 @@ import morphlib class StagingArea(object): '''Represent the staging area for building software. - + The build dependencies of what will be built will be installed in the staging area. The staging area may be a dedicated part of the filesystem, used with chroot, or it can be the actual root of the filesystem, which is needed when bootstrap building Baserock. The caller chooses this by providing the root directory of the staging area when the object is created. The directory must already exist. - + The staging area can also install build artifacts. - + ''' - + def __init__(self, app, dirname, tempdir): self._app = app self.dirname = dirname self.tempdir = tempdir # Wrapper to be overridden by unit tests. - def _mkdir(self, dirname): # pragma: no cover + def _mkdir(self, dirname): # pragma: no cover os.mkdir(dirname) def _dir_for_source(self, source, suffix): - dirname = os.path.join(self.tempdir, + dirname = os.path.join(self.tempdir, '%s.%s' % (source.morphology['name'], suffix)) self._mkdir(dirname) return dirname def builddir(self, source): '''Create a build directory for a given source project. - + Return path to directory. - + ''' return self._dir_for_source(source, 'build') - + def destdir(self, source): '''Create an installation target directory for a given source project. - + This is meant to be used as $DESTDIR when installing chunks. Return path to directory. - + ''' return self._dir_for_source(source, 'inst') @@ -78,32 +78,32 @@ class StagingArea(object): dirname += '/' assert filename.startswith(dirname) - return filename[len(dirname)-1:] # include leading slash + return filename[len(dirname) - 1:] # include leading slash def install_artifact(self, handle): '''Install a build artifact into the staging area. - + We access the artifact via an open file handle. For now, we assume the artifact is a tarball. - + ''' - logging.debug('Installing artifact %s' % - getattr(handle, 'name', 'unknown name')) + logging.debug('Installing artifact %s' % + getattr(handle, 'name', 'unknown name')) morphlib.bins.unpack_binary_from_file(handle, self.dirname) def remove(self): '''Remove the entire staging area. - + Do not expect anything with the staging area to work after this method is called. Be careful about calling this method if the filesystem root directory was given as the dirname. - + ''' - + shutil.rmtree(self.dirname) - def runcmd(self, argv, **kwargs): # pragma: no cover + def runcmd(self, argv, **kwargs): # pragma: no cover '''Run a command in a chroot in the staging area.''' cwd = kwargs.get('cwd') or '/' if 'cwd' in kwargs: @@ -114,4 +114,3 @@ class StagingArea(object): real_argv = ['chroot', self.dirname, 'sh', '-c', 'cd "$1" && shift && exec "$@"', '--', cwd] + argv return self._app.runcmd(real_argv, **kwargs) - diff --git a/morphlib/stagingarea_tests.py b/morphlib/stagingarea_tests.py index 4fbea487..3230b9e3 100644 --- a/morphlib/stagingarea_tests.py +++ b/morphlib/stagingarea_tests.py @@ -1,14 +1,14 @@ # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -42,7 +42,7 @@ class StagingAreaTests(unittest.TestCase): def tearDown(self): shutil.rmtree(self.tempdir) - + def create_chunk(self): chunkdir = os.path.join(self.tempdir, 'chunk') os.mkdir(chunkdir) @@ -52,9 +52,9 @@ class StagingAreaTests(unittest.TestCase): tf = tarfile.TarFile(name=chunk_tar, mode='w') tf.add(chunkdir, arcname='.') tf.close() - + return chunk_tar - + def list_tree(self, root): files = [] for dirname, subdirs, basenames in os.walk(root): @@ -72,25 +72,25 @@ class StagingAreaTests(unittest.TestCase): def test_accepts_root_directory(self): sa = morphlib.stagingarea.StagingArea(object(), '/', '/tmp') self.assertEqual(sa.dirname, '/') - + def test_creates_build_directory(self): source = FakeSource() self.sa._mkdir = self.fake_mkdir dirname = self.sa.builddir(source) self.assertEqual(self.created_dirs, [dirname]) self.assertTrue(dirname.startswith(self.staging)) - + def test_creates_install_directory(self): source = FakeSource() self.sa._mkdir = self.fake_mkdir dirname = self.sa.destdir(source) self.assertEqual(self.created_dirs, [dirname]) self.assertTrue(dirname.startswith(self.staging)) - + def test_makes_relative_name(self): filename = os.path.join(self.staging, 'foobar') self.assertEqual(self.sa.relative(filename), '/foobar') - + def test_installs_artifact(self): chunk_tar = self.create_chunk() with open(chunk_tar, 'rb') as f: @@ -103,4 +103,3 @@ class StagingAreaTests(unittest.TestCase): self.sa.install_artifact(f) self.sa.remove() self.assertFalse(os.path.exists(self.staging)) - diff --git a/morphlib/stopwatch.py b/morphlib/stopwatch.py index 2554a1c9..29e584bd 100644 --- a/morphlib/stopwatch.py +++ b/morphlib/stopwatch.py @@ -1,14 +1,14 @@ # Copyright (C) 2011-2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -48,25 +48,24 @@ class Stopwatch(object): return self.ticks[reference_object]['stop'] def start_stop_delta(self, reference_object): - return (self.stop_time(reference_object) - + return (self.stop_time(reference_object) - self.start_time(reference_object)) def start_stop_seconds(self, reference_object): delta = self.start_stop_delta(reference_object) return (delta.days * 24 * 3600 + delta.seconds + - operator.truediv(delta.microseconds, 10**6)) + operator.truediv(delta.microseconds, 10 ** 6)) def __call__(self, reference_object): self.context_stack.append(reference_object) return self - + def __enter__(self): self.start(self.context_stack[-1]) return self - + def __exit__(self, *args): self.stop(self.context_stack[-1]) self.context_stack.pop() - return False # cause any exception to be re-raised - + return False # cause any exception to be re-raised diff --git a/morphlib/stopwatch_tests.py b/morphlib/stopwatch_tests.py index 76490de8..deb528d5 100644 --- a/morphlib/stopwatch_tests.py +++ b/morphlib/stopwatch_tests.py @@ -1,14 +1,14 @@ # Copyright (C) 2011-2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -24,15 +24,15 @@ class StopwatchTests(unittest.TestCase): def setUp(self): self.stopwatch = morphlib.stopwatch.Stopwatch() - + def test_tick(self): self.stopwatch.tick('tick', 'a') self.assertTrue(self.stopwatch.times('tick')) self.assertTrue(self.stopwatch.time('tick', 'a')) self.assertTrue('a' in self.stopwatch.times('tick')) self.assertEqual(self.stopwatch.time('tick', 'a'), - self.stopwatch.times('tick')['a']) - + self.stopwatch.times('tick')['a']) + now = datetime.datetime.now() self.assertTrue(self.stopwatch.time('tick', 'a') < now) @@ -53,12 +53,12 @@ class StopwatchTests(unittest.TestCase): self.assertEqual(our_delta, watch_delta) self.assertTrue(self.stopwatch.start_stop_seconds('start-stop') > 0) - + def test_with(self): with self.stopwatch('foo'): pass self.assertTrue(self.stopwatch.start_stop_seconds('foo') < 1.0) - + def test_with_within_with(self): with self.stopwatch('foo'): with self.stopwatch('bar'): @@ -69,4 +69,3 @@ class StopwatchTests(unittest.TestCase): self.assertTrue(self.stopwatch.stop_time('bar') is not None) self.assertTrue(self.stopwatch.start_stop_seconds('foo') < 1.0) self.assertTrue(self.stopwatch.start_stop_seconds('bar') < 1.0) - diff --git a/morphlib/tempdir.py b/morphlib/tempdir.py index 6377fdbc..bfbabded 100644 --- a/morphlib/tempdir.py +++ b/morphlib/tempdir.py @@ -1,14 +1,14 @@ # Copyright (C) 2011-2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -36,14 +36,13 @@ class Tempdir(object): def join(self, relative): '''Return full path to file in temporary directory. - + The relative path is given appended to the name of the temporary directory. If the relative path is actually absolute, it is forced to become relative. - + The returned path is normalized. - + ''' - - return os.path.normpath(os.path.join(self.dirname, './' + relative)) + return os.path.normpath(os.path.join(self.dirname, './' + relative)) diff --git a/morphlib/tempdir_tests.py b/morphlib/tempdir_tests.py index ae2b6a59..64ed214a 100644 --- a/morphlib/tempdir_tests.py +++ b/morphlib/tempdir_tests.py @@ -1,14 +1,14 @@ # Copyright (C) 2011-2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -49,10 +49,9 @@ class TempdirTests(unittest.TestCase): self.assertFalse(os.path.exists(dirname)) def test_joins_filename(self): - self.assertEqual(self.tempdir.join('foo'), + self.assertEqual(self.tempdir.join('foo'), os.path.join(self.tempdir.dirname, 'foo')) def test_joins_absolute_filename(self): - self.assertEqual(self.tempdir.join('/foo'), + self.assertEqual(self.tempdir.join('/foo'), os.path.join(self.tempdir.dirname, 'foo')) - diff --git a/morphlib/util.py b/morphlib/util.py index 42dbb3af..e4681ec9 100644 --- a/morphlib/util.py +++ b/morphlib/util.py @@ -1,14 +1,14 @@ # Copyright (C) 2011-2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -18,8 +18,8 @@ try: - from multiprocessing import cpu_count -except NotImplementedError: # pragma: no cover + from multiprocessing import cpu_count +except NotImplementedError: # pragma: no cover cpu_count = lambda: 1 import os @@ -29,30 +29,28 @@ def arch(): return os.uname()[4] - def indent(string, spaces=4): '''Return ``string`` indented by ``spaces`` spaces. - + The final line is not terminated by a newline. This makes it easy to use this function for indenting long text for logging: the logging library adds a newline, so not including it in the indented text avoids a spurious empty line in the log file. - + This also makes the result be a plain ASCII encoded string. - + ''' - if type(string) == unicode: # pragma: no cover + if type(string) == unicode: # pragma: no cover string = string.decode('utf-8') lines = string.splitlines() lines = ['%*s%s' % (spaces, '', line) for line in lines] return '\n'.join(lines) - def make_concurrency(cores=None): '''Return the number of concurrent jobs for make. - + This will be given to make as the -j argument. ''' @@ -62,4 +60,3 @@ def make_concurrency(cores=None): # gives about the optimal result for build times, since much of # builds are I/O bound, not CPU bound. return max(int(n * 1.5 + 0.5), 1) - diff --git a/morphlib/util_tests.py b/morphlib/util_tests.py index 8fdb051a..4f472d4e 100644 --- a/morphlib/util_tests.py +++ b/morphlib/util_tests.py @@ -1,14 +1,14 @@ # Copyright (C) 2011-2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -31,7 +31,7 @@ class IndentTests(unittest.TestCase): def test_returns_empty_string_for_empty_string(self): self.assertEqual(morphlib.util.indent(''), '') - + def test_indents_single_line(self): self.assertEqual(morphlib.util.indent('foo'), ' foo') @@ -39,7 +39,7 @@ class IndentTests(unittest.TestCase): self.assertEqual(morphlib.util.indent('foo', spaces=2), ' foo') def test_indents_multiple_lines(self): - self.assertEqual(morphlib.util.indent('foo\nbar\n'), + self.assertEqual(morphlib.util.indent('foo\nbar\n'), ' foo\n bar') @@ -56,4 +56,3 @@ class MakeConcurrencyTests(unittest.TestCase): def test_returns_6_for_4_cores(self): self.assertEqual(morphlib.util.make_concurrency(cores=4), 6) - diff --git a/scripts/assemble-stratum b/scripts/assemble-stratum index ef81c7f6..9addd36d 100755 --- a/scripts/assemble-stratum +++ b/scripts/assemble-stratum @@ -1,16 +1,16 @@ #!/usr/bin/env python # # Copyright (C) 2011-2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -24,6 +24,7 @@ import os import cliapp + class AssembleStratum(cliapp.Application): def add_settings(self): diff --git a/scripts/check-copyright-year b/scripts/check-copyright-year index f77d3459..99a6df94 100755 --- a/scripts/check-copyright-year +++ b/scripts/check-copyright-year @@ -3,16 +3,16 @@ # Does the copyright statement include the year of the latest git commit? # # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -33,7 +33,7 @@ class CheckCopyrightYear(cliapp.Application): def setup(self): self.all_ok = True - + def cleanup(self): if not self.all_ok: raise cliapp.AppException('Some copyright years need fixing') @@ -60,8 +60,8 @@ class CheckCopyrightYear(cliapp.Application): if ok: self.output.write('OK %s\n' % filename) else: - self.output.write('BAD %s:%s:%s\n' % - (filename, self.lineno, line.strip())) + self.output.write('BAD %s:%s:%s\n' % + (filename, self.lineno, line.strip())) elif not ok: self.output.write('%s\n' % filename) @@ -90,4 +90,3 @@ class CheckCopyrightYear(cliapp.Application): CheckCopyrightYear().run() - diff --git a/scripts/list-overlaps b/scripts/list-overlaps index 3d92d4ad..5fe45a0e 100755 --- a/scripts/list-overlaps +++ b/scripts/list-overlaps @@ -1,16 +1,16 @@ #!/usr/bin/env python # # Copyright (C) 2011-2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -22,6 +22,7 @@ import json import cliapp + class ListOverlaps(cliapp.Application): @staticmethod @@ -1,15 +1,15 @@ #!/usr/bin/python # Copyright (C) 2011, 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -55,7 +55,7 @@ class Clean(clean): def run(self): clean.run(self) - itemses = ([self.clean_files] + + itemses = ([self.clean_files] + [glob.glob(x) for x in self.clean_globs]) for items in itemses: for filename in items: @@ -83,23 +83,22 @@ class Check(Command): os.remove('.coverage') - setup(name='morph', description='FIXME', long_description='''\ FIXME ''', classifiers=[ - 'Development Status :: 2 - Pre-Alpha', - 'Environment :: Console', - 'Intended Audience :: Developers', - 'License :: OSI Approved :: GNU General Public License (GPL)', - 'Operating System :: POSIX :: Linux', - 'Programming Language :: Python', - 'Topic :: Software Development :: Build Tools', - 'Topic :: Software Development :: Embedded Systems', - 'Topic :: System :: Archiving :: Packaging', - 'Topic :: System :: Software Distribution', + 'Development Status :: 2 - Pre-Alpha', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: GNU General Public License (GPL)', + 'Operating System :: POSIX :: Linux', + 'Programming Language :: Python', + 'Topic :: Software Development :: Build Tools', + 'Topic :: Software Development :: Embedded Systems', + 'Topic :: System :: Archiving :: Packaging', + 'Topic :: System :: Software Distribution', ], author='Lars Wirzenius', author_email='lars.wirzenius@codethink.co.uk', @@ -108,9 +107,7 @@ FIXME packages=['morphlib'], data_files=[('share/man/man1', glob.glob('*.[1-8]'))], cmdclass={ - 'build': GenerateManpage, - 'check': Check, - 'clean': Clean, - }, - ) - + 'build': GenerateManpage, + 'check': Check, + 'clean': Clean, + }) diff --git a/source-stats b/source-stats index 429d9e9c..c246bab9 100755 --- a/source-stats +++ b/source-stats @@ -1,15 +1,15 @@ #!/usr/bin/python # Copyright (C) 2012 Codethink Limited -# +# # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; version 2 of the License. -# +# # This program is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. -# +# # You should have received a copy of the GNU General Public License along # with this program; if not, write to the Free Software Foundation, Inc., # 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. @@ -29,77 +29,77 @@ import time class SourceStats(cliapp.Application): '''Compute some basic statistics about Baserock components. - + * name of component * total source lines, excluding blank lines * number of commits over last 12 months * lines added over 12 months - * lines removed over 12 months - + * lines removed over 12 months + Usage: ./source-stat $HOME/baserock/gits/* ''' - + def add_settings(self): self.settings.string(['gitsdir'], 'base directory for git repos') - + def setup(self): self.writer = csv.writer(sys.stdout) self.cols = ['name', 'lines', 'commits', 'added', 'deleted'] self.writer.writerow(self.cols) - + def process_input(self, gitdir): name = os.path.basename(gitdir) stats = self.compute_stats(name, gitdir) row = [stats[x] for x in self.cols] self.writer.writerow(row) sys.stdout.flush() - + def compute_stats(self, name, gitdir): stats = { 'name': name, } - + t = time.time() - 365 * 86400 tt = time.localtime(t) start_date = time.strftime('%Y-%m-%d', tt) - + stats['branch'] = self.pick_branch(gitdir) self.get_sources(gitdir, stats['branch']) stats['lines'] = self.count_source_lines(gitdir) - + start, end = self.find_commit_range(gitdir, start_date) stats['commits'] = self.count_commits(gitdir, start, end) stats['added'], stats['deleted'] = self.diffstat(gitdir, start, end) return stats - + def pick_branch(self, gitdir): out = self.runcmd(['git', 'branch', '-r'], cwd=gitdir) lines = [x.split()[-1] for x in out.splitlines()] - + candidates = [ 'origin/master', 'origin/trunk', 'origin/blead', ] - + for x in candidates: if x in lines: return x raise Exception('Cannot decide on branch in %s' % gitdir) - + def get_sources(self, gitdir, branch): self.runcmd(['git', 'checkout', branch], cwd=gitdir) - + def count_source_lines(self, tempdir): numlines = 0 for dirname, subdirs, basenames in os.walk(tempdir): if '.git' in subdirs: subdirs.remove('.git') - + for basename in basenames: filename = os.path.join(dirname, basename) if os.path.isfile(filename) and not os.path.islink(filename): @@ -107,9 +107,9 @@ class SourceStats(cliapp.Application): for line in f: if line.strip(): numlines += 1 - + return numlines - + def find_commit_range(self, gitdir, start_date): out = self.runcmd(['git', 'log', '--format=oneline', '--since=%s' % start_date], @@ -120,17 +120,18 @@ class SourceStats(cliapp.Application): end = lines[0].split()[0] start = lines[-1].split()[0] return start, end - + def count_commits(self, gitdir, start, end): - out = self.runcmd(['git', 'log', '--format=oneline', + out = self.runcmd(['git', 'log', '--format=oneline', '%s..%s' % (start, end)], cwd=gitdir) return len(out.splitlines()) - + def diffstat(self, gitdir, start, end): - out = self.runcmd(['git', 'diff', '--numstat', start, end], + out = self.runcmd(['git', 'diff', '--numstat', start, end], cwd=gitdir) tuples = [line.split() for line in out.splitlines()] + def toint(s): try: return int(s) |