diff options
| -rwxr-xr-x | morph | 72 | ||||
| -rw-r--r-- | morphlib/builder2.py | 57 |
2 files changed, 124 insertions, 5 deletions
@@ -21,6 +21,7 @@ import collections import json import logging import os +import tempfile import morphlib from morphlib import buildworker @@ -191,8 +192,7 @@ class Morph(cliapp.Application): return pool - - def cmd_build(self, args): + def cmd_build_old(self, args): '''Build a binary from a morphology. Command line arguments are the repository, git tree-ish reference, @@ -236,6 +236,74 @@ class Morph(cliapp.Application): tempdir.remove() + def cmd_build(self, args): + '''Build a binary from a morphology. + + Command line arguments are the repository, git tree-ish reference, + and morphology filename. Morph takes care of building all dependencies + before building the morphology. All generated binaries are put into the + cache. + + (The triplet of command line arguments may be repeated as many + times as necessary.) + + ''' + + logging.debug('cmd_build starting') + + cachedir = self.settings['cachedir'] + if not os.path.exists(cachedir): + os.mkdir(cachedir) + + build_env = morphlib.buildenvironment.BuildEnvironment(self.settings) + ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) + lac = morphlib.localartifactcache.LocalArtifactCache(cachedir) + lrc = morphlib.localrepocache.LocalRepoCache( + cachedir, + self.settings['git-base-url'], + bundle_base_url=self.settings['bundle-server']) + + for repo_name, ref, filename in self._itertriplets(args): + logging.debug('cmd_build: %s %s %s' % (repo_name, ref, filename)) + srcpool = self._create_source_pool(lrc, repo_name, ref, filename) + ar = morphlib.artifactresolver.ArtifactResolver(ckc) + artifacts = ar.resolve_artifacts(srcpool) + order = morphlib.buildorder.BuildOrder(artifacts) + + needed = [] + for group in order.groups: + for artifact in group: + if not lac.has(artifact): + needed.append(artifact) + + for artifact in needed: + repo = lrc.cache_repo(repo_name) + repo.update() + + if self.settings['bootstrap']: + staging_root = '/' + else: + staging_root = tempfile.mkdtemp() + staging_area = morphlib.stagingarea.StagingArea(staging_root) + + builder = morphlib.builder2.Builder(staging_area, lac, + build_env, + self.settings['max-jobs']) + for group in order.groups: + for artifact in group: + if artifact in needed: + builder.build_and_cache(artifact) + # install chunks only + chunk_artifacts = [x + for x in group + if x.source.morphology['kind'] == 'chunk'] + for artifact in chunk_artifacts: + handle = lac.get(artifact) + staging_area.install_artifact(handle) + + if staging_root != '/': + staging_area.remove() + def cmd_show_dependencies(self, args): '''Dumps the dependency tree of all input morphologies.''' diff --git a/morphlib/builder2.py b/morphlib/builder2.py index 7eff7066..394805c6 100644 --- a/morphlib/builder2.py +++ b/morphlib/builder2.py @@ -17,6 +17,7 @@ import json import logging import os +import time import morphlib @@ -87,6 +88,7 @@ class BuilderBase(object): self.artifact.cache_key) def runcmd(self, *args, **kwargs): + kwargs['env'] = self.build_env.env return self.staging_area.runcmd(*args, **kwargs) @@ -103,14 +105,63 @@ class ChunkBuilder(BuilderBase): return morphology[which] def build_and_cache(self): # pragma: no cover + builddir = self.staging_area.builddir(self.artifact.source) + self.get_sources(builddir) destdir = self.staging_area.destdir(self.artifact.source) - self.run_commands(destdir) + self.run_commands(builddir, destdir) self.assemble_chunk_artifacts(destdir) - def run_commands(self, destdir): # pragma: no cover + def get_sources(self, srcdir): # pragma: no cover + '''Get sources from git to a source directory, for building.''' + + def extract_treeish(source, destdir): + logging.debug('Extracting %s into %s' % (source.repo, destdir)) + if not os.path.exists(destdir): + os.mkdir(destdir) + morphlib.git.copy_repository(source.repo.path, destdir, + logging.debug) + morphlib.git.checkout_ref(destdir, source.sha1, logging.debug) + morphlib.git.reset_workdir(destdir, logging.debug) + submodules = morphlib.git.Submodules(source.repo.path, source.sha1) + try: + submodules.load() + except morphlib.git.NoModulesFileError: + return [] + else: + return [(sub.url, os.path.join(destdir, sub.path)) + for sub in submodules] + + todo = [(self.artifact.source, srcdir)] + while todo: + source, srcdir = todo.pop() + todo += extract_treeish(source, srcdir) + self.set_mtime_recursively(srcdir) + + def set_mtime_recursively(self, root): # pragma: no cover + '''Set the mtime for every file in a directory tree to the same. + + We do this because git checkout does not set the mtime to anything, + and some projects (binutils, gperf for example) include formatted + documentation and try to randomly build things or not because of + the timestamps. This should help us get more reliable builds. + + ''' + + now = time.time() + for dirname, subdirs, basenames in os.walk(root, topdown=False): + for basename in basenames: + pathname = os.path.join(dirname, basename) + # we need the following check to ignore broken symlinks + if os.path.exists(pathname): + os.utime(pathname, (now, now)) + os.utime(dirname, (now, now)) + + + def run_commands(self, builddir, destdir): # pragma: no cover m = self.artifact.source.morphology bs = morphlib.buildsystem.lookup_build_system(m['build-system']) + relative_builddir = self.staging_area.relative(builddir) relative_destdir = self.staging_area.relative(destdir) self.build_env.env['DESTDIR'] = relative_destdir @@ -128,7 +179,7 @@ class ChunkBuilder(BuilderBase): self.build_env.env['MAKEFLAGS'] = '-j%s' % max_jobs else: self.build_env.env['MAKEFLAGS'] = '-j1' - self.runcmd(['sh', '-c', cmd], cwd=relative_destdir) + self.runcmd(['sh', '-c', cmd], cwd=relative_builddir) def assemble_chunk_artifacts(self, destdir): # pragma: no cover ex = None # create_chunk doesn't actually use this |