diff options
-rw-r--r-- | morphlib/buildcommand.py | 61 | ||||
-rw-r--r-- | morphlib/buildenvironment.py | 50 | ||||
-rw-r--r-- | morphlib/buildenvironment_tests.py | 45 | ||||
-rw-r--r-- | morphlib/cachekeycomputer.py | 3 | ||||
-rw-r--r-- | morphlib/cachekeycomputer_tests.py | 7 | ||||
-rw-r--r-- | morphlib/morphologyfactory.py | 10 | ||||
-rw-r--r-- | morphlib/morphologyfactory_tests.py | 15 | ||||
-rw-r--r-- | morphlib/util.py | 15 | ||||
-rw-r--r-- | morphlib/util_tests.py | 10 |
9 files changed, 120 insertions, 96 deletions
diff --git a/morphlib/buildcommand.py b/morphlib/buildcommand.py index 55bb0158..189db8b0 100644 --- a/morphlib/buildcommand.py +++ b/morphlib/buildcommand.py @@ -34,11 +34,8 @@ class BuildCommand(object): def __init__(self, app): self.supports_local_build = True - self.target = morphlib.util.target(app.runcmd) self.app = app - self.build_env = self.new_build_env() - self.ckc = self.new_cache_key_computer(self.build_env) self.lac, self.rac = self.new_artifact_caches() self.lrc, self.rrc = self.new_repo_caches() @@ -55,15 +52,6 @@ class BuildCommand(object): self.app.status(msg='Build ends successfully') - def new_build_env(self): - '''Create a new BuildEnvironment instance.''' - return morphlib.buildenvironment.BuildEnvironment(self.app.settings, - self.target) - - def new_cache_key_computer(self, build_env): - '''Create a new cache key computer.''' - return morphlib.cachekeycomputer.CacheKeyComputer(build_env) - def new_artifact_caches(self): '''Create interfaces for the build artifact caches. @@ -75,6 +63,11 @@ class BuildCommand(object): def new_repo_caches(self): return morphlib.util.new_repo_caches(self.app) + def new_build_env(self, arch): + '''Create a new BuildEnvironment instance.''' + return morphlib.buildenvironment.BuildEnvironment(self.app.settings, + arch) + def get_artifact_object(self, repo_name, ref, filename): '''Create an Artifact object representing the given triplet.''' @@ -84,11 +77,6 @@ class BuildCommand(object): srcpool = self.app.create_source_pool( self.lrc, self.rrc, (repo_name, ref, filename)) - root_kind = srcpool.lookup(repo_name, ref, filename).morphology['kind'] - if root_kind is not 'system': - raise morphlib.Error( - 'Building a %s directly is not supported' % root_kind) - self.app.status( msg='Validating cross-morphology references', chatty=True) self._validate_cross_morphology_references(srcpool) @@ -99,14 +87,26 @@ class BuildCommand(object): self.app.status(msg='Resolving artifacts', chatty=True) artifacts = ar.resolve_artifacts(srcpool) - self.app.status(msg='Computing cache keys', chatty=True) - for artifact in artifacts: - artifact.cache_key = self.ckc.compute_key(artifact) - artifact.cache_id = self.ckc.get_cache_id(artifact) - self.app.status(msg='Computing build order', chatty=True) root_artifact = self._find_root_artifact(artifacts) + root_kind = root_artifact.source.morphology['kind'] + if root_kind != 'system': + raise morphlib.Error( + 'Building a %s directly is not supported' % root_kind) + arch = root_artifact.source.morphology['arch'] + + self.app.status(msg='Creating build environment for %(arch)s', + arch=arch, chatty=True) + build_env = self.new_build_env(arch) + + self.app.status(msg='Computing cache keys', chatty=True) + ckc = morphlib.cachekeycomputer.CacheKeyComputer(build_env) + for artifact in artifacts: + artifact.cache_key = ckc.compute_key(artifact) + artifact.cache_id = ckc.get_cache_id(artifact) + + root_artifact.build_env = build_env return root_artifact def _validate_cross_morphology_references(self, srcpool): @@ -188,6 +188,7 @@ class BuildCommand(object): '''Build everything specified in a build order.''' self.app.status(msg='Building a set of artifacts', chatty=True) + build_env = root_artifact.build_env artifacts = root_artifact.walk() old_prefix = self.app.status_prefix for i, a in enumerate(artifacts): @@ -204,7 +205,7 @@ class BuildCommand(object): else: self.app.status(msg='Building %(kind)s %(name)s', kind=a.source.morphology['kind'], name=a.name) - self.build_artifact(a) + self.build_artifact(a, build_env) self.app.status(msg='%(kind)s %(name)s is cached at %(cachepath)s', kind=a.source.morphology['kind'], name=a.name, @@ -216,7 +217,7 @@ class BuildCommand(object): '''Does either cache already have the artifact?''' return self.lac.has(artifact) or (self.rac and self.rac.has(artifact)) - def build_artifact(self, artifact): + def build_artifact(self, artifact, build_env): '''Build one artifact. All the dependencies are assumed to be built and available @@ -242,12 +243,14 @@ class BuildCommand(object): build_mode = 'staging' use_chroot = build_mode=='staging' - staging_area = self.create_staging_area( - use_chroot, extra_env=extra_env, extra_path=extra_path) + staging_area = self.create_staging_area(build_env, + use_chroot, + extra_env=extra_env, + extra_path=extra_path) self.install_fillers(staging_area) self.install_dependencies(staging_area, deps, artifact) else: - staging_area = self.create_staging_area() + staging_area = self.create_staging_area(build_env, False) self.build_and_cache(staging_area, artifact, setup_mounts) self.remove_staging_area(staging_area) @@ -318,14 +321,14 @@ class BuildCommand(object): copy(self.rac.get_artifact_metadata(artifact, 'meta'), self.lac.put_artifact_metadata(artifact, 'meta')) - def create_staging_area(self, use_chroot=True, extra_env={}, + def create_staging_area(self, build_env, use_chroot=True, extra_env={}, extra_path=[]): '''Create the staging area for building a single artifact.''' self.app.status(msg='Creating staging area') staging_dir = tempfile.mkdtemp(dir=self.app.settings['tempdir']) staging_area = morphlib.stagingarea.StagingArea( - self.app, staging_dir, self.build_env, use_chroot, extra_env, + self.app, staging_dir, build_env, use_chroot, extra_env, extra_path) return staging_area diff --git a/morphlib/buildenvironment.py b/morphlib/buildenvironment.py index e6dccb04..68e7e756 100644 --- a/morphlib/buildenvironment.py +++ b/morphlib/buildenvironment.py @@ -32,13 +32,14 @@ class BuildEnvironment(): ''' - def __init__(self, settings, target, arch=None): + def __init__(self, settings, arch): '''Create a new BuildEnvironment object''' self.extra_path = [] - self.target = target - self.arch = morphlib.util.arch() if arch is None else arch + self.env = self._clean_env(settings) + self.env.update(self._env_for_arch(arch)) + _osenv = os.environ _ccache_path = '/usr/lib/ccache' @@ -48,15 +49,6 @@ class BuildEnvironment(): _override_term = 'dumb' _override_username = 'tomjon' - def get_bootstrap_target(self, target): - '''Set 'vendor' field of the given machine triplet as 'bootstrap' ''' - - parts = target.split('-') - if len(parts) < 2: - raise morphlib.Error('Failed to parse machine triplet returned by ' - 'host compiler: %s' % target) - return '-'.join([parts[0], 'bootstrap'] + parts[2:]) - def _clean_env(self, settings): '''Create a fresh set of environment variables for a clean build. @@ -91,11 +83,6 @@ class BuildEnvironment(): env['LC_ALL'] = self._override_locale env['HOME'] = self._override_home - env['BUILD'] = self.target - env['TARGET'] = self.target - env['TARGET_STAGE1'] = self.get_bootstrap_target(self.target) - env['TARGET_GCC_CONFIG'] = '' - if not settings['no-ccache']: self.extra_path.append(self._ccache_path) # FIXME: we should set CCACHE_BASEDIR so any objects that refer to their @@ -111,3 +98,32 @@ class BuildEnvironment(): env['CCACHE_PREFIX'] = 'distcc' return env + + def _env_for_arch(self, arch): + '''Set build environment variables specific to the target machine + + These are entirely controlled by the 'arch' field in the system + morphology, which is passed to the morphologies as MORPH_ARCH to + do what they like with. + + ''' + + env = {} + env['MORPH_ARCH'] = arch + + # GNU triplets are widely used, so we handle these in Morph rather + # than leaving it up to individual morphologies. + if arch == 'x86_32': + cpu = 'i686' + else: + cpu = arch + + if arch.startswith('arm'): + abi = 'eabi' + else: + abi = '' + + env['TARGET'] = cpu + '-baserock-linux-gnu' + abi + env['TARGET_STAGE1'] = cpu + '-bootstrap-linux-gnu' + abi + + return env diff --git a/morphlib/buildenvironment_tests.py b/morphlib/buildenvironment_tests.py index 1995923b..7ae7c2d5 100644 --- a/morphlib/buildenvironment_tests.py +++ b/morphlib/buildenvironment_tests.py @@ -29,7 +29,6 @@ class BuildEnvironmentTests(unittest.TestCase): 'no-ccache': True, 'no-distcc': True } - self.target = '%s-baserock-linux-gnu' % morphlib.util.arch() self.fake_env = { 'PATH': '/fake_bin', } @@ -39,21 +38,9 @@ class BuildEnvironmentTests(unittest.TestCase): target = target or self.target return buildenvironment.BuildEnvironment(settings, target, **kws) - def new_build_env(self, settings=None, target=None, **kws): + def new_build_env(self, settings=None, arch='x86_64'): settings = settings or self.settings - target = target or self.target - return buildenvironment.BuildEnvironment(settings, target, **kws) - - def test_arch_defaults_to_host(self): - buildenv = self.new_build_env() - self.assertEqual(buildenv.arch, morphlib.util.arch()) - - def test_arch_overridable(self): - buildenv = self.new_build_env(arch='noarch') - self.assertEqual(buildenv.arch, 'noarch') - - def test_target_always_valid(self): - self.assertRaises(morphlib.Error, self.new_build_env, target="invalid") + return buildenvironment.BuildEnvironment(settings, arch) def test_copies_whitelist_vars(self): env = self.fake_env @@ -90,9 +77,31 @@ class BuildEnvironmentTests(unittest.TestCase): self.assertEqual(buildenv.env['LC_ALL'], buildenv._override_locale) self.assertEqual(buildenv.env['HOME'], buildenv._override_home) - def test_environment_settings_set(self): - buildenv = self.new_build_env() - self.assertEqual(buildenv.env['TARGET'], self.target) + def test_arch_x86_64(self): + b = self.new_build_env(arch='x86_64') + self.assertEqual(b.env['MORPH_ARCH'], 'x86_64') + self.assertEqual(b.env['TARGET'], 'x86_64-baserock-linux-gnu') + self.assertEqual(b.env['TARGET_STAGE1'], 'x86_64-bootstrap-linux-gnu') + + def test_arch_x86_32(self): + b = self.new_build_env(arch='x86_32') + self.assertEqual(b.env['MORPH_ARCH'], 'x86_32') + self.assertEqual(b.env['TARGET'], 'i686-baserock-linux-gnu') + self.assertEqual(b.env['TARGET_STAGE1'], 'i686-bootstrap-linux-gnu') + + def test_arch_armv7l(self): + b = self.new_build_env(arch='armv7l') + self.assertEqual(b.env['MORPH_ARCH'], 'armv7l') + self.assertEqual(b.env['TARGET'], 'armv7l-baserock-linux-gnueabi') + self.assertEqual(b.env['TARGET_STAGE1'], + 'armv7l-bootstrap-linux-gnueabi') + + def test_arch_armv7b(self): + b = self.new_build_env(arch='armv7b') + self.assertEqual(b.env['MORPH_ARCH'], 'armv7b') + self.assertEqual(b.env['TARGET'], 'armv7b-baserock-linux-gnueabi') + self.assertEqual(b.env['TARGET_STAGE1'], + 'armv7b-bootstrap-linux-gnueabi') def test_ccache_vars_set(self): new_settings = copy.copy(self.settings) diff --git a/morphlib/cachekeycomputer.py b/morphlib/cachekeycomputer.py index 244257a0..6acf654b 100644 --- a/morphlib/cachekeycomputer.py +++ b/morphlib/cachekeycomputer.py @@ -27,7 +27,7 @@ class CacheKeyComputer(object): self._calculated = {} def _filterenv(self, env): - keys = ["LOGNAME", "TARGET", "TARGET_STAGE1", "TARGET_GCC_CONFIG", + keys = ["LOGNAME", "MORPH_ARCH", "TARGET", "TARGET_STAGE1", "USER", "USERNAME"] return dict([(k, env[k]) for k in keys]) @@ -79,7 +79,6 @@ class CacheKeyComputer(object): def _calculate(self, artifact): keys = { - 'arch': self._build_env.arch, 'env': self._filterenv(self._build_env.env), 'filename': artifact.source.filename, 'kids': [self.compute_key(x) for x in artifact.dependencies] diff --git a/morphlib/cachekeycomputer_tests.py b/morphlib/cachekeycomputer_tests.py index ec4c9d22..2f033a7a 100644 --- a/morphlib/cachekeycomputer_tests.py +++ b/morphlib/cachekeycomputer_tests.py @@ -25,8 +25,7 @@ class DummyBuildEnvironment: settings to pick the environment, it just gets passed a dict representing it ''' - def __init__(self, env, arch=None): - self.arch = morphlib.util.arch() if arch is None else arch + def __init__(self, env, arch): self.env = env @@ -105,11 +104,11 @@ class CacheKeyComputerTests(unittest.TestCase): m.builds_artifacts = [m['name']] self.build_env = DummyBuildEnvironment({ "LOGNAME": "foouser", + "MORPH_ARCH": "dummy", "TARGET": "dummy-baserock-linux-gnu", "TARGET_STAGE1": "dummy-baserock-linux-gnu", - "TARGET_GCC_CONFIG": "", "USER": "foouser", - "USERNAME": "foouser"}) + "USERNAME": "foouser"}, 'dummy') self.artifact_resolver = morphlib.artifactresolver.ArtifactResolver() self.artifacts = self.artifact_resolver.resolve_artifacts( self.source_pool) diff --git a/morphlib/morphologyfactory.py b/morphlib/morphologyfactory.py index 7ae68697..e9d2b1c5 100644 --- a/morphlib/morphologyfactory.py +++ b/morphlib/morphologyfactory.py @@ -106,6 +106,16 @@ class MorphologyFactory(object): '(arch is a mandatory field)' % filename) + valid_archs = ['armv7l', 'armv7b', 'x86_32', 'x86_64'] + + if morphology['arch'] == 'armv7': + morphology._dict['arch'] = 'armv7l' + + if morphology['arch'] not in valid_archs: + raise morphlib.Error('Unknown arch %s. This version of Morph ' + 'supports the following architectures: %s' % + (morphology['arch'], ', '.join(valid_archs))) + if not morphology['system-kind']: raise morphlib.Error('No system-kind defined in system %s ' '(it is a mandatory field)' % filename) diff --git a/morphlib/morphologyfactory_tests.py b/morphlib/morphologyfactory_tests.py index 30cfb8fb..08adcae2 100644 --- a/morphlib/morphologyfactory_tests.py +++ b/morphlib/morphologyfactory_tests.py @@ -73,7 +73,7 @@ class FakeLocalRepo(object): } def __init__(self): - self.arch = 'unknown' + self.arch = 'x86_64' self.system_kind = 'unknown' def cat(self, sha1, filename): @@ -219,7 +219,7 @@ class MorphologyFactoryTests(unittest.TestCase): self.assertEqual(morph.builds_artifacts, ['stratum']) def test_sets_build_artifacts_for_system(self): - self.lr.arch = 'x86_64' + self.lr.arch = 'x86_32' morph = self.mf.get_morphology('reponame', 'sha1', 'system.morph') self.assertEqual(morph.builds_artifacts, ['system-rootfs']) @@ -235,6 +235,17 @@ class MorphologyFactoryTests(unittest.TestCase): morph = self.mf.get_morphology('reponame', 'sha1', 'system.morph') self.assertEqual(morph.needs_artifact_metadata_cached, False) + + def test_arch_is_validated(self): + self.lr.arch = 'unknown' + self.assertRaises(morphlib.Error, self.mf.get_morphology, + 'reponame', 'sha1', 'system.morph') + + def test_arch_arm_defaults_to_le(self): + self.lr.arch = 'armv7' + morph = self.mf.get_morphology('reponame', 'sha1', 'system.morph') + self.assertEqual(morph['arch'], 'armv7l') + def test_fails_if_system_does_not_define_system_kind(self): self.lr.system_kind = '' self.assertRaises(morphlib.Error, self.mf.get_morphology, diff --git a/morphlib/util.py b/morphlib/util.py index c3a7ac9f..c77ac8d3 100644 --- a/morphlib/util.py +++ b/morphlib/util.py @@ -38,21 +38,6 @@ except NotImplementedError: # pragma: no cover import os -def arch(): - '''Return the CPU architecture of the current host.''' - return os.uname()[4] - - -def target(runcmd): # pragma: no cover - '''Returns the machine triplet of the current host''' - try: - target = runcmd(['cc', '-dumpmachine']).strip() - except cliapp.AppException as e: - raise morphlib.Error( - 'Failed to execute host compiler \'cc\': %s' % e) - return target - - def indent(string, spaces=4): '''Return ``string`` indented by ``spaces`` spaces. diff --git a/morphlib/util_tests.py b/morphlib/util_tests.py index 4f472d4e..89fe184e 100644 --- a/morphlib/util_tests.py +++ b/morphlib/util_tests.py @@ -1,4 +1,4 @@ -# Copyright (C) 2011-2012 Codethink Limited +# Copyright (C) 2011-2013 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -19,14 +19,6 @@ import unittest import morphlib -class ArchTests(unittest.TestCase): - - def test(self): - arch = morphlib.util.arch() - self.assertEqual(type(arch), str) - self.assertNotEqual(arch, '') - - class IndentTests(unittest.TestCase): def test_returns_empty_string_for_empty_string(self): |