morphlib: add buildenvironment and tests

BuildEnvironment should contain all the information about whatever
settings affect the build. A subset of these will affect the cache
key, but CacheKeyComputer filters them itself.

BuildEnvironment has consumed the purpose of Morph.clean_env() because
the build environment is a good place to store this information.

The environment variables are decided based on the current settings.
The current environment can be accessed by the env attribute.

The tests are a little invasive because it reads environment variables
and most of the work is done in the constructor, but they should test
everything useful.

3 files changed, 235 insertions, 0 deletions

diff --git a/morphlib/__init__.py b/morphlib/__init__.py
index 26727929..adb0fa1e 100644
--- a/morphlib/__init__.py
+++ b/morphlib/__init__.py
@@ -23,6 +23,7 @@ import blobs
 import buildcontroller
 import builddependencygraph
 import buildgraph
+import buildenvironment
 import buildsystem
 import buildworker
 import builder
diff --git a/morphlib/buildenvironment.py b/morphlib/buildenvironment.py
new file mode 100644
index 00000000..a9395ec4
--- /dev/null
+++ b/morphlib/buildenvironment.py
@@ -0,0 +1,97 @@
+# Copyright (C) 2012  Codethink Limited
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+import os
+
+import morphlib
+
+class BuildEnvironment():
+
+    def __init__(self, settings, arch=None):
+        self.arch = morphlib.util.arch() if arch == None else arch
+        self.env = self._clean_env(settings)
+
+    _osenv = os.environ
+    _default_path = '/sbin:/usr/sbin:/bin:/usr/bin'
+    _override_term = 'dumb'
+    _override_shell = '/bin/sh'
+    _override_username = 'tomjon'
+    _override_locale = 'C'
+    _override_home = '/tmp'
+    _ccache_path = '/usr/lib/ccache'
+
+    def _clean_env(self, settings):
+        '''Create a fresh set of environment variables for a clean build.
+        
+        Return a dict with the new environment.
+        
+        '''
+        
+        path = self._osenv['PATH']
+
+        # copy a set of white-listed variables from the original env
+        copied_vars = dict.fromkeys([
+            'DISTCC_HOSTS',
+            'TMPDIR',
+            'LD_PRELOAD',
+            'LD_LIBRARY_PATH',
+            'FAKEROOTKEY',
+            'FAKED_MODE',
+            'FAKEROOT_FD_BASE',
+        ])
+        for name in copied_vars:
+            copied_vars[name] = self._osenv.get(name, None)
+
+        env = {}
+        
+        # apply the copied variables to the clean env
+        for name in copied_vars:
+            if copied_vars[name] is not None:
+                env[name] = copied_vars[name]
+
+        env['TERM'] = self._override_term
+        env['SHELL'] = self._override_shell
+        env['USER'] = \
+            env['USERNAME'] = \
+            env['LOGNAME'] = self._override_username
+        env['LC_ALL'] = self._override_locale
+        env['HOME'] = self._override_home
+
+        if settings['keep-path'] or settings['bootstrap']:
+            env['PATH'] = path
+        else:
+            env['PATH'] = self._default_path
+
+        env['TOOLCHAIN_TARGET'] = settings['toolchain-target']
+        env['CFLAGS'] = settings['target-cflags']
+        env['PREFIX'] = settings['prefix']
+        env['BOOTSTRAP'] = 'true' if settings['bootstrap'] else 'false'
+        if not settings['no-ccache']:
+            env['PATH'] = ('%s:%s' % (self._ccache_path, env['PATH']))
+# FIXME: we should set CCACHE_BASEDIR so any objects that refer to their
+#        current directory get corrected. This improve the cache hit rate
+#            env['CCACHE_BASEDIR'] = self.tempdir.dirname
+            env['CCACHE_EXTRAFILES'] = ':'.join(
+                f for f in ('/baserock/binutils.meta',
+                            '/baserock/eglibc.meta',
+                            '/baserock/gcc.meta') if os.path.exists(f)
+            )
+            if settings['ccache-remotedir'] != '':
+                env['CCACHE_REMOTEDIR'] = settings['ccache-remotedir']
+            env['CCACHE_REMOTENLEVELS'] = str(settings['ccache-remotenlevels'])
+            if not settings['no-distcc']:
+                env['CCACHE_PREFIX'] = 'distcc'
+            
+        return env
diff --git a/morphlib/buildenvironment_tests.py b/morphlib/buildenvironment_tests.py
new file mode 100644
index 00000000..f99484f0
--- /dev/null
+++ b/morphlib/buildenvironment_tests.py
@@ -0,0 +1,137 @@
+# Copyright (C) 2012  Codethink Limited
+# 
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+# 
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+# 
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import unittest
+
+import morphlib
+from morphlib import buildenvironment
+
+class BuildEnvironmentTests(unittest.TestCase):
+
+    def setUp(self):
+        self.settings = {
+            'keep-path': False,
+            'bootstrap': False,
+            'toolchain-target': '%s-baserock-linux-gnu' % morphlib.util.arch(),
+            'target-cflags': '',
+            'prefix': '/usr',
+            'no-ccache': True,
+            'ccache-remotedir': '',
+            'ccache-remotenlevels': 2,
+            'no-distcc': True,
+        }
+        self.fake_env = {
+            'PATH': '/fake_bin',
+        }
+        self.default_path = 'no:such:path'
+        self.remote_ccache = 'http://example.com/ccache'
+
+    def test_arch_defaults_to_host(self):
+        buildenv = buildenvironment.BuildEnvironment(self.settings)
+        self.assertEqual(buildenv.arch, morphlib.util.arch())
+
+    def test_arch_overridable(self):
+        buildenv = buildenvironment.BuildEnvironment(self.settings,
+                                                     arch='noarch')
+        self.assertEqual(buildenv.arch, 'noarch')
+
+    def test_sets_default_path(self):
+        self.settings['keep-path'] = False
+        self.settings['bootstrap'] = False
+        olddefaultpath = buildenvironment.BuildEnvironment._default_path
+        buildenvironment.BuildEnvironment._default_path = self.default_path
+        buildenv = buildenvironment.BuildEnvironment(self.settings)
+        buildenvironment.BuildEnvironment._default_path = olddefaultpath
+        self.assertTrue(self.default_path in buildenv.env['PATH'])
+
+    def test_uses_env_path_with_keep_path(self):
+        self.settings['keep-path'] = True
+
+        old_osenv = buildenvironment.BuildEnvironment._osenv
+        buildenvironment.BuildEnvironment._osenv = self.fake_env
+        buildenv = buildenvironment.BuildEnvironment(self.settings)
+        buildenvironment.BuildEnvironment._osenv = old_osenv
+
+        self.assertEqual(buildenv.env['PATH'], self.fake_env['PATH'])
+
+    def test_uses_env_path_with_bootstrap(self):
+        self.settings['bootstrap'] = True
+
+        old_osenv = buildenvironment.BuildEnvironment._osenv
+        buildenvironment.BuildEnvironment._osenv = self.fake_env
+        buildenv = buildenvironment.BuildEnvironment(self.settings)
+        buildenvironment.BuildEnvironment._osenv = old_osenv
+
+        self.assertEqual(buildenv.env['PATH'], self.fake_env['PATH'])
+
+    def test_copies_whitelist_vars(self):
+        env = self.fake_env
+        safe = {
+            'DISTCC_HOSTS': 'example.com:example.co.uk',
+            'TMPDIR': '/buildenv/tmp/dir',
+            'LD_PRELOAD': '/buildenv/lib/libbuildenv.so',
+            'LD_LIBRARY_PATH': '/buildenv/lib:/buildenv/lib64',
+            'FAKEROOTKEY': 'b011de73',
+            'FAKED_MODE': 'non-fakeroot',
+            'FAKEROOT_FD_BASE': '-1',
+        }
+        env.update(safe)
+
+        old_osenv = buildenvironment.BuildEnvironment._osenv
+        buildenvironment.BuildEnvironment._osenv = env
+        buildenv = buildenvironment.BuildEnvironment(self.settings)
+        buildenvironment.BuildEnvironment._osenv = old_osenv
+
+        self.assertEqual(sorted(safe.items()),
+                         sorted([(k, buildenv.env[k]) for k in safe.keys()]))
+
+    def test_user_spellings_equal(self):
+        buildenv = buildenvironment.BuildEnvironment(self.settings)
+        self.assertTrue(buildenv.env['USER'] == buildenv.env['USERNAME'] ==
+                        buildenv.env['LOGNAME'])
+
+    def test_environment_overrides(self):
+        buildenv = buildenvironment.BuildEnvironment(self.settings)
+        self.assertEqual(buildenv.env['TERM'], buildenv._override_term)
+        self.assertEqual(buildenv.env['SHELL'], buildenv._override_shell)
+        self.assertEqual(buildenv.env['USER'], buildenv._override_username)
+        self.assertEqual(buildenv.env['USERNAME'], buildenv._override_username)
+        self.assertEqual(buildenv.env['LOGNAME'], buildenv._override_username)
+        self.assertEqual(buildenv.env['LC_ALL'], buildenv._override_locale)
+        self.assertEqual(buildenv.env['HOME'], buildenv._override_home)
+
+    def test_environment_settings_set(self):
+        buildenv = buildenvironment.BuildEnvironment(self.settings)
+        self.assertEqual(buildenv.env['TOOLCHAIN_TARGET'],
+                         self.settings['toolchain-target'])
+        self.assertEqual(buildenv.env['CFLAGS'],
+                         self.settings['target-cflags'])
+        self.assertEqual(buildenv.env['PREFIX'],
+                         self.settings['prefix'])
+        self.assertEqual(buildenv.env['BOOTSTRAP'],
+                         'true' if self.settings['bootstrap'] else 'false')
+
+    def test_ccache_vars_set(self):
+        self.settings['no-ccache'] = False
+        self.settings['ccache-remotedir'] = self.remote_ccache
+        self.settings['no-distcc'] = False
+        buildenv = buildenvironment.BuildEnvironment(self.settings)
+        self.assertTrue(buildenv._ccache_path in buildenv.env['PATH'])
+        self.assertEqual(buildenv.env['CCACHE_REMOTEDIR'],
+                         self.remote_ccache)
+        self.assertEqual(buildenv.env['CCACHE_REMOTENLEVELS'],
+                         str(self.settings['ccache-remotenlevels']))
+        self.assertEqual(buildenv.env['CCACHE_PREFIX'], 'distcc')