Merge branch 'richardmaw/login'

6 files changed, 94 insertions, 44 deletions

diff --git a/check b/check
index d2513696..794d035c 100755
--- a/check
+++ b/check
@@ -20,4 +20,4 @@
 set -e
 
 python setup.py clean check
-$HOME/cmdtest/cmdtest/cmdtest -c ./morph tests
+cmdtest -c ./morph tests
diff --git a/morph b/morph
index 84ccefbc..cbee2305 100755
--- a/morph
+++ b/morph
@@ -88,7 +88,9 @@ class Morph(cliapp.Application):
             self.msg('Building %s - %s - %s' % (repo, ref, filename))
             builder.build(repo, ref, filename)
         
-        tempdir.remove()
+        # we may not have permission to tempdir.remove()
+        ex = morphlib.execute.Execute('.', lambda msg: None)
+        ex.runv(["rm", "-rf", tempdir.dirname], as_root=True)
 
         if args:
             raise cliapp.AppException('Extra args on command line: %s' % args)
diff --git a/morphlib/bins.py b/morphlib/bins.py
index d8cbf8e0..0f628210 100644
--- a/morphlib/bins.py
+++ b/morphlib/bins.py
@@ -29,7 +29,8 @@ import tempfile
 import morphlib
 
 
-def create_chunk(rootdir, chunk_filename, regexps, dump_memory_profile=None):
+def create_chunk(rootdir, chunk_filename, regexps, ex,
+                 dump_memory_profile=None):
     '''Create a chunk from the contents of a directory.
     
     Only files and directories that match at least one of the regular
@@ -85,9 +86,9 @@ def create_chunk(rootdir, chunk_filename, regexps, dump_memory_profile=None):
     with open(include_filename, 'w') as f:
         for name in include:
             f.write('%s\0' % mkrel(name))
-    ex = morphlib.execute.Execute('.', lambda msg: None)
     ex.runv(['tar', '-C', rootdir, '-caf', chunk_filename,
-             '--null', '-T', include_filename, '--no-recursion'])
+             '--null', '-T', include_filename, '--no-recursion'],
+             as_fakeroot=True)
     os.remove(include_filename)
     dump_memory_profile('after creating tarball')
 
@@ -101,22 +102,24 @@ def create_chunk(rootdir, chunk_filename, regexps, dump_memory_profile=None):
     dump_memory_profile('after removing in create_chunks')
 
 
-def create_stratum(rootdir, stratum_filename):
+def create_stratum(rootdir, stratum_filename, ex):
     '''Create a stratum from the contents of a directory.'''
     logging.debug('Creating stratum file %s from %s' % 
                     (stratum_filename, rootdir))
-    ex = morphlib.execute.Execute('.', lambda msg: None)
-    ex.runv(['tar', '-C', rootdir, '-caf', stratum_filename, '.'])
+    ex.runv(['tar', '-C', rootdir, '-caf', stratum_filename, '.'],
+             as_fakeroot=True)
 
 
-def unpack_binary(filename, dirname):
+def unpack_binary(filename, dirname, ex, as_fakeroot=False, as_root=False):
     '''Unpack a binary into a directory.
     
     The directory must exist already.
+    If the binary will be packed up again by tar with the same Execute
+    object then as_fakeroot will suffice
+    If it will be creating a system image as_root will be needed
     
     '''
 
     logging.debug('Unpacking %s into %s' % (filename, dirname))
-    ex = morphlib.execute.Execute(dirname, msg=lambda s: None)
-    ex.runv(['tar', '-xvf', filename])
+    ex.runv(['tar', '-C', dirname, '-xvf', filename], as_fakeroot=as_fakeroot, as_root=as_root)
 
diff --git a/morphlib/bins_tests.py b/morphlib/bins_tests.py
index 171b9652..6fe48aa7 100644
--- a/morphlib/bins_tests.py
+++ b/morphlib/bins_tests.py
@@ -56,6 +56,7 @@ def recursive_lstat(root):
 class ChunkTests(unittest.TestCase):
 
     def setUp(self):
+        self.ex = morphlib.execute.Execute('.', lambda s: None)
         self.tempdir = tempfile.mkdtemp()
         self.instdir = os.path.join(self.tempdir, 'inst')
         self.chunk_file = os.path.join(self.tempdir, 'chunk')
@@ -79,7 +80,8 @@ class ChunkTests(unittest.TestCase):
 
     def test_empties_everything(self):
         self.populate_instdir()
-        morphlib.bins.create_chunk(self.instdir, self.chunk_file, ['.'])
+        morphlib.bins.create_chunk(self.instdir, self.chunk_file, ['.'],
+                                   self.ex)
         empty = os.path.join(self.tempdir, 'empty')
         os.mkdir(empty)
         self.assertEqual([x for x,y in recursive_lstat(self.instdir)], ['.'])
@@ -87,16 +89,19 @@ class ChunkTests(unittest.TestCase):
     def test_creates_and_unpacks_chunk_exactly(self):
         self.populate_instdir()
         orig_files = recursive_lstat(self.instdir)
-        morphlib.bins.create_chunk(self.instdir, self.chunk_file, ['.'])
+        morphlib.bins.create_chunk(self.instdir, self.chunk_file, ['.'],
+                                   self.ex)
         os.mkdir(self.unpacked)
-        morphlib.bins.unpack_binary(self.chunk_file, self.unpacked)
+        morphlib.bins.unpack_binary(self.chunk_file, self.unpacked, self.ex,
+                                    as_fakeroot=True)
         self.assertEqual(orig_files, recursive_lstat(self.unpacked))
 
     def test_uses_only_matching_names(self):
         self.populate_instdir()
-        morphlib.bins.create_chunk(self.instdir, self.chunk_file, ['bin'])
+        morphlib.bins.create_chunk(self.instdir, self.chunk_file, ['bin'],
+                                   self.ex)
         os.mkdir(self.unpacked)
-        morphlib.bins.unpack_binary(self.chunk_file, self.unpacked)
+        morphlib.bins.unpack_binary(self.chunk_file, self.unpacked, self.ex)
         self.assertEqual([x for x,y in recursive_lstat(self.unpacked)],
                          ['.', 'bin', 'bin/foo'])
         self.assertEqual([x for x,y in recursive_lstat(self.instdir)],
@@ -105,6 +110,7 @@ class ChunkTests(unittest.TestCase):
 class StratumTests(unittest.TestCase):
 
     def setUp(self):
+        self.ex = morphlib.execute.Execute('.', lambda s: None)
         self.tempdir = tempfile.mkdtemp()
         self.instdir = os.path.join(self.tempdir, 'inst')
         self.stratum_file = os.path.join(self.tempdir, 'stratum')
@@ -119,9 +125,10 @@ class StratumTests(unittest.TestCase):
 
     def test_creates_and_unpacks_stratum_exactly(self):
         self.populate_instdir()
-        morphlib.bins.create_stratum(self.instdir, self.stratum_file)
+        morphlib.bins.create_stratum(self.instdir, self.stratum_file, self.ex)
         os.mkdir(self.unpacked)
-        morphlib.bins.unpack_binary(self.stratum_file, self.unpacked)
+        morphlib.bins.unpack_binary(self.stratum_file, self.unpacked, self.ex,
+                                    as_fakeroot=True)
         self.assertEqual(recursive_lstat(self.instdir),
                          recursive_lstat(self.unpacked))
 
diff --git a/morphlib/builder.py b/morphlib/builder.py
index 4f7dc432..d1ef7c32 100644
--- a/morphlib/builder.py
+++ b/morphlib/builder.py
@@ -74,6 +74,7 @@ class BinaryBlob(object):
         filename = os.path.join(dirname, '%s.meta' % blob_name)
         if not os.path.exists(dirname):
             os.mkdir(dirname)
+            
         with open(filename, 'w') as f:
             json.dump(meta, f, indent=4)
             f.write('\n')
@@ -243,13 +244,13 @@ class Chunk(BinaryBlob):
         self.ex.run(commands)
         self.build_watch.stop(what)
 
-    def run_sequentially(self, what, commands, as_fakeroot=False):
+    def run_sequentially(self, what, commands, as_fakeroot=False, as_root=False):
         self.msg ('commands: %s' % what)
         self.build_watch.start(what)
         flags = self.ex.env['MAKEFLAGS']
         self.ex.env['MAKEFLAGS'] = '-j1'
         logging.debug('Setting MAKEFLAGS=%s' % self.ex.env['MAKEFLAGS'])
-        self.ex.run(commands, as_fakeroot=as_fakeroot)
+        self.ex.run(commands, as_fakeroot=as_fakeroot, as_root=as_root)
         self.ex.env['MAKEFLAGS'] = flags
         logging.debug('Restore MAKEFLAGS=%s' % self.ex.env['MAKEFLAGS'])
         self.build_watch.stop(what)
@@ -265,7 +266,7 @@ class Chunk(BinaryBlob):
             filename = self.filename(chunk_name)
             self.msg('Creating binary for %s' % chunk_name)
             morphlib.bins.create_chunk(self.destdir, filename, patterns,
-                                       self.dump_memory_profile)
+                                       self.ex, self.dump_memory_profile)
             ret[chunk_name] = filename
         self.build_watch.stop('create-chunks')
         files = os.listdir(self.destdir)
@@ -292,16 +293,18 @@ class Stratum(BinaryBlob):
 
     def build(self):
         os.mkdir(self.destdir)
+        ex = morphlib.execute.Execute(self.destdir, self.msg)
         self.build_watch.start('unpack-chunks')
         for chunk_name, filename in self.built:
             self.msg('Unpacking chunk %s' % chunk_name)
-            morphlib.bins.unpack_binary(filename, self.destdir)
+            morphlib.bins.unpack_binary(filename, self.destdir, ex,
+                                        as_fakeroot=True)
         self.build_watch.stop('unpack-chunks')
         self.prepare_binary_metadata(self.morph.name)
         self.build_watch.start('create-binary')
         self.msg('Creating binary for %s' % self.morph.name)
         filename = self.filename(self.morph.name)
-        morphlib.bins.create_stratum(self.destdir, filename)
+        morphlib.bins.create_stratum(self.destdir, filename, ex)
         self.build_watch.stop('create-binary')
         return { self.morph.name: filename }
 
@@ -375,26 +378,26 @@ class System(BinaryBlob):
             # Create fstab.
             self.build_watch.start('create-fstab')
             fstab = self.tempdir.join('mnt/etc/fstab')
-            with open(fstab, 'w') as f:
-                f.write('proc /proc proc defaults 0 0\n')
-                f.write('sysfs /sys sysfs defaults 0 0\n')
-                f.write('/dev/sda1 / ext4 errors=remount-ro 0 1\n')
+            # sorry about the hack, I wish I knew a better way
+            self.ex.runv(['tee', fstab], feed_stdin='''
+proc      /proc proc  defaults          0 0
+sysfs     /sys  sysfs defaults          0 0
+/dev/sda1 /     ext4  errors=remount-ro 0 1
+''', as_root=True, stdout=open(os.devnull,'w'))
             self.build_watch.stop('create-fstab')
 
             # Install extlinux bootloader.
             self.build_watch.start('install-bootloader')
             conf = os.path.join(mount_point, 'extlinux.conf')
             logging.debug('configure extlinux %s' % conf)
-            f = open(conf, 'w')
-            f.write('''
+            self.ex.runv(['tee', conf], feed_stdin='''
 default linux
 timeout 1
 
 label linux
 kernel /vmlinuz
-append root=/dev/sda1 init=/bin/sh quiet rw
-''')
-            f.close()
+append root=/dev/sda1 init=/sbin/init quiet rw
+''', as_root=True, stdout=open(os.devnull, 'w'))
 
             self.ex.runv(['extlinux', '--install', mount_point], as_root=True)
             
@@ -544,10 +547,13 @@ class Builder(object):
             return
         if self.settings['bootstrap']:
             self.msg('Unpacking chunk %s onto system' % chunk_name)
-            morphlib.bins.unpack_binary(chunk_filename, '/')
+            ex = morphlib.execute.Execute('/', self.msg)
+            morphlib.bins.unpack_binary(chunk_filename, '/', ex, as_root=True)
         else:
             self.msg('Unpacking chunk %s into staging' % chunk_name)
-            morphlib.bins.unpack_binary(chunk_filename, staging_dir)
+            ex = morphlib.execute.Execute(staging_dir, self.msg)
+            morphlib.bins.unpack_binary(chunk_filename, staging_dir, ex,
+                                        as_root=True)
             
     def get_morph_from_git(self, repo, ref, filename):
         morph_text = morphlib.git.get_morph_text(repo, ref, filename)
diff --git a/morphlib/execute.py b/morphlib/execute.py
index afad1317..b502cb11 100644
--- a/morphlib/execute.py
+++ b/morphlib/execute.py
@@ -18,6 +18,7 @@ import cliapp
 import logging
 import os
 import subprocess
+import tempfile
 
 import morphlib
 
@@ -38,6 +39,15 @@ class Execute(object):
         self._setup_env()
         self.dirname = dirname
         self.msg = msg
+        self._fakeroot_session = None
+
+    def __del__(self): # pragma: no cover
+        try:
+            object.__del__(self)
+        except AttributeError:
+            pass
+        if self._fakeroot_session:
+            os.remove(self._fakeroot_session)
 
     def _setup_env(self):
         self.env = dict(os.environ)
@@ -55,9 +65,14 @@ class Execute(object):
             self.msg('# %s' % command)
             argv = ['sh', '-c', command]
             if as_root:
-                argv = ['sudo'] + argv # pragma: no cover
+                argv = (['sudo'] +
+                        ["%s=%s" % x for x in self.env.iteritems()] +
+                        argv) # pragma: no cover
             elif as_fakeroot:
-                argv = ['fakeroot'] + argv
+                if not self._fakeroot_session:
+                    self._fakeroot_session = tempfile.mkstemp()[1]
+                argv = ['fakeroot', '-i', self._fakeroot_session, '-s',
+                        self._fakeroot_session, '--'] + argv
             logging.debug('run: argv=%s' % repr(argv))
             logging.debug('run: env=%s' % repr(self.env))
             logging.debug('run: cwd=%s' % repr(self.dirname))
@@ -76,31 +91,48 @@ class Execute(object):
             stdouts.append(out)
         return stdouts
 
-    def runv(self, argv, as_root=False, as_fakeroot=False, _log=True):
+    def runv(self, argv, feed_stdin=None, as_root=False, as_fakeroot=False,
+             _log=True, **kwargs):
         '''Run a command given as a list of argv elements.
         
         Return standard output. Raise ``CommandFailure`` if the command
         fails. Log standard output and error in any case.
         
         '''
+        if 'stdout' not in kwargs:
+            kwargs['stdout'] = subprocess.PIPE
+        if feed_stdin is not None and 'stdin' not in kwargs: 
+            kwargs['stdin'] = subprocess.PIPE # pragma: no cover
+        if 'stderr' not in kwargs:
+            kwargs['stderr'] = subprocess.STDOUT
+        if 'cwd' not in kwargs:
+            kwargs['cwd'] = self.dirname
+        if 'env' not in kwargs:
+            kwargs['env'] = self.env
 
         if as_root:
-            argv = ['sudo'] + argv # pragma: no cover
+            argv = (['sudo'] +
+                    ["%s=%s" % x for x in self.env.iteritems()] +
+                    argv) # pragma: no cover
         elif as_fakeroot:
-            argv = ['fakeroot'] + argv
+            if not self._fakeroot_session:
+                self._fakeroot_session = tempfile.mkstemp()[1]
+            argv = ['fakeroot', '-i', self._fakeroot_session, '-s',
+                    self._fakeroot_session, '--'] + argv
         logging.debug('runv: argv=%s' % repr(argv))
         logging.debug('runv: env=%s' % repr(self.env))
         logging.debug('runv: cwd=%s' % repr(self.dirname))
         self.msg('# %s' % ' '.join(argv))
-        p = subprocess.Popen(argv, stdout=subprocess.PIPE, 
-                             stderr=subprocess.STDOUT, cwd=self.dirname)
-        out, err = p.communicate()
+        p = subprocess.Popen(argv, **kwargs)
+        out, err = p.communicate(feed_stdin)
         
         if p.returncode != 0:
             if _log: # pragma: no cover
                 logging.error('Exit code: %d' % p.returncode)
-                logging.error('Standard output and error:\n%s' % 
-                                morphlib.util.indent(out))
+                logging.error('Standard output:\n%s' %
+                                morphlib.util.indent(out or ''))
+                logging.error('Standard error:\n%s' %
+                                morphlib.util.indent(err or ''))
             raise CommandFailure(' '.join(argv), out)
         return out