diff options
author | Richard Maw <richard.maw@codethink.co.uk> | 2011-12-12 09:16:34 +0000 |
---|---|---|
committer | Richard Maw <richard.maw@codethink.co.uk> | 2011-12-12 09:16:34 +0000 |
commit | 308d0e324c88777fd71a34eb3665c48ea019ff5e (patch) | |
tree | b7c7146c284e0ad6c7ff1299ebf5e1d7bf6503b7 /morphlib | |
parent | 72ac2db983130cdcf3ccf83fbf6a4040e93d776b (diff) | |
parent | a4da32f5a81c8bc6d660404724cedc3bc0914a75 (diff) | |
download | morph-308d0e324c88777fd71a34eb3665c48ea019ff5e.tar.gz |
Merge branch 'richardmaw/login'
Diffstat (limited to 'morphlib')
-rw-r--r-- | morphlib/bins.py | 21 | ||||
-rw-r--r-- | morphlib/bins_tests.py | 21 | ||||
-rw-r--r-- | morphlib/builder.py | 38 | ||||
-rw-r--r-- | morphlib/execute.py | 52 |
4 files changed, 90 insertions, 42 deletions
diff --git a/morphlib/bins.py b/morphlib/bins.py index d8cbf8e0..0f628210 100644 --- a/morphlib/bins.py +++ b/morphlib/bins.py @@ -29,7 +29,8 @@ import tempfile import morphlib -def create_chunk(rootdir, chunk_filename, regexps, dump_memory_profile=None): +def create_chunk(rootdir, chunk_filename, regexps, ex, + dump_memory_profile=None): '''Create a chunk from the contents of a directory. Only files and directories that match at least one of the regular @@ -85,9 +86,9 @@ def create_chunk(rootdir, chunk_filename, regexps, dump_memory_profile=None): with open(include_filename, 'w') as f: for name in include: f.write('%s\0' % mkrel(name)) - ex = morphlib.execute.Execute('.', lambda msg: None) ex.runv(['tar', '-C', rootdir, '-caf', chunk_filename, - '--null', '-T', include_filename, '--no-recursion']) + '--null', '-T', include_filename, '--no-recursion'], + as_fakeroot=True) os.remove(include_filename) dump_memory_profile('after creating tarball') @@ -101,22 +102,24 @@ def create_chunk(rootdir, chunk_filename, regexps, dump_memory_profile=None): dump_memory_profile('after removing in create_chunks') -def create_stratum(rootdir, stratum_filename): +def create_stratum(rootdir, stratum_filename, ex): '''Create a stratum from the contents of a directory.''' logging.debug('Creating stratum file %s from %s' % (stratum_filename, rootdir)) - ex = morphlib.execute.Execute('.', lambda msg: None) - ex.runv(['tar', '-C', rootdir, '-caf', stratum_filename, '.']) + ex.runv(['tar', '-C', rootdir, '-caf', stratum_filename, '.'], + as_fakeroot=True) -def unpack_binary(filename, dirname): +def unpack_binary(filename, dirname, ex, as_fakeroot=False, as_root=False): '''Unpack a binary into a directory. The directory must exist already. + If the binary will be packed up again by tar with the same Execute + object then as_fakeroot will suffice + If it will be creating a system image as_root will be needed ''' logging.debug('Unpacking %s into %s' % (filename, dirname)) - ex = morphlib.execute.Execute(dirname, msg=lambda s: None) - ex.runv(['tar', '-xvf', filename]) + ex.runv(['tar', '-C', dirname, '-xvf', filename], as_fakeroot=as_fakeroot, as_root=as_root) diff --git a/morphlib/bins_tests.py b/morphlib/bins_tests.py index 171b9652..6fe48aa7 100644 --- a/morphlib/bins_tests.py +++ b/morphlib/bins_tests.py @@ -56,6 +56,7 @@ def recursive_lstat(root): class ChunkTests(unittest.TestCase): def setUp(self): + self.ex = morphlib.execute.Execute('.', lambda s: None) self.tempdir = tempfile.mkdtemp() self.instdir = os.path.join(self.tempdir, 'inst') self.chunk_file = os.path.join(self.tempdir, 'chunk') @@ -79,7 +80,8 @@ class ChunkTests(unittest.TestCase): def test_empties_everything(self): self.populate_instdir() - morphlib.bins.create_chunk(self.instdir, self.chunk_file, ['.']) + morphlib.bins.create_chunk(self.instdir, self.chunk_file, ['.'], + self.ex) empty = os.path.join(self.tempdir, 'empty') os.mkdir(empty) self.assertEqual([x for x,y in recursive_lstat(self.instdir)], ['.']) @@ -87,16 +89,19 @@ class ChunkTests(unittest.TestCase): def test_creates_and_unpacks_chunk_exactly(self): self.populate_instdir() orig_files = recursive_lstat(self.instdir) - morphlib.bins.create_chunk(self.instdir, self.chunk_file, ['.']) + morphlib.bins.create_chunk(self.instdir, self.chunk_file, ['.'], + self.ex) os.mkdir(self.unpacked) - morphlib.bins.unpack_binary(self.chunk_file, self.unpacked) + morphlib.bins.unpack_binary(self.chunk_file, self.unpacked, self.ex, + as_fakeroot=True) self.assertEqual(orig_files, recursive_lstat(self.unpacked)) def test_uses_only_matching_names(self): self.populate_instdir() - morphlib.bins.create_chunk(self.instdir, self.chunk_file, ['bin']) + morphlib.bins.create_chunk(self.instdir, self.chunk_file, ['bin'], + self.ex) os.mkdir(self.unpacked) - morphlib.bins.unpack_binary(self.chunk_file, self.unpacked) + morphlib.bins.unpack_binary(self.chunk_file, self.unpacked, self.ex) self.assertEqual([x for x,y in recursive_lstat(self.unpacked)], ['.', 'bin', 'bin/foo']) self.assertEqual([x for x,y in recursive_lstat(self.instdir)], @@ -105,6 +110,7 @@ class ChunkTests(unittest.TestCase): class StratumTests(unittest.TestCase): def setUp(self): + self.ex = morphlib.execute.Execute('.', lambda s: None) self.tempdir = tempfile.mkdtemp() self.instdir = os.path.join(self.tempdir, 'inst') self.stratum_file = os.path.join(self.tempdir, 'stratum') @@ -119,9 +125,10 @@ class StratumTests(unittest.TestCase): def test_creates_and_unpacks_stratum_exactly(self): self.populate_instdir() - morphlib.bins.create_stratum(self.instdir, self.stratum_file) + morphlib.bins.create_stratum(self.instdir, self.stratum_file, self.ex) os.mkdir(self.unpacked) - morphlib.bins.unpack_binary(self.stratum_file, self.unpacked) + morphlib.bins.unpack_binary(self.stratum_file, self.unpacked, self.ex, + as_fakeroot=True) self.assertEqual(recursive_lstat(self.instdir), recursive_lstat(self.unpacked)) diff --git a/morphlib/builder.py b/morphlib/builder.py index 4f7dc432..d1ef7c32 100644 --- a/morphlib/builder.py +++ b/morphlib/builder.py @@ -74,6 +74,7 @@ class BinaryBlob(object): filename = os.path.join(dirname, '%s.meta' % blob_name) if not os.path.exists(dirname): os.mkdir(dirname) + with open(filename, 'w') as f: json.dump(meta, f, indent=4) f.write('\n') @@ -243,13 +244,13 @@ class Chunk(BinaryBlob): self.ex.run(commands) self.build_watch.stop(what) - def run_sequentially(self, what, commands, as_fakeroot=False): + def run_sequentially(self, what, commands, as_fakeroot=False, as_root=False): self.msg ('commands: %s' % what) self.build_watch.start(what) flags = self.ex.env['MAKEFLAGS'] self.ex.env['MAKEFLAGS'] = '-j1' logging.debug('Setting MAKEFLAGS=%s' % self.ex.env['MAKEFLAGS']) - self.ex.run(commands, as_fakeroot=as_fakeroot) + self.ex.run(commands, as_fakeroot=as_fakeroot, as_root=as_root) self.ex.env['MAKEFLAGS'] = flags logging.debug('Restore MAKEFLAGS=%s' % self.ex.env['MAKEFLAGS']) self.build_watch.stop(what) @@ -265,7 +266,7 @@ class Chunk(BinaryBlob): filename = self.filename(chunk_name) self.msg('Creating binary for %s' % chunk_name) morphlib.bins.create_chunk(self.destdir, filename, patterns, - self.dump_memory_profile) + self.ex, self.dump_memory_profile) ret[chunk_name] = filename self.build_watch.stop('create-chunks') files = os.listdir(self.destdir) @@ -292,16 +293,18 @@ class Stratum(BinaryBlob): def build(self): os.mkdir(self.destdir) + ex = morphlib.execute.Execute(self.destdir, self.msg) self.build_watch.start('unpack-chunks') for chunk_name, filename in self.built: self.msg('Unpacking chunk %s' % chunk_name) - morphlib.bins.unpack_binary(filename, self.destdir) + morphlib.bins.unpack_binary(filename, self.destdir, ex, + as_fakeroot=True) self.build_watch.stop('unpack-chunks') self.prepare_binary_metadata(self.morph.name) self.build_watch.start('create-binary') self.msg('Creating binary for %s' % self.morph.name) filename = self.filename(self.morph.name) - morphlib.bins.create_stratum(self.destdir, filename) + morphlib.bins.create_stratum(self.destdir, filename, ex) self.build_watch.stop('create-binary') return { self.morph.name: filename } @@ -375,26 +378,26 @@ class System(BinaryBlob): # Create fstab. self.build_watch.start('create-fstab') fstab = self.tempdir.join('mnt/etc/fstab') - with open(fstab, 'w') as f: - f.write('proc /proc proc defaults 0 0\n') - f.write('sysfs /sys sysfs defaults 0 0\n') - f.write('/dev/sda1 / ext4 errors=remount-ro 0 1\n') + # sorry about the hack, I wish I knew a better way + self.ex.runv(['tee', fstab], feed_stdin=''' +proc /proc proc defaults 0 0 +sysfs /sys sysfs defaults 0 0 +/dev/sda1 / ext4 errors=remount-ro 0 1 +''', as_root=True, stdout=open(os.devnull,'w')) self.build_watch.stop('create-fstab') # Install extlinux bootloader. self.build_watch.start('install-bootloader') conf = os.path.join(mount_point, 'extlinux.conf') logging.debug('configure extlinux %s' % conf) - f = open(conf, 'w') - f.write(''' + self.ex.runv(['tee', conf], feed_stdin=''' default linux timeout 1 label linux kernel /vmlinuz -append root=/dev/sda1 init=/bin/sh quiet rw -''') - f.close() +append root=/dev/sda1 init=/sbin/init quiet rw +''', as_root=True, stdout=open(os.devnull, 'w')) self.ex.runv(['extlinux', '--install', mount_point], as_root=True) @@ -544,10 +547,13 @@ class Builder(object): return if self.settings['bootstrap']: self.msg('Unpacking chunk %s onto system' % chunk_name) - morphlib.bins.unpack_binary(chunk_filename, '/') + ex = morphlib.execute.Execute('/', self.msg) + morphlib.bins.unpack_binary(chunk_filename, '/', ex, as_root=True) else: self.msg('Unpacking chunk %s into staging' % chunk_name) - morphlib.bins.unpack_binary(chunk_filename, staging_dir) + ex = morphlib.execute.Execute(staging_dir, self.msg) + morphlib.bins.unpack_binary(chunk_filename, staging_dir, ex, + as_root=True) def get_morph_from_git(self, repo, ref, filename): morph_text = morphlib.git.get_morph_text(repo, ref, filename) diff --git a/morphlib/execute.py b/morphlib/execute.py index afad1317..b502cb11 100644 --- a/morphlib/execute.py +++ b/morphlib/execute.py @@ -18,6 +18,7 @@ import cliapp import logging import os import subprocess +import tempfile import morphlib @@ -38,6 +39,15 @@ class Execute(object): self._setup_env() self.dirname = dirname self.msg = msg + self._fakeroot_session = None + + def __del__(self): # pragma: no cover + try: + object.__del__(self) + except AttributeError: + pass + if self._fakeroot_session: + os.remove(self._fakeroot_session) def _setup_env(self): self.env = dict(os.environ) @@ -55,9 +65,14 @@ class Execute(object): self.msg('# %s' % command) argv = ['sh', '-c', command] if as_root: - argv = ['sudo'] + argv # pragma: no cover + argv = (['sudo'] + + ["%s=%s" % x for x in self.env.iteritems()] + + argv) # pragma: no cover elif as_fakeroot: - argv = ['fakeroot'] + argv + if not self._fakeroot_session: + self._fakeroot_session = tempfile.mkstemp()[1] + argv = ['fakeroot', '-i', self._fakeroot_session, '-s', + self._fakeroot_session, '--'] + argv logging.debug('run: argv=%s' % repr(argv)) logging.debug('run: env=%s' % repr(self.env)) logging.debug('run: cwd=%s' % repr(self.dirname)) @@ -76,31 +91,48 @@ class Execute(object): stdouts.append(out) return stdouts - def runv(self, argv, as_root=False, as_fakeroot=False, _log=True): + def runv(self, argv, feed_stdin=None, as_root=False, as_fakeroot=False, + _log=True, **kwargs): '''Run a command given as a list of argv elements. Return standard output. Raise ``CommandFailure`` if the command fails. Log standard output and error in any case. ''' + if 'stdout' not in kwargs: + kwargs['stdout'] = subprocess.PIPE + if feed_stdin is not None and 'stdin' not in kwargs: + kwargs['stdin'] = subprocess.PIPE # pragma: no cover + if 'stderr' not in kwargs: + kwargs['stderr'] = subprocess.STDOUT + if 'cwd' not in kwargs: + kwargs['cwd'] = self.dirname + if 'env' not in kwargs: + kwargs['env'] = self.env if as_root: - argv = ['sudo'] + argv # pragma: no cover + argv = (['sudo'] + + ["%s=%s" % x for x in self.env.iteritems()] + + argv) # pragma: no cover elif as_fakeroot: - argv = ['fakeroot'] + argv + if not self._fakeroot_session: + self._fakeroot_session = tempfile.mkstemp()[1] + argv = ['fakeroot', '-i', self._fakeroot_session, '-s', + self._fakeroot_session, '--'] + argv logging.debug('runv: argv=%s' % repr(argv)) logging.debug('runv: env=%s' % repr(self.env)) logging.debug('runv: cwd=%s' % repr(self.dirname)) self.msg('# %s' % ' '.join(argv)) - p = subprocess.Popen(argv, stdout=subprocess.PIPE, - stderr=subprocess.STDOUT, cwd=self.dirname) - out, err = p.communicate() + p = subprocess.Popen(argv, **kwargs) + out, err = p.communicate(feed_stdin) if p.returncode != 0: if _log: # pragma: no cover logging.error('Exit code: %d' % p.returncode) - logging.error('Standard output and error:\n%s' % - morphlib.util.indent(out)) + logging.error('Standard output:\n%s' % + morphlib.util.indent(out or '')) + logging.error('Standard error:\n%s' % + morphlib.util.indent(err or '')) raise CommandFailure(' '.join(argv), out) return out |