Merge branch 'master' into rm/morph-pass1-merge

6 files changed, 99 insertions, 43 deletions

diff --git a/baserock-bootstrap b/baserock-bootstrap
index 45062f9d..1187b747 100755
--- a/baserock-bootstrap
+++ b/baserock-bootstrap
@@ -12,7 +12,12 @@ tools="$LFS/tools"
 buildwhat="$1"
 
 export LC_ALL=C
-export LFS_TGT=$(uname -m)-lfs-linux-gnu
+
+if (uname -m | grep arm) >/dev/null; then
+    export LFS_TGT=$(uname -m)-lfs-linux-gnueabi
+else
+    export LFS_TGT=$(uname -m)-lfs-linux-gnu
+fi
 
 HOST_CAT=`which cat`
 HOST_CP=`which cp`
@@ -20,7 +25,7 @@ HOST_MKDIR=`which mkdir`
 HOST_CHMOD=`which chmod`
 
 export CCACHE_DIR="/var/tmp/ccache"
-export PATH="$tools/bin:$tools/sbin:/usr/lib/ccache:/usr/bin:/bin"
+export PATH="$tools/lib/ccache:$tools/bin:$tools/sbin:/usr/lib/ccache:/usr/bin:/bin"
 
 pass1_directories()
 {
@@ -45,6 +50,27 @@ pass1_directories()
         mknod -m 644 "$LFS/dev/urandom" c 1 9
 }
 
+pass1_ccache()
+{
+    echo "Building ccache"
+    if [ ! -e "$tools/bin/ccache" ]
+    then
+        unpack ccache-3.1.7
+        cd "$sources/ccache-3.1.7"
+        ./configure --prefix="$tools"
+        make -j1
+        make install
+
+        mkdir "$tools/lib/ccache"
+        for name in cc c++ gcc gcc-4.6.1 g++ g++-4.6.1; do
+            ln -s "$tools/bin/ccache" "$tools/lib/ccache/$name"
+            ln -s "$tools/bin/ccache" "$tools/lib/ccache/$LFS_TGT-$name"
+        done
+
+        rm -rf "$sources/ccache-3.1.7"
+    fi
+}
+
 pass1_get_sources_with_morph()
 {
     cd $LFS/baserock/gits/morph
@@ -374,31 +400,31 @@ EOF
 echo "Bootstrapping Baserock development environment"
 echo "LFS_TGT=$LFS_TGT"
 
-pass1_directories
+time pass1_directories
 
 case "$buildwhat" in
     pass1a)
-        pass1_get_sources_with_morph
+        time pass1_get_sources_with_morph
         ;;
     pass1b)
-        pass1_build_with_morph
+        time pass1_build_with_morph
         ;;
     pass2a)
-        pass2_get_morph
-        pass2_prepare_for_chroot
-        pass2_get_sources_with_morph_in_chroot
-        pass2a_cleanup_at_end
+        time pass2_get_morph
+        time pass2_prepare_for_chroot
+        time pass2_get_sources_with_morph_in_chroot
+        time pass2a_cleanup_at_end
         ;;
     pass2b)
-        pass2_build_with_morph_in_chroot
-        pass2b_cleanup_at_end
+        time pass2_build_with_morph_in_chroot
+        time pass2b_cleanup_at_end
         ;;
     pass3a)
-        pass3_remove_tools
-        pass3_get_sources_with_morph_in_chroot
+        time pass3_remove_tools
+        time pass3_get_sources_with_morph_in_chroot
         ;;
     pass3b)
-        pass3_build_with_morph_in_chroot
+        time pass3_build_with_morph_in_chroot
         ;;
     *)  echo "Usage! (sorry, I'm unhelpful)" 1>&2
         exit 1
diff --git a/morphlib/builder.py b/morphlib/builder.py
index ac21df70..735e4334 100644
--- a/morphlib/builder.py
+++ b/morphlib/builder.py
@@ -411,9 +411,11 @@ class StratumBuilder(BlobBuilder): # pragma: no cover
 class SystemBuilder(BlobBuilder): # pragma: no cover
 
     def builds(self):
-        return {}
+        filename = self.filename(self.blob.morph.name)
+        return { self.blob.morph.name: filename }
 
     def do_build(self):
+        logging.debug('SystemBuilder.do_build called')
         self.ex = morphlib.execute.Execute(self.tempdir.dirname, self.msg)
         
         image_name = self.tempdir.join('%s.img' % self.blob.morph.name)
@@ -432,7 +434,7 @@ class SystemBuilder(BlobBuilder): # pragma: no cover
             self._install_extlinux(mount_point)
             self._unmount(mount_point)
         except BaseException:
-            self._umount(mount_point)
+            self._unmount(mount_point)
             self._undo_device_mapping(image_name)
             raise
 
@@ -441,34 +443,47 @@ class SystemBuilder(BlobBuilder): # pragma: no cover
 
     def _create_image(self, image_name):
         with self.build_watch('create-image'):
-            self.ex.runv(['dd', 'if=/dev/zero', 'of=' + image_name, 'bs=1024',
-                          'seek=' + (self.blob.morph.disk_size / 1024),
+            # FIXME: This could be done in pure python, no need to run dd
+            self.ex.runv(['dd', 'if=/dev/zero', 'of=' + image_name, 'bs=1',
+                          'seek=%d' % self.blob.morph.disk_size,
                           'count=0'])
 
     def _partition_image(self, image_name):
         with self.build_watch('partition-image'):
-            self.ex.runv(['sfdisk', image_name], feed_stdin='1,,83,*')
+            self.ex.runv(['sfdisk', image_name], feed_stdin='1,,83,*\n')
 
     def _install_mbr(self, image_name):
         with self.build_watch('install-mbr'):
             for path in ['/usr/lib/extlinux/mbr.bin',
                          '/usr/share/syslinux/mbr.bin']:
                 if os.path.exists(path):
-                    os.ex.runv(['dd', 'if=' + path, 'of=' + image_name,
-                                'conv=notrunc'])
+                    self.ex.runv(['dd', 'if=' + path, 'of=' + image_name,
+                                  'conv=notrunc'])
                     break
 
     def _setup_device_mapping(self, image_name):
         with self.build_watch('setup-device-mapper'):
-            out = self.ex.runv(['kpartx', '-av', image_name])
-            devices = [line.split()[2]
-                       for line in out.splitlines()
-                       if line.startswith('add map ')]
-            partition = '/dev/mapper/%s' % devices[0]
+            out = self.ex.runv(['sfdisk', '-d', image_name])
+            for line in out.splitlines():
+                words = line.split()
+                if (len(words) >= 4 and 
+                    words[2] == 'start=' and 
+                    words[3] != '0,'):
+                    n = int(words[3][:-1]) # skip trailing comma
+                    start = n * 512
+                    break
+            
+            self.ex.runv(['losetup', '-o', str(start), '-f', image_name])
+            
+            out = self.ex.runv(['losetup', '-j', image_name])
+            line = out.strip()
+            i = line.find(':')
+            return line[:i]
 
     def _create_fs(self, partition):
         with self.build_watch('create-filesystem'):
-            self.ex.runv(['mkfs', '-t', 'ext4', '-q', partition, 4194304])
+            # FIXME: the hardcoded size is icky but the default broke
+            self.ex.runv(['mkfs', '-t', 'ext4', '-q', partition, '4194304'])
 
     def _mount(self, partition, mount_point):
         with self.build_watch('mount-filesystem'):
@@ -497,11 +512,11 @@ class SystemBuilder(BlobBuilder): # pragma: no cover
             conf = os.path.join(mount_point, 'extlinux.conf')
             logging.debug('configure extlinux %s' % conf)
             with open(conf, 'w') as f:
-                f.write('default linux')
-                f.write('timeout 1')
-                f.write('label linux')
-                f.write('kernel /boot/vmlinuz')
-                f.write('append root=/dev/sda1 init=/sbin/init quiet rw')
+                f.write('default linux\n')
+                f.write('timeout 1\n')
+                f.write('label linux\n')
+                f.write('kernel /boot/vmlinuz\n')
+                f.write('append root=/dev/sda1 init=/sbin/init quiet rw\n')
 
             self.ex.runv(['extlinux', '--install', mount_point])
             
@@ -517,7 +532,11 @@ class SystemBuilder(BlobBuilder): # pragma: no cover
 
     def _undo_device_mapping(self, image_name):
         with self.build_watch('undo-device-mapper'):
-            self.ex.runv(['kpartx', '-d', image_name])
+            out = self.ex.runv(['losetup', '-j', image_name])
+            for line in out.splitlines():
+                i = line.find(':')
+                device = line[:i]
+                self.ex.runv(['losetup', '-d', device])
 
     def _move_image_to_cache(self, image_name):
         with self.build_watch('cache-image'):
@@ -562,6 +581,7 @@ class Builder(object): # pragma: no cover
         '''Build a list of groups of morphologies. Items in a group
            can be built in parallel.'''
 
+        logging.debug('Builder.build called')
         self.indent_more()
 
         # first pass: create builders for all blobs
@@ -574,6 +594,7 @@ class Builder(object): # pragma: no cover
         # second pass: build group by group, item after item
         for group in build_order:
             for blob in group:
+                logging.debug('Building blob %s' % repr(blob))
                 self.msg('Building %s' % blob)
                 self.indent_more()
 
@@ -591,12 +612,15 @@ class Builder(object): # pragma: no cover
                 #    depbuilder.unstage()
 
                 builder = builders[blob]
+                logging.debug('builder = %s' % repr(builder))
 
                 # get a list of all the items we have to build for this blob
                 builds = builder.builds()
+                logging.debug('builds = %s' % repr(builds))
 
                 # if not all build items are in the cache, rebuild the blob
                 if not self.all_built(builds):
+                    logging.debug('calling builders build method')
                     builders[blob].build()
                 
                 # check again, fail if not all build items were actually built
@@ -703,7 +727,7 @@ class Builder(object): # pragma: no cover
                                       blob.morph.kind)
 
         dict_key = {
-            'name': blob.morph.name,
+            'filename': blob.morph.filename,
             'arch': morphlib.util.arch(),
             'ref': blob.morph.treeish.sha1,
             'kids': ''.join(self.cachedir.key(k) for k in kids),
diff --git a/morphlib/morphology.py b/morphlib/morphology.py
index 734c4e3d..e44e59b3 100644
--- a/morphlib/morphology.py
+++ b/morphlib/morphology.py
@@ -102,7 +102,17 @@ class Morphology(object):
 
     @property
     def disk_size(self):
-        return self._dict['disk-size']
+        size = self._dict['disk-size']
+        size = size.lower()
+        if size.endswith('g'):
+            size = int(size[:-1]) * 1024**3
+        elif size.endswith('m'): # pragma: no cover
+            size = int(size[:-1]) * 1024**2
+        elif size.endswith('k'): # pragma: no cover
+            size = int(size[:-1]) * 1024
+        else: # pragma: no cover
+            size = int(size)
+        return size
 
     @property
     def test_stories(self):
diff --git a/morphlib/morphology_tests.py b/morphlib/morphology_tests.py
index 575aaeec..560677b5 100644
--- a/morphlib/morphology_tests.py
+++ b/morphlib/morphology_tests.py
@@ -169,7 +169,7 @@ class MorphologyTests(unittest.TestCase):
                                 ]
                             }'''))
         self.assertEqual(morph.kind, 'system')
-        self.assertEqual(morph.disk_size, '1G')
+        self.assertEqual(morph.disk_size, 1024**3)
         self.assertEqual(morph.strata, ['foo', 'bar'])
         self.assertEqual(morph.test_stories, ['test-1', 'test-2'])
 
diff --git a/run-bootstrap-in-chroot b/run-bootstrap-in-chroot
index 9fec89d2..460e480b 100755
--- a/run-bootstrap-in-chroot
+++ b/run-bootstrap-in-chroot
@@ -97,7 +97,6 @@ then
     echo You have to set DEBIAN_MIRROR and other environment variables 1>&2
     exit 1
 fi
-#DEBIAN_MIRROR="http://192.168.1.185/debian"
 
 # prepare the ccache directory in the chroot, if necessary
 if [ "x$CCACHE_HOST_DIR" = "x" ]; then
@@ -125,10 +124,10 @@ squeeze "$dir" "$DEBIAN_MIRROR"
 
     # We need a backport of tar 1.26. On ARM, older versions (possibly
     # prior to 1.25) have a bug unpacking things onto an NFS filesystem.
-    # code.liw.fi has the backport, so we'll add that to apt's sources.list
+    # We have a custom apt repository, so we'll add that to apt's sources.list
     # and install tar manually.
     cat <<EOF >> "$dir/etc/apt/sources.list"
-deb http://code.liw.fi/debian squeeze main
+deb http://roadtrain.codethink.co.uk/debrock/ squeeze main
 EOF
     ./do-squeeze-chroot apt-get update
     ./do-squeeze-chroot apt-get install --allow-unauthenticated tar python-cliapp
diff --git a/wget-list b/wget-list
index 72a329ad..775ddd6f 100644
--- a/wget-list
+++ b/wget-list
@@ -1,3 +1,4 @@
+http://samba.org/ftp/ccache/ccache-3.1.7.tar.xz
 http://roadtrain.codethink.co.uk/tarballs/eglibc-2.14.tar.bz2
 http://ftp.gnu.org/gnu/autoconf/autoconf-2.68.tar.bz2
 http://ftp.gnu.org/gnu/automake/automake-1.11.1.tar.bz2
@@ -18,7 +19,6 @@ http://ftp.gnu.org/gnu/gawk/gawk-4.0.0.tar.bz2
 http://ftp.gnu.org/gnu/gcc/gcc-4.6.1/gcc-4.6.1.tar.bz2
 http://ftp.gnu.org/gnu/gdbm/gdbm-1.9.1.tar.gz
 http://ftp.gnu.org/gnu/gettext/gettext-0.18.1.1.tar.gz
-http://ftp.gnu.org/gnu/glibc/glibc-2.14.1.tar.bz2
 http://ftp.gnu.org/gnu/gmp/gmp-5.0.2.tar.bz2
 http://ftp.gnu.org/gnu/grep/grep-2.9.tar.gz
 http://ftp.gnu.org/gnu/groff/groff-1.21.tar.gz
@@ -66,9 +66,6 @@ http://www.linuxfromscratch.org/patches/lfs/7.0/coreutils-8.14-uname-1.patch
 http://www.linuxfromscratch.org/patches/lfs/7.0/flex-2.5.35-gcc44-1.patch
 http://www.linuxfromscratch.org/patches/lfs/7.0/gcc-4.6.1-cross_compile-1.patch
 http://www.linuxfromscratch.org/patches/lfs/7.0/gcc-4.6.1-startfiles_fix-1.patch
-http://www.linuxfromscratch.org/patches/lfs/7.0/glibc-2.14.1-fixes-1.patch
-http://www.linuxfromscratch.org/patches/lfs/7.0/glibc-2.14.1-gcc_fix-1.patch
-http://www.linuxfromscratch.org/patches/lfs/7.0/glibc-2.14.1-cpuid-1.patch
 http://www.linuxfromscratch.org/patches/lfs/7.0/gcc-4.6.1-locale-1.patch
 http://www.linuxfromscratch.org/patches/lfs/7.0/kbd-1.15.2-backspace-1.patch
 http://www.linuxfromscratch.org/patches/lfs/7.0/module-init-tools-3.16-man_pages-1.patch