Further work on making build-essential stratum bootstrappable

2 files changed, 172 insertions, 96 deletions

diff --git a/build-essential.morph b/build-essential.morph
index e53ead3..3153bb8 100644
--- a/build-essential.morph
+++ b/build-essential.morph
@@ -8,32 +8,25 @@
             "repo": "upstream:binutils-redhat",
             "ref": "baserock/samthursfield/build-essential-2",
             "build-mode": "bootstrap",
-            "build-depends": [],
-            "prefix": "/toolchain"
+            "prefix": "/tools",
+            "build-depends": []
         },
         {
             "name": "stage1-gcc",
             "repo": "upstream:gcc-tarball",
             "ref": "baserock/samthursfield/build-essential-2",
             "build-mode": "bootstrap",
-            "build-depends": [],
-            "prefix": "/toolchain"
-        },
-        {
-            "name": "stage2-binutils",
-            "repo": "upstream:binutils-redhat",
-            "ref": "baserock/samthursfield/build-essential-2",
-            "build-mode": "bootstrap",
+            "prefix": "/tools",
             "build-depends": [
-                "stage1-binutils",
-                "stage1-gcc"
+                "stage1-binutils"
             ]
         },
         {
-            "name": "stage2-busybox",
-            "repo": "upstream:busybox",
+            "name": "stage2-linux-api-headers",
+            "repo": "upstream:linux",
             "ref": "baserock/samthursfield/build-essential-2",
             "build-mode": "bootstrap",
+            "prefix": "/tools",
             "build-depends": [
                 "stage1-binutils",
                 "stage1-gcc"
@@ -44,49 +37,67 @@
             "repo": "upstream:eglibc2",
             "ref": "baserock/samthursfield/build-essential-2",
             "build-mode": "bootstrap",
+            "prefix": "/tools",
             "build-depends": [
                 "stage1-binutils",
-                "stage1-gcc"
+                "stage1-gcc",
+                "stage2-linux-api-headers"
             ]
         },
         {
-            "name": "stage2-fhs-dirs",
-            "repo": "baserock:baserock/fhs-dirs",
+            "name": "stage2-binutils",
+            "repo": "upstream:binutils-redhat",
             "ref": "baserock/samthursfield/build-essential-2",
             "build-mode": "bootstrap",
+            "prefix": "/tools",
             "build-depends": [
                 "stage1-binutils",
-                "stage1-gcc"
+                "stage1-gcc",
+                "stage2-eglibc"
             ]
         },
         {
-            "name": "stage2-gawk",
-            "repo": "upstream:gawk",
+            "name": "stage2-gcc",
+            "repo": "upstream:gcc-tarball",
             "ref": "baserock/samthursfield/build-essential-2",
             "build-mode": "bootstrap",
+            "prefix": "/tools",
             "build-depends": [
                 "stage1-binutils",
-                "stage1-gcc"
+                "stage1-gcc",
+                "stage2-eglibc"
             ]
         },
         {
-            "name": "stage2-gcc",
-            "repo": "upstream:gcc-tarball",
+            "name": "stage2-busybox",
+            "repo": "upstream:busybox",
             "ref": "baserock/samthursfield/build-essential-2",
             "build-mode": "bootstrap",
+            "prefix": "/tools",
             "build-depends": [
                 "stage1-binutils",
-                "stage1-gcc"
+                "stage1-gcc",
+                "stage2-eglibc"
             ]
         },
         {
-            "name": "stage2-linux-api-headers",
-            "repo": "upstream:linux",
+            "name": "stage2-fhs-dirs",
+            "repo": "baserock:baserock/fhs-dirs",
             "ref": "baserock/samthursfield/build-essential-2",
             "build-mode": "bootstrap",
+            "prefix": "/tools",
+            "build-depends": []
+        },
+        {
+            "name": "stage2-gawk",
+            "repo": "upstream:gawk",
+            "ref": "baserock/samthursfield/build-essential-2",
+            "build-mode": "bootstrap",
+            "prefix": "/tools",
             "build-depends": [
                 "stage1-binutils",
-                "stage1-gcc"
+                "stage1-gcc",
+                "stage2-eglibc"
             ]
         },
         {
@@ -94,16 +105,19 @@
             "repo": "upstream:make",
             "ref": "baserock/samthursfield/build-essential-2",
             "build-mode": "bootstrap",
+            "prefix": "/tools",
             "build-depends": [
                 "stage1-binutils",
-                "stage1-gcc"
+                "stage1-gcc",
+                "stage2-eglibc"
             ]
         },
         {
-            "name": "binutils",
-            "repo": "upstream:binutils-redhat",
+            "name": "fhs-dirs",
+            "repo": "baserock:baserock/fhs-dirs",
             "ref": "baserock/samthursfield/build-essential-2",
             "build-mode": "staging",
+            "prefix": "/usr",
             "build-depends": [
                 "stage2-binutils",
                 "stage2-busybox",
@@ -116,10 +130,11 @@
             ]
         },
         {
-            "name": "busybox",
-            "repo": "upstream:busybox",
+            "name": "linux-api-headers",
+            "repo": "upstream:linux",
             "ref": "baserock/samthursfield/build-essential-2",
             "build-mode": "staging",
+            "prefix": "/usr",
             "build-depends": [
                 "stage2-binutils",
                 "stage2-busybox",
@@ -136,6 +151,7 @@
             "repo": "upstream:eglibc2",
             "ref": "baserock/2.15-build-essential",
             "build-mode": "staging",
+            "prefix": "/usr",
             "build-depends": [
                 "stage2-binutils",
                 "stage2-busybox",
@@ -144,14 +160,16 @@
                 "stage2-gawk",
                 "stage2-gcc",
                 "stage2-linux-api-headers",
-                "stage2-make"
+                "stage2-make",
+                "linux-api-headers"
             ]
         },
         {
-            "name": "fhs-dirs",
-            "repo": "baserock:baserock/fhs-dirs",
+            "name": "binutils",
+            "repo": "upstream:binutils-redhat",
             "ref": "baserock/samthursfield/build-essential-2",
             "build-mode": "staging",
+            "prefix": "/usr",
             "build-depends": [
                 "stage2-binutils",
                 "stage2-busybox",
@@ -160,14 +178,16 @@
                 "stage2-gawk",
                 "stage2-gcc",
                 "stage2-linux-api-headers",
-                "stage2-make"
+                "stage2-make",
+                "eglibc"
             ]
         },
         {
-            "name": "gawk",
-            "repo": "upstream:gawk",
+            "name": "busybox",
+            "repo": "upstream:busybox",
             "ref": "baserock/samthursfield/build-essential-2",
             "build-mode": "staging",
+            "prefix": "/usr",
             "build-depends": [
                 "stage2-binutils",
                 "stage2-busybox",
@@ -176,14 +196,16 @@
                 "stage2-gawk",
                 "stage2-gcc",
                 "stage2-linux-api-headers",
-                "stage2-make"
+                "stage2-make",
+                "eglibc"
             ]
         },
         {
-            "name": "gcc",
-            "repo": "upstream:gcc-tarball",
+            "name": "gawk",
+            "repo": "upstream:gawk",
             "ref": "baserock/samthursfield/build-essential-2",
             "build-mode": "staging",
+            "prefix": "/usr",
             "build-depends": [
                 "stage2-binutils",
                 "stage2-busybox",
@@ -192,14 +214,16 @@
                 "stage2-gawk",
                 "stage2-gcc",
                 "stage2-linux-api-headers",
-                "stage2-make"
+                "stage2-make",
+                "eglibc"
             ]
         },
         {
-            "name": "linux-api-headers",
-            "repo": "upstream:linux",
+            "name": "gcc",
+            "repo": "upstream:gcc-tarball",
             "ref": "baserock/samthursfield/build-essential-2",
             "build-mode": "staging",
+            "prefix": "/usr",
             "build-depends": [
                 "stage2-binutils",
                 "stage2-busybox",
@@ -208,7 +232,8 @@
                 "stage2-gawk",
                 "stage2-gcc",
                 "stage2-linux-api-headers",
-                "stage2-make"
+                "stage2-make",
+                "eglibc"
             ]
         },
         {
@@ -216,6 +241,7 @@
             "repo": "upstream:make",
             "ref": "baserock/samthursfield/build-essential-2",
             "build-mode": "staging",
+            "prefix": "/usr",
             "build-depends": [
                 "stage2-binutils",
                 "stage2-busybox",
@@ -224,7 +250,8 @@
                 "stage2-gawk",
                 "stage2-gcc",
                 "stage2-linux-api-headers",
-                "stage2-make"
+                "stage2-make",
+                "eglibc"
             ]
         },
         {
@@ -232,15 +259,17 @@
             "repo": "upstream:ccache",
             "ref": "baserock/samthursfield/build-essential-2",
             "build-mode": "staging",
+            "prefix": "/usr",
             "build-depends": [
-                "binutils",
-                "busybox",
-                "eglibc",
-                "fhs-dirs",
-                "gawk",
-                "gcc",
-                "linux-api-headers",
-                "make"
+                "stage2-binutils",
+                "stage2-busybox",
+                "stage2-eglibc",
+                "stage2-fhs-dirs",
+                "stage2-gawk",
+                "stage2-gcc",
+                "stage2-linux-api-headers",
+                "stage2-make",
+                "eglibc"
             ]
         }
     ]
diff --git a/build-essential.morph.yaml b/build-essential.morph.yaml
index b9cc1fb..873a30a 100644
--- a/build-essential.morph.yaml
+++ b/build-essential.morph.yaml
@@ -4,6 +4,7 @@ kind: stratum
 
 chunks:
     # Stage 1: build a minimal cross compiler with the host's tools.
+    #
     # Starting with a cross compiler ensures that (a) nothing from the host
     # can leak into the build-essential artifacts, and (b) cross-compiling
     # build-essential is fully tested and supported, since we always use the
@@ -13,31 +14,30 @@ chunks:
       repo: upstream:binutils-redhat
       ref: baserock/samthursfield/build-essential-2
       build-mode: bootstrap
+      prefix: /tools
       build-depends: []
-      prefix: /toolchain
 
     - name: stage1-gcc
       repo: upstream:gcc-tarball
       ref: baserock/samthursfield/build-essential-2
       build-mode: bootstrap
-      build-depends: []
-      prefix: /toolchain
-
-    # Stage 2: build the whole of build-essential, using the host's tools
-    # but the cross-compiler toolchain.
-
-    - name: stage2-binutils
-      repo: upstream:binutils-redhat
-      ref: baserock/samthursfield/build-essential-2
-      build-mode: bootstrap
+      prefix: /tools
       build-depends:
           - stage1-binutils
-          - stage1-gcc
 
-    - name: stage2-busybox
-      repo: upstream:busybox
+    # Stage 2: cross-build the whole of build-essential, using the host's tools
+    # but the cross-compiler toolchain.
+    #
+    # Stage 2 GCC outputs code for the same 'bootstrap' machine as stage 1 GCC,
+    # but because stage 2 GCC is also built to *run* on the bootstrap machine
+    # it can only execute inside the stage 3 chroot (due to being built against
+    # a libc with a non-standard prefix).
+
+    - name: stage2-linux-api-headers
+      repo: upstream:linux
       ref: baserock/samthursfield/build-essential-2
       build-mode: bootstrap
+      prefix: /tools
       build-depends:
           - stage1-binutils
           - stage1-gcc
@@ -46,58 +46,90 @@ chunks:
       repo: upstream:eglibc2
       ref: baserock/samthursfield/build-essential-2
       build-mode: bootstrap
+      prefix: /tools
       build-depends:
           - stage1-binutils
           - stage1-gcc
+          - stage2-linux-api-headers
 
-    - name: stage2-fhs-dirs
-      repo: baserock:baserock/fhs-dirs
-      ref: baserock/samthursfield/build-essential-2 
+    - name: stage2-binutils
+      repo: upstream:binutils-redhat
+      ref: baserock/samthursfield/build-essential-2
       build-mode: bootstrap
+      prefix: /tools
       build-depends:
           - stage1-binutils
           - stage1-gcc
+          - stage2-eglibc
 
-    - name: stage2-gawk
-      repo: upstream:gawk
+    - name: stage2-gcc
+      repo: upstream:gcc-tarball
       ref: baserock/samthursfield/build-essential-2
       build-mode: bootstrap
+      prefix: /tools
       build-depends:
           - stage1-binutils
           - stage1-gcc
+          - stage2-eglibc
 
-    - name: stage2-gcc
-      repo: upstream:gcc-tarball
+    - name: stage2-busybox
+      repo: upstream:busybox
       ref: baserock/samthursfield/build-essential-2
       build-mode: bootstrap
+      prefix: /tools
       build-depends:
           - stage1-binutils
           - stage1-gcc
+          - stage2-eglibc
 
-    - name: stage2-linux-api-headers
-      repo: upstream:linux
+    - name: stage2-fhs-dirs
+      repo: baserock:baserock/fhs-dirs
+      ref: baserock/samthursfield/build-essential-2 
+      build-mode: bootstrap
+      prefix: /tools
+      build-depends: []
+
+    - name: stage2-gawk
+      repo: upstream:gawk
       ref: baserock/samthursfield/build-essential-2
       build-mode: bootstrap
+      prefix: /tools
       build-depends:
           - stage1-binutils
           - stage1-gcc
+          - stage2-eglibc
 
     - name: stage2-make
       repo: upstream:make
       ref: baserock/samthursfield/build-essential-2
       build-mode: bootstrap
+      prefix: /tools
       build-depends:
           - stage1-binutils
           - stage1-gcc
+          - stage2-eglibc
 
     # Stage 3: build the whole of build-essential again, this time using a
     # staging area containing only the output of stage 2. The result of this
     # build is fully reproducible.
+    #
+    # We do a switch-a-roo between stage 2 and 3: stages 2 chunks are all built
+    # to run on a host *-bootstrap-* while stage 3 chunks are built are native
+    # built for a *-baserock-* machine. This works, because the cross build was
+    # all for show (and cleanliness) and the binaries actually still run on the
+    # the host.
+    #
+    # After build-essential is built we do another trick: stage2-fhs-dirs
+    # symlinks /bin to /tools/bin and /lib to /tools/lib, while the stage 3
+    # chunks actually install things to those directories. FIXME: which breaks
+    # everything!!! This will have to change.
+    #
 
-    - name: binutils
-      repo: upstream:binutils-redhat
+    - name: fhs-dirs
+      repo: baserock:baserock/fhs-dirs
       ref: baserock/samthursfield/build-essential-2
       build-mode: staging
+      prefix: /usr
       build-depends:
           - stage2-binutils
           - stage2-busybox
@@ -108,10 +140,11 @@ chunks:
           - stage2-linux-api-headers
           - stage2-make
 
-    - name: busybox
-      repo: upstream:busybox
+    - name: linux-api-headers
+      repo: upstream:linux
       ref: baserock/samthursfield/build-essential-2
       build-mode: staging
+      prefix: /usr
       build-depends:
           - stage2-binutils
           - stage2-busybox
@@ -126,6 +159,7 @@ chunks:
       repo: upstream:eglibc2
       ref: baserock/2.15-build-essential
       build-mode: staging
+      prefix: /usr
       build-depends:
           - stage2-binutils
           - stage2-busybox
@@ -135,11 +169,13 @@ chunks:
           - stage2-gcc
           - stage2-linux-api-headers
           - stage2-make
+          - linux-api-headers
 
-    - name: fhs-dirs
-      repo: baserock:baserock/fhs-dirs
+    - name: binutils
+      repo: upstream:binutils-redhat
       ref: baserock/samthursfield/build-essential-2
       build-mode: staging
+      prefix: /usr
       build-depends:
           - stage2-binutils
           - stage2-busybox
@@ -149,11 +185,13 @@ chunks:
           - stage2-gcc
           - stage2-linux-api-headers
           - stage2-make
+          - eglibc
 
-    - name: gawk
-      repo: upstream:gawk
+    - name: busybox
+      repo: upstream:busybox
       ref: baserock/samthursfield/build-essential-2
       build-mode: staging
+      prefix: /usr
       build-depends:
           - stage2-binutils
           - stage2-busybox
@@ -163,11 +201,13 @@ chunks:
           - stage2-gcc
           - stage2-linux-api-headers
           - stage2-make
+          - eglibc
 
-    - name: gcc
-      repo: upstream:gcc-tarball
+    - name: gawk
+      repo: upstream:gawk
       ref: baserock/samthursfield/build-essential-2
       build-mode: staging
+      prefix: /usr
       build-depends:
           - stage2-binutils
           - stage2-busybox
@@ -177,11 +217,13 @@ chunks:
           - stage2-gcc
           - stage2-linux-api-headers
           - stage2-make
+          - eglibc
 
-    - name: linux-api-headers
-      repo: upstream:linux
+    - name: gcc
+      repo: upstream:gcc-tarball
       ref: baserock/samthursfield/build-essential-2
       build-mode: staging
+      prefix: /usr
       build-depends:
           - stage2-binutils
           - stage2-busybox
@@ -191,11 +233,13 @@ chunks:
           - stage2-gcc
           - stage2-linux-api-headers
           - stage2-make
+          - eglibc
 
     - name: make
       repo: upstream:make
       ref: baserock/samthursfield/build-essential-2
       build-mode: staging
+      prefix: /usr
       build-depends:
           - stage2-binutils
           - stage2-busybox
@@ -205,6 +249,7 @@ chunks:
           - stage2-gcc
           - stage2-linux-api-headers
           - stage2-make
+          - eglibc
 
     # Extras that need to be in build-essential but don't need bootstrapping.
 
@@ -212,12 +257,14 @@ chunks:
       repo: upstream:ccache
       ref: baserock/samthursfield/build-essential-2
       build-mode: staging
+      prefix: /usr
       build-depends:
-          - binutils
-          - busybox
+          - stage2-binutils
+          - stage2-busybox
+          - stage2-eglibc
+          - stage2-fhs-dirs
+          - stage2-gawk
+          - stage2-gcc
+          - stage2-linux-api-headers
+          - stage2-make
           - eglibc
-          - fhs-dirs
-          - gawk
-          - gcc
-          - linux-api-headers
-          - make