Add --host-arch and --target-arch, and 'host-arches' conditional

This command adds initial cross-compilation support to BuildStream.

It has been tested against a converted version of the Baserock
compiler bootstrap and used to cross build sysroots for armv8l64 and ppc64l
from an x86_64 host.

For example, to build a sysroot for ARM v8 64-bit you can do this:

    bst build --target-arch=armv8b64 gnu-toolchain/stage2.bst

This would cause the adapted Baserock definitions to produce a stage1 simple
cross compiler that runs on the native architecture and produces armv8b64
binaries, and then cross build a stage2 sysroot that executes on armv8b64.

Currently the --host-arch option does nothing of use. It will one day
enable host-incompatible builds using a QEMU-powered cross sandbox.

The `--arch=` option is now shorthand for `--host-arch= --target-arch=`.

Elements have 2 new variables available, %{bst-host-arch} and
%{bst-target-arch}. The 'arches' conditional now follows %{bst-target-arch},
while the new 'host-arches' conditional follows %{bst-host-arch}. All
of --arch, --host-arch and --target-arch default to the output of `uname -a`.

There's no magic here that would make all BuildStream elements suddenly
able to cross compile. It is up to an individual element to support this by
honouring %{bst-target-arch} in whatever way makes sense.

5 files changed, 114 insertions, 42 deletions

diff --git a/tests/loader/arches.py b/tests/loader/arches.py
index 99602d181..bc69d5b8f 100644
--- a/tests/loader/arches.py
+++ b/tests/loader/arches.py
@@ -19,7 +19,7 @@ def test_simple_conditional_nomatch(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
     loader = Loader(
-        basedir, 'elements/simple-conditional.bst', None, 'arm')
+        basedir, 'elements/simple-conditional.bst', None, 'arm', None)
 
     element = loader.load()
     assert(isinstance(element, MetaElement))
@@ -34,7 +34,7 @@ def test_simple_conditional_x86_64(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
     loader = Loader(
-        basedir, 'elements/simple-conditional.bst', None, 'x86_64')
+        basedir, 'elements/simple-conditional.bst', None, 'x86_64', None)
 
     element = loader.load()
     assert(isinstance(element, MetaElement))
@@ -49,7 +49,7 @@ def test_simple_conditional_x86_32(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
     loader = Loader(
-        basedir, 'elements/simple-conditional.bst', None, 'x86_32')
+        basedir, 'elements/simple-conditional.bst', None, 'x86_32', None)
 
     element = loader.load()
     assert(isinstance(element, MetaElement))
@@ -60,6 +60,58 @@ def test_simple_conditional_x86_32(datafiles):
 
 
 ##############################################################
+#            Test Arch and Host-Arch Conditionals            #
+##############################################################
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_host_arch_conditional_armv8(datafiles):
+
+    basedir = os.path.join(datafiles.dirname, datafiles.basename)
+    loader = Loader(
+        basedir, 'elements/host-arch-conditional.bst', None, 'armv8', None)
+
+    element = loader.load()
+    assert(isinstance(element, MetaElement))
+    number = element.config.get('number')
+
+    # armv8 host-arch overrides the number to 88
+    assert(number == 88)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_host_arch_conditional_ignores_target_arch(datafiles):
+
+    basedir = os.path.join(datafiles.dirname, datafiles.basename)
+    loader = Loader(
+        basedir, 'elements/host-arch-conditional.bst', None, 'armv8', 'x86_32')
+
+    element = loader.load()
+    assert(isinstance(element, MetaElement))
+    number = element.config.get('number')
+
+    # The setting a target-arch has no effect on host-arches: the number is
+    # still 88
+    assert(number == 88)
+
+
+@pytest.mark.datafiles(DATA_DIR)
+def test_host_arch_conditional_overridden(datafiles):
+
+    basedir = os.path.join(datafiles.dirname, datafiles.basename)
+    loader = Loader(
+        basedir, 'elements/host-arch-conditional.bst', None, 'armv8', 'x86_64')
+
+    element = loader.load()
+    assert(isinstance(element, MetaElement))
+    number = element.config.get('number')
+
+    # The 'arches' conditional follows the target architecture, and overrides
+    # anything specified in 'host-arches'.
+    assert(number == 6)
+
+
+##############################################################
 #            Test Arch Conditionals inside Variants          #
 ##############################################################
 @pytest.mark.datafiles(DATA_DIR)
@@ -67,7 +119,7 @@ def test_variant_arch_default(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
     loader = Loader(
-        basedir, 'elements/variant-arch-conditional.bst', None, 'arm')
+        basedir, 'elements/variant-arch-conditional.bst', None, 'arm', None)
 
     element = loader.load()
     assert(isinstance(element, MetaElement))
@@ -82,7 +134,7 @@ def test_variant_arch_default_x86_64(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
     loader = Loader(
-        basedir, 'elements/variant-arch-conditional.bst', None, 'x86_64')
+        basedir, 'elements/variant-arch-conditional.bst', None, 'x86_64', None)
 
     element = loader.load()
     assert(isinstance(element, MetaElement))
@@ -96,7 +148,7 @@ def test_variant_arch_default_x86_32(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
     loader = Loader(
-        basedir, 'elements/variant-arch-conditional.bst', None, 'x86_32')
+        basedir, 'elements/variant-arch-conditional.bst', None, 'x86_32', None)
 
     element = loader.load()
     assert(isinstance(element, MetaElement))
@@ -110,7 +162,7 @@ def test_variant_arch_pink_pony(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
     loader = Loader(
-        basedir, 'elements/variant-arch-conditional.bst', 'pink', 'arm')
+        basedir, 'elements/variant-arch-conditional.bst', 'pink', 'arm', None)
 
     element = loader.load()
     assert(isinstance(element, MetaElement))
@@ -125,7 +177,7 @@ def test_variant_arch_pink_pony_x86_64(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
     loader = Loader(
-        basedir, 'elements/variant-arch-conditional.bst', 'pink', 'x86_64')
+        basedir, 'elements/variant-arch-conditional.bst', 'pink', 'x86_64', None)
 
     element = loader.load()
     assert(isinstance(element, MetaElement))
@@ -139,7 +191,7 @@ def test_variant_arch_pink_pony_x86_32(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
     loader = Loader(
-        basedir, 'elements/variant-arch-conditional.bst', 'pink', 'x86_32')
+        basedir, 'elements/variant-arch-conditional.bst', 'pink', 'x86_32', None)
 
     element = loader.load()
     assert(isinstance(element, MetaElement))
@@ -153,7 +205,7 @@ def test_variant_arch_blue_pony(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
     loader = Loader(
-        basedir, 'elements/variant-arch-conditional.bst', 'blue', 'arm')
+        basedir, 'elements/variant-arch-conditional.bst', 'blue', 'arm', None)
 
     element = loader.load()
     assert(isinstance(element, MetaElement))
@@ -168,7 +220,7 @@ def test_variant_arch_blue_pony_x86_64(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
     loader = Loader(
-        basedir, 'elements/variant-arch-conditional.bst', 'blue', 'x86_64')
+        basedir, 'elements/variant-arch-conditional.bst', 'blue', 'x86_64', None)
 
     element = loader.load()
     assert(isinstance(element, MetaElement))
@@ -182,7 +234,7 @@ def test_variant_arch_blue_pony_x86_32(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
     loader = Loader(
-        basedir, 'elements/variant-arch-conditional.bst', 'blue', 'x86_32')
+        basedir, 'elements/variant-arch-conditional.bst', 'blue', 'x86_32', None)
 
     element = loader.load()
     assert(isinstance(element, MetaElement))
diff --git a/tests/loader/arches/elements/host-arch-conditional.bst b/tests/loader/arches/elements/host-arch-conditional.bst
new file mode 100644
index 000000000..4c9272feb
--- /dev/null
+++ b/tests/loader/arches/elements/host-arch-conditional.bst
@@ -0,0 +1,20 @@
+kind: pony
+description: |
+  An element with host-arches and arches conditionals.
+
+  The 'arches' conditional follows --target-arch if specified, while the
+  'host-arches' conditional does not. The 'arches' conditional overrides
+  'host-arches' if both for some reason try to override the same thing.
+config:
+  number: 5
+arches:
+  x86_64:
+    config:
+      number: 6
+host-arches:
+  x86_64:
+    config:
+      number: 66
+  armv8:
+    config:
+      number: 88
diff --git a/tests/loader/basics.py b/tests/loader/basics.py
index f706d06ec..d7f85b0fe 100644
--- a/tests/loader/basics.py
+++ b/tests/loader/basics.py
@@ -18,7 +18,7 @@ DATA_DIR = os.path.join(
 def test_one_file(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = Loader(basedir, 'elements/onefile.bst', None, None)
+    loader = Loader(basedir, 'elements/onefile.bst', None, None, None)
 
     element = loader.load()
 
@@ -30,7 +30,7 @@ def test_one_file(datafiles):
 def test_missing_file(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = Loader(basedir, 'elements/missing.bst', None, None)
+    loader = Loader(basedir, 'elements/missing.bst', None, None, None)
 
     with pytest.raises(LoadError) as exc:
         element = loader.load()
@@ -42,7 +42,7 @@ def test_missing_file(datafiles):
 def test_invalid_reference(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = Loader(basedir, 'elements/badreference.bst', None, None)
+    loader = Loader(basedir, 'elements/badreference.bst', None, None, None)
 
     with pytest.raises(LoadError) as exc:
         element = loader.load()
@@ -54,7 +54,7 @@ def test_invalid_reference(datafiles):
 def test_invalid_yaml(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = Loader(basedir, 'elements/badfile.bst', None, None)
+    loader = Loader(basedir, 'elements/badfile.bst', None, None, None)
 
     with pytest.raises(LoadError) as exc:
         element = loader.load()
@@ -69,6 +69,6 @@ def test_fail_fullpath_target(datafiles):
     fullpath = os.path.join(basedir, 'elements', 'onefile.bst')
 
     with pytest.raises(LoadError) as exc:
-        loader = Loader(basedir, fullpath, None, None)
+        loader = Loader(basedir, fullpath, None, None, None)
 
     assert (exc.value.reason == LoadErrorReason.INVALID_DATA)
diff --git a/tests/loader/dependencies.py b/tests/loader/dependencies.py
index 8a88bcaf8..a701d4723 100644
--- a/tests/loader/dependencies.py
+++ b/tests/loader/dependencies.py
@@ -18,7 +18,7 @@ DATA_DIR = os.path.join(
 def test_two_files(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = Loader(basedir, 'elements/target.bst', None, None)
+    loader = Loader(basedir, 'elements/target.bst', None, None, None)
     element = loader.load()
 
     assert(isinstance(element, MetaElement))
@@ -34,7 +34,7 @@ def test_two_files(datafiles):
 def test_shared_dependency(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = Loader(basedir, 'elements/shareddeptarget.bst', None, None)
+    loader = Loader(basedir, 'elements/shareddeptarget.bst', None, None, None)
     element = loader.load()
 
     # Toplevel is 'pony' with 2 dependencies
@@ -77,7 +77,7 @@ def test_shared_dependency(datafiles):
 def test_dependency_dict(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = Loader(basedir, 'elements/target-depdict.bst', None, None)
+    loader = Loader(basedir, 'elements/target-depdict.bst', None, None, None)
     element = loader.load()
 
     assert(isinstance(element, MetaElement))
@@ -92,7 +92,7 @@ def test_dependency_dict(datafiles):
 @pytest.mark.datafiles(DATA_DIR)
 def test_invalid_dependency_declaration(datafiles):
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = Loader(basedir, 'elements/invaliddep.bst', None, None)
+    loader = Loader(basedir, 'elements/invaliddep.bst', None, None, None)
 
     with pytest.raises(LoadError) as exc:
         element = loader.load()
@@ -103,7 +103,7 @@ def test_invalid_dependency_declaration(datafiles):
 @pytest.mark.datafiles(DATA_DIR)
 def test_circular_dependency(datafiles):
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = Loader(basedir, 'elements/circulartarget.bst', None, None)
+    loader = Loader(basedir, 'elements/circulartarget.bst', None, None, None)
 
     with pytest.raises(LoadError) as exc:
         element = loader.load()
@@ -114,7 +114,7 @@ def test_circular_dependency(datafiles):
 @pytest.mark.datafiles(DATA_DIR)
 def test_invalid_dependency_type(datafiles):
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = Loader(basedir, 'elements/invaliddeptype.bst', None, None)
+    loader = Loader(basedir, 'elements/invaliddeptype.bst', None, None, None)
 
     with pytest.raises(LoadError) as exc:
         element = loader.load()
@@ -125,7 +125,7 @@ def test_invalid_dependency_type(datafiles):
 @pytest.mark.datafiles(DATA_DIR)
 def test_build_dependency(datafiles):
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = Loader(basedir, 'elements/builddep.bst', None, None)
+    loader = Loader(basedir, 'elements/builddep.bst', None, None, None)
     element = loader.load()
 
     assert(isinstance(element, MetaElement))
@@ -141,7 +141,7 @@ def test_build_dependency(datafiles):
 @pytest.mark.datafiles(DATA_DIR)
 def test_runtime_dependency(datafiles):
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = Loader(basedir, 'elements/runtimedep.bst', None, None)
+    loader = Loader(basedir, 'elements/runtimedep.bst', None, None, None)
     element = loader.load()
 
     assert(isinstance(element, MetaElement))
@@ -157,7 +157,7 @@ def test_runtime_dependency(datafiles):
 @pytest.mark.datafiles(DATA_DIR)
 def test_build_runtime_dependency(datafiles):
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = Loader(basedir, 'elements/target.bst', None, None)
+    loader = Loader(basedir, 'elements/target.bst', None, None, None)
     element = loader.load()
 
     assert(isinstance(element, MetaElement))
diff --git a/tests/loader/variants.py b/tests/loader/variants.py
index 0e20faf93..865ffa268 100644
--- a/tests/loader/variants.py
+++ b/tests/loader/variants.py
@@ -18,7 +18,7 @@ DATA_DIR = os.path.join(
 def test_variant_not_list(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = Loader(basedir, 'elements/variants-not-list.bst', None, None)
+    loader = Loader(basedir, 'elements/variants-not-list.bst', None, None, None)
 
     with pytest.raises(LoadError) as exc:
         element = loader.load()
@@ -30,7 +30,7 @@ def test_variant_not_list(datafiles):
 def test_variant_unnamed(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = Loader(basedir, 'elements/unnamed-variant.bst', None, None)
+    loader = Loader(basedir, 'elements/unnamed-variant.bst', None, None, None)
 
     with pytest.raises(LoadError) as exc:
         element = loader.load()
@@ -42,7 +42,7 @@ def test_variant_unnamed(datafiles):
 def test_variant_bad_name(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = Loader(basedir, 'elements/variant-bad-name.bst', None, None)
+    loader = Loader(basedir, 'elements/variant-bad-name.bst', None, None, None)
 
     with pytest.raises(LoadError) as exc:
         element = loader.load()
@@ -54,7 +54,7 @@ def test_variant_bad_name(datafiles):
 def test_variant_only_one(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = Loader(basedir, 'elements/only-one-variant.bst', None, None)
+    loader = Loader(basedir, 'elements/only-one-variant.bst', None, None, None)
 
     with pytest.raises(LoadError) as exc:
         element = loader.load()
@@ -67,7 +67,7 @@ def test_variant_illegal_composite(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
     loader = Loader(
-        basedir, 'elements/variant-illegal-composite.bst', None, None)
+        basedir, 'elements/variant-illegal-composite.bst', None, None, None)
 
     with pytest.raises(LoadError) as exc:
         element = loader.load()
@@ -83,7 +83,7 @@ def test_variant_simple_composite_default(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
     loader = Loader(
-        basedir, 'elements/simple-variant-compositing.bst', None, None)
+        basedir, 'elements/simple-variant-compositing.bst', None, None, None)
 
     element = loader.load()
 
@@ -100,7 +100,7 @@ def test_variant_simple_composite_pink_pony(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
     loader = Loader(
-        basedir, 'elements/simple-variant-compositing.bst', 'pink', None)
+        basedir, 'elements/simple-variant-compositing.bst', 'pink', None, None)
 
     element = loader.load()
 
@@ -117,7 +117,7 @@ def test_variant_simple_composite_blue_pony(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
     loader = Loader(
-        basedir, 'elements/simple-variant-compositing.bst', 'blue', None)
+        basedir, 'elements/simple-variant-compositing.bst', 'blue', None, None)
 
     element = loader.load()
 
@@ -155,7 +155,7 @@ def test_variant_simple_dependency_default(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
     loader = Loader(
-        basedir, 'elements/simple-dependency-variants.bst', None, None)
+        basedir, 'elements/simple-dependency-variants.bst', None, None, None)
     element = loader.load()
 
     assert(isinstance(element, MetaElement))
@@ -170,7 +170,7 @@ def test_variant_simple_dependency_pink_pony(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
     loader = Loader(
-        basedir, 'elements/simple-dependency-variants.bst', 'pink', None)
+        basedir, 'elements/simple-dependency-variants.bst', 'pink', None, None)
     element = loader.load()
 
     assert(isinstance(element, MetaElement))
@@ -185,7 +185,7 @@ def test_variant_simple_dependency_blue_pony(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
     loader = Loader(
-        basedir, 'elements/simple-dependency-variants.bst', 'blue', None)
+        basedir, 'elements/simple-dependency-variants.bst', 'blue', None, None)
     element = loader.load()
 
     assert(isinstance(element, MetaElement))
@@ -200,7 +200,7 @@ def test_variant_indirect_dependency_default(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
     loader = Loader(
-        basedir, 'elements/indirect-dependency-variants.bst', None, None)
+        basedir, 'elements/indirect-dependency-variants.bst', None, None, None)
     element = loader.load()
 
     assert(isinstance(element, MetaElement))
@@ -219,7 +219,7 @@ def test_variant_indirect_dependency_blue_pony(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
     loader = Loader(
-        basedir, 'elements/indirect-dependency-variants.bst', 'blue', None)
+        basedir, 'elements/indirect-dependency-variants.bst', 'blue', None, None)
     element = loader.load()
 
     assert(isinstance(element, MetaElement))
@@ -238,7 +238,7 @@ def test_variant_indirect_dependency_pink_pony(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
     loader = Loader(
-        basedir, 'elements/indirect-dependency-variants.bst', 'pink', None)
+        basedir, 'elements/indirect-dependency-variants.bst', 'pink', None, None)
     element = loader.load()
 
     assert(isinstance(element, MetaElement))
@@ -256,7 +256,7 @@ def test_variant_indirect_dependency_pink_pony(datafiles):
 def test_engine_resolve_agreement(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = Loader(basedir, 'elements/tricky.bst', None, None)
+    loader = Loader(basedir, 'elements/tricky.bst', None, None, None)
     element = loader.load()
 
     assert(isinstance(element, MetaElement))
@@ -271,7 +271,7 @@ def test_engine_resolve_agreement(datafiles):
 def test_engine_disagreement(datafiles):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    loader = Loader(basedir, 'elements/disagreement.bst', None, None)
+    loader = Loader(basedir, 'elements/disagreement.bst', None, None, None)
 
     with pytest.raises(LoadError) as exc:
         element = loader.load()