Mark buildbox tests as xfail if they require non implemented features

9 files changed, 32 insertions, 2 deletions

diff --git a/tests/integration/artifact.py b/tests/integration/artifact.py
index 56c516e67..4180bf6fd 100644
--- a/tests/integration/artifact.py
+++ b/tests/integration/artifact.py
@@ -45,6 +45,8 @@ DATA_DIR = os.path.join(
 # A test to capture the integration of the cachebuildtrees
 # behaviour, which by default is to include the buildtree
 # content of an element on caching.
+
+# Dse this really need a sandbox?
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
 def test_cache_buildtrees(cli, tmpdir, datafiles):
diff --git a/tests/integration/autotools.py b/tests/integration/autotools.py
index c4bf429f5..c985e8e6c 100644
--- a/tests/integration/autotools.py
+++ b/tests/integration/autotools.py
@@ -44,6 +44,7 @@ def test_autotools_build(cli, datafiles):
 # amhello project for this.
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_autotools_confroot_build(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
diff --git a/tests/integration/cachedfail.py b/tests/integration/cachedfail.py
index a7509ab3b..e3b5b2796 100644
--- a/tests/integration/cachedfail.py
+++ b/tests/integration/cachedfail.py
@@ -139,6 +139,7 @@ def test_build_depend_on_cached_fail(cli, datafiles):
 
 
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.parametrize("on_error", ("continue", "quit"))
 def test_push_cached_fail(cli, tmpdir, datafiles, on_error):
diff --git a/tests/integration/cmake.py b/tests/integration/cmake.py
index 84ea96af2..0fe91ed6d 100644
--- a/tests/integration/cmake.py
+++ b/tests/integration/cmake.py
@@ -36,6 +36,7 @@ def test_cmake_build(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_cmake_confroot_build(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
diff --git a/tests/integration/make.py b/tests/integration/make.py
index 664e7ca7a..4678c5319 100644
--- a/tests/integration/make.py
+++ b/tests/integration/make.py
@@ -22,6 +22,7 @@ DATA_DIR = os.path.join(
 # makehello project for this.
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 def test_make_build(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
@@ -40,6 +41,7 @@ def test_make_build(cli, datafiles):
 # Test running an executable built with make
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 def test_make_run(cli, datafiles):
     project = str(datafiles)
     element_name = 'make/makehello.bst'
diff --git a/tests/integration/script.py b/tests/integration/script.py
index fc57e8744..dc893f679 100644
--- a/tests/integration/script.py
+++ b/tests/integration/script.py
@@ -100,6 +100,7 @@ def test_script_root(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_script_no_root(cli, datafiles):
     project = str(datafiles)
     element_path = os.path.join(project, 'elements')
@@ -123,6 +124,7 @@ def test_script_no_root(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_script_cwd(cli, datafiles):
     project = str(datafiles)
     checkout = os.path.join(cli.directory, 'checkout')
diff --git a/tests/integration/shell.py b/tests/integration/shell.py
index a1f38d879..b0eca2471 100644
--- a/tests/integration/shell.py
+++ b/tests/integration/shell.py
@@ -80,6 +80,8 @@ def test_executable(cli, datafiles):
 @pytest.mark.parametrize("animal", [("Horse"), ("Pony")])
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
+# This test seems to fail or pass depending on if this file is run or the hole test suite
 def test_env_assign(cli, datafiles, animal):
     project = str(datafiles)
     expected = animal + '\n'
@@ -100,6 +102,8 @@ def test_env_assign(cli, datafiles, animal):
 @pytest.mark.parametrize("animal", [("Horse"), ("Pony")])
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
+# This test seems to fail or pass depending on if this file is run or the hole test suite
 def test_env_assign_expand_host_environ(cli, datafiles, animal):
     project = str(datafiles)
     expected = 'The animal is: {}\n'.format(animal)
@@ -123,6 +127,8 @@ def test_env_assign_expand_host_environ(cli, datafiles, animal):
 @pytest.mark.parametrize("animal", [("Horse"), ("Pony")])
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
+# This test seems to faili or pass depending on if this file is run or the hole test suite
 def test_env_assign_isolated(cli, datafiles, animal):
     project = str(datafiles)
     result = execute_shell(cli, project, ['/bin/sh', '-c', 'echo ${ANIMAL}'], isolate=True, config={
@@ -174,6 +180,7 @@ def test_no_shell(cli, datafiles):
 @pytest.mark.parametrize("path", [("/etc/pony.conf"), ("/usr/share/pony/pony.txt")])
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_host_files(cli, datafiles, path):
     project = str(datafiles)
     ponyfile = os.path.join(project, 'files', 'shell-mount', 'pony.txt')
@@ -195,6 +202,7 @@ def test_host_files(cli, datafiles, path):
 @pytest.mark.parametrize("path", [("/etc"), ("/usr/share/pony")])
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_host_files_expand_environ(cli, datafiles, path):
     project = str(datafiles)
     hostpath = os.path.join(project, 'files', 'shell-mount')
@@ -221,6 +229,7 @@ def test_host_files_expand_environ(cli, datafiles, path):
 @pytest.mark.parametrize("path", [("/etc/pony.conf"), ("/usr/share/pony/pony.txt")])
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 def test_isolated_no_mount(cli, datafiles, path):
     project = str(datafiles)
     ponyfile = os.path.join(project, 'files', 'shell-mount', 'pony.txt')
@@ -244,6 +253,7 @@ def test_isolated_no_mount(cli, datafiles, path):
 @pytest.mark.parametrize("optional", [("mandatory"), ("optional")])
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox')
 def test_host_files_missing(cli, datafiles, optional):
     project = str(datafiles)
     ponyfile = os.path.join(project, 'files', 'shell-mount', 'horsy.txt')
@@ -277,6 +287,7 @@ def test_host_files_missing(cli, datafiles, optional):
 @pytest.mark.parametrize("path", [("/etc/pony.conf"), ("/usr/share/pony/pony.txt")])
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_cli_mount(cli, datafiles, path):
     project = str(datafiles)
     ponyfile = os.path.join(project, 'files', 'shell-mount', 'pony.txt')
@@ -289,6 +300,7 @@ def test_cli_mount(cli, datafiles, path):
 # Test that we can see the workspace files in a shell
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_workspace_visible(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -323,6 +335,7 @@ def test_workspace_visible(cli, datafiles):
 # Test that '--sysroot' works
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_sysroot(cli, tmpdir, datafiles):
     project = str(datafiles)
     base_element = "base/base-alpine.bst"
@@ -366,6 +379,7 @@ def test_integration_devices(cli, datafiles):
 @pytest.mark.parametrize("build_shell", [("build"), ("nobuild")])
 @pytest.mark.parametrize("guess_element", [True, False], ids=["guess", "no-guess"])
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_integration_external_workspace(cli, tmpdir_factory, datafiles, build_shell, guess_element):
     tmpdir = tmpdir_factory.mktemp("")
     project = str(datafiles)
diff --git a/tests/integration/shellbuildtrees.py b/tests/integration/shellbuildtrees.py
index a1eecb1eb..81a279479 100644
--- a/tests/integration/shellbuildtrees.py
+++ b/tests/integration/shellbuildtrees.py
@@ -59,8 +59,8 @@ def test_buildtree_staged_forced_true(cli_integration, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
-def test_buildtree_staged_warn_non_cached(cli_integration, tmpdir, datafiles):
-    # Test that if we attempt to stage a buildtree that was never cached, we warn the user.
+def test_buildtree_staged_warn_empty_cached(cli_integration, tmpdir, datafiles):
+    # Test that if we stage a cached and empty buildtree, we warn the user.
     project = str(datafiles)
     element_name = 'build-shell/buildtree.bst'
 
diff --git a/tests/integration/workspace.py b/tests/integration/workspace.py
index 78379912c..045f8c490 100644
--- a/tests/integration/workspace.py
+++ b/tests/integration/workspace.py
@@ -20,6 +20,7 @@ DATA_DIR = os.path.join(
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_workspace_mount(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -36,6 +37,7 @@ def test_workspace_mount(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_workspace_commanddir(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -53,6 +55,7 @@ def test_workspace_commanddir(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_workspace_updated_dependency(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -107,6 +110,7 @@ def test_workspace_updated_dependency(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_workspace_update_dependency_failed(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -182,6 +186,7 @@ def test_workspace_update_dependency_failed(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_updated_dependency_nested(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -235,6 +240,7 @@ def test_updated_dependency_nested(cli, datafiles):
 
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_incremental_configure_commands_run_only_once(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')
@@ -287,6 +293,7 @@ def test_incremental_configure_commands_run_only_once(cli, datafiles):
 #
 @pytest.mark.datafiles(DATA_DIR)
 @pytest.mark.skipif(not HAVE_SANDBOX, reason='Only available with a functioning sandbox')
+@pytest.mark.xfail(HAVE_SANDBOX == 'buildbox', reason='Not working with BuildBox', strict=True)
 def test_workspace_missing_last_successful(cli, datafiles):
     project = str(datafiles)
     workspace = os.path.join(cli.directory, 'workspace')