tests/sources/tar.py: Converted tar test to use the CLI and enhanced

Now test to also ensure that base-dir expressions always behave
the same way regardless of whether the tarball was created with
a leading '.' or not.

14 files changed, 134 insertions, 107 deletions

diff --git a/tests/sources/tar.py b/tests/sources/tar.py
index 21e44469d..f3f90d0f2 100644
--- a/tests/sources/tar.py
+++ b/tests/sources/tar.py
@@ -2,10 +2,9 @@ import os
 import pytest
 import tarfile
 
-from buildstream import SourceError, Consistency
-from buildstream import utils
-
-from .fixture import Setup
+from buildstream._pipeline import PipelineError
+from buildstream import utils, _yaml
+from tests.testutils import cli
 
 DATA_DIR = os.path.join(
     os.path.dirname(os.path.realpath(__file__)),
@@ -21,92 +20,74 @@ def _assemble_tar(workingdir, srcdir, dstfile):
     os.chdir(old_dir)
 
 
-# Test that the source can be parsed meaningfully.
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'basic'))
-def test_create_source(tmpdir, datafiles):
-    setup = Setup(datafiles, 'target.bst', tmpdir)
-    assert(setup.source.get_kind() == 'tar')
-    assert(setup.source.url == 'http://www.example.com')
-    assert(setup.source.get_ref() == 'foo')
+def generate_project(project_dir, tmpdir):
+    project_file = os.path.join(project_dir, "project.conf")
+    _yaml.dump({
+        'name': 'foo',
+        'aliases': {
+            'tmpdir': "file:///" + str(tmpdir)
+        }
+    }, project_file)
 
 
 # Test that without ref, consistency is set appropriately.
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref'))
-def test_no_ref(tmpdir, datafiles):
-    setup = Setup(datafiles, 'target.bst', tmpdir)
-    assert(setup.source.get_consistency() == Consistency.INCONSISTENT)
-
-
-# Test that when I fetch, it ends up in the cache.
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
-def test_fetch(tmpdir, datafiles):
-    setup = Setup(datafiles, 'target.bst', tmpdir)
-    # Create a local tar
-    src_tar = os.path.join(str(tmpdir), "a.tar.gz")
-    _assemble_tar(str(datafiles), "a", src_tar)
-    setup.source.ref = utils.sha256sum(src_tar)
-
-    # Fetch the source
-    setup.source.fetch()
-
-    # File was fetched into mirror
-    assert(os.path.isfile(setup.source._get_mirror_file()))
-
-    # Fetched file is a tar
-    assert(tarfile.is_tarfile(setup.source._get_mirror_file()))
-
-    # Fetched file has the same contents as the source tar.
-    with tarfile.open(src_tar) as tar:
-        source_contents = tar.getnames()
-    with tarfile.open(setup.source._get_mirror_file()) as tar:
-        fetched_contents = tar.getnames()
-    assert(source_contents == fetched_contents)
+def test_no_ref(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    generate_project(project, tmpdir)
+    assert cli.get_element_state(project, 'target.bst') == 'no reference'
 
 
 # Test that when I fetch a nonexistent URL, errors are handled gracefully.
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
-def test_fetch_bad_url(tmpdir, datafiles):
-    setup = Setup(datafiles, 'target.bst', tmpdir)
-    with pytest.raises(SourceError):
-        setup.source.fetch()
+def test_fetch_bad_url(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    generate_project(project, tmpdir)
+
+    # Try to fetch it
+    result = cli.run(project=project, args=[
+        'fetch', 'target.bst'
+    ])
+    assert result.exit_code != 0
+    assert result.exception
+    assert isinstance(result.exception, PipelineError)
 
 
 # Test that when I fetch with an invalid ref, it fails.
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
-def test_fetch_bad_ref(tmpdir, datafiles):
-    setup = Setup(datafiles, 'target.bst', tmpdir)
-    # Create a local tar
-    src_tar = os.path.join(str(tmpdir), "a.tar.gz")
-    _assemble_tar(str(datafiles), "a", src_tar)
+def test_fetch_bad_ref(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    generate_project(project, tmpdir)
 
-    # Fetch the source
-    with pytest.raises(SourceError):
-        setup.source.fetch()
-
-
-# Test that when I track, it gives me the sha256sum of the downloaded file.
-@pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-ref'))
-def test_track(tmpdir, datafiles):
-    setup = Setup(datafiles, 'target.bst', tmpdir)
     # Create a local tar
     src_tar = os.path.join(str(tmpdir), "a.tar.gz")
-    _assemble_tar(str(datafiles), "a", src_tar)
-    tar_sha = utils.sha256sum(src_tar)
+    _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar)
 
-    assert(tar_sha == setup.source.track())
+    # Try to fetch it
+    result = cli.run(project=project, args=[
+        'fetch', 'target.bst'
+    ])
+    assert result.exit_code != 0
+    assert result.exception
+    assert isinstance(result.exception, PipelineError)
 
 
 # Test that when tracking with a ref set, there is a warning
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
-def test_track_with_ref(tmpdir, datafiles, capfd):
-    setup = Setup(datafiles, 'target.bst', tmpdir)
+def test_track_warning(cli, tmpdir, datafiles):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    generate_project(project, tmpdir)
+
     # Create a local tar
     src_tar = os.path.join(str(tmpdir), "a.tar.gz")
-    _assemble_tar(str(datafiles), "a", src_tar)
+    _assemble_tar(os.path.join(str(datafiles), "content"), "a", src_tar)
 
-    setup.source.track()
-    out, _ = capfd.readouterr()
-    assert("Potential man-in-the-middle attack!" in out)
+    # Track it
+    result = cli.run(project=project, args=[
+        'track', 'target.bst'
+    ])
+    assert result.exit_code == 0
+    assert "Potential man-in-the-middle attack!" in result.output
 
 
 def _list_dir_contents(srcdir):
@@ -121,43 +102,86 @@ def _list_dir_contents(srcdir):
 
 # Test that a staged checkout matches what was tarred up, with the default first subdir
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'fetch'))
-def test_stage_default_basedir(tmpdir, datafiles):
-    setup = Setup(datafiles, 'target.bst', tmpdir)
+@pytest.mark.parametrize("srcdir", ["a", "./a"])
+def test_stage_default_basedir(cli, tmpdir, datafiles, srcdir):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    generate_project(project, tmpdir)
+    checkoutdir = os.path.join(str(tmpdir), "checkout")
+
     # Create a local tar
     src_tar = os.path.join(str(tmpdir), "a.tar.gz")
-    _assemble_tar(str(datafiles), "a", src_tar)
-    setup.source.ref = utils.sha256sum(src_tar)
-
-    # Fetch the source
-    setup.source.fetch()
-
-    # Unpack the source
-    stage_dir = os.path.join(str(tmpdir), "stage")
-    os.makedirs(stage_dir)
-    setup.source.stage(stage_dir)
-    original_dir = os.path.join(str(datafiles), "a")
-    stage_contents = _list_dir_contents(stage_dir)
+    _assemble_tar(os.path.join(str(datafiles), "content"), srcdir, src_tar)
+
+    # Track, fetch, build, checkout
+    result = cli.run(project=project, args=['track', 'target.bst'])
+    assert result.exit_code == 0
+    result = cli.run(project=project, args=['fetch', 'target.bst'])
+    assert result.exit_code == 0
+    result = cli.run(project=project, args=['build', 'target.bst'])
+    assert result.exit_code == 0
+    result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir])
+    assert result.exit_code == 0
+
+    # Check that the content of the first directory is checked out (base-dir: '*')
+    original_dir = os.path.join(str(datafiles), "content", "a")
     original_contents = _list_dir_contents(original_dir)
-    assert(stage_contents == original_contents)
+    checkout_contents = _list_dir_contents(checkoutdir)
+    assert(checkout_contents == original_contents)
 
 
-# Test that a staged checkout matches what was tarred up, with the full tarball
+# Test that a staged checkout matches what was tarred up, with an empty base-dir
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'no-basedir'))
-def test_stage_no_basedir(tmpdir, datafiles):
-    setup = Setup(datafiles, 'target.bst', tmpdir)
+@pytest.mark.parametrize("srcdir", ["a", "./a"])
+def test_stage_no_basedir(cli, tmpdir, datafiles, srcdir):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    generate_project(project, tmpdir)
+    checkoutdir = os.path.join(str(tmpdir), "checkout")
+
+    # Create a local tar
+    src_tar = os.path.join(str(tmpdir), "a.tar.gz")
+    _assemble_tar(os.path.join(str(datafiles), "content"), srcdir, src_tar)
+
+    # Track, fetch, build, checkout
+    result = cli.run(project=project, args=['track', 'target.bst'])
+    assert result.exit_code == 0
+    result = cli.run(project=project, args=['fetch', 'target.bst'])
+    assert result.exit_code == 0
+    result = cli.run(project=project, args=['build', 'target.bst'])
+    assert result.exit_code == 0
+    result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir])
+    assert result.exit_code == 0
+
+    # Check that the full content of the tarball is checked out (base-dir: '')
+    original_dir = os.path.join(str(datafiles), "content")
+    original_contents = _list_dir_contents(original_dir)
+    checkout_contents = _list_dir_contents(checkoutdir)
+    assert(checkout_contents == original_contents)
+
+
+# Test that a staged checkout matches what was tarred up, with an explicit basedir
+@pytest.mark.datafiles(os.path.join(DATA_DIR, 'explicit-basedir'))
+@pytest.mark.parametrize("srcdir", ["a", "./a"])
+def test_stage_explicit_basedir(cli, tmpdir, datafiles, srcdir):
+    project = os.path.join(datafiles.dirname, datafiles.basename)
+    generate_project(project, tmpdir)
+    checkoutdir = os.path.join(str(tmpdir), "checkout")
+
     # Create a local tar
     src_tar = os.path.join(str(tmpdir), "a.tar.gz")
-    _assemble_tar(str(datafiles), "a", src_tar)
-    setup.source.ref = utils.sha256sum(src_tar)
-
-    # Fetch the source
-    setup.source.fetch()
-
-    # Unpack the source
-    stage_dir = os.path.join(str(tmpdir), "stage")
-    os.makedirs(stage_dir)
-    setup.source.stage(stage_dir)
-    original_dir = os.path.join(str(datafiles), "a")
-    stage_contents = _list_dir_contents(os.path.join(stage_dir, "a"))
+    _assemble_tar(os.path.join(str(datafiles), "content"), srcdir, src_tar)
+
+    # Track, fetch, build, checkout
+    result = cli.run(project=project, args=['track', 'target.bst'])
+    assert result.exit_code == 0
+    result = cli.run(project=project, args=['fetch', 'target.bst'])
+    assert result.exit_code == 0
+    result = cli.run(project=project, args=['build', 'target.bst'])
+    assert result.exit_code == 0
+    result = cli.run(project=project, args=['checkout', 'target.bst', checkoutdir])
+    assert result.exit_code == 0
+
+    # Check that the content of the first directory is checked out (base-dir: '*')
+    original_dir = os.path.join(str(datafiles), "content", "a")
     original_contents = _list_dir_contents(original_dir)
-    assert(stage_contents == original_contents)
+    checkout_contents = _list_dir_contents(checkoutdir)
+    assert(checkout_contents == original_contents)
diff --git a/tests/sources/tar/fetch/a/b/d b/tests/sources/tar/explicit-basedir/content/a/b/d
index 4bcfe98e6..4bcfe98e6 100644
--- a/tests/sources/tar/fetch/a/b/d
+++ b/tests/sources/tar/explicit-basedir/content/a/b/d
diff --git a/tests/sources/tar/fetch/a/c b/tests/sources/tar/explicit-basedir/content/a/c
index f2ad6c76f..f2ad6c76f 100644
--- a/tests/sources/tar/fetch/a/c
+++ b/tests/sources/tar/explicit-basedir/content/a/c
diff --git a/tests/sources/tar/explicit-basedir/target.bst b/tests/sources/tar/explicit-basedir/target.bst
new file mode 100644
index 000000000..fb85bdf42
--- /dev/null
+++ b/tests/sources/tar/explicit-basedir/target.bst
@@ -0,0 +1,7 @@
+kind: import
+description: The kind of this element is irrelevant.
+sources:
+- kind: tar
+  url: tmpdir:/a.tar.gz
+  ref: foo
+  base-dir: 'a'
diff --git a/tests/sources/tar/no-basedir/a/b/d b/tests/sources/tar/fetch/content/a/b/d
index 4bcfe98e6..4bcfe98e6 100644
--- a/tests/sources/tar/no-basedir/a/b/d
+++ b/tests/sources/tar/fetch/content/a/b/d
diff --git a/tests/sources/tar/no-basedir/a/c b/tests/sources/tar/fetch/content/a/c
index f2ad6c76f..f2ad6c76f 100644
--- a/tests/sources/tar/no-basedir/a/c
+++ b/tests/sources/tar/fetch/content/a/c
diff --git a/tests/sources/tar/fetch/project.conf b/tests/sources/tar/fetch/project.conf
deleted file mode 100644
index afa0f5475..000000000
--- a/tests/sources/tar/fetch/project.conf
+++ /dev/null
@@ -1,2 +0,0 @@
-# Basic project
-name: foo
diff --git a/tests/sources/tar/fetch/target.bst b/tests/sources/tar/fetch/target.bst
index 8bdf8997a..ad413a2f4 100644
--- a/tests/sources/tar/fetch/target.bst
+++ b/tests/sources/tar/fetch/target.bst
@@ -1,4 +1,4 @@
-kind: foo
+kind: import
 description: The kind of this element is irrelevant.
 sources:
 - kind: tar
diff --git a/tests/sources/tar/no-basedir/content/a/b/d b/tests/sources/tar/no-basedir/content/a/b/d
new file mode 100644
index 000000000..4bcfe98e6
--- /dev/null
+++ b/tests/sources/tar/no-basedir/content/a/b/d
@@ -0,0 +1 @@
+d
diff --git a/tests/sources/tar/no-basedir/content/a/c b/tests/sources/tar/no-basedir/content/a/c
new file mode 100644
index 000000000..f2ad6c76f
--- /dev/null
+++ b/tests/sources/tar/no-basedir/content/a/c
@@ -0,0 +1 @@
+c
diff --git a/tests/sources/tar/no-basedir/project.conf b/tests/sources/tar/no-basedir/project.conf
deleted file mode 100644
index afa0f5475..000000000
--- a/tests/sources/tar/no-basedir/project.conf
+++ /dev/null
@@ -1,2 +0,0 @@
-# Basic project
-name: foo
diff --git a/tests/sources/tar/no-basedir/target.bst b/tests/sources/tar/no-basedir/target.bst
index e6ace2796..5a34d87a3 100644
--- a/tests/sources/tar/no-basedir/target.bst
+++ b/tests/sources/tar/no-basedir/target.bst
@@ -1,4 +1,4 @@
-kind: foo
+kind: import
 description: The kind of this element is irrelevant.
 sources:
 - kind: tar
diff --git a/tests/sources/tar/no-ref/project.conf b/tests/sources/tar/no-ref/project.conf
deleted file mode 100644
index afa0f5475..000000000
--- a/tests/sources/tar/no-ref/project.conf
+++ /dev/null
@@ -1,2 +0,0 @@
-# Basic project
-name: foo
diff --git a/tests/sources/tar/no-ref/target.bst b/tests/sources/tar/no-ref/target.bst
index 84978ef23..5a73fa25b 100644
--- a/tests/sources/tar/no-ref/target.bst
+++ b/tests/sources/tar/no-ref/target.bst
@@ -1,4 +1,4 @@
-kind: foo
+kind: autotools
 description: The kind of this element is irrelevant.
 sources:
 - kind: tar