_stream.py, _pipeline.py: Refactoring of the pipeline itself

Here the pipeline becomes essentially stateless, some dangling
state remains to be factored out because of frontend accesses
which will be changed in a later commit.

Essentially, the Pipeline.load() method no longer has any knowledge
of the specific purposes of the loaded targets, and now takes
a list of target groups and returns a corresponding list of element
groups.

The Stream() business logic methods now use other pipeline helper
methods to create and filter lists from the loaded target elements.

The Stream() also finally absorbs the Scheduler frontend facing
APIs. However Queues are still exposed on the Stream object for
logging purposes and through callbacks such that the frontend can
retry elements.

2 files changed, 36 insertions, 66 deletions

diff --git a/tests/frontend/buildtrack.py b/tests/frontend/buildtrack.py
index 84d543e52..3f0a3adbe 100644
--- a/tests/frontend/buildtrack.py
+++ b/tests/frontend/buildtrack.py
@@ -31,40 +31,24 @@ def create_element(repo, name, path, dependencies, ref=None):
 
 @pytest.mark.datafiles(os.path.join(DATA_DIR))
 @pytest.mark.parametrize("ref_storage", [('inline'), ('project.refs')])
-@pytest.mark.parametrize("exceptions,excepted", [
+@pytest.mark.parametrize("track_targets,exceptions,tracked", [
     # Test with no exceptions
-    ([], []),
+    (['0.bst'], [], ['0.bst', '2.bst', '3.bst', '4.bst', '5.bst', '6.bst', '7.bst']),
+    (['3.bst'], [], ['3.bst', '4.bst', '5.bst', '6.bst']),
+    (['2.bst', '3.bst'], [], ['2.bst', '3.bst', '4.bst', '5.bst', '6.bst', '7.bst']),
 
     # Test excepting '2.bst'
-    (['2.bst'], ['2.bst', '7.bst']),
+    (['0.bst'], ['2.bst'], ['0.bst', '3.bst', '4.bst', '5.bst', '6.bst']),
+    (['3.bst'], ['2.bst'], []),
+    (['2.bst', '3.bst'], ['2.bst'], ['3.bst', '4.bst', '5.bst', '6.bst']),
 
     # Test excepting '2.bst' and '3.bst'
-    (['2.bst', '3.bst'], [
-        '2.bst', '3.bst', '4.bst',
-        '5.bst', '6.bst', '7.bst'
-    ])
+    (['0.bst'], ['2.bst', '3.bst'], ['0.bst']),
+    (['3.bst'], ['2.bst', '3.bst'], []),
+    (['2.bst', '3.bst'], ['2.bst', '3.bst'], [])
 ])
-@pytest.mark.parametrize("track_targets,tracked", [
-    # Test tracking the main target element
-    (['0.bst'], [
-        '0.bst', '2.bst', '3.bst',
-        '4.bst', '5.bst', '6.bst', '7.bst'
-    ]),
-
-    # Test tracking a child element
-    (['3.bst'], [
-        '3.bst', '4.bst', '5.bst',
-        '6.bst'
-    ]),
-
-    # Test tracking multiple children
-    (['2.bst', '3.bst'], [
-        '2.bst', '3.bst', '4.bst',
-        '5.bst', '6.bst', '7.bst'
-    ])
-])
-def test_build_track(cli, datafiles, tmpdir, ref_storage, track_targets,
-                     exceptions, tracked, excepted):
+def test_build_track(cli, datafiles, tmpdir, ref_storage,
+                     track_targets, exceptions, tracked):
     project = os.path.join(datafiles.dirname, datafiles.basename)
     dev_files_path = os.path.join(project, 'files', 'dev-files')
     element_path = os.path.join(project, 'elements')
@@ -102,7 +86,7 @@ def test_build_track(cli, datafiles, tmpdir, ref_storage, track_targets,
     for element, dependencies in create_elements.items():
         # Test the element inconsistency resolution by ensuring that
         # only elements that aren't tracked have refs
-        if element in set(tracked) - set(excepted):
+        if element in set(tracked):
             # Elements which should not have a ref set
             #
             create_element(repo, element, element_path, dependencies)
@@ -133,14 +117,14 @@ def test_build_track(cli, datafiles, tmpdir, ref_storage, track_targets,
     result = cli.run(project=project, silent=True, args=args)
     tracked_elements = result.get_tracked_elements()
 
-    assert set(tracked_elements) == set(tracked) - set(excepted)
+    assert set(tracked_elements) == set(tracked)
 
     # Delete element sources
     source_dir = os.path.join(project, 'cache', 'sources')
     shutil.rmtree(source_dir)
 
     # Delete artifacts one by one and assert element states
-    for target in set(tracked) - set(excepted):
+    for target in set(tracked):
         cli.remove_artifact_from_cache(project, target)
 
         # Assert that it's tracked
@@ -154,40 +138,24 @@ def test_build_track(cli, datafiles, tmpdir, ref_storage, track_targets,
 
 
 @pytest.mark.datafiles(os.path.join(DATA_DIR))
-@pytest.mark.parametrize("exceptions,excepted", [
+@pytest.mark.parametrize("track_targets,exceptions,tracked", [
     # Test with no exceptions
-    ([], []),
+    (['0.bst'], [], ['0.bst', '2.bst', '3.bst', '4.bst', '5.bst', '6.bst', '7.bst']),
+    (['3.bst'], [], ['3.bst', '4.bst', '5.bst', '6.bst']),
+    (['2.bst', '3.bst'], [], ['2.bst', '3.bst', '4.bst', '5.bst', '6.bst', '7.bst']),
 
     # Test excepting '2.bst'
-    (['2.bst'], ['2.bst', '7.bst']),
+    (['0.bst'], ['2.bst'], ['0.bst', '3.bst', '4.bst', '5.bst', '6.bst']),
+    (['3.bst'], ['2.bst'], []),
+    (['2.bst', '3.bst'], ['2.bst'], ['3.bst', '4.bst', '5.bst', '6.bst']),
 
     # Test excepting '2.bst' and '3.bst'
-    (['2.bst', '3.bst'], [
-        '2.bst', '3.bst', '4.bst',
-        '5.bst', '6.bst', '7.bst'
-    ])
-])
-@pytest.mark.parametrize("track_targets,tracked", [
-    # Test tracking the main target element
-    (['0.bst'], [
-        '0.bst', '2.bst', '3.bst',
-        '4.bst', '5.bst', '6.bst', '7.bst'
-    ]),
-
-    # Test tracking a child element
-    (['3.bst'], [
-        '3.bst', '4.bst', '5.bst',
-        '6.bst'
-    ]),
-
-    # Test tracking multiple children
-    (['2.bst', '3.bst'], [
-        '2.bst', '3.bst', '4.bst',
-        '5.bst', '6.bst', '7.bst'
-    ])
+    (['0.bst'], ['2.bst', '3.bst'], ['0.bst']),
+    (['3.bst'], ['2.bst', '3.bst'], []),
+    (['2.bst', '3.bst'], ['2.bst', '3.bst'], [])
 ])
 def test_build_track_update(cli, datafiles, tmpdir, track_targets,
-                            exceptions, tracked, excepted):
+                            exceptions, tracked):
     project = os.path.join(datafiles.dirname, datafiles.basename)
     dev_files_path = os.path.join(project, 'files', 'dev-files')
     element_path = os.path.join(project, 'elements')
@@ -231,7 +199,7 @@ def test_build_track_update(cli, datafiles, tmpdir, track_targets,
     result = cli.run(project=project, silent=True, args=args)
     tracked_elements = result.get_tracked_elements()
 
-    assert set(tracked_elements) == set(tracked) - set(excepted)
+    assert set(tracked_elements) == set(tracked)
 
 
 @pytest.mark.datafiles(os.path.join(DATA_DIR))
diff --git a/tests/plugins/pipeline.py b/tests/plugins/pipeline.py
index db683094b..65929cf50 100644
--- a/tests/plugins/pipeline.py
+++ b/tests/plugins/pipeline.py
@@ -23,23 +23,25 @@ def create_pipeline(tmpdir, basedir, target):
 
     context.set_message_handler(dummy_handler)
 
-    return Pipeline(context, project, None, [target], [])
+    pipeline = Pipeline(context, project, None)
+    targets, = pipeline.load([(target,)])
+    return targets
 
 
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'customsource'))
 def test_customsource(datafiles, tmpdir):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    pipeline = create_pipeline(tmpdir, basedir, 'simple.bst')
-    assert(pipeline.targets[0].get_kind() == "autotools")
+    targets = create_pipeline(tmpdir, basedir, 'simple.bst')
+    assert(targets[0].get_kind() == "autotools")
 
 
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'customelement'))
 def test_customelement(datafiles, tmpdir):
 
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
-    pipeline = create_pipeline(tmpdir, basedir, 'simple.bst')
-    assert(pipeline.targets[0].get_kind() == "foo")
+    targets = create_pipeline(tmpdir, basedir, 'simple.bst')
+    assert(targets[0].get_kind() == "foo")
 
 
 @pytest.mark.datafiles(os.path.join(DATA_DIR, 'badversionsource'))
@@ -47,7 +49,7 @@ def test_badversionsource(datafiles, tmpdir):
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
 
     with pytest.raises(LoadError) as exc:
-        pipeline = create_pipeline(tmpdir, basedir, 'simple.bst')
+        targets = create_pipeline(tmpdir, basedir, 'simple.bst')
 
     assert exc.value.reason == LoadErrorReason.UNSUPPORTED_PLUGIN
 
@@ -57,6 +59,6 @@ def test_badversionelement(datafiles, tmpdir):
     basedir = os.path.join(datafiles.dirname, datafiles.basename)
 
     with pytest.raises(LoadError) as exc:
-        pipeline = create_pipeline(tmpdir, basedir, 'simple.bst')
+        targets = create_pipeline(tmpdir, basedir, 'simple.bst')
 
     assert exc.value.reason == LoadErrorReason.UNSUPPORTED_PLUGIN