_scheduler: Fix bookkeeping of terminated jobs

 * Enhanced the base Job class to bookkeep which jobs have
   been terminated, and to consider them as `skipped` when
   asked via the `skipped` property.

 * Enhanced the base Queue class to bookkeep the job statuses
   more carefully.

This fixes #479

2 files changed, 24 insertions, 9 deletions

diff --git a/buildstream/_scheduler/jobs/job.py b/buildstream/_scheduler/jobs/job.py
index d77fa0c82..d2f5f6536 100644
--- a/buildstream/_scheduler/jobs/job.py
+++ b/buildstream/_scheduler/jobs/job.py
@@ -119,6 +119,8 @@ class Job():
         self._result = None                    # Return value of child action in the parent
         self._tries = 0                        # Try count, for retryable jobs
         self._skipped_flag = False             # Indicate whether the job was skipped.
+        self._terminated = False               # Whether this job has been explicitly terminated
+
         # If False, a retry will not be attempted regardless of whether _tries is less than _max_retries.
         #
         self._retry_flag = True
@@ -188,6 +190,8 @@ class Job():
         # Terminate the process using multiprocessing API pathway
         self._process.terminate()
 
+        self._terminated = True
+
     # terminate_wait()
     #
     # Wait for terminated jobs to complete
@@ -271,18 +275,22 @@ class Job():
     # running the integration commands).
     #
     # Args:
-    #     (int): The plugin identifier for this task
+    #     task_id (int): The plugin identifier for this task
     #
     def set_task_id(self, task_id):
         self._task_id = task_id
 
     # skipped
     #
+    # This will evaluate to True if the job was skipped
+    # during processing, or if it was forcefully terminated.
+    #
     # Returns:
-    #    bool: True if the job was skipped while processing.
+    #    (bool): Whether the job should appear as skipped
+    #
     @property
     def skipped(self):
-        return self._skipped_flag
+        return self._skipped_flag or self._terminated
 
     #######################################################
     #                  Abstract Methods                   #
diff --git a/buildstream/_scheduler/queues/queue.py b/buildstream/_scheduler/queues/queue.py
index 15467ca67..af4698350 100644
--- a/buildstream/_scheduler/queues/queue.py
+++ b/buildstream/_scheduler/queues/queue.py
@@ -325,15 +325,22 @@ class Queue():
                           detail=traceback.format_exc())
             self.failed_elements.append(element)
         else:
-
-            # No exception occured, handle the success/failure state in the normal way
             #
+            # No exception occured in post processing
+            #
+
+            # Only place in the output done queue if the job
+            # was considered successful
             if success:
                 self._done_queue.append(job)
-                if not job.skipped:
-                    self.processed_elements.append(element)
-                else:
-                    self.skipped_elements.append(element)
+
+            # A Job can be skipped whether or not it has failed,
+            # we want to only bookkeep them as processed or failed
+            # if they are not skipped.
+            if job.skipped:
+                self.skipped_elements.append(element)
+            elif success:
+                self.processed_elements.append(element)
             else:
                 self.failed_elements.append(element)