Merge remote-tracking branch 'origin/baserock/liw/lc-list-failed-jobs'

Reviewed-By: Francisco Redondo Marchena <francisco.marchena@codethink.co.uk>
Reviewed-By: Sam Thursfield <sam.thursfield@codethink.co.uk>

4 files changed, 115 insertions, 15 deletions

diff --git a/lorrycontroller/showlorry.py b/lorrycontroller/showlorry.py
index fc336a5..19f9429 100644
--- a/lorrycontroller/showlorry.py
+++ b/lorrycontroller/showlorry.py
@@ -24,32 +24,43 @@ import bottle
 import lorrycontroller
 
 
-class ShowLorry(lorrycontroller.LorryControllerRoute):
+class ShowLorryBase(object):
+
+    def get_lorry_info_with_job_lists(self, statedb, path):
+        obj= statedb.get_lorry_info(path)
+        obj['jobs'] = statedb.get_jobs_for_lorry(path)
+        obj['failed_jobs'] = statedb.get_failed_jobs_for_lorry(path)
+        return obj
+
+
+class ShowLorry(ShowLorryBase, lorrycontroller.LorryControllerRoute):
 
     http_method = 'GET'
     path = '/1.0/lorry/<path:path>'
 
     def run(self, **kwargs):
         logging.info('%s %s called', self.http_method, self.path)
-        statedb = self.open_statedb()
-        try:
-            return statedb.get_lorry_info(kwargs['path'])
-        except lorrycontroller.LorryNotFoundError as e:
-            bottle.abort(404, str(e))
+        with self.open_statedb() as statedb:
+            try:
+                return self.get_lorry_info_with_job_lists(statedb, kwargs['path'])
+            except lorrycontroller.LorryNotFoundError as e:
+                bottle.abort(404, str(e))
 
 
-class ShowLorryHTML(lorrycontroller.LorryControllerRoute):
+class ShowLorryHTML(ShowLorryBase, lorrycontroller.LorryControllerRoute):
 
     http_method = 'GET'
     path = '/1.0/lorry-html/<path:path>'
 
     def run(self, **kwargs):
         logging.info('%s %s called', self.http_method, self.path)
-        statedb = self.open_statedb()
-        try:
-            lorry_info = statedb.get_lorry_info(kwargs['path'])
-        except lorrycontroller.LorryNotFoundError as e:
-            bottle.abort(404, str(e))
+
+        with self.open_statedb() as statedb:
+            try:
+                lorry_info = self.get_lorry_info_with_job_lists(
+                    statedb, kwargs['path'])
+            except lorrycontroller.LorryNotFoundError as e:
+                bottle.abort(404, str(e))
 
         renderer = lorrycontroller.StatusRenderer()
         shower = lorrycontroller.JobShower()
diff --git a/lorrycontroller/statedb.py b/lorrycontroller/statedb.py
index fd7857d..7a26098 100644
--- a/lorrycontroller/statedb.py
+++ b/lorrycontroller/statedb.py
@@ -473,6 +473,19 @@ class StateDB(object):
             'output': row[10],
             }
 
+    def get_jobs_for_lorry(self, path):
+        c = self.get_cursor()
+        c.execute('SELECT job_id FROM jobs WHERE path=?', (path,))
+        return [row[0] for row in c.fetchall()]
+
+    def get_failed_jobs_for_lorry(self, path):
+        c = self.get_cursor()
+        c.execute(
+            'SELECT job_id FROM jobs '
+            'WHERE path=? AND exit != \'no\' AND exit != 0',
+            (path,))
+        return [row[0] for row in c.fetchall()]
+
     def add_new_job(self, job_id, host, pid, path, started):
         logging.debug(
             'StateDB.add_new_job(%r, %r, %r, %r, %r) called',
diff --git a/templates/lorry.tpl b/templates/lorry.tpl
index fad85cd..7c475e0 100644
--- a/templates/lorry.tpl
+++ b/templates/lorry.tpl
@@ -40,5 +40,30 @@
 
         <p>Updated: {{timestamp}}</p>
 
+<h2>Failed jobs for this Lorry</h2>
+
+% if lorry['failed_jobs']:
+<p>
+%   for job_id in lorry['failed_jobs']:
+<a href="/1.0/job-html/{{job_id}}">{{job_id}}</a>
+%   end
+</p>
+% else:
+<p>No failed jobs.</p>
+% end
+
+<h2>All jobs for this lorry</h2>
+
+% if lorry['jobs']:
+<p>
+%   for job_id in lorry['jobs']:
+<a href="/1.0/job-html/{{job_id}}">{{job_id}}</a>
+%   end
+</p>
+% else:
+<p>No jobs.</p>
+% end
+
+
     </body>
 </html>
diff --git a/yarns.webapp/040-running-jobs.yarn b/yarns.webapp/040-running-jobs.yarn
index cbc8f75..1c9a4de 100644
--- a/yarns.webapp/040-running-jobs.yarn
+++ b/yarns.webapp/040-running-jobs.yarn
@@ -28,13 +28,18 @@ Then make sure we don't get a job when we request one.
     WHEN admin makes request GET /1.0/list-running-jobs
     THEN response has running_jobs set to []
 
-Add a Lorry spec to the run-queue, and request a job. We still
-shouldn't get a job, since the queue isn't set to run yet.
+Add a Lorry spec to the run-queue, and check that it looks OK.
 
     GIVEN Lorry file CONFGIT/foo.lorry with {"foo":{"type":"git","url":"git://foo"}}
 
     WHEN admin makes request POST /1.0/read-configuration
-    AND admin makes request POST /1.0/give-me-job with host=testhost&pid=123
+    AND admin makes request GET /1.0/lorry/upstream/foo
+    THEN response has jobs set to []
+
+Request a job. We still shouldn't get a job, since the queue isn't set
+to run yet.
+
+    WHEN admin makes request POST /1.0/give-me-job with host=testhost&pid=123
     THEN response has job_id set to null
 
 Enable the queue, and off we go.
@@ -46,6 +51,7 @@ Enable the queue, and off we go.
 
     WHEN admin makes request GET /1.0/lorry/upstream/foo
     THEN response has running_job set to 1
+    AND response has jobs set to [1]
     
     WHEN admin makes request GET /1.0/list-running-jobs
     THEN response has running_jobs set to [1]
@@ -61,6 +67,8 @@ Inform WEBAPP the job is finished.
     THEN response has kill set to false
     WHEN admin makes request GET /1.0/lorry/upstream/foo
     THEN response has running_job set to null
+    AND response has jobs set to [1]
+    AND response has failed_jobs set to []
     WHEN admin makes request GET /1.0/list-running-jobs
     THEN response has running_jobs set to []
 
@@ -69,6 +77,49 @@ Cleanup.
     FINALLY WEBAPP terminates
 
 
+Run a job that fails
+--------------------
+
+Lorry Controller needs to be able to deal with jobs that fail. It also
+needs to be able to list them correctly to the user.
+
+    SCENARIO run a job that fails
+    GIVEN a new git repository in CONFGIT
+    AND an empty lorry-controller.conf in CONFGIT
+    AND lorry-controller.conf in CONFGIT adds lorries *.lorry using prefix upstream
+    AND WEBAPP uses CONFGIT as its configuration directory
+    AND a running WEBAPP
+    AND Lorry file CONFGIT/foo.lorry with {"foo":{"type":"git","url":"git://foo"}}
+    WHEN admin makes request POST /1.0/read-configuration
+    AND admin makes request POST /1.0/start-queue
+
+Initially, the lorry spec should have no jobs or failed jobs listed.
+
+    WHEN admin makes request GET /1.0/lorry/upstream/foo
+    THEN response has jobs set to []
+    AND response has failed_jobs set to []
+
+MINION requests a job.
+
+    WHEN MINION makes request POST /1.0/give-me-job with host=testhost&pid=123
+    THEN response has job_id set to 1
+    AND response has path set to "upstream/foo"
+
+Now, when MINION updates WEBAPP about the job, indicating that it has
+failed, and admin will then see that the lorry spec lists the job in
+failed jobs.
+
+    WHEN MINION makes request POST /1.0/job-update with job_id=1&exit=1
+    AND admin makes request GET /1.0/lorry/upstream/foo
+    THEN response has jobs set to [1]
+    AND response has failed_jobs set to [1]
+
+Cleanup.
+
+    FINALLY WEBAPP terminates
+
+
+
 Limit number of jobs running at the same time
 ---------------------------------------------