From 1f9facecce6a0ff2887810feb6223f63f82dba11 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Mon, 8 Sep 2014 16:45:40 +0000 Subject: Add tests for jobs, failed_jobs attributes in lorry specs When getting information about a lorry spec from WEBAPP (/1.0/lorry/PATH), we now need it to return two lists of job ids: one for all jobs for that lorry spec, one with failed jobs only. We test here the version of the request that returns JSON. The version that returns HTML is similar (or so we assume), but it's harder to test the HTML output so we're happy with this. --- yarns.webapp/040-running-jobs.yarn | 57 ++++++++++++++++++++++++++++++++++++-- 1 file changed, 54 insertions(+), 3 deletions(-) diff --git a/yarns.webapp/040-running-jobs.yarn b/yarns.webapp/040-running-jobs.yarn index 571afd6..74c6516 100644 --- a/yarns.webapp/040-running-jobs.yarn +++ b/yarns.webapp/040-running-jobs.yarn @@ -28,13 +28,18 @@ Then make sure we don't get a job when we request one. WHEN admin makes request GET /1.0/list-running-jobs THEN response has running_jobs set to [] -Add a Lorry spec to the run-queue, and request a job. We still -shouldn't get a job, since the queue isn't set to run yet. +Add a Lorry spec to the run-queue, and check that it looks OK. GIVEN Lorry file CONFGIT/foo.lorry with {"foo":{"type":"git","url":"git://foo"}} WHEN admin makes request POST /1.0/read-configuration - AND admin makes request POST /1.0/give-me-job with host=testhost&pid=123 + AND admin makes request GET /1.0/lorry/upstream/foo + THEN response has jobs set to [] + +Request a job. We still shouldn't get a job, since the queue isn't set +to run yet. + + WHEN admin makes request POST /1.0/give-me-job with host=testhost&pid=123 THEN response has job_id set to null Enable the queue, and off we go. @@ -46,6 +51,7 @@ Enable the queue, and off we go. WHEN admin makes request GET /1.0/lorry/upstream/foo THEN response has running_job set to 1 + AND response has jobs set to [1] WHEN admin makes request GET /1.0/list-running-jobs THEN response has running_jobs set to [1] @@ -61,6 +67,8 @@ Inform WEBAPP the job is finished. THEN response has kill set to false WHEN admin makes request GET /1.0/lorry/upstream/foo THEN response has running_job set to null + AND response has jobs set to [1] + AND response has failed_jobs set to [] WHEN admin makes request GET /1.0/list-running-jobs THEN response has running_jobs set to [] @@ -69,6 +77,49 @@ Cleanup. FINALLY WEBAPP terminates +Run a job that fails +-------------------- + +Lorry Controller needs to be able to deal with jobs that fail. It also +needs to be able to list them correctly to the user. + + SCENARIO run a job that fails + GIVEN a new git repository in CONFGIT + AND an empty lorry-controller.conf in CONFGIT + AND lorry-controller.conf in CONFGIT adds lorries *.lorry using prefix upstream + AND WEBAPP uses CONFGIT as its configuration directory + AND a running WEBAPP + AND Lorry file CONFGIT/foo.lorry with {"foo":{"type":"git","url":"git://foo"}} + WHEN admin makes request POST /1.0/read-configuration + AND admin makes request POST /1.0/start-queue + +Initially, the lorry spec should have no jobs or failed jobs listed. + + WHEN admin makes request GET /1.0/lorry/upstream/foo + THEN response has jobs set to [] + AND response has failed_jobs set to [] + +MINION requests a job. + + WHEN MINION makes request POST /1.0/give-me-job with host=testhost&pid=123 + THEN response has job_id set to 1 + AND response has path set to "upstream/foo" + +Now, when MINION updates WEBAPP about the job, indicating that it has +failed, and admin will then see that the lorry spec lists the job in +failed jobs. + + WHEN MINION makes request POST /1.0/job-update with job_id=1&exit=1 + AND admin makes request GET /1.0/lorry/upstream/foo + THEN response has jobs set to [1] + AND response has failed_jobs set to [1] + +Cleanup. + + FINALLY WEBAPP terminates + + + Limit number of jobs running at the same time --------------------------------------------- -- cgit v1.2.1 From c491acc03665213e01e40fc9ef81f5e326cf9ff9 Mon Sep 17 00:00:00 2001 From: Lars Wirzenius Date: Mon, 8 Sep 2014 17:11:24 +0000 Subject: Add lists of failed, all jobs to lorry spec info This adds them to both the JSON and HTML outputs. --- lorrycontroller/showlorry.py | 35 +++++++++++++++++++++++------------ lorrycontroller/statedb.py | 13 +++++++++++++ templates/lorry.tpl | 25 +++++++++++++++++++++++++ 3 files changed, 61 insertions(+), 12 deletions(-) diff --git a/lorrycontroller/showlorry.py b/lorrycontroller/showlorry.py index fc336a5..19f9429 100644 --- a/lorrycontroller/showlorry.py +++ b/lorrycontroller/showlorry.py @@ -24,32 +24,43 @@ import bottle import lorrycontroller -class ShowLorry(lorrycontroller.LorryControllerRoute): +class ShowLorryBase(object): + + def get_lorry_info_with_job_lists(self, statedb, path): + obj= statedb.get_lorry_info(path) + obj['jobs'] = statedb.get_jobs_for_lorry(path) + obj['failed_jobs'] = statedb.get_failed_jobs_for_lorry(path) + return obj + + +class ShowLorry(ShowLorryBase, lorrycontroller.LorryControllerRoute): http_method = 'GET' path = '/1.0/lorry/' def run(self, **kwargs): logging.info('%s %s called', self.http_method, self.path) - statedb = self.open_statedb() - try: - return statedb.get_lorry_info(kwargs['path']) - except lorrycontroller.LorryNotFoundError as e: - bottle.abort(404, str(e)) + with self.open_statedb() as statedb: + try: + return self.get_lorry_info_with_job_lists(statedb, kwargs['path']) + except lorrycontroller.LorryNotFoundError as e: + bottle.abort(404, str(e)) -class ShowLorryHTML(lorrycontroller.LorryControllerRoute): +class ShowLorryHTML(ShowLorryBase, lorrycontroller.LorryControllerRoute): http_method = 'GET' path = '/1.0/lorry-html/' def run(self, **kwargs): logging.info('%s %s called', self.http_method, self.path) - statedb = self.open_statedb() - try: - lorry_info = statedb.get_lorry_info(kwargs['path']) - except lorrycontroller.LorryNotFoundError as e: - bottle.abort(404, str(e)) + + with self.open_statedb() as statedb: + try: + lorry_info = self.get_lorry_info_with_job_lists( + statedb, kwargs['path']) + except lorrycontroller.LorryNotFoundError as e: + bottle.abort(404, str(e)) renderer = lorrycontroller.StatusRenderer() shower = lorrycontroller.JobShower() diff --git a/lorrycontroller/statedb.py b/lorrycontroller/statedb.py index fd7857d..7a26098 100644 --- a/lorrycontroller/statedb.py +++ b/lorrycontroller/statedb.py @@ -473,6 +473,19 @@ class StateDB(object): 'output': row[10], } + def get_jobs_for_lorry(self, path): + c = self.get_cursor() + c.execute('SELECT job_id FROM jobs WHERE path=?', (path,)) + return [row[0] for row in c.fetchall()] + + def get_failed_jobs_for_lorry(self, path): + c = self.get_cursor() + c.execute( + 'SELECT job_id FROM jobs ' + 'WHERE path=? AND exit != \'no\' AND exit != 0', + (path,)) + return [row[0] for row in c.fetchall()] + def add_new_job(self, job_id, host, pid, path, started): logging.debug( 'StateDB.add_new_job(%r, %r, %r, %r, %r) called', diff --git a/templates/lorry.tpl b/templates/lorry.tpl index fad85cd..7c475e0 100644 --- a/templates/lorry.tpl +++ b/templates/lorry.tpl @@ -40,5 +40,30 @@

Updated: {{timestamp}}

+

Failed jobs for this Lorry

+ +% if lorry['failed_jobs']: +

+% for job_id in lorry['failed_jobs']: +{{job_id}} +% end +

+% else: +

No failed jobs.

+% end + +

All jobs for this lorry

+ +% if lorry['jobs']: +

+% for job_id in lorry['jobs']: +{{job_id}} +% end +

+% else: +

No jobs.

+% end + + -- cgit v1.2.1