Merge branch 'pedro/publish-failures' into 'master'

pedro/publish-failures

See merge request !1

9 files changed, 114 insertions, 27 deletions

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 0000000..28c664c
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,13 @@
+image: debian:stretch
+
+before_script:
+  - apt-get update -y
+  - apt-get install -y -qq python-dev python-pip
+  # Deps for running tests
+  - apt-get install -y -qq cmdtest curl git
+  # Deps for lorry-controller
+  - apt-get install -y -qq python-bottle python-flup python-requests
+  - pip install yoyo-migrations
+run-check:
+  script:
+  - sh check
diff --git a/check b/check
index b449f2b..c86a848 100755
--- a/check
+++ b/check
@@ -2,4 +2,4 @@
 
 set -eu
 
-yarn -s yarns.webapp/yarn.sh yarns.webapp/*.yarn --env PYTHONPATH="$PYTHONPATH" "$@"
+yarn -s yarns.webapp/yarn.sh yarns.webapp/*.yarn --env PYTHONPATH="${PYTHONPATH:=''}" "$@"
diff --git a/lorry-controller-webapp b/lorry-controller-webapp
index 7d4479c..43cff0d 100755
--- a/lorry-controller-webapp
+++ b/lorry-controller-webapp
@@ -1,6 +1,6 @@
 #!/usr/bin/env python
 #
-# Copyright (C) 2014-2016  Codethink Limited
+# Copyright (C) 2014-2017  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -138,6 +138,10 @@ class WEBAPP(cliapp.Application):
             ['gitlab-private-token'],
             'private token for GitLab API access')
 
+        self.settings.boolean(
+            ['publish-failures'],
+            'make the status page show failure logs from lorry')
+
     def find_routes(self):
         '''Return all classes that are API routes.
 
@@ -181,6 +185,10 @@ class WEBAPP(cliapp.Application):
                 method=route.http_method,
                 callback=route.run)
 
+        logging.info('Initialising database')
+        statedb = lorrycontroller.StateDB(self.settings['statedb'])
+        statedb.initialise_db()
+
         logging.info('Starting server')
         if self.settings['wsgi']:
             self.run_wsgi_server(webapp)
diff --git a/lorrycontroller/jobupdate.py b/lorrycontroller/jobupdate.py
index efc9ce1..ec7e533 100644
--- a/lorrycontroller/jobupdate.py
+++ b/lorrycontroller/jobupdate.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2014  Codethink Limited
+# Copyright (C) 2014-2017  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -51,6 +51,11 @@ class JobUpdate(lorrycontroller.LorryControllerRoute):
             lorry_info = statedb.get_lorry_info(path)
 
             if exit is not None and exit != 'no':
+                if exit != '0':
+                    job_output = statedb.get_job_output(job_id)
+                else:
+                    job_output = ''
+                statedb.set_lorry_last_run_exit_and_output(path, exit, job_output)
                 statedb.set_lorry_last_run(path, int(now))
                 statedb.set_running_job(path, None)
                 statedb.set_job_exit(job_id, exit, int(now), disk_usage)
diff --git a/lorrycontroller/migrations/0002-add-last_run-status-columns-to-lorries.py b/lorrycontroller/migrations/0002-add-last_run-status-columns-to-lorries.py
new file mode 100644
index 0000000..ce30edf
--- /dev/null
+++ b/lorrycontroller/migrations/0002-add-last_run-status-columns-to-lorries.py
@@ -0,0 +1,20 @@
+# Copyright (C) 2017  Codethink Limited
+#
+# This program is free software; you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation; version 2 of the License.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with this program; if not, write to the Free Software Foundation, Inc.,
+# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA.
+
+
+import yoyo
+
+yoyo.step('ALTER TABLE lorries ADD COLUMN last_run_exit')
+yoyo.step('ALTER TABLE lorries ADD COLUMN last_run_error')
diff --git a/lorrycontroller/statedb.py b/lorrycontroller/statedb.py
index 1b885d9..17b31dd 100644
--- a/lorrycontroller/statedb.py
+++ b/lorrycontroller/statedb.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2014-2016  Codethink Limited
+# Copyright (C) 2014-2017  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -56,8 +56,7 @@ class StateDB(object):
         self._conn = None
         self._transaction_started = None
 
-    def _open(self):
-        self.lorries_fields = [
+        self.initial_lorries_fields = [
             ('path', 'TEXT PRIMARY KEY'),
             ('text', 'TEXT'),
             ('from_trovehost', 'TEXT'),
@@ -68,28 +67,44 @@ class StateDB(object):
             ('lorry_timeout', 'INT'),
             ('disk_usage', 'INT'),
             ]
+        self.lorries_fields = list(self.initial_lorries_fields)
+        self.lorries_fields.extend([
+            ('last_run_exit', 'TEXT'),
+            ('last_run_error', 'TEXT'),
+            ])
         self.lorries_booleans = [
             ]
 
+    def _open(self):
         if self._conn is None:
-            existed = os.path.exists(self._filename)
-            logging.debug(
-                'Connecting to %r (existed=%r)', self._filename, existed)
-            self._conn = sqlite3.connect(
-                self._filename,
-                timeout=100000,
-                isolation_level="IMMEDIATE")
-            logging.debug('New connection is %r', self._conn)
-            if not existed:
-                self._initialise_tables()
-
-            self.perform_any_migrations()
-
-    def perform_any_migrations(self):
+            db_exists = os.path.exists(self._filename)
+            assert db_exists
+            self._create_or_connect_to_db()
+
+    def _create_or_connect_to_db(self):
+        logging.debug(
+            'Connecting to %r', self._filename)
+        self._conn = sqlite3.connect(
+            self._filename,
+            timeout=100000,
+            isolation_level="IMMEDIATE")
+        logging.debug('New connection is %r', self._conn)
+
+    def initialise_db(self):
+        db_exists = os.path.exists(self._filename)
+        if self._conn is None:
+            self._create_or_connect_to_db()
+        if not db_exists:
+            self._initialise_tables()
+        self._perform_any_migrations()
+
+    def _perform_any_migrations(self):
+        logging.debug('Performing database migrations needed')
         backend = yoyo.get_backend('sqlite:///' + self._filename)
         migrations_dir = os.path.join(os.path.dirname(__file__), 'migrations')
         migrations = yoyo.read_migrations(migrations_dir)
         backend.apply_migrations(backend.to_apply(migrations))
+        logging.debug('Database migrated')
 
     def _initialise_tables(self):
         logging.debug('Initialising tables in database')
@@ -118,7 +133,7 @@ class StateDB(object):
         # Table for all the known lorries (the "run queue").
 
         fields_sql = ', '.join(
-            '%s %s' % (name, info) for name, info in self.lorries_fields
+            '%s %s' % (name, info) for name, info in self.initial_lorries_fields
             )
 
         c.execute('CREATE TABLE lorries (%s)' % fields_sql)
@@ -435,6 +450,16 @@ class StateDB(object):
             'UPDATE lorries SET last_run=? WHERE path=?',
             (last_run, path))
 
+    def set_lorry_last_run_exit_and_output(self, path, exit, output):
+        logging.debug(
+            'StateDB.set_lorry_last_run_exit_and_output(%r, %r, %r) called',
+            path, exit, output)
+        assert self.in_transaction
+        c = self.get_cursor()
+        c.execute(
+            'UPDATE lorries SET last_run_exit=?, last_run_error=? WHERE path=?',
+            (exit, output, path))
+
     def set_lorry_disk_usage(self, path, disk_usage):
         logging.debug(
             'StateDB.set_lorry_disk_usage(%r, %r) called', path, disk_usage)
diff --git a/lorrycontroller/status.py b/lorrycontroller/status.py
index 40bf964..9c907bf 100644
--- a/lorrycontroller/status.py
+++ b/lorrycontroller/status.py
@@ -1,4 +1,4 @@
-# Copyright (C) 2014  Codethink Limited
+# Copyright (C) 2014-2017  Codethink Limited
 #
 # This program is free software; you can redistribute it and/or modify
 # it under the terms of the GNU General Public License as published by
@@ -39,6 +39,7 @@ class StatusRenderer(object):
             'warning_msg': '',
             'max_jobs': self.get_max_jobs(statedb),
             'links': True,
+            'publish_failures': True,
             }
         status.update(self.get_free_disk_space(work_directory))
         return status
@@ -46,9 +47,11 @@ class StatusRenderer(object):
     def render_status_as_html(self, template, status):
         return bottle.template(template, **status)
 
-    def write_status_as_html(self, template, status, filename):
+    def write_status_as_html(self, template, status, filename,
+            publish_failures):
         modified_status = dict(status)
         modified_status['links'] = False
+        modified_status['publish_failures'] = publish_failures
         html = self.render_status_as_html(template, modified_status)
 
         # We write the file first to a temporary file and then
@@ -58,7 +61,7 @@ class StatusRenderer(object):
         try:
             temp_filename = self.temp_filename_in_same_dir_as(filename)
             with open(temp_filename, 'w') as f:
-                f.write(html)
+                f.write(html.encode("UTF-8"))
             os.rename(temp_filename, filename)
         except (OSError, IOError) as e:
             self.remove_temp_file(temp_filename)
@@ -167,7 +170,8 @@ class Status(lorrycontroller.LorryControllerRoute):
         renderer.write_status_as_html(
             self._templates['status'],
             status,
-            self.app_settings['status-html'])
+            self.app_settings['status-html'],
+            self.app_settings['publish-failures'])
         return status
 
 
@@ -185,6 +189,7 @@ class StatusHTML(lorrycontroller.LorryControllerRoute):
         renderer.write_status_as_html(
             self._templates['status'],
             status,
-            self.app_settings['status-html'])
+            self.app_settings['status-html'],
+            self.app_settings['publish-failures'])
         return renderer.render_status_as_html(
             self._templates['status'], status)
diff --git a/static/style.css b/static/style.css
index 8a6937d..4998b2a 100644
--- a/static/style.css
+++ b/static/style.css
@@ -15,4 +15,5 @@ td {
     font-family: monospace;
     border-top: 1px solid black;
     text-align: left;
+    vertical-align: top;
 }
diff --git a/templates/status.tpl b/templates/status.tpl
index 2665861..939f77d 100644
--- a/templates/status.tpl
+++ b/templates/status.tpl
@@ -104,7 +104,9 @@
 <th>Path</th>
 <th>Interval</th>
 <th>Due</th>
+<th>Last run exit</th>
 <th>Job?</th>
+
 </tr>
 % for i, spec in enumerate(run_queue):
 %   obj = json.loads(spec['text'])
@@ -118,7 +120,15 @@
 <td>{{spec['path']}}</td>
 %   end
 <td>{{spec['interval_nice']}}</td>
-<td>{{spec['due_nice']}}</td>
+<td nowrap>{{spec['due_nice']}}</td>
+%   if publish_failures and spec['last_run_exit'] is not None and spec['last_run_error'] != "":
+<td><details>
+  <summary>{{spec['last_run_exit']}}: Show log</summary>
+  <p><pre>{{spec['last_run_error']}}</pre></p>
+</details></td>
+%   else:
+<td>{{spec['last_run_exit']}}</td>
+%   end
 %   if spec['running_job'] and links:
 <td><a href="/1.0/job-html/{{spec['running_job']}}">{{spec['running_job']}}</a></td>
 %   else: