From 0cc8dff0f435df5d445dd6253a5d1dbfd8d159d1 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Piedehierro Date: Fri, 7 Jul 2017 01:02:45 +0100 Subject: Create extra setting to publish lorry failures --- lorry-controller-webapp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lorry-controller-webapp b/lorry-controller-webapp index 7d4479c..7a3e3b5 100755 --- a/lorry-controller-webapp +++ b/lorry-controller-webapp @@ -138,6 +138,10 @@ class WEBAPP(cliapp.Application): ['gitlab-private-token'], 'private token for GitLab API access') + self.settings.boolean( + ['publish-failures'], + 'make the status page show failure logs from lorry') + def find_routes(self): '''Return all classes that are API routes. -- cgit v1.2.1 From d2cbc43175eeb4a770f7dae2e984df3f994dcfb8 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Piedehierro Date: Sat, 8 Jul 2017 03:21:17 +0100 Subject: statedb: add columns for last failure information --- .../0002-add-last_run-status-columns-to-lorries.py | 20 ++++++++++++++++++++ lorrycontroller/statedb.py | 11 ++++++++--- 2 files changed, 28 insertions(+), 3 deletions(-) create mode 100644 lorrycontroller/migrations/0002-add-last_run-status-columns-to-lorries.py diff --git a/lorrycontroller/migrations/0002-add-last_run-status-columns-to-lorries.py b/lorrycontroller/migrations/0002-add-last_run-status-columns-to-lorries.py new file mode 100644 index 0000000..ce30edf --- /dev/null +++ b/lorrycontroller/migrations/0002-add-last_run-status-columns-to-lorries.py @@ -0,0 +1,20 @@ +# Copyright (C) 2017 Codethink Limited +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program; if not, write to the Free Software Foundation, Inc., +# 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. + + +import yoyo + +yoyo.step('ALTER TABLE lorries ADD COLUMN last_run_exit') +yoyo.step('ALTER TABLE lorries ADD COLUMN last_run_error') diff --git a/lorrycontroller/statedb.py b/lorrycontroller/statedb.py index 1b885d9..8c62a31 100644 --- a/lorrycontroller/statedb.py +++ b/lorrycontroller/statedb.py @@ -1,4 +1,4 @@ -# Copyright (C) 2014-2016 Codethink Limited +# Copyright (C) 2014-2017 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -57,7 +57,7 @@ class StateDB(object): self._transaction_started = None def _open(self): - self.lorries_fields = [ + self.initial_lorries_fields = [ ('path', 'TEXT PRIMARY KEY'), ('text', 'TEXT'), ('from_trovehost', 'TEXT'), @@ -68,6 +68,11 @@ class StateDB(object): ('lorry_timeout', 'INT'), ('disk_usage', 'INT'), ] + self.lorries_fields = list(self.initial_lorries_fields) + self.lorries_fields.extend([ + ('last_run_exit', 'TEXT'), + ('last_run_error', 'TEXT'), + ]) self.lorries_booleans = [ ] @@ -118,7 +123,7 @@ class StateDB(object): # Table for all the known lorries (the "run queue"). fields_sql = ', '.join( - '%s %s' % (name, info) for name, info in self.lorries_fields + '%s %s' % (name, info) for name, info in self.initial_lorries_fields ) c.execute('CREATE TABLE lorries (%s)' % fields_sql) -- cgit v1.2.1 From 7ca915a812db8662362bfec4b335ba781531ae88 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Piedehierro Date: Fri, 7 Jul 2017 01:04:18 +0100 Subject: jobupdate: Store job information in lorry if job failed --- lorrycontroller/jobupdate.py | 7 ++++++- lorrycontroller/statedb.py | 10 ++++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/lorrycontroller/jobupdate.py b/lorrycontroller/jobupdate.py index efc9ce1..ec7e533 100644 --- a/lorrycontroller/jobupdate.py +++ b/lorrycontroller/jobupdate.py @@ -1,4 +1,4 @@ -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2017 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -51,6 +51,11 @@ class JobUpdate(lorrycontroller.LorryControllerRoute): lorry_info = statedb.get_lorry_info(path) if exit is not None and exit != 'no': + if exit != '0': + job_output = statedb.get_job_output(job_id) + else: + job_output = '' + statedb.set_lorry_last_run_exit_and_output(path, exit, job_output) statedb.set_lorry_last_run(path, int(now)) statedb.set_running_job(path, None) statedb.set_job_exit(job_id, exit, int(now), disk_usage) diff --git a/lorrycontroller/statedb.py b/lorrycontroller/statedb.py index 8c62a31..99ea7fc 100644 --- a/lorrycontroller/statedb.py +++ b/lorrycontroller/statedb.py @@ -440,6 +440,16 @@ class StateDB(object): 'UPDATE lorries SET last_run=? WHERE path=?', (last_run, path)) + def set_lorry_last_run_exit_and_output(self, path, exit, output): + logging.debug( + 'StateDB.set_lorry_last_run_exit_and_output(%r, %r, %r) called', + path, exit, output) + assert self.in_transaction + c = self.get_cursor() + c.execute( + 'UPDATE lorries SET last_run_exit=?, last_run_error=? WHERE path=?', + (exit, output, path)) + def set_lorry_disk_usage(self, path, disk_usage): logging.debug( 'StateDB.set_lorry_disk_usage(%r, %r) called', path, disk_usage) -- cgit v1.2.1 From 79bdd7d34958d5b84dce7d44c16154eb20cbfd1f Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Piedehierro Date: Sat, 8 Jul 2017 00:26:42 +0100 Subject: status: send 'publish-failures' option to template --- lorrycontroller/status.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/lorrycontroller/status.py b/lorrycontroller/status.py index 40bf964..58669fb 100644 --- a/lorrycontroller/status.py +++ b/lorrycontroller/status.py @@ -1,4 +1,4 @@ -# Copyright (C) 2014 Codethink Limited +# Copyright (C) 2014-2017 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -39,6 +39,7 @@ class StatusRenderer(object): 'warning_msg': '', 'max_jobs': self.get_max_jobs(statedb), 'links': True, + 'publish_failures': True, } status.update(self.get_free_disk_space(work_directory)) return status @@ -46,9 +47,11 @@ class StatusRenderer(object): def render_status_as_html(self, template, status): return bottle.template(template, **status) - def write_status_as_html(self, template, status, filename): + def write_status_as_html(self, template, status, filename, + publish_failures): modified_status = dict(status) modified_status['links'] = False + modified_status['publish_failures'] = publish_failures html = self.render_status_as_html(template, modified_status) # We write the file first to a temporary file and then @@ -167,7 +170,8 @@ class Status(lorrycontroller.LorryControllerRoute): renderer.write_status_as_html( self._templates['status'], status, - self.app_settings['status-html']) + self.app_settings['status-html'], + self.app_settings['publish-failures']) return status @@ -185,6 +189,7 @@ class StatusHTML(lorrycontroller.LorryControllerRoute): renderer.write_status_as_html( self._templates['status'], status, - self.app_settings['status-html']) + self.app_settings['status-html'], + self.app_settings['publish-failures']) return renderer.render_status_as_html( self._templates['status'], status) -- cgit v1.2.1 From f91dc87d4d79ae8af213ffa8cfc8aa268d7bcf83 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Piedehierro Date: Fri, 7 Jul 2017 01:06:11 +0100 Subject: templates/status.tpl: Add column to show failures Make all columns align to the top, so that the content stays in the same place when expanding the log output column. --- static/style.css | 1 + templates/status.tpl | 12 +++++++++++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/static/style.css b/static/style.css index 8a6937d..4998b2a 100644 --- a/static/style.css +++ b/static/style.css @@ -15,4 +15,5 @@ td { font-family: monospace; border-top: 1px solid black; text-align: left; + vertical-align: top; } diff --git a/templates/status.tpl b/templates/status.tpl index 2665861..939f77d 100644 --- a/templates/status.tpl +++ b/templates/status.tpl @@ -104,7 +104,9 @@ Path Interval Due +Last run exit Job? + % for i, spec in enumerate(run_queue): % obj = json.loads(spec['text']) @@ -118,7 +120,15 @@ {{spec['path']}} % end {{spec['interval_nice']}} -{{spec['due_nice']}} +{{spec['due_nice']}} +% if publish_failures and spec['last_run_exit'] is not None and spec['last_run_error'] != "": +
+ {{spec['last_run_exit']}}: Show log + 

{{spec['last_run_error']}}

+
+% else: +{{spec['last_run_exit']}} +% end % if spec['running_job'] and links: {{spec['running_job']}} % else: -- cgit v1.2.1 From fbf2a7cd7422dedf87d5f005f2e6204c0b202d20 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Piedehierro Date: Sat, 8 Jul 2017 13:40:32 +0100 Subject: check: set default PYTHONPATH --- check | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/check b/check index b449f2b..c86a848 100755 --- a/check +++ b/check @@ -2,4 +2,4 @@ set -eu -yarn -s yarns.webapp/yarn.sh yarns.webapp/*.yarn --env PYTHONPATH="$PYTHONPATH" "$@" +yarn -s yarns.webapp/yarn.sh yarns.webapp/*.yarn --env PYTHONPATH="${PYTHONPATH:=''}" "$@" -- cgit v1.2.1 From 7f8d636caa4e0120320b7088f78292d54c99efd7 Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Piedehierro Date: Sat, 8 Jul 2017 00:51:26 +0100 Subject: Setup CI with .gitlab-ci.yml --- .gitlab-ci.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 .gitlab-ci.yml diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000..28c664c --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,13 @@ +image: debian:stretch + +before_script: + - apt-get update -y + - apt-get install -y -qq python-dev python-pip + # Deps for running tests + - apt-get install -y -qq cmdtest curl git + # Deps for lorry-controller + - apt-get install -y -qq python-bottle python-flup python-requests + - pip install yoyo-migrations +run-check: + script: + - sh check -- cgit v1.2.1 From 170b2ce981f6e7be834be56bee3c81da503555ee Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Piedehierro Date: Sun, 9 Jul 2017 00:59:03 +0100 Subject: status: Encode generated html as UTF-8 Some job outputs include Unicode characters. This made the status.html generation fail given that some job logs are included in the status page. --- lorrycontroller/status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lorrycontroller/status.py b/lorrycontroller/status.py index 58669fb..9c907bf 100644 --- a/lorrycontroller/status.py +++ b/lorrycontroller/status.py @@ -61,7 +61,7 @@ class StatusRenderer(object): try: temp_filename = self.temp_filename_in_same_dir_as(filename) with open(temp_filename, 'w') as f: - f.write(html) + f.write(html.encode("UTF-8")) os.rename(temp_filename, filename) except (OSError, IOError) as e: self.remove_temp_file(temp_filename) -- cgit v1.2.1 From 905007e5df80c9fc0b7fe1b696b3182068586eda Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Piedehierro Date: Sun, 9 Jul 2017 18:31:40 +0100 Subject: Make migrations run only once per execution Yoyo migration libraries were failing in some cases due to "database is locked" errors. It was difficult to track down what parallel operations were causing the problems, so I dediced it was better to run the migrations once per execution instead of executing them everytime we opened a connection with the database (for every request). --- lorry-controller-webapp | 6 +++++- lorrycontroller/statedb.py | 40 +++++++++++++++++++++++++--------------- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/lorry-controller-webapp b/lorry-controller-webapp index 7a3e3b5..43cff0d 100755 --- a/lorry-controller-webapp +++ b/lorry-controller-webapp @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Copyright (C) 2014-2016 Codethink Limited +# Copyright (C) 2014-2017 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -185,6 +185,10 @@ class WEBAPP(cliapp.Application): method=route.http_method, callback=route.run) + logging.info('Initialising database') + statedb = lorrycontroller.StateDB(self.settings['statedb']) + statedb.initialise_db() + logging.info('Starting server') if self.settings['wsgi']: self.run_wsgi_server(webapp) diff --git a/lorrycontroller/statedb.py b/lorrycontroller/statedb.py index 99ea7fc..17b31dd 100644 --- a/lorrycontroller/statedb.py +++ b/lorrycontroller/statedb.py @@ -56,7 +56,6 @@ class StateDB(object): self._conn = None self._transaction_started = None - def _open(self): self.initial_lorries_fields = [ ('path', 'TEXT PRIMARY KEY'), ('text', 'TEXT'), @@ -76,25 +75,36 @@ class StateDB(object): self.lorries_booleans = [ ] + def _open(self): if self._conn is None: - existed = os.path.exists(self._filename) - logging.debug( - 'Connecting to %r (existed=%r)', self._filename, existed) - self._conn = sqlite3.connect( - self._filename, - timeout=100000, - isolation_level="IMMEDIATE") - logging.debug('New connection is %r', self._conn) - if not existed: - self._initialise_tables() - - self.perform_any_migrations() - - def perform_any_migrations(self): + db_exists = os.path.exists(self._filename) + assert db_exists + self._create_or_connect_to_db() + + def _create_or_connect_to_db(self): + logging.debug( + 'Connecting to %r', self._filename) + self._conn = sqlite3.connect( + self._filename, + timeout=100000, + isolation_level="IMMEDIATE") + logging.debug('New connection is %r', self._conn) + + def initialise_db(self): + db_exists = os.path.exists(self._filename) + if self._conn is None: + self._create_or_connect_to_db() + if not db_exists: + self._initialise_tables() + self._perform_any_migrations() + + def _perform_any_migrations(self): + logging.debug('Performing database migrations needed') backend = yoyo.get_backend('sqlite:///' + self._filename) migrations_dir = os.path.join(os.path.dirname(__file__), 'migrations') migrations = yoyo.read_migrations(migrations_dir) backend.apply_migrations(backend.to_apply(migrations)) + logging.debug('Database migrated') def _initialise_tables(self): logging.debug('Initialising tables in database') -- cgit v1.2.1