From 905007e5df80c9fc0b7fe1b696b3182068586eda Mon Sep 17 00:00:00 2001 From: Pedro Alvarez Piedehierro Date: Sun, 9 Jul 2017 18:31:40 +0100 Subject: Make migrations run only once per execution Yoyo migration libraries were failing in some cases due to "database is locked" errors. It was difficult to track down what parallel operations were causing the problems, so I dediced it was better to run the migrations once per execution instead of executing them everytime we opened a connection with the database (for every request). --- lorry-controller-webapp | 6 +++++- lorrycontroller/statedb.py | 40 +++++++++++++++++++++++++--------------- 2 files changed, 30 insertions(+), 16 deletions(-) diff --git a/lorry-controller-webapp b/lorry-controller-webapp index 7a3e3b5..43cff0d 100755 --- a/lorry-controller-webapp +++ b/lorry-controller-webapp @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Copyright (C) 2014-2016 Codethink Limited +# Copyright (C) 2014-2017 Codethink Limited # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -185,6 +185,10 @@ class WEBAPP(cliapp.Application): method=route.http_method, callback=route.run) + logging.info('Initialising database') + statedb = lorrycontroller.StateDB(self.settings['statedb']) + statedb.initialise_db() + logging.info('Starting server') if self.settings['wsgi']: self.run_wsgi_server(webapp) diff --git a/lorrycontroller/statedb.py b/lorrycontroller/statedb.py index 99ea7fc..17b31dd 100644 --- a/lorrycontroller/statedb.py +++ b/lorrycontroller/statedb.py @@ -56,7 +56,6 @@ class StateDB(object): self._conn = None self._transaction_started = None - def _open(self): self.initial_lorries_fields = [ ('path', 'TEXT PRIMARY KEY'), ('text', 'TEXT'), @@ -76,25 +75,36 @@ class StateDB(object): self.lorries_booleans = [ ] + def _open(self): if self._conn is None: - existed = os.path.exists(self._filename) - logging.debug( - 'Connecting to %r (existed=%r)', self._filename, existed) - self._conn = sqlite3.connect( - self._filename, - timeout=100000, - isolation_level="IMMEDIATE") - logging.debug('New connection is %r', self._conn) - if not existed: - self._initialise_tables() - - self.perform_any_migrations() - - def perform_any_migrations(self): + db_exists = os.path.exists(self._filename) + assert db_exists + self._create_or_connect_to_db() + + def _create_or_connect_to_db(self): + logging.debug( + 'Connecting to %r', self._filename) + self._conn = sqlite3.connect( + self._filename, + timeout=100000, + isolation_level="IMMEDIATE") + logging.debug('New connection is %r', self._conn) + + def initialise_db(self): + db_exists = os.path.exists(self._filename) + if self._conn is None: + self._create_or_connect_to_db() + if not db_exists: + self._initialise_tables() + self._perform_any_migrations() + + def _perform_any_migrations(self): + logging.debug('Performing database migrations needed') backend = yoyo.get_backend('sqlite:///' + self._filename) migrations_dir = os.path.join(os.path.dirname(__file__), 'migrations') migrations = yoyo.read_migrations(migrations_dir) backend.apply_migrations(backend.to_apply(migrations)) + logging.debug('Database migrated') def _initialise_tables(self): logging.debug('Initialising tables in database') -- cgit v1.2.1