diff options
| -rw-r--r-- | etc/nova/nova.conf.sample | 4 | ||||
| -rw-r--r-- | nova/db/sqlalchemy/session.py | 61 | ||||
| -rw-r--r-- | nova/flags.py | 4 |
3 files changed, 51 insertions, 18 deletions
diff --git a/etc/nova/nova.conf.sample b/etc/nova/nova.conf.sample index ea4e014a9f..b2607a9c2f 100644 --- a/etc/nova/nova.conf.sample +++ b/etc/nova/nova.conf.sample @@ -215,6 +215,8 @@ # sql_connection="sqlite:///$state_path/$sqlite_db" ###### (IntOpt) timeout before idle sql connections are reaped # sql_idle_timeout=3600 +###### (IntOpt) maximum db connection retries during startup. (setting -1 implies an infinite retry count) +# sql_max_retries=10 ###### (IntOpt) interval between retries of opening a sql connection # sql_retry_interval=10 ###### (StrOpt) the filename to use with sqlite @@ -1105,4 +1107,4 @@ ###### (StrOpt) The ZFS path under which to create zvols for volumes. # san_zfs_volume_base="rpool/" -# Total option count: 466 +# Total option count: 467 diff --git a/nova/db/sqlalchemy/session.py b/nova/db/sqlalchemy/session.py index bf67a397dd..3f206323ef 100644 --- a/nova/db/sqlalchemy/session.py +++ b/nova/db/sqlalchemy/session.py @@ -22,7 +22,7 @@ import time import sqlalchemy.interfaces import sqlalchemy.orm -from sqlalchemy.exc import DisconnectionError +from sqlalchemy.exc import DisconnectionError, OperationalError from sqlalchemy.pool import NullPool, StaticPool import nova.exception @@ -39,11 +39,11 @@ _MAKER = None def get_session(autocommit=True, expire_on_commit=False): """Return a SQLAlchemy session.""" - global _ENGINE, _MAKER + global _MAKER - if _MAKER is None or _ENGINE is None: - _ENGINE = get_engine() - _MAKER = get_maker(_ENGINE, autocommit, expire_on_commit) + if _MAKER is None: + engine = get_engine() + _MAKER = get_maker(engine, autocommit, expire_on_commit) session = _MAKER() session.query = nova.exception.wrap_db_error(session.query) @@ -80,6 +80,17 @@ class MySQLPingListener(object): raise +def is_db_connection_error(args): + """Return True if error in connecting to db.""" + # NOTE(adam_g): This is currently MySQL specific and needs to be extended + # to support Postgres and others. + conn_err_codes = ('2002', '2003', '2006') + for err_code in conn_err_codes: + if args.find(err_code) != -1: + return True + return False + + def get_engine(): """Return a SQLAlchemy engine.""" global _ENGINE @@ -105,21 +116,37 @@ def get_engine(): engine_args["poolclass"] = StaticPool engine_args["connect_args"] = {'check_same_thread': False} - engine_args = { - "pool_recycle": FLAGS.sql_idle_timeout, - "echo": False, - 'convert_unicode': True, - } + if not FLAGS.sqlite_synchronous: + engine_args["listeners"] = [SynchronousSwitchListener()] + + if 'mysql' in connection_dict.drivername: + engine_args['listeners'] = [MySQLPingListener()] - if "sqlite" in connection_dict.drivername: - engine_args["poolclass"] = sqlalchemy.pool.NullPool - if not FLAGS.sqlite_synchronous: - engine_args["listeners"] = [SynchronousSwitchListener()] + _ENGINE = sqlalchemy.create_engine(FLAGS.sql_connection, **engine_args) - if 'mysql' in connection_dict.drivername: - engine_args['listeners'] = [MySQLPingListener()] + try: + _ENGINE.connect() + except OperationalError, e: + if not is_db_connection_error(e.args[0]): + raise - return sqlalchemy.create_engine(FLAGS.sql_connection, **engine_args) + remaining = FLAGS.sql_max_retries + if remaining == -1: + remaining = 'infinite' + while True: + msg = _('SQL connection failed. %s attempts left.') + LOG.warn(msg % remaining) + if remaining != 'infinite': + remaining -= 1 + time.sleep(FLAGS.sql_retry_interval) + try: + _ENGINE.connect() + break + except OperationalError, e: + if (remaining != 'infinite' and remaining == 0) or \ + not is_db_connection_error(e.args[0]): + raise + return _ENGINE def get_maker(engine, autocommit=True, expire_on_commit=False): diff --git a/nova/flags.py b/nova/flags.py index 430abe67ee..652d3465d7 100644 --- a/nova/flags.py +++ b/nova/flags.py @@ -325,6 +325,10 @@ global_opts = [ cfg.IntOpt('sql_idle_timeout', default=3600, help='timeout before idle sql connections are reaped'), + cfg.IntOpt('sql_max_retries', + default=10, + help='maximum db connection retries during startup. ' + '(setting -1 implies an infinite retry count)'), cfg.IntOpt('sql_retry_interval', default=10, help='interval between retries of opening a sql connection'), |