db: retry instance_info_cache_update() on deadlock

If a Galera cluster is used in multi-writer mode it's possible, that
instance_info_cache_update() will be executed concurrently on two
different MySQL hosts for the very same row, which causes a deadlock
exception for one of the callers due to how Galera works internally.

This can affect operations like association or disassociation of
floating IPs, which will fail, if instance_info_cache_update() does
not handle deadlocks gracefully, i.e. is not retried.

Closes-Bug: #1567336

Change-Id: Ib5abffd94d2480dfbcc8b6cca7b1c73ce39e7d10

2 files changed, 23 insertions, 0 deletions

diff --git a/nova/db/sqlalchemy/api.py b/nova/db/sqlalchemy/api.py
index 91a4626884..6ecba17ab6 100644
--- a/nova/db/sqlalchemy/api.py
+++ b/nova/db/sqlalchemy/api.py
@@ -3005,6 +3005,7 @@ def instance_info_cache_get(context, instance_uuid):
 
 
 @require_context
+@oslo_db_api.wrap_db_retry(max_retries=5, retry_on_deadlock=True)
 @pick_context_manager_writer
 def instance_info_cache_update(context, instance_uuid, values):
     """Update an instance info cache record in the table.
diff --git a/nova/tests/unit/db/test_db_api.py b/nova/tests/unit/db/test_db_api.py
index 8ad9ed2f57..0328a20b15 100644
--- a/nova/tests/unit/db/test_db_api.py
+++ b/nova/tests/unit/db/test_db_api.py
@@ -9849,6 +9849,28 @@ class TestInstanceInfoCache(test.TestCase):
         self.assertEqual(network_info, info_cache.network_info)
         self.assertEqual(instance_uuid, info_cache.instance_uuid)
 
+    @mock.patch.object(models.InstanceInfoCache, 'update')
+    def test_instance_info_cache_retried_on_deadlock(self, update):
+        update.side_effect = [db_exc.DBDeadlock(), db_exc.DBDeadlock(), None]
+
+        instance = db.instance_create(self.context, {})
+        network_info = 'net'
+        updated = db.instance_info_cache_update(self.context, instance.uuid,
+                                                {'network_info': network_info})
+        self.assertEqual(instance.uuid, updated.instance_uuid)
+
+    @mock.patch.object(models.InstanceInfoCache, 'update')
+    def test_instance_info_cache_not_retried_on_deadlock_forever(self, update):
+        update.side_effect = db_exc.DBDeadlock
+
+        instance = db.instance_create(self.context, {})
+        network_info = 'net'
+
+        self.assertRaises(db_exc.DBDeadlock,
+                          db.instance_info_cache_update,
+                          self.context, instance.uuid,
+                          {'network_info': network_info})
+
 
 class TestInstanceTagsFiltering(test.TestCase):
     sample_data = {