Raise InstanceMappingNotFound if StaleDataError is encountered

We have a race where if a user issues a delete request while an
instance is in the middle of booting, we could fail to update the
'queued_for_delete' field on the instance mapping with:

  sqlalchemy.orm.exc.StaleDataError: UPDATE statement on table
  'instance_mappings' expected to update 1 row(s); 0 were matched.

This happens if we've retrieved the instance mapping record from the
database and then it gets deleted by nova-conductor before we attempt
to save() it.

This handles the situation by adding try-except around the update call
to catch StaleDataError and raise InstanceMappingNotFound instead,
which the caller does know how to handle.

Closes-Bug: #1882608

Change-Id: I2cdcad7226312ed81f4242c8d9ac919715524b48
(cherry picked from commit 16df22dcd57a73fe3be15c64c41b4081b4826ef2)
(cherry picked from commit 812ce632d50bfc32de62d544746e0b9a83d90ab7)

2 files changed, 20 insertions, 2 deletions

diff --git a/nova/objects/instance_mapping.py b/nova/objects/instance_mapping.py
index 0392f04770..8b5f6ba92e 100644
--- a/nova/objects/instance_mapping.py
+++ b/nova/objects/instance_mapping.py
@@ -15,6 +15,7 @@ import collections
 from oslo_log import log as logging
 from oslo_utils import versionutils
 import six
+from sqlalchemy.orm import exc as orm_exc
 from sqlalchemy.orm import joinedload
 from sqlalchemy.sql import false
 from sqlalchemy.sql import func
@@ -161,8 +162,16 @@ class InstanceMapping(base.NovaTimestampObject, base.NovaObject):
     def save(self):
         changes = self.obj_get_changes()
         changes = self._update_with_cell_id(changes)
-        db_mapping = self._save_in_db(self._context, self.instance_uuid,
-                changes)
+        try:
+            db_mapping = self._save_in_db(self._context, self.instance_uuid,
+                    changes)
+        except orm_exc.StaleDataError:
+            # NOTE(melwitt): If the instance mapping has been deleted out from
+            # under us by conductor (delete requested while booting), we will
+            # encounter a StaleDataError after we retrieved the row and try to
+            # update it after it's been deleted. We can treat this like an
+            # instance mapping not found and allow the caller to handle it.
+            raise exception.InstanceMappingNotFound(uuid=self.instance_uuid)
         self._from_db_object(self._context, self, db_mapping)
         self.obj_reset_changes()
 
diff --git a/nova/tests/unit/objects/test_instance_mapping.py b/nova/tests/unit/objects/test_instance_mapping.py
index ec50517a20..2c877c0a1f 100644
--- a/nova/tests/unit/objects/test_instance_mapping.py
+++ b/nova/tests/unit/objects/test_instance_mapping.py
@@ -12,6 +12,7 @@
 
 import mock
 from oslo_utils import uuidutils
+from sqlalchemy.orm import exc as orm_exc
 
 from nova import exception
 from nova import objects
@@ -151,6 +152,14 @@ class _TestInstanceMappingObject(object):
                          comparators={
                              'cell_mapping': self._check_cell_map_value})
 
+    @mock.patch.object(instance_mapping.InstanceMapping, '_save_in_db')
+    def test_save_stale_data_error(self, save_in_db):
+        save_in_db.side_effect = orm_exc.StaleDataError
+        mapping_obj = objects.InstanceMapping(self.context)
+        mapping_obj.instance_uuid = uuidutils.generate_uuid()
+
+        self.assertRaises(exception.InstanceMappingNotFound, mapping_obj.save)
+
     @mock.patch.object(instance_mapping.InstanceMapping, '_destroy_in_db')
     def test_destroy(self, destroy_in_db):
         uuid = uuidutils.generate_uuid()