Handle InstanceNotFound when sending instance update notification

When Tempest (or any automated client) is waiting for an instance to be
ACTIVE but goes to ERROR state, it immediately fails a test and deletes
the instance.  The instance could be deleted by the time the instance
update notification processing happens so there is a race where that
could fail with InstanceNotFound.  That will get traced to the conductor
logs and we should try to keep the conductor logs as clean as possible
from expected exceptions.

This handles InstanceNotFound and just logs it at debug rather than
trace out the exception.

Closes-Bug: #1455640

Change-Id: Iaee147051afb59f7991f8dfef3bd3072c28f829d

2 files changed, 39 insertions, 0 deletions

diff --git a/nova/notifications.py b/nova/notifications.py
index 4eff275838..09332938bd 100644
--- a/nova/notifications.py
+++ b/nova/notifications.py
@@ -28,6 +28,7 @@ from oslo_utils import timeutils
 import six
 
 import nova.context
+from nova import exception
 from nova.i18n import _LE
 from nova.image import glance
 from nova import network
@@ -150,6 +151,10 @@ def send_update(context, old_instance, new_instance, service="compute",
             _send_instance_update_notification(context, new_instance,
                     service=service, host=host,
                     old_display_name=old_display_name)
+        except exception.InstanceNotFound:
+            LOG.debug('Failed to send instance update notification. The '
+                      'instance could not be found and was most likely '
+                      'deleted.', instance=new_instance)
         except Exception:
             LOG.exception(_LE("Failed to send state update notification"),
                     instance=new_instance)
@@ -189,6 +194,10 @@ def send_update_with_states(context, instance, old_vm_state, new_vm_state,
                     old_vm_state=old_vm_state, old_task_state=old_task_state,
                     new_vm_state=new_vm_state, new_task_state=new_task_state,
                     service=service, host=host)
+        except exception.InstanceNotFound:
+            LOG.debug('Failed to send instance update notification. The '
+                      'instance could not be found and was most likely '
+                      'deleted.', instance=instance)
         except Exception:
             LOG.exception(_LE("Failed to send state update notification"),
                     instance=instance)
diff --git a/nova/tests/unit/test_notifications.py b/nova/tests/unit/test_notifications.py
index e5fcab7a65..0b2db10a2d 100644
--- a/nova/tests/unit/test_notifications.py
+++ b/nova/tests/unit/test_notifications.py
@@ -26,6 +26,7 @@ from nova.compute import flavors
 from nova.compute import task_states
 from nova.compute import vm_states
 from nova import context
+from nova import exception
 from nova.network import api as network_api
 from nova import notifications
 from nova import objects
@@ -432,6 +433,35 @@ class NotificationsTestCase(test.TestCase):
         notifications.send_update(self.context, self.instance, self.instance)
         self.assertEqual(0, len(fake_notifier.NOTIFICATIONS))
 
+    @mock.patch.object(notifications.LOG, 'exception')
+    def test_fail_sending_update_instance_not_found(self, mock_log_exception):
+        # Tests that InstanceNotFound is handled as an expected exception and
+        # not logged as an error.
+        notfound = exception.InstanceNotFound(instance_id=self.instance.uuid)
+        with mock.patch.object(notifications,
+                               '_send_instance_update_notification',
+                               side_effect=notfound):
+            notifications.send_update(
+                self.context, self.instance, self.instance)
+        self.assertEqual(0, len(fake_notifier.NOTIFICATIONS))
+        self.assertEqual(0, mock_log_exception.call_count)
+
+    @mock.patch.object(notifications.LOG, 'exception')
+    def test_fail_send_update_with_states_inst_not_found(self,
+                                                         mock_log_exception):
+        # Tests that InstanceNotFound is handled as an expected exception and
+        # not logged as an error.
+        notfound = exception.InstanceNotFound(instance_id=self.instance.uuid)
+        with mock.patch.object(notifications,
+                               '_send_instance_update_notification',
+                               side_effect=notfound):
+            notifications.send_update_with_states(
+                self.context, self.instance,
+                vm_states.BUILDING, vm_states.ERROR,
+                task_states.NETWORKING, new_task_state=None)
+        self.assertEqual(0, len(fake_notifier.NOTIFICATIONS))
+        self.assertEqual(0, mock_log_exception.call_count)
+
 
 class NotificationsFormatTestCase(test.NoDBTestCase):