poll_for_events fall back to stack get

This checks the stack status with a stack get if there have been two
event polls which return no events. This handles the cases where an
event may have been lost, or older heat versions which didn't generate
"resource" events for stack state changes.

Change-Id: I0db71aecbd828c9b7dba0df8e548ba48f83436c2
Blueprint: heat-support-python-openstackclient

2 files changed, 84 insertions, 9 deletions

diff --git a/heatclient/common/event_utils.py b/heatclient/common/event_utils.py
index 03bb8b2..3e6b197 100644
--- a/heatclient/common/event_utils.py
+++ b/heatclient/common/event_utils.py
@@ -130,17 +130,26 @@ def _get_stack_events(hc, stack_id, event_args):
         return events
 
 
-def poll_for_events(hc, stack_name, action, poll_period):
+def poll_for_events(hc, stack_name, action=None, poll_period=5, marker=None):
     """Continuously poll events and logs for performed action on stack."""
 
-    marker = None
-    stop_status = ('%s_FAILED' % action, '%s_COMPLETE' % action)
+    if action:
+        stop_status = ('%s_FAILED' % action, '%s_COMPLETE' % action)
+        stop_check = lambda a: a in stop_status
+    else:
+        stop_check = lambda a: a.endswith('_COMPLETE') or a.endswith('_FAILED')
+
+    no_event_polls = 0
+    msg_template = _("\n Stack %(name)s %(status)s \n")
     while True:
         events = get_events(hc, stack_id=stack_name,
                             event_args={'sort_dir': 'asc',
                                         'marker': marker})
 
-        if len(events) >= 1:
+        if len(events) == 0:
+            no_event_polls += 1
+        else:
+            no_event_polls = 0
             # set marker to last event that was received.
             marker = getattr(events[-1], 'id', None)
             events_log = utils.event_log_formatter(events)
@@ -150,9 +159,20 @@ def poll_for_events(hc, stack_name, action, poll_period):
                 # check if stack event was also received
                 if getattr(event, 'resource_name', '') == stack_name:
                     stack_status = getattr(event, 'resource_status', '')
-                    msg = _("\n Stack %(name)s %(status)s \n") % dict(
+                    msg = msg_template % dict(
                         name=stack_name, status=stack_status)
-                    if stack_status in stop_status:
+                    if stop_check(stack_status):
                         return stack_status, msg
 
+        if no_event_polls >= 2:
+            # after 2 polls with no events, fall back to a stack get
+            stack = hc.stacks.get(stack_name)
+            stack_status = stack.stack_status
+            msg = msg_template % dict(
+                name=stack_name, status=stack_status)
+            if stop_check(stack_status):
+                return stack_status, msg
+            # go back to event polling again
+            no_event_polls = 0
+
         time.sleep(poll_period)
diff --git a/heatclient/tests/unit/test_event_utils.py b/heatclient/tests/unit/test_event_utils.py
index d2350e5..098d2b5 100644
--- a/heatclient/tests/unit/test_event_utils.py
+++ b/heatclient/tests/unit/test_event_utils.py
@@ -146,7 +146,7 @@ class ShellTestEventUtils(testtools.TestCase):
         ]]
 
         stack_status, msg = event_utils.poll_for_events(
-            None, 'astack', 'CREATE', 0)
+            None, 'astack', action='CREATE', poll_period=0)
         self.assertEqual('CREATE_COMPLETE', stack_status)
         self.assertEqual('\n Stack astack CREATE_COMPLETE \n', msg)
         ge.assert_has_calls([
@@ -159,6 +159,25 @@ class ShellTestEventUtils(testtools.TestCase):
         ])
 
     @mock.patch('heatclient.common.event_utils.get_events')
+    def test_poll_for_events_with_marker(self, ge):
+        ge.side_effect = [[
+            self._mock_event('5', 'res_child1', 'CREATE_COMPLETE'),
+            self._mock_event('6', 'res_child2', 'CREATE_COMPLETE'),
+            self._mock_event('7', 'res_child3', 'CREATE_COMPLETE'),
+            self._mock_event('8', 'astack', 'CREATE_COMPLETE')
+        ]]
+
+        stack_status, msg = event_utils.poll_for_events(
+            None, 'astack', action='CREATE', poll_period=0, marker='4')
+        self.assertEqual('CREATE_COMPLETE', stack_status)
+        self.assertEqual('\n Stack astack CREATE_COMPLETE \n', msg)
+        ge.assert_has_calls([
+            mock.call(None, stack_id='astack', event_args={
+                'sort_dir': 'asc', 'marker': '4'
+            })
+        ])
+
+    @mock.patch('heatclient.common.event_utils.get_events')
     def test_poll_for_events_in_progress_resource(self, ge):
         ge.side_effect = [[
             self._mock_event('1', 'astack', 'CREATE_IN_PROGRESS'),
@@ -167,7 +186,7 @@ class ShellTestEventUtils(testtools.TestCase):
         ]]
 
         stack_status, msg = event_utils.poll_for_events(
-            None, 'astack', 'CREATE', 0)
+            None, 'astack', action='CREATE', poll_period=0)
         self.assertEqual('CREATE_COMPLETE', stack_status)
         self.assertEqual('\n Stack astack CREATE_COMPLETE \n', msg)
 
@@ -186,6 +205,42 @@ class ShellTestEventUtils(testtools.TestCase):
         ]]
 
         stack_status, msg = event_utils.poll_for_events(
-            None, 'astack', 'CREATE', 0)
+            None, 'astack', action='CREATE', poll_period=0)
+        self.assertEqual('CREATE_FAILED', stack_status)
+        self.assertEqual('\n Stack astack CREATE_FAILED \n', msg)
+
+    @mock.patch('heatclient.common.event_utils.get_events')
+    def test_poll_for_events_no_action(self, ge):
+        ge.side_effect = [[
+            self._mock_event('1', 'astack', 'CREATE_IN_PROGRESS'),
+            self._mock_event('2', 'res_child1', 'CREATE_IN_PROGRESS'),
+            self._mock_event('3', 'res_child2', 'CREATE_IN_PROGRESS'),
+            self._mock_event('4', 'res_child3', 'CREATE_IN_PROGRESS')
+        ], [
+            self._mock_event('5', 'res_child1', 'CREATE_COMPLETE'),
+            self._mock_event('6', 'res_child2', 'CREATE_FAILED'),
+            self._mock_event('7', 'res_child3', 'CREATE_COMPLETE'),
+            self._mock_event('8', 'astack', 'FOO_FAILED')
+        ]]
+
+        stack_status, msg = event_utils.poll_for_events(
+            None, 'astack', action=None, poll_period=0)
+        self.assertEqual('FOO_FAILED', stack_status)
+        self.assertEqual('\n Stack astack FOO_FAILED \n', msg)
+
+    @mock.patch('heatclient.common.event_utils.get_events')
+    def test_poll_for_events_stack_get(self, ge):
+        mock_client = mock.MagicMock()
+        mock_client.stacks.get.return_value.stack_status = 'CREATE_FAILED'
+
+        ge.side_effect = [[
+            self._mock_event('1', 'astack', 'CREATE_IN_PROGRESS'),
+            self._mock_event('2', 'res_child1', 'CREATE_IN_PROGRESS'),
+            self._mock_event('3', 'res_child2', 'CREATE_IN_PROGRESS'),
+            self._mock_event('4', 'res_child3', 'CREATE_IN_PROGRESS')
+        ], [], []]
+
+        stack_status, msg = event_utils.poll_for_events(
+            mock_client, 'astack', action='CREATE', poll_period=0)
         self.assertEqual('CREATE_FAILED', stack_status)
         self.assertEqual('\n Stack astack CREATE_FAILED \n', msg)