Check for server in attachements when checking for detach complete

It's possible that the volume is already attached to another
server, which can happen by marking a  VolumeAttachment resource
unhealthy after removing the volume from the server.

Change-Id: Ia77e0462bf732224b491591d22584cbcfa779956
Story: #2003850
Task: 26643
(cherry picked from commit 3b59f9ad98427ea152e0d55397d102998883d429)

4 files changed, 14 insertions, 6 deletions

diff --git a/heat/engine/clients/os/cinder.py b/heat/engine/clients/os/cinder.py
index 6e51c2b9f..bbcd886c8 100644
--- a/heat/engine/clients/os/cinder.py
+++ b/heat/engine/clients/os/cinder.py
@@ -128,13 +128,18 @@ class CinderClientPlugin(client_plugin.ClientPlugin):
         return (isinstance(ex, exceptions.ClientException) and
                 ex.code == 409)
 
-    def check_detach_volume_complete(self, vol_id):
+    def check_detach_volume_complete(self, vol_id, server_id=None):
         try:
             vol = self.client().volumes.get(vol_id)
         except Exception as ex:
             self.ignore_not_found(ex)
             return True
 
+        server_ids = [
+            a['server_id'] for a in vol.attachments if 'server_id' in a]
+        if server_id and server_id not in server_ids:
+            return True
+
         if vol.status in ('in-use', 'detaching'):
             LOG.debug('%s - volume still in use', vol_id)
             return False
diff --git a/heat/engine/resources/openstack/cinder/volume.py b/heat/engine/resources/openstack/cinder/volume.py
index de05fe4de..7fb4294ae 100644
--- a/heat/engine/resources/openstack/cinder/volume.py
+++ b/heat/engine/resources/openstack/cinder/volume.py
@@ -507,9 +507,9 @@ class CinderVolume(vb.BaseVolume, sh.SchedulerHintsMixin):
             prg_detach.called = True
             return False
         if not prg_detach.cinder_complete:
-            cinder_complete_res = self.client_plugin(
-            ).check_detach_volume_complete(prg_detach.vol_id)
-            prg_detach.cinder_complete = cinder_complete_res
+            prg_detach.cinder_complete = self.client_plugin(
+            ).check_detach_volume_complete(prg_detach.vol_id,
+                                           prg_detach.srv_id)
             return False
         if not prg_detach.nova_complete:
             prg_detach.nova_complete = self.client_plugin(
@@ -770,7 +770,8 @@ class CinderVolumeAttachment(vb.BaseVolumeAttachment):
             return True
         if not prg_detach.cinder_complete:
             prg_detach.cinder_complete = self.client_plugin(
-            ).check_detach_volume_complete(prg_detach.vol_id)
+            ).check_detach_volume_complete(prg_detach.vol_id,
+                                           prg_detach.srv_id)
             return False
         if not prg_detach.nova_complete:
             prg_detach.nova_complete = self.client_plugin(
diff --git a/heat/engine/resources/volume_base.py b/heat/engine/resources/volume_base.py
index f9079a2a4..f959ce6b7 100644
--- a/heat/engine/resources/volume_base.py
+++ b/heat/engine/resources/volume_base.py
@@ -211,7 +211,7 @@ class BaseVolumeAttachment(resource.Resource):
 
         if not prg.cinder_complete:
             prg.cinder_complete = self.client_plugin(
-            ).check_detach_volume_complete(prg.vol_id)
+            ).check_detach_volume_complete(prg.vol_id, prg.srv_id)
             return False
         if not prg.nova_complete:
             prg.nova_complete = self.client_plugin(
diff --git a/heat/tests/openstack/cinder/test_volume_utils.py b/heat/tests/openstack/cinder/test_volume_utils.py
index ce4df1b90..e37804798 100644
--- a/heat/tests/openstack/cinder/test_volume_utils.py
+++ b/heat/tests/openstack/cinder/test_volume_utils.py
@@ -114,6 +114,8 @@ class FakeVolume(object):
             setattr(self, key, value)
         if 'id' not in attrs:
             self.id = self._ID
+        if 'attachments' not in attrs:
+            self.attachments = [{'server_id': 'WikiDatabase'}]
 
 
 class FakeBackup(FakeVolume):