Merge "hyperv: Retry after WMI query fails to find dev"

3 files changed, 78 insertions, 2 deletions

diff --git a/nova/tests/virt/hyperv/test_hypervapi.py b/nova/tests/virt/hyperv/test_hypervapi.py
index f7137b76fd..d162d3ed21 100644
--- a/nova/tests/virt/hyperv/test_hypervapi.py
+++ b/nova/tests/virt/hyperv/test_hypervapi.py
@@ -18,6 +18,7 @@
 Test suite for the Hyper-V driver and related APIs.
 """
 
+import contextlib
 import io
 import os
 import platform
@@ -60,6 +61,7 @@ from nova.virt.hyperv import vhdutils
 from nova.virt.hyperv import vhdutilsv2
 from nova.virt.hyperv import vmutils
 from nova.virt.hyperv import vmutilsv2
+from nova.virt.hyperv import volumeops
 from nova.virt.hyperv import volumeutils
 from nova.virt.hyperv import volumeutilsv2
 from nova.virt import images
@@ -1636,3 +1638,52 @@ class HyperVAPITestCase(test.NoDBTestCase):
                     self._test_spawn_instance, [], None)
             mock_destroy.assert_called_once_with(self._context,
                     self._test_spawn_instance, [], None)
+
+
+class VolumeOpsTestCase(HyperVAPITestCase):
+    """Unit tests for VolumeOps class."""
+
+    def setUp(self):
+        super(VolumeOpsTestCase, self).setUp()
+        self.volumeops = volumeops.VolumeOps()
+
+    def test_get_mounted_disk_from_lun(self):
+        with contextlib.nested(
+            mock.patch.object(self.volumeops._volutils,
+                              'get_device_number_for_target'),
+            mock.patch.object(self.volumeops._vmutils,
+                              'get_mounted_disk_by_drive_number')
+            ) as (mock_get_device_number_for_target,
+                  mock_get_mounted_disk_by_drive_number):
+
+            mock_get_device_number_for_target.return_value = 0
+            mock_get_mounted_disk_by_drive_number.return_value = 'disk_path'
+
+            block_device_info = db_fakes.get_fake_block_device_info(
+                self._volume_target_portal, self._volume_id)
+
+            mapping = driver.block_device_info_get_mapping(block_device_info)
+            data = mapping[0]['connection_info']['data']
+            target_lun = data['target_lun']
+            target_iqn = data['target_iqn']
+
+            disk = self.volumeops._get_mounted_disk_from_lun(target_iqn,
+                                                             target_lun)
+            self.assertEqual(disk, 'disk_path')
+
+    def test_get_mounted_disk_from_lun_failure(self):
+        with mock.patch.object(self.volumeops._volutils,
+                               'get_device_number_for_target') as m_device_num:
+            m_device_num.return_value = None
+
+            block_device_info = db_fakes.get_fake_block_device_info(
+                self._volume_target_portal, self._volume_id)
+
+            mapping = driver.block_device_info_get_mapping(block_device_info)
+            data = mapping[0]['connection_info']['data']
+            target_lun = data['target_lun']
+            target_iqn = data['target_iqn']
+
+            self.assertRaises(exception.NotFound,
+                              self.volumeops._get_mounted_disk_from_lun,
+                              target_iqn, target_lun)
diff --git a/nova/virt/hyperv/basevolumeutils.py b/nova/virt/hyperv/basevolumeutils.py
index 5125b483af..ff8e6722f2 100644
--- a/nova/virt/hyperv/basevolumeutils.py
+++ b/nova/virt/hyperv/basevolumeutils.py
@@ -127,6 +127,10 @@ class BaseVolumeUtils(object):
             return None
 
         devices = initiator_sessions[0].Devices
+
+        if not devices:
+            return None
+
         for device in devices:
             if device.ScsiLun == target_lun:
                 return device.DeviceNumber
diff --git a/nova/virt/hyperv/volumeops.py b/nova/virt/hyperv/volumeops.py
index 5b67067f36..47728d1286 100644
--- a/nova/virt/hyperv/volumeops.py
+++ b/nova/virt/hyperv/volumeops.py
@@ -39,6 +39,14 @@ hyper_volumeops_opts = [
     cfg.IntOpt('volume_attach_retry_interval',
                default=5,
                help='Interval between volume attachment attempts, in seconds'),
+    cfg.IntOpt('mounted_disk_query_retry_count',
+               default=10,
+               help='The number of times to retry checking for a disk mounted '
+                    'via iSCSI.'),
+    cfg.IntOpt('mounted_disk_query_retry_interval',
+               default=5,
+               help='Interval between checks for a mounted iSCSI '
+                    'disk, in seconds.'),
 ]
 
 CONF = cfg.CONF
@@ -192,8 +200,21 @@ class VolumeOps(object):
 
     def _get_mounted_disk_from_lun(self, target_iqn, target_lun,
                                    wait_for_device=False):
-        device_number = self._volutils.get_device_number_for_target(target_iqn,
-                                                                    target_lun)
+        # The WMI query in get_device_number_for_target can incorrectly
+        # return no data when the system is under load.  This issue can
+        # be avoided by adding a retry.
+        for i in xrange(CONF.hyperv.mounted_disk_query_retry_count):
+            device_number = self._volutils.get_device_number_for_target(
+                            target_iqn, target_lun)
+            if device_number is None:
+                attempt = i + 1
+                LOG.debug(_('Attempt %d to get device_number '
+                          'from get_device_number_for_target failed. '
+                          'Retrying...') % attempt)
+                time.sleep(CONF.hyperv.mounted_disk_query_retry_interval)
+            else:
+                break
+
         if device_number is None:
             raise exception.NotFound(_('Unable to find a mounted disk for '
                                        'target_iqn: %s') % target_iqn)