Add ability to download Glance images into the libvirt image cache via RBD

This change allows compute hosts to quickly download and cache images on the
local compute host directly from Ceph rather than slow dowloads from the
Glance API.

New '[glance]/enable_rbd_download' option is introduced to enable this
behavior. This is slight change compared to the original idea described in
the relevant blueprint where it was discussed to use (now obsolete)
'[glance]/allowed_direct_url_schemes' option.

Additionally, when an image signature verification is requested, it should
be done also for the image fetched by the new download handler. This was
completely missing so far.

New '[glance]/rbd_{user,pool,ceph_conf,connect_timeout}' configurables
are introduced to allow operators to configure access to the cluster
hosting Glance without the need to use the existing '[libvirt]'
specific configurables. nova.storage.rbd_utils.RBDDriver has also been
modified to accept these but continues to default to the '[libvirt]'
specific configurables for now.

Change-Id: I3032bbe6bd2d6acc9ba0f0cac4d00ed4b4464ceb
Implements: blueprint nova-image-download-via-rbd

5 files changed, 319 insertions, 13 deletions

diff --git a/nova/conf/glance.py b/nova/conf/glance.py
index 3391621c53..8838160411 100644
--- a/nova/conf/glance.py
+++ b/nova/conf/glance.py
@@ -148,9 +148,64 @@ Related options:
 * The value of this option may be used if both verify_glance_signatures and
   enable_certificate_validation are enabled.
 """),
+    cfg.BoolOpt('enable_rbd_download',
+        default=False,
+        help="""
+Enable download of Glance images directly via RBD.
+
+Allow compute hosts to quickly download and cache images localy directly
+from Ceph rather than slow dowloads from the Glance API. This can
+reduce download time for images in the ten to hundreds of GBs from tens of
+minutes to tens of seconds, but requires a Ceph-based deployment and access
+from the compute nodes to Ceph.
+
+Related options:
+
+* ``[glance] rbd_user``
+* ``[glance] rbd_connect_timeout``
+* ``[glance] rbd_pool``
+* ``[glance] rbd_ceph_conf``
+"""),
+    cfg.StrOpt('rbd_user',
+        default='',
+        help="""
+The RADOS client name for accessing Glance images stored as rbd volumes.
+
+Related options:
+
+* This option is only used if ``[glance] enable_rbd_download`` is set to True.
+"""),
+    cfg.IntOpt('rbd_connect_timeout',
+        default=5,
+        help="""
+The RADOS client timeout in seconds when initially connecting to the cluster.
+
+Related options:
+
+* This option is only used if ``[glance] enable_rbd_download`` is set to True.
+"""),
+    cfg.StrOpt('rbd_pool',
+        default='',
+        help="""
+The RADOS pool in which the Glance images are stored as rbd volumes.
+
+Related options:
+
+* This option is only used if ``[glance] enable_rbd_download`` is set to True.
+"""),
+    cfg.StrOpt('rbd_ceph_conf',
+        default='',
+        help="""
+Path to the ceph configuration file to use.
+
+Related options:
+
+* This option is only used if ``[glance] enable_rbd_download`` is set to True.
+"""),
+
     cfg.BoolOpt('debug',
-         default=False,
-         help='Enable or disable debug logging with glanceclient.')
+        default=False,
+        help='Enable or disable debug logging with glanceclient.')
 ]
 
 deprecated_ksa_opts = {
diff --git a/nova/image/glance.py b/nova/image/glance.py
index be0c1eeccd..5e72679a2a 100644
--- a/nova/image/glance.py
+++ b/nova/image/glance.py
@@ -24,12 +24,14 @@ import re
 import stat
 import sys
 import time
+import urllib.parse as urlparse
 
 import cryptography
 from cursive import certificate_utils
 from cursive import exception as cursive_exception
 from cursive import signature_utils
 import glanceclient
+from glanceclient.common import utils as glance_utils
 import glanceclient.exc
 from glanceclient.v2 import schemas
 from keystoneauth1 import loading as ks_loading
@@ -39,7 +41,6 @@ from oslo_utils import excutils
 from oslo_utils import timeutils
 import six
 from six.moves import range
-import six.moves.urllib.parse as urlparse
 
 import nova.conf
 from nova import exception
@@ -221,6 +222,51 @@ class GlanceImageServiceV2(object):
         # to be added here.
         self._download_handlers = {}
 
+        if CONF.glance.enable_rbd_download:
+            self._download_handlers['rbd'] = self.rbd_download
+
+    def rbd_download(self, context, url_parts, dst_path, metadata=None):
+        """Use an explicit rbd call to download an image.
+
+        :param context: The `nova.context.RequestContext` object for the
+                        request
+        :param url_parts: Parts of URL pointing to the image location
+        :param dst_path: Filepath to transfer the image file to.
+        :param metadata: Image location metadata (currently unused)
+        """
+
+        # avoid circular import
+        from nova.storage import rbd_utils
+        try:
+            # Parse the RBD URL from url_parts, it should consist of 4
+            # sections and be in the format of:
+            # <cluster_uuid>/<pool_name>/<image_uuid>/<snapshot_name>
+            url_path = str(urlparse.unquote(url_parts.path))
+            cluster_uuid, pool_name, image_uuid, snapshot_name = (
+                url_path.split('/'))
+        except ValueError as e:
+            msg = f"Invalid RBD URL format: {e}"
+            LOG.error(msg)
+            raise nova.exception.InvalidParameterValue(msg)
+
+        rbd_driver = rbd_utils.RBDDriver(
+            user=CONF.glance.rbd_user,
+            pool=CONF.glance.rbd_pool,
+            ceph_conf=CONF.glance.rbd_ceph_conf,
+            connect_timeout=CONF.glance.rbd_connect_timeout)
+
+        try:
+            LOG.debug("Attempting to export RBD image: "
+                      "[pool_name: %s] [image_uuid: %s] "
+                      "[snapshot_name: %s] [dst_path: %s]",
+                      pool_name, image_uuid, snapshot_name, dst_path)
+
+            rbd_driver.export_image(dst_path, image_uuid,
+                                    snapshot_name, pool_name)
+        except Exception as e:
+            LOG.error("Error during RBD image export: %s", e)
+            raise nova.exception.CouldNotFetchImage(image_id=image_uuid)
+
     def show(self, context, image_id, include_locations=False,
              show_deleted=True):
         """Returns a dict with image data for the given opaque image id.
@@ -299,7 +345,13 @@ class GlanceImageServiceV2(object):
     def download(self, context, image_id, data=None, dst_path=None,
                  trusted_certs=None):
         """Calls out to Glance for data and writes data."""
-        if CONF.glance.allowed_direct_url_schemes and dst_path is not None:
+
+        # First, check if image could be directly downloaded by special handler
+        # TODO(stephenfin): Remove check for 'allowed_direct_url_schemes' when
+        # we clean up tests since it's not used elsewhere
+        if ((CONF.glance.allowed_direct_url_schemes or
+             self._download_handlers) and dst_path is not None
+        ):
             image = self.show(context, image_id, include_locations=True)
             for entry in image.get('locations', []):
                 loc_url = entry['url']
@@ -310,10 +362,21 @@ class GlanceImageServiceV2(object):
                     try:
                         xfer_method(context, o, dst_path, loc_meta)
                         LOG.info("Successfully transferred using %s", o.scheme)
+
+                        # Load chunks from the downloaded image file
+                        # for verification (if required)
+                        with open(dst_path, 'rb') as fh:
+                            downloaded_length = os.path.getsize(dst_path)
+                            image_chunks = glance_utils.IterableWithLength(fh,
+                                downloaded_length)
+                            self._verify_and_write(context, image_id,
+                                trusted_certs, image_chunks, None, None)
                         return
                     except Exception:
                         LOG.exception("Download image error")
 
+        # By default (or if direct download has failed), use glance client call
+        # to fetch the image and fill image_chunks
         try:
             image_chunks = self._client.call(
                 context, 2, 'data', args=(image_id,))
diff --git a/nova/storage/rbd_utils.py b/nova/storage/rbd_utils.py
index bda1b5d542..22bafe5053 100644
--- a/nova/storage/rbd_utils.py
+++ b/nova/storage/rbd_utils.py
@@ -122,14 +122,16 @@ class RADOSClient(object):
 
 class RBDDriver(object):
 
-    def __init__(self):
+    def __init__(self, pool=None, user=None, ceph_conf=None,
+                 connect_timeout=None):
         if rbd is None:
             raise RuntimeError(_('rbd python libraries not found'))
 
-        self.pool = CONF.libvirt.images_rbd_pool
-        self.rbd_user = CONF.libvirt.rbd_user
-        self.rbd_connect_timeout = CONF.libvirt.rbd_connect_timeout
-        self.ceph_conf = CONF.libvirt.images_rbd_ceph_conf
+        self.pool = pool or CONF.libvirt.images_rbd_pool
+        self.rbd_user = user or CONF.libvirt.rbd_user
+        self.rbd_connect_timeout = (
+            connect_timeout or CONF.libvirt.rbd_connect_timeout)
+        self.ceph_conf = ceph_conf or CONF.libvirt.images_rbd_ceph_conf
 
     def _connect_to_rados(self, pool=None):
         client = rados.Rados(rados_id=self.rbd_user,
@@ -335,6 +337,25 @@ class RBDDriver(object):
         args += self.ceph_args()
         processutils.execute('rbd', 'import', *args)
 
+    def export_image(self, base, name, snap, pool=None):
+        """Export RBD volume to image file.
+
+        Uses the command line export to export rbd volume snapshot to
+        local image file.
+
+        :base: Path to image file
+        :name: Name of RBD volume
+        :snap: Name of RBD snapshot
+        :pool: Name of RBD pool
+        """
+        if pool is None:
+            pool = self.pool
+
+        args = ['--pool', pool, '--image', name, '--path', base,
+                '--snap', snap]
+        args += self.ceph_args()
+        processutils.execute('rbd', 'export', *args)
+
     def _destroy_volume(self, client, volume, pool=None):
         """Destroy an RBD volume, retrying as needed.
         """
diff --git a/nova/tests/unit/image/test_glance.py b/nova/tests/unit/image/test_glance.py
index a672a967ed..170b2282f3 100644
--- a/nova/tests/unit/image/test_glance.py
+++ b/nova/tests/unit/image/test_glance.py
@@ -16,6 +16,7 @@
 
 import copy
 import datetime
+import urllib.parse as urlparse
 
 import cryptography
 from cursive import exception as cursive_exception
@@ -36,6 +37,7 @@ from nova import exception
 from nova.image import glance
 from nova import objects
 from nova import service_auth
+from nova.storage import rbd_utils
 from nova import test
 
 
@@ -686,9 +688,14 @@ class TestDownloadNoDirectUri(test.NoDBTestCase):
         with testtools.ExpectedException(exception.ImageUnacceptable):
             service.download(ctx, mock.sentinel.image_id)
 
+    # TODO(stephenfin): Drop this test since it's not possible to run in
+    # production
+    @mock.patch('os.path.getsize', return_value=1)
+    @mock.patch.object(six.moves.builtins, 'open')
     @mock.patch('nova.image.glance.GlanceImageServiceV2._get_transfer_method')
     @mock.patch('nova.image.glance.GlanceImageServiceV2.show')
-    def test_download_direct_file_uri_v2(self, show_mock, get_tran_mock):
+    def test_download_direct_file_uri_v2(
+            self, show_mock, get_tran_mock, open_mock, getsize_mock):
         self.flags(allowed_direct_url_schemes=['file'], group='glance')
         show_mock.return_value = {
             'locations': [
@@ -702,6 +709,8 @@ class TestDownloadNoDirectUri(test.NoDBTestCase):
         get_tran_mock.return_value = tran_mod
         client = mock.MagicMock()
         ctx = mock.sentinel.ctx
+        writer = mock.MagicMock()
+        open_mock.return_value = writer
         service = glance.GlanceImageServiceV2(client)
         res = service.download(ctx, mock.sentinel.image_id,
                                dst_path=mock.sentinel.dst_path)
@@ -716,6 +725,76 @@ class TestDownloadNoDirectUri(test.NoDBTestCase):
                                          mock.sentinel.dst_path,
                                          mock.sentinel.loc_meta)
 
+    @mock.patch('glanceclient.common.utils.IterableWithLength')
+    @mock.patch('os.path.getsize', return_value=1)
+    @mock.patch.object(six.moves.builtins, 'open')
+    @mock.patch('nova.image.glance.LOG')
+    @mock.patch('nova.image.glance.GlanceImageServiceV2._get_verifier')
+    @mock.patch('nova.image.glance.GlanceImageServiceV2._get_transfer_method')
+    @mock.patch('nova.image.glance.GlanceImageServiceV2.show')
+    def test_download_direct_rbd_uri_v2(
+            self, show_mock, get_tran_mock, get_verifier_mock, log_mock,
+            open_mock, getsize_mock, iterable_with_length_mock):
+        self.flags(enable_rbd_download=True, group='glance')
+        show_mock.return_value = {
+            'locations': [
+                {
+                    'url': 'rbd://cluster_uuid/pool_name/image_uuid/snapshot',
+                    'metadata': mock.sentinel.loc_meta
+                }
+            ]
+        }
+        tran_mod = mock.MagicMock()
+        get_tran_mock.return_value = tran_mod
+        client = mock.MagicMock()
+        ctx = mock.sentinel.ctx
+        writer = mock.MagicMock()
+        open_mock.return_value = writer
+        iterable_with_length_mock.return_value = ["rbd1", "rbd2"]
+        service = glance.GlanceImageServiceV2(client)
+
+        verifier = mock.MagicMock()
+        get_verifier_mock.return_value = verifier
+
+        res = service.download(ctx, mock.sentinel.image_id,
+                               dst_path=mock.sentinel.dst_path,
+                               trusted_certs=mock.sentinel.trusted_certs)
+
+        self.assertIsNone(res)
+        show_mock.assert_called_once_with(ctx,
+                                          mock.sentinel.image_id,
+                                          include_locations=True)
+        tran_mod.assert_called_once_with(ctx, mock.ANY,
+                                         mock.sentinel.dst_path,
+                                         mock.sentinel.loc_meta)
+        open_mock.assert_called_once_with(mock.sentinel.dst_path, 'rb')
+        get_tran_mock.assert_called_once_with('rbd')
+
+        # no client call, chunks were read right after xfer_mod.download:
+        client.call.assert_not_called()
+
+        # verifier called with the value we got from rbd download
+        verifier.update.assert_has_calls(
+                [
+                    mock.call("rbd1"),
+                    mock.call("rbd2")
+                ]
+        )
+        verifier.verify.assert_called()
+        log_mock.info.assert_has_calls(
+            [
+                mock.call('Successfully transferred using %s', 'rbd'),
+                mock.call(
+                    'Image signature verification succeeded for image %s',
+                    mock.sentinel.image_id)
+            ]
+        )
+
+        # not opened for writing (already written)
+        self.assertFalse(open_mock(mock.sentinel.dst_path, 'rw').called)
+        # write not called (written by rbd download)
+        writer.write.assert_not_called()
+
     @mock.patch('nova.image.glance.GlanceImageServiceV2._get_transfer_method')
     @mock.patch('nova.image.glance.GlanceImageServiceV2.show')
     @mock.patch('nova.image.glance.GlanceImageServiceV2._safe_fsync')
@@ -1249,6 +1328,60 @@ class TestIsImageAvailable(test.NoDBTestCase):
         self.assertTrue(res)
 
 
+class TestRBDDownload(test.NoDBTestCase):
+
+    def setUp(self):
+        super(TestRBDDownload, self).setUp()
+        loc_url = "rbd://ce2d1ace/images/b86d6d06-faac/snap"
+        self.url_parts = urlparse.urlparse(loc_url)
+        self.image_uuid = "b86d6d06-faac"
+        self.pool_name = "images"
+        self.snapshot_name = "snap"
+
+    @mock.patch.object(rbd_utils.RBDDriver, 'export_image')
+    @mock.patch.object(rbd_utils, 'rbd')
+    def test_rbd_download_success(self, mock_rbd, mock_export_image):
+        client = mock.MagicMock()
+        ctx = mock.sentinel.ctx
+        service = glance.GlanceImageServiceV2(client)
+
+        service.rbd_download(ctx, self.url_parts, mock.sentinel.dst_path)
+
+        # Assert that we attempt to export using the correct rbd pool, volume
+        # and snapshot given the provided URL
+        mock_export_image.assert_called_once_with(mock.sentinel.dst_path,
+            self.image_uuid,
+            self.snapshot_name,
+            self.pool_name)
+
+    def test_rbd_download_broken_url(self):
+        client = mock.MagicMock()
+        ctx = mock.sentinel.ctx
+        service = glance.GlanceImageServiceV2(client)
+
+        wrong_url = "http://www.example.com"
+        wrong_url_parts = urlparse.urlparse(wrong_url)
+
+        # Assert InvalidParameterValue is raised when we can't parse the URL
+        self.assertRaises(
+            exception.InvalidParameterValue, service.rbd_download, ctx,
+            wrong_url_parts, mock.sentinel.dst_path)
+
+    @mock.patch('nova.storage.rbd_utils.RBDDriver.export_image')
+    @mock.patch.object(rbd_utils, 'rbd')
+    def test_rbd_download_export_failure(self, mock_rbd, mock_export_image):
+        client = mock.MagicMock()
+        ctx = mock.sentinel.ctx
+        service = glance.GlanceImageServiceV2(client)
+
+        mock_export_image.side_effect = Exception
+
+        # Assert CouldNotFetchImage is raised when the export fails
+        self.assertRaisesRegex(
+            exception.CouldNotFetchImage, self.image_uuid,
+            service.rbd_download, ctx, self.url_parts, mock.sentinel.dst_path)
+
+
 class TestShow(test.NoDBTestCase):
 
     """Tests the show method of the GlanceImageServiceV2."""
diff --git a/nova/tests/unit/storage/test_rbd.py b/nova/tests/unit/storage/test_rbd.py
index fd9372b870..5a39bdbd5a 100644
--- a/nova/tests/unit/storage/test_rbd.py
+++ b/nova/tests/unit/storage/test_rbd.py
@@ -109,9 +109,11 @@ class RbdTestCase(test.NoDBTestCase):
 
         self.rbd_pool = 'rbd'
         self.rbd_connect_timeout = 5
-        self.flags(images_rbd_pool=self.rbd_pool, group='libvirt')
-        self.flags(rbd_connect_timeout=self.rbd_connect_timeout,
-                    group='libvirt')
+        self.flags(
+            images_rbd_pool=self.images_rbd_pool,
+            images_rbd_ceph_conf='/foo/bar.conf',
+            rbd_connect_timeout=self.rbd_connect_timeout,
+            rbd_user='foo', group='libvirt')
 
         rados_patcher = mock.patch.object(rbd_utils, 'rados')
         self.mock_rados = rados_patcher.start()
@@ -657,3 +659,35 @@ class RbdTestCase(test.NoDBTestCase):
         ceph_df_not_found = CEPH_DF.replace('rbd', 'vms')
         mock_execute.return_value = (ceph_df_not_found, '')
         self.assertRaises(exception.NotFound, self.driver.get_pool_info)
+
+    @mock.patch('oslo_concurrency.processutils.execute')
+    def test_export_image(self, mock_execute):
+        self.driver.rbd_user = 'foo'
+        self.driver.export_image(mock.sentinel.dst_path,
+                                 mock.sentinel.name,
+                                 mock.sentinel.snap,
+                                 mock.sentinel.pool)
+
+        mock_execute.assert_called_once_with(
+            'rbd', 'export',
+            '--pool', mock.sentinel.pool,
+            '--image', mock.sentinel.name,
+            '--path', mock.sentinel.dst_path,
+            '--snap', mock.sentinel.snap,
+            '--id', 'foo',
+            '--conf', '/foo/bar.conf')
+
+    @mock.patch('oslo_concurrency.processutils.execute')
+    def test_export_image_default_pool(self, mock_execute):
+        self.driver.export_image(mock.sentinel.dst_path,
+                                 mock.sentinel.name,
+                                 mock.sentinel.snap)
+
+        mock_execute.assert_called_once_with(
+            'rbd', 'export',
+            '--pool', self.rbd_pool,
+            '--image', mock.sentinel.name,
+            '--path', mock.sentinel.dst_path,
+            '--snap', mock.sentinel.snap,
+            '--id', 'foo',
+            '--conf', '/foo/bar.conf')