Handle sparse images in glance_store

Add new configuration option ``rbd_thin_provisioning`` and
``filesystem_thin_provisioning`` to rbd and filesystem
store to enable or not sparse upload.

A sparse file means that we do not actually write null byte
sequences but only the data itself at a given offset, the
"holes" which can appear will automatically be interpreted by
the storage backend as null bytes, and do not really consume
your storage.

Change-Id: I129e30f490e3920e9093c2b793f89b70ce310a50
Co-Authored-By: Grégoire Unbekandt <gregoire.unbekandt@gmail.com>
Partially Implements: blueprint handle-sparse-image

7 files changed, 243 insertions, 7 deletions

diff --git a/glance_store/_drivers/filesystem.py b/glance_store/_drivers/filesystem.py
index 57204de..aa77e24 100644
--- a/glance_store/_drivers/filesystem.py
+++ b/glance_store/_drivers/filesystem.py
@@ -165,7 +165,29 @@ Possible Values:
 Related options:
     * None
 
-""")]
+"""),
+    cfg.BoolOpt('filesystem_thin_provisioning',
+                default=False,
+                help="""
+Enable or not thin provisioning in this backend.
+
+This configuration option enable the feature of not really write null byte
+sequences on the filesystem, the holes who can appear will automatically
+be interpreted by the filesystem as null bytes, and do not really consume
+your storage.
+Enabling this feature will also speed up image upload and save network trafic
+in addition to save space in the backend, as null bytes sequences are not
+sent over the network.
+
+Possible Values:
+    * True
+    * False
+
+Related options:
+    * None
+
+"""),
+]
 
 MULTI_FILESYSTEM_METADATA_SCHEMA = {
     "type": "array",
@@ -408,6 +430,8 @@ class Store(glance_store.driver.Store):
         fstore_perm = store_conf.filesystem_store_file_perm
         meta_file = store_conf.filesystem_store_metadata_file
 
+        self.thin_provisioning = store_conf.\
+            filesystem_thin_provisioning
         self.chunk_size = store_conf.filesystem_store_chunk_size
         self.READ_CHUNKSIZE = self.chunk_size
         self.WRITE_CHUNKSIZE = self.READ_CHUNKSIZE
@@ -725,7 +749,11 @@ class Store(glance_store.driver.Store):
                     checksum.update(buf)
                     if verifier:
                         verifier.update(buf)
-                    f.write(buf)
+                    if self.thin_provisioning and not any(buf):
+                        f.truncate(bytes_written)
+                        f.seek(0, os.SEEK_END)
+                    else:
+                        f.write(buf)
         except IOError as e:
             if e.errno != errno.EACCES:
                 self._delete_partial(filepath, image_id)
diff --git a/glance_store/_drivers/rbd.py b/glance_store/_drivers/rbd.py
index 44d2df7..7c71075 100644
--- a/glance_store/_drivers/rbd.py
+++ b/glance_store/_drivers/rbd.py
@@ -151,6 +151,26 @@ Related options:
     * None
 
 """),
+    cfg.BoolOpt('rbd_thin_provisioning',
+                default=False,
+                help="""
+Enable or not thin provisioning in this backend.
+
+This configuration option enable the feature of not really write null byte
+sequences on the RBD backend, the holes who can appear will automatically
+be interpreted by Ceph as null bytes, and do not really consume your storage.
+Enabling this feature will also speed up image upload and save network trafic
+in addition to save space in the backend, as null bytes sequences are not
+sent over the network.
+
+Possible Values:
+    * True
+    * False
+
+Related options:
+    * None
+
+"""),
 ]
 
 
@@ -302,13 +322,19 @@ class Store(driver.Store):
                                     self.backend_group).rbd_store_ceph_conf
                 connect_timeout = getattr(
                     self.conf, self.backend_group).rados_connect_timeout
+                thin_provisioning = getattr(self.conf,
+                                            self.backend_group).\
+                    rbd_thin_provisioning
             else:
                 chunk = self.conf.glance_store.rbd_store_chunk_size
                 pool = self.conf.glance_store.rbd_store_pool
                 user = self.conf.glance_store.rbd_store_user
                 conf_file = self.conf.glance_store.rbd_store_ceph_conf
                 connect_timeout = self.conf.glance_store.rados_connect_timeout
+                thin_provisioning = \
+                    self.conf.glance_store.rbd_thin_provisioning
 
+            self.thin_provisioning = thin_provisioning
             self.chunk_size = chunk * units.Mi
             self.READ_CHUNKSIZE = self.chunk_size
             self.WRITE_CHUNKSIZE = self.READ_CHUNKSIZE
@@ -555,10 +581,10 @@ class Store(driver.Store):
                                                               image_size,
                                                               bytes_written,
                                                               chunk_length)
-                            LOG.debug(_("writing chunk at offset %s") %
-                                      (offset))
-                            offset += image.write(chunk, offset)
                             bytes_written += chunk_length
+                            if not (self.thin_provisioning and not any(chunk)):
+                                image.write(chunk, offset)
+                            offset += chunk_length
                             os_hash_value.update(chunk)
                             checksum.update(chunk)
                             if verifier:
diff --git a/glance_store/multi_backend.py b/glance_store/multi_backend.py
index 4223e28..cfd80c0 100644
--- a/glance_store/multi_backend.py
+++ b/glance_store/multi_backend.py
@@ -143,7 +143,10 @@ def register_store_opts(conf, reserved_stores=None):
                 cfg.IntOpt('filesystem_store_chunk_size',
                            default=64 * units.Ki,
                            min=1,
-                           help=FS_CONF_CHUNKSIZE_HELP.format(key))]
+                           help=FS_CONF_CHUNKSIZE_HELP.format(key)),
+                cfg.BoolOpt('filesystem_thin_provisioning',
+                            default=False,
+                            help="""Not used""")]
             LOG.debug("Registering options for reserved store: {}".format(key))
             conf.register_opts(fs_conf_template, group=key)
 
diff --git a/glance_store/tests/unit/test_filesystem_store.py b/glance_store/tests/unit/test_filesystem_store.py
index 6f2661e..bb98f41 100644
--- a/glance_store/tests/unit/test_filesystem_store.py
+++ b/glance_store/tests/unit/test_filesystem_store.py
@@ -143,8 +143,13 @@ class TestStore(base.StoreBaseTest,
                           self.store.get,
                           loc)
 
-    def test_add(self):
+    def _do_test_add(self, enable_thin_provisoning):
         """Test that we can add an image via the filesystem backend."""
+        self.config(filesystem_store_chunk_size=units.Ki,
+                    filesystem_thin_provisioning=enable_thin_provisoning,
+                    group='glance_store')
+        self.store.configure()
+
         filesystem.ChunkedFile.CHUNKSIZE = units.Ki
         expected_image_id = str(uuid.uuid4())
         expected_file_size = 5 * units.Ki  # 5K
@@ -176,6 +181,86 @@ class TestStore(base.StoreBaseTest,
         self.assertEqual(expected_file_contents, new_image_contents)
         self.assertEqual(expected_file_size, new_image_file_size)
 
+    def test_thin_provisioning_is_disabled_by_default(self):
+        self.assertEqual(self.store.thin_provisioning, False)
+
+    def test_add_with_thick_provisioning(self):
+        self._do_test_add(enable_thin_provisoning=False)
+
+    def test_add_with_thin_provisioning(self):
+        self._do_test_add(enable_thin_provisoning=True)
+
+    def test_add_thick_provisioning_with_holes_in_file(self):
+        """
+        Tests that a file which contains null bytes chunks is fully
+        written with a thick provisioning configuration.
+        """
+        chunk_size = units.Ki  # 1K
+        content = b"*" * chunk_size + b"\x00" * chunk_size + b"*" * chunk_size
+        self._do_test_thin_provisioning(content, 3 * chunk_size, 0, 3, False)
+
+    def test_add_thin_provisioning_with_holes_in_file(self):
+        """
+        Tests that a file which contains null bytes chunks is sparsified
+        with a thin provisioning configuration.
+        """
+        chunk_size = units.Ki  # 1K
+        content = b"*" * chunk_size + b"\x00" * chunk_size + b"*" * chunk_size
+        self._do_test_thin_provisioning(content, 3 * chunk_size, 1, 2, True)
+
+    def test_add_thick_provisioning_without_holes_in_file(self):
+        """
+        Tests that a file which not contain null bytes chunks is fully
+        written with a thick provisioning configuration.
+        """
+        chunk_size = units.Ki  # 1K
+        content = b"*" * 3 * chunk_size
+        self._do_test_thin_provisioning(content, 3 * chunk_size, 0, 3, False)
+
+    def test_add_thin_provisioning_without_holes_in_file(self):
+        """
+        Tests that a file which not contain null bytes chunks is fully
+        written with a thin provisioning configuration.
+        """
+        chunk_size = units.Ki  # 1K
+        content = b"*" * 3 * chunk_size
+        self._do_test_thin_provisioning(content, 3 * chunk_size, 0, 3, True)
+
+    def test_add_thick_provisioning_with_partial_holes_in_file(self):
+        """
+        Tests that a file which contains null bytes not aligned with
+        chunk size is fully written with a thick provisioning configuration.
+        """
+        chunk_size = units.Ki  # 1K
+        my_chunk = int(chunk_size * 1.5)
+        content = b"*" * my_chunk + b"\x00" * my_chunk + b"*" * my_chunk
+        self._do_test_thin_provisioning(content, 3 * my_chunk, 0, 5, False)
+
+    def test_add_thin_provisioning_with_partial_holes_in_file(self):
+        """
+        Tests that a file which contains null bytes not aligned with
+        chunk size is sparsified with a thin provisioning configuration.
+        """
+        chunk_size = units.Ki  # 1K
+        my_chunk = int(chunk_size * 1.5)
+        content = b"*" * my_chunk + b"\x00" * my_chunk + b"*" * my_chunk
+        self._do_test_thin_provisioning(content, 3 * my_chunk, 1, 4, True)
+
+    def _do_test_thin_provisioning(self, content, size, truncate, write, thin):
+        self.config(filesystem_store_chunk_size=units.Ki,
+                    filesystem_thin_provisioning=thin,
+                    group='glance_store')
+        self.store.configure()
+
+        image_file = six.BytesIO(content)
+        image_id = str(uuid.uuid4())
+        with mock.patch.object(builtins, 'open') as popen:
+            self.store.add(image_id, image_file, size, self.hash_algo)
+            write_count = popen.return_value.__enter__().write.call_count
+            truncate_count = popen.return_value.__enter__().truncate.call_count
+            self.assertEqual(write_count, write)
+            self.assertEqual(truncate_count, truncate)
+
     def test_add_with_verifier(self):
         """Test that 'verifier.update' is called when verifier is provided."""
         verifier = mock.MagicMock(name='mock_verifier')
diff --git a/glance_store/tests/unit/test_opts.py b/glance_store/tests/unit/test_opts.py
index 5cc184b..6f46d60 100644
--- a/glance_store/tests/unit/test_opts.py
+++ b/glance_store/tests/unit/test_opts.py
@@ -89,12 +89,14 @@ class OptsTestCase(base.StoreBaseTest):
             'filesystem_store_datadirs',
             'filesystem_store_file_perm',
             'filesystem_store_metadata_file',
+            'filesystem_thin_provisioning',
             'http_proxy_information',
             'https_ca_certificates_file',
             'rbd_store_ceph_conf',
             'rbd_store_chunk_size',
             'rbd_store_pool',
             'rbd_store_user',
+            'rbd_thin_provisioning',
             'rados_connect_timeout',
             'rootwrap_config',
             's3_store_access_key',
diff --git a/glance_store/tests/unit/test_rbd_store.py b/glance_store/tests/unit/test_rbd_store.py
index 132fe8e..4f44bc2 100644
--- a/glance_store/tests/unit/test_rbd_store.py
+++ b/glance_store/tests/unit/test_rbd_store.py
@@ -247,6 +247,9 @@ class TestStore(base.StoreBaseTest,
         self.data_iter = six.BytesIO(b'*' * self.data_len)
         self.hash_algo = 'sha256'
 
+    def test_thin_provisioning_is_disabled_by_default(self):
+        self.assertEqual(self.store.thin_provisioning, False)
+
     def test_add_w_image_size_zero(self):
         """Assert that correct size is returned even though 0 was provided."""
         self.store.chunk_size = units.Ki
@@ -359,6 +362,80 @@ class TestStore(base.StoreBaseTest,
         self.assertEqual(expected_checksum, checksum)
         self.assertEqual(expected_multihash, multihash)
 
+    def test_add_thick_provisioning_with_holes_in_file(self):
+        """
+        Tests that a file which contains null bytes chunks is fully
+        written to rbd backend in a thick provisioning configuration.
+        """
+        chunk_size = units.Mi
+        content = b"*" * chunk_size + b"\x00" * chunk_size + b"*" * chunk_size
+        self._do_test_thin_provisioning(content, 3 * chunk_size, 3, False)
+
+    def test_add_thin_provisioning_with_holes_in_file(self):
+        """
+        Tests that a file which contains null bytes chunks is sparsified
+        in rbd backend with a thin provisioning configuration.
+        """
+        chunk_size = units.Mi
+        content = b"*" * chunk_size + b"\x00" * chunk_size + b"*" * chunk_size
+        self._do_test_thin_provisioning(content, 3 * chunk_size, 2, True)
+
+    def test_add_thick_provisioning_without_holes_in_file(self):
+        """
+        Tests that a file which not contain null bytes chunks is fully
+        written to rbd backend in a thick provisioning configuration.
+        """
+        chunk_size = units.Mi
+        content = b"*" * 3 * chunk_size
+        self._do_test_thin_provisioning(content, 3 * chunk_size, 3, False)
+
+    def test_add_thin_provisioning_without_holes_in_file(self):
+        """
+        Tests that a file which not contain null bytes chunks is fully
+        written to rbd backend in a thin provisioning configuration.
+        """
+        chunk_size = units.Mi
+        content = b"*" * 3 * chunk_size
+        self._do_test_thin_provisioning(content, 3 * chunk_size, 3, True)
+
+    def test_add_thick_provisioning_with_partial_holes_in_file(self):
+        """
+        Tests that a file which contains null bytes not aligned with
+        chunk size is fully written with a thick provisioning configuration.
+        """
+        chunk_size = units.Mi
+        my_chunk = int(chunk_size * 1.5)
+        content = b"*" * my_chunk + b"\x00" * my_chunk + b"*" * my_chunk
+        self._do_test_thin_provisioning(content, 3 * my_chunk, 5, False)
+
+    def test_add_thin_provisioning_with_partial_holes_in_file(self):
+        """
+        Tests that a file which contains null bytes not aligned with
+        chunk size is sparsified with a thin provisioning configuration.
+        """
+        chunk_size = units.Mi
+        my_chunk = int(chunk_size * 1.5)
+        content = b"*" * my_chunk + b"\x00" * my_chunk + b"*" * my_chunk
+        self._do_test_thin_provisioning(content, 3 * my_chunk, 4, True)
+
+    def _do_test_thin_provisioning(self, content, size, write, thin):
+        self.config(rbd_store_chunk_size=1,
+                    rbd_thin_provisioning=thin)
+        self.store.configure()
+
+        image_id = 'fake_image_id'
+        image_file = six.BytesIO(content)
+        expected_checksum = hashlib.md5(content).hexdigest()
+        expected_multihash = hashlib.sha256(content).hexdigest()
+
+        with mock.patch.object(rbd_store.rbd.Image, 'write') as mock_write:
+            loc, size, checksum, multihash, _ = self.store.add(
+                image_id, image_file, size, self.hash_algo)
+            self.assertEqual(mock_write.call_count, write)
+
+        self.assertEqual(expected_checksum, checksum)
+        self.assertEqual(expected_multihash, multihash)
+
     def test_delete(self):
         def _fake_remove(*args, **kwargs):
             self.called_commands_actual.append('remove')
diff --git a/releasenotes/notes/handle-sparse-image-a3ecfc4ae1c00d48.yaml b/releasenotes/notes/handle-sparse-image-a3ecfc4ae1c00d48.yaml
new file mode 100644
index 0000000..6122051
--- /dev/null
+++ b/releasenotes/notes/handle-sparse-image-a3ecfc4ae1c00d48.yaml
@@ -0,0 +1,15 @@
+---
+features:
+  - |
+    Add new configuration option ``rbd_thin_provisioning`` and
+    ``filesystem_thin_provisioning`` to rbd and filesystem
+    store to enable or not sparse upload, default are False.
+
+    A sparse file means that we do not actually write null byte sequences
+    but only the data itself at a given offset, the "holes" which can
+    appear will automatically be interpreted by the storage backend as
+    null bytes, and do not really consume your storage.
+
+    Enabling this feature will also speed up image upload and save
+    network traffic in addition to save space in the backend, as null
+    bytes sequences are not sent over the network.