Correct such logic in store.get() when chunk_size param provided

The change corrected the logic of image content reading function in
get() for some store drivers. if the chunk_size param provided from
outside, it means client want to read partial image content from
storage, current implementation is incomplete and has some problem.

The change didn't try to make drivers fully support random access from
backend with chunk_size (and offset) param, that stuff will be
implementated by a dedicated patch, this one only to make default logic
be correct.

Change-Id: I76fd3746a2c449e257ea2a19c2dc6118be207eec
Signed-off-by: Zhi Yan Liu <zhiyanl@cn.ibm.com>

5 files changed, 20 insertions, 13 deletions

diff --git a/glance_store/_drivers/filesystem.py b/glance_store/_drivers/filesystem.py
index 5460102..ae2b29e 100644
--- a/glance_store/_drivers/filesystem.py
+++ b/glance_store/_drivers/filesystem.py
@@ -116,10 +116,12 @@ class ChunkedFile(object):
     something that can iterate over a large file
     """
 
-    def __init__(self, filepath, offset=0, chunk_size=None, partial=False):
-        self.partial = partial
+    def __init__(self, filepath, offset=0, chunk_size=4096,
+                 partial_length=None):
         self.filepath = filepath
         self.chunk_size = chunk_size
+        self.partial_length = partial_length
+        self.partial = self.partial_length is not None
         self.fp = open(self.filepath, 'rb')
         if offset:
             self.fp.seek(offset)
@@ -129,12 +131,19 @@ class ChunkedFile(object):
         try:
             if self.fp:
                 while True:
-                    chunk = self.fp.read(self.chunk_size)
+                    if self.partial:
+                        size = min(self.chunk_size, self.partial_length)
+                    else:
+                        size = self.chunk_size
+
+                    chunk = self.fp.read(size)
                     if chunk:
                         yield chunk
 
                         if self.partial:
-                            break
+                            self.partial_length -= len(chunk)
+                            if self.partial_length <= 0:
+                                break
                     else:
                         break
         finally:
@@ -451,8 +460,8 @@ class Store(glance_store.driver.Store):
         LOG.debug(msg)
         return (ChunkedFile(filepath,
                             offset=offset,
-                            chunk_size=chunk_size or self.READ_CHUNKSIZE,
-                            partial=chunk_size is not None),
+                            chunk_size=self.READ_CHUNKSIZE,
+                            partial_length=chunk_size),
                 chunk_size or filesize)
 
     def get_size(self, location, context=None):
diff --git a/glance_store/_drivers/http.py b/glance_store/_drivers/http.py
index eef578d..a2073de 100644
--- a/glance_store/_drivers/http.py
+++ b/glance_store/_drivers/http.py
@@ -133,8 +133,7 @@ class Store(glance_store.driver.Store):
             LOG.error(reason)
             raise exceptions.RemoteServiceUnavailable()
 
-        cs = chunk_size or self.READ_CHUNKSIZE
-        iterator = http_response_iterator(conn, resp, cs)
+        iterator = http_response_iterator(conn, resp, self.READ_CHUNKSIZE)
 
         class ResponseIndexable(glance_store.Indexable):
             def another(self):
diff --git a/glance_store/_drivers/rbd.py b/glance_store/_drivers/rbd.py
index 09d2468..767a4ff 100644
--- a/glance_store/_drivers/rbd.py
+++ b/glance_store/_drivers/rbd.py
@@ -220,9 +220,8 @@ class Store(driver.Store):
         :raises `glance_store.exceptions.NotFound` if image does not exist
         """
         loc = location.store_location
-        return (ImageIterator(loc.pool, loc.image, loc.snapshot,
-                              self, chunk_size=chunk_size),
-                chunk_size or self.get_size(location))
+        return (ImageIterator(loc.pool, loc.image, loc.snapshot, self),
+                self.get_size(location))
 
     def get_size(self, location, context=None):
         """
diff --git a/glance_store/_drivers/s3.py b/glance_store/_drivers/s3.py
index 2fcb550..1f82813 100644
--- a/glance_store/_drivers/s3.py
+++ b/glance_store/_drivers/s3.py
@@ -378,7 +378,7 @@ class Store(glance_store.driver.Store):
         :raises `glance_store.exceptions.NotFound` if image does not exist
         """
         key = self._retrieve_key(location)
-        cs = chunk_size or self.READ_CHUNKSIZE
+        cs = self.READ_CHUNKSIZE
         key.BufferSize = cs
 
         class ChunkedIndexable(glance_store.Indexable):
diff --git a/glance_store/_drivers/sheepdog.py b/glance_store/_drivers/sheepdog.py
index 597f857..18a5b29 100644
--- a/glance_store/_drivers/sheepdog.py
+++ b/glance_store/_drivers/sheepdog.py
@@ -226,7 +226,7 @@ class Store(glance_store.driver.Store):
 
         loc = location.store_location
         image = SheepdogImage(self.addr, self.port, loc.image,
-                              chunk_size or self.READ_CHUNKSIZE)
+                              self.READ_CHUNKSIZE)
         if not image.exist():
             raise exceptions.NotFound(_("Sheepdog image %s does not exist")
                                       % image.name)