diff options
-rw-r--r-- | glance/store/_drivers/filesystem.py | 15 | ||||
-rw-r--r-- | glance/store/_drivers/http.py | 3 | ||||
-rw-r--r-- | glance/store/_drivers/rbd.py | 13 | ||||
-rw-r--r-- | glance/store/_drivers/s3.py | 21 | ||||
-rw-r--r-- | glance/store/_drivers/sheepdog.py | 19 | ||||
-rw-r--r-- | glance/store/driver.py | 3 | ||||
-rw-r--r-- | tests/unit/test_filesystem_store.py | 4 | ||||
-rw-r--r-- | tests/unit/test_http_store.py | 2 |
8 files changed, 47 insertions, 33 deletions
diff --git a/glance/store/_drivers/filesystem.py b/glance/store/_drivers/filesystem.py index 3b683e9..7e582d2 100644 --- a/glance/store/_drivers/filesystem.py +++ b/glance/store/_drivers/filesystem.py @@ -88,10 +88,9 @@ class ChunkedFile(object): something that can iterate over a large file """ - CHUNKSIZE = 65536 - - def __init__(self, filepath): + def __init__(self, filepath, chunk_size=None): self.filepath = filepath + self.chunk_size = chunk_size self.fp = open(self.filepath, 'rb') def __iter__(self): @@ -99,7 +98,7 @@ class ChunkedFile(object): try: if self.fp: while True: - chunk = self.fp.read(ChunkedFile.CHUNKSIZE) + chunk = self.fp.read(self.chunk_size) if chunk: yield chunk else: @@ -117,6 +116,8 @@ class ChunkedFile(object): class Store(glance.store.driver.Store): OPTIONS = _FILESYSTEM_CONFIGS + READ_CHUNKSIZE = 64 * units.Ki + WRITE_CHUNKSIZE = READ_CHUNKSIZE def get_schemes(self): return ('file', 'filesystem') @@ -313,7 +314,9 @@ class Store(glance.store.driver.Store): filepath, filesize = self._resolve_location(location) msg = _("Found image at %s. Returning in ChunkedFile.") % filepath LOG.debug(msg) - return (ChunkedFile(filepath), filesize) + return (ChunkedFile(filepath, + chunk_size=chunk_size or self.READ_CHUNKSIZE), + filesize) def get_size(self, location, context=None): """ @@ -434,7 +437,7 @@ class Store(glance.store.driver.Store): try: with open(filepath, 'wb') as f: for buf in utils.chunkreadable(image_file, - ChunkedFile.CHUNKSIZE): + self.WRITE_CHUNKSIZE): bytes_written += len(buf) checksum.update(buf) f.write(buf) diff --git a/glance/store/_drivers/http.py b/glance/store/_drivers/http.py index 571ad91..b621b6d 100644 --- a/glance/store/_drivers/http.py +++ b/glance/store/_drivers/http.py @@ -121,7 +121,8 @@ class Store(glance.store.driver.Store): """ conn, resp, content_length = self._query(location, 'GET') - iterator = http_response_iterator(conn, resp, self.CHUNKSIZE) + cs = chunk_size or self.READ_CHUNKSIZE + iterator = http_response_iterator(conn, resp, cs) class ResponseIndexable(glance.store.Indexable): def another(self): diff --git a/glance/store/_drivers/rbd.py b/glance/store/_drivers/rbd.py index 666798e..0e48440 100644 --- a/glance/store/_drivers/rbd.py +++ b/glance/store/_drivers/rbd.py @@ -141,12 +141,12 @@ class ImageIterator(object): Reads data from an RBD image, one chunk at a time. """ - def __init__(self, name, store): + def __init__(self, name, store, chunk_size=None): self.name = name self.pool = store.pool self.user = store.user self.conf_file = store.conf_file - self.chunk_size = store.chunk_size + self.chunk_size = chunk_size or store.chunk_size def __iter__(self): try: @@ -188,6 +188,8 @@ class Store(driver.Store): try: chunk = self.conf.glance_store.rbd_store_chunk_size self.chunk_size = chunk * (1024 ^ 2) + self.READ_CHUNKSIZE = self.chunk_size + self.WRITE_CHUNKSIZE = self.READ_CHUNKSIZE # these must not be unicode since they will be passed to a # non-unicode-aware C library @@ -211,7 +213,8 @@ class Store(driver.Store): :raises `glance.store.exceptions.NotFound` if image does not exist """ loc = location.store_location - return (ImageIterator(loc.image, self), self.get_size(location)) + return (ImageIterator(loc.image, self), + self.get_size(location), chunk_size) def get_size(self, location, context=None): """ @@ -323,7 +326,7 @@ class Store(driver.Store): if hasattr(conn, 'get_fsid'): fsid = conn.get_fsid() with conn.open_ioctx(self.pool) as ioctx: - order = int(math.log(self.chunk_size, 2)) + order = int(math.log(self.WRITE_CHUNKSIZE, 2)) LOG.debug('creating image %s with order %d and size %d', image_name, order, image_size) if image_size == 0: @@ -343,7 +346,7 @@ class Store(driver.Store): bytes_written = 0 offset = 0 chunks = utils.chunkreadable(image_file, - self.chunk_size) + self.WRITE_CHUNKSIZE) for chunk in chunks: # If the image size provided is zero we need to do # a resize for the amount we are writing. This will diff --git a/glance/store/_drivers/s3.py b/glance/store/_drivers/s3.py index 936aea5..e141a75 100644 --- a/glance/store/_drivers/s3.py +++ b/glance/store/_drivers/s3.py @@ -30,6 +30,7 @@ import glance.store.driver from glance.store import exceptions from glance.store.i18n import _ import glance.store.location +from glance.store.openstack.common import units LOG = logging.getLogger(__name__) @@ -173,17 +174,16 @@ class ChunkedFile(object): something that can iterate over a ``boto.s3.key.Key`` """ - CHUNKSIZE = 65536 - - def __init__(self, fp): + def __init__(self, fp, chunk_size): self.fp = fp + self.chunk_size = chunk_size def __iter__(self): """Return an iterator over the image file""" try: if self.fp: while True: - chunk = self.fp.read(ChunkedFile.CHUNKSIZE) + chunk = self.fp.read(self.chunk_size) if chunk: yield chunk else: @@ -214,6 +214,9 @@ class Store(glance.store.driver.Store): OPTIONS = _S3_OPTS EXAMPLE_URL = "s3://<ACCESS_KEY>:<SECRET_KEY>@<S3_URL>/<BUCKET>/<OBJ>" + READ_CHUNKSIZE = 64 * units.Ki + WRITE_CHUNKSIZE = READ_CHUNKSIZE + def get_schemes(self): return ('s3', 's3+http', 's3+https') @@ -268,15 +271,15 @@ class Store(glance.store.driver.Store): :raises `glance.store.exceptions.NotFound` if image does not exist """ key = self._retrieve_key(location) - - key.BufferSize = self.CHUNKSIZE + cs = chunk_size or self.READ_CHUNKSIZE + key.BufferSize = cs class ChunkedIndexable(glance.store.Indexable): def another(self): - return (self.wrapped.fp.read(ChunkedFile.CHUNKSIZE) + return (self.wrapped.fp.read(cs) if self.wrapped.fp else None) - return (ChunkedIndexable(ChunkedFile(key), key.size), key.size) + return (ChunkedIndexable(ChunkedFile(key, cs), key.size), key.size) def get_size(self, location, context=None): """ @@ -404,7 +407,7 @@ class Store(glance.store.driver.Store): tmpdir = self.s3_store_object_buffer_dir temp_file = tempfile.NamedTemporaryFile(dir=tmpdir) checksum = hashlib.md5() - for chunk in utils.chunkreadable(image_file, self.CHUNKSIZE): + for chunk in utils.chunkreadable(image_file, self.WRITE_CHUNKSIZE): checksum.update(chunk) temp_file.write(chunk) temp_file.flush() diff --git a/glance/store/_drivers/sheepdog.py b/glance/store/_drivers/sheepdog.py index dc4edd0..9fbb1eb 100644 --- a/glance/store/_drivers/sheepdog.py +++ b/glance/store/_drivers/sheepdog.py @@ -192,6 +192,9 @@ class Store(glance.store.driver.Store): try: self.chunk_size = CONF.sheepdog_store_chunk_size * units.Mi + self.READ_CHUNKSIZE = self.chunk_size + self.WRITE_CHUNKSIZE = self.READ_CHUNKSIZE + self.addr = CONF.sheepdog_store_address self.port = CONF.sheepdog_store_port except cfg.ConfigFileValueError as e: @@ -208,7 +211,7 @@ class Store(glance.store.driver.Store): raise exceptions.BadStoreConfiguration(store_name='sheepdog', reason=reason) - def get(self, location): + def get(self, location, offset=0, chunk_size=None, context=None): """ Takes a `glance.store.location.Location` object that indicates where to find the image file, and returns a generator for reading @@ -221,13 +224,13 @@ class Store(glance.store.driver.Store): loc = location.store_location image = SheepdogImage(self.addr, self.port, loc.image, - self.chunk_size) + chunk_size or self.READ_CHUNKSIZE) if not image.exist(): raise exceptions.NotFound(_("Sheepdog image %s does not exist") % image.name) return (ImageIterator(image), image.get_size()) - def get_size(self, location): + def get_size(self, location, context=None): """ Takes a `glance.store.location.Location` object that indicates where to find the image file and returns the image size @@ -240,13 +243,13 @@ class Store(glance.store.driver.Store): loc = location.store_location image = SheepdogImage(self.addr, self.port, loc.image, - self.chunk_size) + self.READ_CHUNKSIZE) if not image.exist(): raise exceptions.NotFound(_("Sheepdog image %s does not exist") % image.name) return image.get_size() - def add(self, image_id, image_file, image_size): + def add(self, image_id, image_file, image_size, context=None): """ Stores an image file with supplied identifier to the backend storage system and returns a tuple containing information @@ -262,7 +265,7 @@ class Store(glance.store.driver.Store): """ image = SheepdogImage(self.addr, self.port, image_id, - self.chunk_size) + self.WRITE_CHUNKSIZE) if image.exist(): raise exceptions.Duplicate(_("Sheepdog image %s already exists") % image_id) @@ -288,7 +291,7 @@ class Store(glance.store.driver.Store): return (location.get_uri(), image_size, checksum.hexdigest(), {}) - def delete(self, location): + def delete(self, location, context=None): """ Takes a `glance.store.location.Location` object that indicates where to find the image file to delete @@ -301,7 +304,7 @@ class Store(glance.store.driver.Store): loc = location.store_location image = SheepdogImage(self.addr, self.port, loc.image, - self.chunk_size) + self.WRITE_CHUNKSIZE) if not image.exist(): raise exceptions.NotFound(_("Sheepdog image %s does not exist") % loc.image) diff --git a/glance/store/driver.py b/glance/store/driver.py index d276cb0..c59038b 100644 --- a/glance/store/driver.py +++ b/glance/store/driver.py @@ -43,7 +43,8 @@ def _exception_to_unicode(exc): class Store(object): OPTIONS = None - CHUNKSIZE = 16 * (1024 * 1024) # 16M + READ_CHUNKSIZE = 16 * (1024 * 1024) # 16M + WRITE_CHUNKSIZE = READ_CHUNKSIZE def __init__(self, conf): """ diff --git a/tests/unit/test_filesystem_store.py b/tests/unit/test_filesystem_store.py index 1492e7e..936d6a5 100644 --- a/tests/unit/test_filesystem_store.py +++ b/tests/unit/test_filesystem_store.py @@ -43,8 +43,8 @@ class TestStore(base.StoreBaseTest): def setUp(self): """Establish a clean test environment.""" super(TestStore, self).setUp() - self.orig_chunksize = ChunkedFile.CHUNKSIZE - ChunkedFile.CHUNKSIZE = 10 + self.orig_chunksize = Store.READ_CHUNKSIZE + Store.READ_CHUNKSIZE = 10 self.store = Store(self.conf) self.config(filesystem_store_datadir=self.test_dir, group="glance_store") diff --git a/tests/unit/test_http_store.py b/tests/unit/test_http_store.py index db79dc3..50f41ad 100644 --- a/tests/unit/test_http_store.py +++ b/tests/unit/test_http_store.py @@ -29,7 +29,7 @@ class TestHttpStore(base.StoreBaseTest): def setUp(self): super(TestHttpStore, self).setUp() self.config(default_store='http', group='glance_store') - http.Store.CHUNKSIZE = 2 + http.Store.READ_CHUNKSIZE = 2 self.store = http.Store(self.conf) response = mock.patch('httplib.HTTPConnection.getresponse') |