diff options
author | Wouter Bolsterlee <uws@xs4all.nl> | 2014-02-25 20:22:15 +0100 |
---|---|---|
committer | Wouter Bolsterlee <uws@xs4all.nl> | 2014-02-25 20:22:15 +0100 |
commit | da109ab7b3c8a6ede332bc0d99e3e71cbecf5154 (patch) | |
tree | 4c688ad469e321f0af1454e61e2a6ec129ac6d1d | |
parent | 91e0f8afac4ed3aeef3c0a79ae69b5df744ba45b (diff) | |
download | happybase-da109ab7b3c8a6ede332bc0d99e3e71cbecf5154.tar.gz |
Revert "Allow batch_size=None in Table.scan() to avoid filter incompatibilities"
This reverts commit 8481d317804e060b12839b571ef22306074fba9c, since it
is not a correct fix. See issue #56.
-rw-r--r-- | NEWS.rst | 5 | ||||
-rw-r--r-- | happybase/table.py | 31 | ||||
-rw-r--r-- | tests/test_api.py | 9 |
3 files changed, 12 insertions, 33 deletions
@@ -18,11 +18,6 @@ Release date: *not yet released* Python 2.6 a separate ```ordereddict``` pacakge has to be installed from PyPI. (`issue #39 <https://github.com/wbolster/happybase/issues/39>`_) -* Allow `None` as a valid value for the `batch_size` argument to - :py:meth:`Table.scan`, since HBase does not support specifying a batch size - when some scanner filters are used. (`issue #54 - <https://github.com/wbolster/happybase/issues/54>`_). - HappyBase 0.7 ------------- diff --git a/happybase/table.py b/happybase/table.py index abcceb0..c8f2a75 100644 --- a/happybase/table.py +++ b/happybase/table.py @@ -254,15 +254,6 @@ class Table(object): this to a low value (or even 1) if your data is large, since a low batch size results in added round-trips to the server. - .. warning:: - - Not all HBase filters can be used in combination with a batch - size. Explicitly specify `None` for the `batch_size` argument - in those cases to override the default value. Failure to do - so can result in hard to debug errors (not HappyBase's - fault), such as a non-responsive connection. The HBase logs - may contain more useful information in these situations. - **Compatibility notes:** * The `filter` argument is only available when using HBase 0.92 @@ -289,11 +280,11 @@ class Table(object): :return: generator yielding the rows matching the scan :rtype: iterable of `(row_key, row_data)` tuples """ - if batch_size is not None and batch_size < 1: - raise ValueError("'batch_size' must be >= 1 (or None)") + if batch_size < 1: + raise ValueError("'batch_size' must be >= 1") if limit is not None and limit < 1: - raise ValueError("'limit' must be >= 1 (or None)") + raise ValueError("'limit' must be >= 1") if sorted_columns and self.connection.compat < '0.96': raise NotImplementedError( @@ -358,16 +349,16 @@ class Table(object): n_returned = n_fetched = 0 try: while True: - if batch_size is None: - how_many = 1 - else: + if limit is None: how_many = batch_size + else: + how_many = min(batch_size, limit - n_returned) - if limit is not None: - how_many = min(how_many, limit - n_returned) - - items = self.connection.client.scannerGetList( - scan_id, how_many) + if how_many == 1: + items = self.connection.client.scannerGet(scan_id) + else: + items = self.connection.client.scannerGetList( + scan_id, how_many) n_fetched += len(items) diff --git a/tests/test_api.py b/tests/test_api.py index ed78f81..2d22717 100644 --- a/tests/test_api.py +++ b/tests/test_api.py @@ -357,7 +357,7 @@ def test_scan(): list(table.scan(row_prefix='foobar', row_start='xyz')) with assert_raises(ValueError): - list(table.scan(batch_size=0)) + list(table.scan(batch_size=None)) if connection.compat == '0.90': with assert_raises(NotImplementedError): @@ -446,13 +446,6 @@ def test_scan_sorting(): row.items()) -def test_scan_filter_and_batch_size(): - # See issue #54 - filter = "SingleColumnValueFilter ('cf1', 'qual1', =, 'binary:val1')" - for k, v in table.scan(filter=filter, batch_size=None): - print v - - def test_delete(): row_key = 'row-test-delete' data = {'cf1:col1': 'v1', |