From 33824a9c06ca555ad208a9925bc7b40fe489fc72 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Tue, 2 Nov 2021 10:58:01 -0400 Subject: ensure soft_close occurs for fetchmany with server side cursor Fixed regression where the :meth:`_engine.CursorResult.fetchmany` method would fail to autoclose a server-side cursor (i.e. when ``stream_results`` or ``yield_per`` is in use, either Core or ORM oriented results) when the results were fully exhausted. All :class:`_result.Result` objects will now consistently raise :class:`_exc.ResourceClosedError` if they are used after a hard close, which includes the "hard close" that occurs after calling "single row or value" methods like :meth:`_result.Result.first` and :meth:`_result.Result.scalar`. This was already the behavior of the most common class of result objects returned for Core statement executions, i.e. those based on :class:`_engine.CursorResult`, so this behavior is not new. However, the change has been extended to properly accommodate for the ORM "filtering" result objects returned when using 2.0 style ORM queries, which would previously behave in "soft closed" style of returning empty results, or wouldn't actually "soft close" at all and would continue yielding from the underlying cursor. As part of this change, also added :meth:`_result.Result.close` to the base :class:`_result.Result` class and implemented it for the filtered result implementations that are used by the ORM, so that it is possible to call the :meth:`_engine.CursorResult.close` method on the underlying :class:`_engine.CursorResult` when the the ``yield_per`` execution option is in use to close a server side cursor before remaining ORM results have been fetched. This was again already available for Core result sets but the change makes it available for 2.0 style ORM results as well. Fixes: #7274 Change-Id: Id4acdfedbcab891582a7f8edd2e2e7d20d868e53 --- lib/sqlalchemy/engine/cursor.py | 10 ++++++-- lib/sqlalchemy/engine/result.py | 56 ++++++++++++++++++++++++++++++++++++----- 2 files changed, 58 insertions(+), 8 deletions(-) (limited to 'lib/sqlalchemy') diff --git a/lib/sqlalchemy/engine/cursor.py b/lib/sqlalchemy/engine/cursor.py index 071f95cff..0099d69ff 100644 --- a/lib/sqlalchemy/engine/cursor.py +++ b/lib/sqlalchemy/engine/cursor.py @@ -923,6 +923,8 @@ class BufferedRowCursorFetchStrategy(CursorFetchStrategy): ) def _buffer_rows(self, result, dbapi_cursor): + """this is currently used only by fetchone().""" + size = self._bufsize try: if size < 1: @@ -975,9 +977,14 @@ class BufferedRowCursorFetchStrategy(CursorFetchStrategy): lb = len(buf) if size > lb: try: - buf.extend(dbapi_cursor.fetchmany(size - lb)) + new = dbapi_cursor.fetchmany(size - lb) except BaseException as e: self.handle_exception(result, dbapi_cursor, e) + else: + if not new: + result._soft_close() + else: + buf.extend(new) result = buf[0:size] self._rowbuffer = collections.deque(buf[size:]) @@ -1216,7 +1223,6 @@ class BaseCursorResult(object): """ - if (not hard and self._soft_closed) or (hard and self.closed): return diff --git a/lib/sqlalchemy/engine/result.py b/lib/sqlalchemy/engine/result.py index 48572c7fe..79b07ab91 100644 --- a/lib/sqlalchemy/engine/result.py +++ b/lib/sqlalchemy/engine/result.py @@ -720,6 +720,28 @@ class Result(_WithKeys, ResultInternal): def _soft_close(self, hard=False): raise NotImplementedError() + def close(self): + """close this :class:`_result.Result`. + + The behavior of this method is implementation specific, and is + not implemented by default. The method should generally end + the resources in use by the result object and also cause any + subsequent iteration or row fetching to raise + :class:`.ResourceClosedError`. + + .. versionadded:: 1.4.27 - ``.close()`` was previously not generally + available for all :class:`_result.Result` classes, instead only + being available on the :class:`_engine.CursorResult` returned for + Core statement executions. As most other result objects, namely the + ones used by the ORM, are proxying a :class:`_engine.CursorResult` + in any case, this allows the underlying cursor result to be closed + from the outside facade for the case when the ORM query is using + the ``yield_per`` execution option where it does not immediately + exhaust and autoclose the database cursor. + + """ + self._soft_close(hard=True) + @_generative def yield_per(self, num): """Configure the row-fetching strategy to fetch num rows at a time. @@ -1593,6 +1615,8 @@ class IteratorResult(Result): """ + _hard_closed = False + def __init__( self, cursor_metadata, @@ -1605,16 +1629,29 @@ class IteratorResult(Result): self.raw = raw self._source_supports_scalars = _source_supports_scalars - def _soft_close(self, **kw): + def _soft_close(self, hard=False, **kw): + if hard: + self._hard_closed = True + if self.raw is not None: + self.raw._soft_close(hard=hard, **kw) self.iterator = iter([]) + self._reset_memoizations() + + def _raise_hard_closed(self): + raise exc.ResourceClosedError("This result object is closed.") def _raw_row_iterator(self): return self.iterator def _fetchiter_impl(self): + if self._hard_closed: + self._raise_hard_closed() return self.iterator def _fetchone_impl(self, hard_close=False): + if self._hard_closed: + self._raise_hard_closed() + row = next(self.iterator, _NO_ROW) if row is _NO_ROW: self._soft_close(hard=hard_close) @@ -1623,12 +1660,18 @@ class IteratorResult(Result): return row def _fetchall_impl(self): + if self._hard_closed: + self._raise_hard_closed() + try: return list(self.iterator) finally: self._soft_close() def _fetchmany_impl(self, size=None): + if self._hard_closed: + self._raise_hard_closed() + return list(itertools.islice(self.iterator, 0, size)) @@ -1677,6 +1720,10 @@ class ChunkedIteratorResult(IteratorResult): self._yield_per = num self.iterator = itertools.chain.from_iterable(self.chunks(num)) + def _soft_close(self, **kw): + super(ChunkedIteratorResult, self)._soft_close(**kw) + self.chunks = lambda size: [] + def _fetchmany_impl(self, size=None): if self.dynamic_yield_per: self.iterator = itertools.chain.from_iterable(self.chunks(size)) @@ -1714,11 +1761,8 @@ class MergedResult(IteratorResult): *[r._attributes for r in results] ) - def close(self): - self._soft_close(hard=True) - - def _soft_close(self, hard=False): + def _soft_close(self, hard=False, **kw): for r in self._results: - r._soft_close(hard=hard) + r._soft_close(hard=hard, **kw) if hard: self.closed = True -- cgit v1.2.1