diff options
Diffstat (limited to 'lib/sqlalchemy')
| -rw-r--r-- | lib/sqlalchemy/dialects/mysql/base.py | 31 | ||||
| -rw-r--r-- | lib/sqlalchemy/dialects/mysql/mariadbconnector.py | 11 | ||||
| -rw-r--r-- | lib/sqlalchemy/dialects/mysql/mysqldb.py | 3 | ||||
| -rw-r--r-- | lib/sqlalchemy/dialects/mysql/pymysql.py | 4 | ||||
| -rw-r--r-- | lib/sqlalchemy/dialects/postgresql/asyncpg.py | 4 | ||||
| -rw-r--r-- | lib/sqlalchemy/dialects/postgresql/base.py | 31 | ||||
| -rw-r--r-- | lib/sqlalchemy/dialects/postgresql/psycopg2.py | 17 | ||||
| -rw-r--r-- | lib/sqlalchemy/engine/base.py | 9 | ||||
| -rw-r--r-- | lib/sqlalchemy/engine/default.py | 25 | ||||
| -rw-r--r-- | lib/sqlalchemy/orm/query.py | 4 | ||||
| -rw-r--r-- | lib/sqlalchemy/testing/suite/test_results.py | 19 |
11 files changed, 113 insertions, 45 deletions
diff --git a/lib/sqlalchemy/dialects/mysql/base.py b/lib/sqlalchemy/dialects/mysql/base.py index 130aeeb0f..b9067f384 100644 --- a/lib/sqlalchemy/dialects/mysql/base.py +++ b/lib/sqlalchemy/dialects/mysql/base.py @@ -271,16 +271,31 @@ multi-column key for some storage engines:: Server Side Cursors ------------------- -Server-side cursor support is available for the MySQLdb and PyMySQL dialects. -From a database driver point of view this means that the ``MySQLdb.cursors.SSCursor`` or -``pymysql.cursors.SSCursor`` class is used when building up the cursor which -will receive results. The most typical way of invoking this feature is via the +Server-side cursor support is available for the mysqlclient, PyMySQL, +maridbconnector dialects and may also be available in others. This makes use +of either the "buffered=True/False" flag if available or by using a class such +as ``MySQLdb.cursors.SSCursor`` or ``pymysql.cursors.SSCursor`` internally. + + +Server side cursors are enabled on a per-statement basis by using the :paramref:`.Connection.execution_options.stream_results` connection execution -option. Server side cursors can also be enabled for all SELECT statements -unconditionally by passing ``server_side_cursors=True`` to -:func:`_sa.create_engine`. +option:: + + with engine.connect() as conn: + result = conn.execution_options(stream_resuls=True).execute(text("select * from table")) + +Note that some kinds of SQL statements may not be supported with +server side cursors; generally, only SQL statements that return rows should be +used with this option. + +.. deprecated:: 1.4 The dialect-level server_side_cursors flag is deprecated + and will be removed in a future release. Please use the + :paramref:`_engine.Connection.stream_results` execution option for + unbuffered cursor support. + +.. seealso:: -.. versionadded:: 1.1.4 - added server-side cursor support. + :ref:`engine_stream_results` .. _mysql_unicode: diff --git a/lib/sqlalchemy/dialects/mysql/mariadbconnector.py b/lib/sqlalchemy/dialects/mysql/mariadbconnector.py index aa28ffc67..4e0b4e0a9 100644 --- a/lib/sqlalchemy/dialects/mysql/mariadbconnector.py +++ b/lib/sqlalchemy/dialects/mysql/mariadbconnector.py @@ -40,7 +40,11 @@ mariadb_cpy_minimum_version = (1, 0, 1) class MySQLExecutionContext_mariadbconnector(MySQLExecutionContext): - pass + def create_server_side_cursor(self): + return self._dbapi_connection.cursor(buffered=False) + + def create_default_cursor(self): + return self._dbapi_connection.cursor(buffered=True) class MySQLCompiler_mariadbconnector(MySQLCompiler): @@ -75,6 +79,8 @@ class MySQLDialect_mariadbconnector(MySQLDialect): statement_compiler = MySQLCompiler_mariadbconnector preparer = MySQLIdentifierPreparer_mariadbconnector + supports_server_side_cursors = True + @util.memoized_property def _dbapi_version(self): if self.dbapi and hasattr(self.dbapi, "__version__"): @@ -89,9 +95,8 @@ class MySQLDialect_mariadbconnector(MySQLDialect): else: return (99, 99, 99) - def __init__(self, server_side_cursors=False, **kwargs): + def __init__(self, **kwargs): super(MySQLDialect_mariadbconnector, self).__init__(**kwargs) - self.server_side_cursors = True self.paramstyle = "qmark" if self.dbapi is not None: if self._dbapi_version < mariadb_cpy_minimum_version: diff --git a/lib/sqlalchemy/dialects/mysql/mysqldb.py b/lib/sqlalchemy/dialects/mysql/mysqldb.py index 03c1779c3..b20e061fb 100644 --- a/lib/sqlalchemy/dialects/mysql/mysqldb.py +++ b/lib/sqlalchemy/dialects/mysql/mysqldb.py @@ -88,9 +88,8 @@ class MySQLDialect_mysqldb(MySQLDialect): statement_compiler = MySQLCompiler_mysqldb preparer = MySQLIdentifierPreparer_mysqldb - def __init__(self, server_side_cursors=False, **kwargs): + def __init__(self, **kwargs): super(MySQLDialect_mysqldb, self).__init__(**kwargs) - self.server_side_cursors = server_side_cursors self._mysql_dbapi_version = ( self._parse_dbapi_version(self.dbapi.__version__) if self.dbapi is not None and hasattr(self.dbapi, "__version__") diff --git a/lib/sqlalchemy/dialects/mysql/pymysql.py b/lib/sqlalchemy/dialects/mysql/pymysql.py index e4d5d6206..7d7770105 100644 --- a/lib/sqlalchemy/dialects/mysql/pymysql.py +++ b/lib/sqlalchemy/dialects/mysql/pymysql.py @@ -44,10 +44,6 @@ class MySQLDialect_pymysql(MySQLDialect_mysqldb): supports_unicode_statements = True supports_unicode_binds = True - def __init__(self, server_side_cursors=False, **kwargs): - super(MySQLDialect_pymysql, self).__init__(**kwargs) - self.server_side_cursors = server_side_cursors - @langhelpers.memoized_property def supports_server_side_cursors(self): try: diff --git a/lib/sqlalchemy/dialects/postgresql/asyncpg.py b/lib/sqlalchemy/dialects/postgresql/asyncpg.py index eb87249b4..780e23844 100644 --- a/lib/sqlalchemy/dialects/postgresql/asyncpg.py +++ b/lib/sqlalchemy/dialects/postgresql/asyncpg.py @@ -723,10 +723,6 @@ class PGDialect_asyncpg(PGDialect): }, ) - def __init__(self, server_side_cursors=False, **kwargs): - PGDialect.__init__(self, **kwargs) - self.server_side_cursors = server_side_cursors - @util.memoized_property def _dbapi_version(self): if self.dbapi and hasattr(self.dbapi, "__version__"): diff --git a/lib/sqlalchemy/dialects/postgresql/base.py b/lib/sqlalchemy/dialects/postgresql/base.py index 3ef87620f..5e81586b4 100644 --- a/lib/sqlalchemy/dialects/postgresql/base.py +++ b/lib/sqlalchemy/dialects/postgresql/base.py @@ -112,6 +112,34 @@ The CREATE TABLE for the above :class:`_schema.Table` object would be: PRIMARY KEY (id) ) +.. _postgresql_ss_cursors: + +Server Side Cursors +------------------- + +Server-side cursor support is available for the psycopg2, asyncpg +dialects and may also be available in others. + +Server side cursors are enabled on a per-statement basis by using the +:paramref:`.Connection.execution_options.stream_results` connection execution +option:: + + with engine.connect() as conn: + result = conn.execution_options(stream_resuls=True).execute(text("select * from table")) + +Note that some kinds of SQL statements may not be supported with +server side cursors; generally, only SQL statements that return rows should be +used with this option. + +.. deprecated:: 1.4 The dialect-level server_side_cursors flag is deprecated + and will be removed in a future release. Please use the + :paramref:`_engine.Connection.stream_results` execution option for + unbuffered cursor support. + +.. seealso:: + + :ref:`engine_stream_results` + .. _postgresql_isolation_level: Transaction Isolation Level @@ -1075,7 +1103,8 @@ E.g.:: ) -""" +""" # noqa E501 + from collections import defaultdict import datetime as dt import re diff --git a/lib/sqlalchemy/dialects/postgresql/psycopg2.py b/lib/sqlalchemy/dialects/postgresql/psycopg2.py index 3cc62fc93..91576c4d2 100644 --- a/lib/sqlalchemy/dialects/postgresql/psycopg2.py +++ b/lib/sqlalchemy/dialects/postgresql/psycopg2.py @@ -17,18 +17,6 @@ psycopg2 Connect Arguments psycopg2-specific keyword arguments which are accepted by :func:`_sa.create_engine()` are: -* ``server_side_cursors``: Enable the usage of "server side cursors" for SQL - statements which support this feature. What this essentially means from a - psycopg2 point of view is that the cursor is created using a name, e.g. - ``connection.cursor('some name')``, which has the effect that result rows - are not immediately pre-fetched and buffered after statement execution, but - are instead left on the server and only retrieved as needed. SQLAlchemy's - :class:`~sqlalchemy.engine.CursorResult` uses special row-buffering - behavior when this feature is enabled, such that groups of 100 rows at a - time are fetched over the wire to reduce conversational overhead. - Note that the :paramref:`.Connection.execution_options.stream_results` - execution option is a more targeted - way of enabling this mode on a per-execution basis. * ``use_native_unicode``: Enable the usage of Psycopg2 "native unicode" mode per connection. True by default. @@ -129,8 +117,7 @@ in addition to those not specific to DBAPIs: * ``stream_results`` - Enable or disable usage of psycopg2 server side cursors - this feature makes use of "named" cursors in combination with special result handling methods so that result rows are not fully buffered. - If ``None`` or not set, the ``server_side_cursors`` option of the - :class:`_engine.Engine` is used. + Defaults to False, meaning cursors are buffered by default. * ``max_row_buffer`` - when using ``stream_results``, an integer value that specifies the maximum number of rows to buffer at a time. This is @@ -691,7 +678,6 @@ class PGDialect_psycopg2(PGDialect): def __init__( self, - server_side_cursors=False, use_native_unicode=True, client_encoding=None, use_native_hstore=True, @@ -702,7 +688,6 @@ class PGDialect_psycopg2(PGDialect): **kwargs ): PGDialect.__init__(self, **kwargs) - self.server_side_cursors = server_side_cursors self.use_native_unicode = use_native_unicode if not use_native_hstore: self._has_native_hstore = False diff --git a/lib/sqlalchemy/engine/base.py b/lib/sqlalchemy/engine/base.py index afab8e7b4..55266fae7 100644 --- a/lib/sqlalchemy/engine/base.py +++ b/lib/sqlalchemy/engine/base.py @@ -307,8 +307,13 @@ class Connection(Connectable): :param stream_results: Available on: Connection, statement. Indicate to the dialect that results should be "streamed" and not pre-buffered, if possible. This is a limitation - of many DBAPIs. The flag is currently understood only by the - psycopg2, mysqldb and pymysql dialects. + of many DBAPIs. The flag is currently understood within a subset + of dialects within the PostgreSQL and MySQL categories, and + may be supported by other third party dialects as well. + + .. seealso:: + + :ref:`engine_stream_results` :param schema_translate_map: Available on: Connection, Engine. A dictionary mapping schema names to schema names, that will be diff --git a/lib/sqlalchemy/engine/default.py b/lib/sqlalchemy/engine/default.py index 9d2eaf606..ff29c3b9d 100644 --- a/lib/sqlalchemy/engine/default.py +++ b/lib/sqlalchemy/engine/default.py @@ -144,6 +144,8 @@ class DefaultDialect(interfaces.Dialect): supports_server_side_cursors = False + server_side_cursors = False + # extra record-level locking features (#4860) supports_for_update_of = False @@ -235,6 +237,14 @@ class DefaultDialect(interfaces.Dialect): "Applications should work with result column names in a case " "sensitive fashion.", ), + server_side_cursors=( + "1.4", + "The :paramref:`_sa.create_engine.server_side_cursors` parameter " + "is deprecated and will be removed in a future release. Please " + "use the " + ":paramref:`_engine.Connection.execution_options.stream_results` " + "parameter.", + ), ) def __init__( self, @@ -250,6 +260,7 @@ class DefaultDialect(interfaces.Dialect): # int() is because the @deprecated_params decorator cannot accommodate # the direct reference to the "NO_LINTING" object compiler_linting=int(compiler.NO_LINTING), + server_side_cursors=False, **kwargs ): @@ -259,6 +270,14 @@ class DefaultDialect(interfaces.Dialect): % self.name ) + if server_side_cursors: + if not self.supports_server_side_cursors: + raise exc.ArgumentError( + "Dialect %s does not support server side cursors" % self + ) + else: + self.server_side_cursors = True + self.convert_unicode = convert_unicode self.encoding = encoding self.positional = False @@ -1189,6 +1208,7 @@ class DefaultExecutionContext(interfaces.ExecutionContext): return False if self.dialect.server_side_cursors: + # this is deprecated use_server_side = self.execution_options.get( "stream_results", True ) and ( @@ -1232,7 +1252,10 @@ class DefaultExecutionContext(interfaces.ExecutionContext): return self.create_server_side_cursor() else: self._is_server_side = False - return self._dbapi_connection.cursor() + return self.create_default_cursor() + + def create_default_cursor(self): + return self._dbapi_connection.cursor() def create_server_side_cursor(self): raise NotImplementedError() diff --git a/lib/sqlalchemy/orm/query.py b/lib/sqlalchemy/orm/query.py index 26a5a89c7..91a638e4b 100644 --- a/lib/sqlalchemy/orm/query.py +++ b/lib/sqlalchemy/orm/query.py @@ -803,7 +803,7 @@ class Query( .. seealso:: - :meth:`_query.Query.enable_eagerloads` + :ref:`engine_stream_results` """ self.load_options += {"_yield_per": count} @@ -1528,6 +1528,8 @@ class Query( .. seealso:: + :ref:`engine_stream_results` + :meth:`_query.Query.get_execution_options` """ diff --git a/lib/sqlalchemy/testing/suite/test_results.py b/lib/sqlalchemy/testing/suite/test_results.py index 6d28a207e..1c1b20cf0 100644 --- a/lib/sqlalchemy/testing/suite/test_results.py +++ b/lib/sqlalchemy/testing/suite/test_results.py @@ -236,15 +236,28 @@ class ServerSideCursorsTest( elif self.engine.dialect.driver == "mysqldb": sscursor = __import__("MySQLdb.cursors").cursors.SSCursor return isinstance(cursor, sscursor) + elif self.engine.dialect.driver == "mariadbconnector": + return not cursor.buffered elif self.engine.dialect.driver == "asyncpg": return cursor.server_side else: return False def _fixture(self, server_side_cursors): - self.engine = engines.testing_engine( - options={"server_side_cursors": server_side_cursors} - ) + if server_side_cursors: + with testing.expect_deprecated( + "The create_engine.server_side_cursors parameter is " + "deprecated and will be removed in a future release. " + "Please use the Connection.execution_options.stream_results " + "parameter." + ): + self.engine = engines.testing_engine( + options={"server_side_cursors": server_side_cursors} + ) + else: + self.engine = engines.testing_engine( + options={"server_side_cursors": server_side_cursors} + ) return self.engine def tearDown(self): |