From df4e59ff557b16202c4c3e47ad48667ba1e143c0 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Tue, 9 Apr 2013 11:52:21 -0400 Subject: Fixed bug when a query of the form: ``query(SubClass).options(subqueryload(Baseclass.attrname))``, where ``SubClass`` is a joined inh of ``BaseClass``, would fail to apply the ``JOIN`` inside the subquery on the attribute load, producing a cartesian product. The populated results still tended to be correct as additional rows are just ignored, so this issue may be present as a performance degradation in applications that are otherwise working correctly. [ticket:2699] --- lib/sqlalchemy/orm/strategies.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/sqlalchemy/orm/strategies.py b/lib/sqlalchemy/orm/strategies.py index 5c79de749..e08bb40cb 100644 --- a/lib/sqlalchemy/orm/strategies.py +++ b/lib/sqlalchemy/orm/strategies.py @@ -719,7 +719,7 @@ class SubqueryLoader(AbstractRelationshipLoader): # produce a subquery from it. left_alias = self._generate_from_original_query( orig_query, leftmost_mapper, - leftmost_attr + leftmost_attr, entity.mapper ) # generate another Query that will join the @@ -772,7 +772,7 @@ class SubqueryLoader(AbstractRelationshipLoader): def _generate_from_original_query(self, orig_query, leftmost_mapper, - leftmost_attr + leftmost_attr, entity_mapper ): # reformat the original query # to look only for significant columns @@ -781,6 +781,8 @@ class SubqueryLoader(AbstractRelationshipLoader): # TODO: why does polymporphic etc. require hardcoding # into _adapt_col_list ? Does query.add_columns(...) work # with polymorphic loading ? + if entity_mapper.isa(leftmost_mapper): + q._set_select_from(entity_mapper) q._set_entities(q._adapt_col_list(leftmost_attr)) if q._order_by is False: @@ -792,6 +794,7 @@ class SubqueryLoader(AbstractRelationshipLoader): # the original query now becomes a subquery # which we'll join onto. + embed_q = q.with_labels().subquery() left_alias = orm_util.AliasedClass(leftmost_mapper, embed_q, use_mapper_path=True) -- cgit v1.2.1 From d84ae4f75468fea6b3345c1275d6a472d0cf7c5f Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Tue, 9 Apr 2013 14:21:40 -0400 Subject: Fixed indirect regression regarding :func:`.has_inherited_table`, where since it considers the current class' ``__table__``, was sensitive to when it was called. This is 0.7's behavior also, but in 0.7 things tended to "work out" within events like ``__mapper_args__()``. :func:`.has_inherited_table` now only considers superclasses, so should return the same answer regarding the current class no matter when it's called (obviously assuming the state of the superclass). [ticket:2656] --- lib/sqlalchemy/ext/declarative/api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/sqlalchemy/ext/declarative/api.py b/lib/sqlalchemy/ext/declarative/api.py index af77919a7..6f3ffddc7 100644 --- a/lib/sqlalchemy/ext/declarative/api.py +++ b/lib/sqlalchemy/ext/declarative/api.py @@ -38,7 +38,7 @@ def has_inherited_table(cls): """Given a class, return True if any of the classes it inherits from has a mapped table, otherwise return False. """ - for class_ in cls.__mro__: + for class_ in cls.__mro__[1:]: if getattr(class_, '__table__', None) is not None: return True return False -- cgit v1.2.1 From a5ede47f1225ac10e69e2624038424b013d6144f Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 11 Apr 2013 16:14:23 -0400 Subject: A major fix to the way in which a select() object produces labeled columns when apply_labels() is used; this mode produces a SELECT where each column is labeled as in _, to remove column name collisions for a multiple table select. The fix is that if two labels collide when combined with the table name, i.e. "foo.bar_id" and "foo_bar.id", anonymous aliasing will be applied to one of the dupes. This allows the ORM to handle both columns independently; previously, 0.7 would in some cases silently emit a second SELECT for the column that was "duped", and in 0.8 an ambiguous column error would be emitted. The "keys" applied to the .c. collection of the select() will also be deduped, so that the "column being replaced" warning will no longer emit for any select() that specifies use_labels, though the dupe key will be given an anonymous label which isn't generally user-friendly. [ticket:2702] --- lib/sqlalchemy/engine/result.py | 2 +- lib/sqlalchemy/sql/compiler.py | 34 ++++++++++++++------------- lib/sqlalchemy/sql/expression.py | 46 ++++++++++++++++++++++++++++++++----- lib/sqlalchemy/util/_collections.py | 1 - 4 files changed, 59 insertions(+), 24 deletions(-) (limited to 'lib') diff --git a/lib/sqlalchemy/engine/result.py b/lib/sqlalchemy/engine/result.py index 7572564bb..1c148e1f0 100644 --- a/lib/sqlalchemy/engine/result.py +++ b/lib/sqlalchemy/engine/result.py @@ -301,7 +301,7 @@ class ResultMetaData(object): # this check isn't currently available if the row # was unpickled. if result is not None and \ - result[1] is not None: + result[1] is not None: for obj in result[1]: if key._compare_name_for_result(obj): break diff --git a/lib/sqlalchemy/sql/compiler.py b/lib/sqlalchemy/sql/compiler.py index 5a3a92a3e..b902f9ffc 100644 --- a/lib/sqlalchemy/sql/compiler.py +++ b/lib/sqlalchemy/sql/compiler.py @@ -992,13 +992,15 @@ class SQLCompiler(engine.Compiled): else: self.result_map[keyname] = name, objects, type_ - def _label_select_column(self, select, column, populate_result_map, + def _label_select_column(self, select, column, + populate_result_map, asfrom, column_clause_args, + name=None, within_columns_clause=True): """produce labeled columns present in a select().""" if column.type._has_column_expression and \ - populate_result_map: + populate_result_map: col_expr = column.type.column_expression(column) add_to_result_map = lambda keyname, name, objects, type_: \ self._add_to_result_map( @@ -1023,13 +1025,11 @@ class SQLCompiler(engine.Compiled): else: result_expr = col_expr - elif select is not None and \ - select.use_labels and \ - column._label: + elif select is not None and name: result_expr = _CompileLabel( col_expr, - column._label, - alt_names=(column._key_label, ) + name, + alt_names=(column._key_label,) ) elif \ @@ -1037,7 +1037,7 @@ class SQLCompiler(engine.Compiled): isinstance(column, sql.ColumnClause) and \ not column.is_literal and \ column.table is not None and \ - not isinstance(column.table, sql.Select): + not isinstance(column.table, sql.Select): result_expr = _CompileLabel(col_expr, sql._as_truncated(column.name), alt_names=(column.key,)) @@ -1098,11 +1098,11 @@ class SQLCompiler(engine.Compiled): # correlate_froms.union(existingfroms) populate_result_map = force_result_map or ( - compound_index == 0 and ( - not entry or \ - entry.get('iswrapper', False) - ) - ) + compound_index == 0 and ( + not entry or \ + entry.get('iswrapper', False) + ) + ) self.stack.append({'from': correlate_froms, 'iswrapper': iswrapper}) @@ -1117,10 +1117,12 @@ class SQLCompiler(engine.Compiled): # the actual list of columns to print in the SELECT column list. inner_columns = [ c for c in [ - self._label_select_column(select, column, + self._label_select_column(select, + column, populate_result_map, asfrom, - column_clause_args) - for column in util.unique_list(select.inner_columns) + column_clause_args, + name=name) + for name, column in select._columns_plus_names ] if c is not None ] diff --git a/lib/sqlalchemy/sql/expression.py b/lib/sqlalchemy/sql/expression.py index 5cef778bb..28b1c6ddd 100644 --- a/lib/sqlalchemy/sql/expression.py +++ b/lib/sqlalchemy/sql/expression.py @@ -5781,13 +5781,47 @@ class Select(HasPrefixes, SelectBase): fromclause = _interpret_as_from(fromclause) self._from_obj = self._from_obj.union([fromclause]) + + @_memoized_property + def _columns_plus_names(self): + if self.use_labels: + names = set() + def name_for_col(c): + if c._label is None: + return (None, c) + name = c._label + if name in names: + name = c.anon_label + else: + names.add(name) + return name, c + + return [ + name_for_col(c) + for c in util.unique_list(_select_iterables(self._raw_columns)) + ] + else: + return [ + (None, c) + for c in util.unique_list(_select_iterables(self._raw_columns)) + ] + def _populate_column_collection(self): - for c in self.inner_columns: - if hasattr(c, '_make_proxy'): - c._make_proxy(self, - name=c._label if self.use_labels else None, - key=c._key_label if self.use_labels else None, - name_is_truncatable=True) + for name, c in self._columns_plus_names: + if not hasattr(c, '_make_proxy'): + continue + if name is None: + key = None + elif self.use_labels: + key = c._key_label + if key is not None and key in self.c: + key = c.anon_label + else: + key = None + + c._make_proxy(self, key=key, + name=name, + name_is_truncatable=True) def _refresh_for_new_column(self, column): for fromclause in self._froms: diff --git a/lib/sqlalchemy/util/_collections.py b/lib/sqlalchemy/util/_collections.py index 2c9c982fb..8e61275e7 100644 --- a/lib/sqlalchemy/util/_collections.py +++ b/lib/sqlalchemy/util/_collections.py @@ -672,7 +672,6 @@ column_dict = dict ordered_column_set = OrderedSet populate_column_dict = PopulateDict - def unique_list(seq, hashfunc=None): seen = {} if not hashfunc: -- cgit v1.2.1 From ee7bb9c1744b4b09263ef32032afd38f694df136 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 11 Apr 2013 19:10:02 -0400 Subject: Improvements to Connection auto-invalidation handling. If a non-disconnect error occurs, but leads to a delayed disconnect error within error handling (happens with MySQL), the disconnect condition is detected. The Connection can now also be closed when in an invalid state, meaning it will raise "closed" on next usage, and additionally the "close with result" feature will work even if the autorollback in an error handling routine fails and regardless of whether the condition is a disconnect or not. [ticket:2695] --- lib/sqlalchemy/engine/base.py | 56 ++++++++++++++++++++++++------------------- 1 file changed, 32 insertions(+), 24 deletions(-) (limited to 'lib') diff --git a/lib/sqlalchemy/engine/base.py b/lib/sqlalchemy/engine/base.py index 6242f0816..e40af6219 100644 --- a/lib/sqlalchemy/engine/base.py +++ b/lib/sqlalchemy/engine/base.py @@ -62,6 +62,7 @@ class Connection(Connectable): self.__savepoint_seq = 0 self.__branch = _branch self.__invalid = False + self.__can_reconnect = True if _dispatch: self.dispatch = _dispatch elif engine._has_events: @@ -213,8 +214,8 @@ class Connection(Connectable): def closed(self): """Return True if this connection is closed.""" - return not self.__invalid and '_Connection__connection' \ - not in self.__dict__ + return '_Connection__connection' not in self.__dict__ \ + and not self.__can_reconnect @property def invalidated(self): @@ -232,7 +233,7 @@ class Connection(Connectable): return self._revalidate_connection() def _revalidate_connection(self): - if self.__invalid: + if self.__can_reconnect and self.__invalid: if self.__transaction is not None: raise exc.InvalidRequestError( "Can't reconnect until invalid " @@ -577,15 +578,15 @@ class Connection(Connectable): and will allow no further operations. """ - try: conn = self.__connection except AttributeError: - return - if not self.__branch: - conn.close() - self.__invalid = False - del self.__connection + pass + else: + if not self.__branch: + conn.close() + del self.__connection + self.__can_reconnect = False self.__transaction = None def scalar(self, object, *multiparams, **params): @@ -972,13 +973,22 @@ class Connection(Connectable): if isinstance(e, (SystemExit, KeyboardInterrupt)): raise + _reentrant_error = False + _is_disconnect = False + def _handle_dbapi_exception(self, e, statement, parameters, cursor, context): - if getattr(self, '_reentrant_error', False): + + if not self._is_disconnect: + self._is_disconnect = isinstance(e, self.dialect.dbapi.Error) and \ + not self.closed and \ + self.dialect.is_disconnect(e, self.__connection, cursor) + + if self._reentrant_error: # Py3K #raise exc.DBAPIError.instance(statement, parameters, e, # self.dialect.dbapi.Error) from e @@ -1006,21 +1016,10 @@ class Connection(Connectable): e) context.handle_dbapi_exception(e) - is_disconnect = isinstance(e, self.dialect.dbapi.Error) and \ - self.dialect.is_disconnect(e, self.__connection, cursor) - - if is_disconnect: - dbapi_conn_wrapper = self.connection - self.invalidate(e) - if not hasattr(dbapi_conn_wrapper, '_pool') or \ - dbapi_conn_wrapper._pool is self.engine.pool: - self.engine.dispose() - else: + if not self._is_disconnect: if cursor: self._safe_close_cursor(cursor) self._autorollback() - if self.should_close_with_result: - self.close() if not should_wrap: return @@ -1031,7 +1030,7 @@ class Connection(Connectable): # parameters, # e, # self.dialect.dbapi.Error, - # connection_invalidated=is_disconnect) \ + # connection_invalidated=self._is_disconnect) \ # from e # Py2K raise exc.DBAPIError.instance( @@ -1039,12 +1038,21 @@ class Connection(Connectable): parameters, e, self.dialect.dbapi.Error, - connection_invalidated=is_disconnect), \ + connection_invalidated=self._is_disconnect), \ None, sys.exc_info()[2] # end Py2K finally: del self._reentrant_error + if self._is_disconnect: + del self._is_disconnect + dbapi_conn_wrapper = self.connection + self.invalidate(e) + if not hasattr(dbapi_conn_wrapper, '_pool') or \ + dbapi_conn_wrapper._pool is self.engine.pool: + self.engine.dispose() + if self.should_close_with_result: + self.close() # poor man's multimethod/generic function thingy executors = { -- cgit v1.2.1 From a0e95869a4a17cc1c162341a05d0c8a04a398f55 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 11 Apr 2013 19:34:46 -0400 Subject: The _Binary base type now converts values through the bytes() callable when run on Python 3; in particular psycopg2 2.5 with Python 3.3 seems to now be returning the "memoryview" type, so this is converted to bytes before return. --- lib/sqlalchemy/types.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'lib') diff --git a/lib/sqlalchemy/types.py b/lib/sqlalchemy/types.py index 08aba4b56..1824a9b3f 100644 --- a/lib/sqlalchemy/types.py +++ b/lib/sqlalchemy/types.py @@ -1711,8 +1711,15 @@ class _Binary(TypeEngine): return process # Python 3 has native bytes() type - # both sqlite3 and pg8000 seem to return it - # (i.e. and not 'memoryview') + # both sqlite3 and pg8000 seem to return it, + # psycopg2 as of 2.5 returns 'memoryview' + # Py3K + #def result_processor(self, dialect, coltype): + # def process(value): + # if value is not None: + # value = bytes(value) + # return value + # return process # Py2K def result_processor(self, dialect, coltype): if util.jython: -- cgit v1.2.1 From 6e2bfaf769c429021d5ab6716f2fcdcd327c2627 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 11 Apr 2013 19:37:56 -0400 Subject: Fixed `__repr__()` on mapper property constructs to work before the object is initialized, so that Sphinx builds with recent Sphinx versions can read them. --- lib/sqlalchemy/orm/interfaces.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib') diff --git a/lib/sqlalchemy/orm/interfaces.py b/lib/sqlalchemy/orm/interfaces.py index 62cdb2710..70743624c 100644 --- a/lib/sqlalchemy/orm/interfaces.py +++ b/lib/sqlalchemy/orm/interfaces.py @@ -289,7 +289,7 @@ class MapperProperty(_MappedAttribute, _InspectionAttr): def __repr__(self): return '<%s at 0x%x; %s>' % ( self.__class__.__name__, - id(self), self.key) + id(self), getattr(self, 'key', 'no key')) class PropComparator(operators.ColumnOperators): """Defines boolean, comparison, and other operators for -- cgit v1.2.1 From ce5cff151ad5f4d807c2655f7e44c3245552c0b6 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Thu, 11 Apr 2013 20:04:57 -0400 Subject: Fixed bug whereby ORM would run the wrong kind of query when refreshing an inheritance-mapped class where the superclass was mapped to a non-Table object, like a custom join() or a select(), running a query that assumed a hierarchy that's mapped to individual Table-per-class. [ticket:2697] --- lib/sqlalchemy/orm/mapper.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib') diff --git a/lib/sqlalchemy/orm/mapper.py b/lib/sqlalchemy/orm/mapper.py index d258a20b6..914c29b7f 100644 --- a/lib/sqlalchemy/orm/mapper.py +++ b/lib/sqlalchemy/orm/mapper.py @@ -1933,6 +1933,8 @@ class Mapper(_InspectionAttr): for mapper in reversed(list(self.iterate_to_root())): if mapper.local_table in tables: start = True + elif not isinstance(mapper.local_table, expression.TableClause): + return None if start and not mapper.single: allconds.append(visitors.cloned_traverse( mapper.inherit_condition, -- cgit v1.2.1