From c691b4cbdf7424964f49ac2fd05057514e5856a3 Mon Sep 17 00:00:00 2001 From: Mike Bayer Date: Sat, 11 Dec 2010 17:44:46 -0500 Subject: - support for cdecimal - add --with-cdecimal flag to tests, monkeypatches cdecimal in - fix mssql/pyodbc.py to not use private '_int' accessor in decimal conversion routines - pyodbc version 2.1.8 is needed for cdecimal in any case as previous versions also called '_int', 2.1.8 adds the same string logic as our own dialect, so that logic is skipped for modern pyodbc version - make the imports for "Decimal" consistent across the whole lib. not sure yet how we should be importing "Decimal" or what the best way forward is that would allow a clean user-invoked swap of cdecimal; for now, added docs suggesting a global monkeypatch - the two decimal libs are not compatible with each other so any chance of mixing produces serious issues. adding adapters to DBAPIs tedious and adds in-python overhead. suggestions welcome on how we should be doing Decimal/cdecimal. --- lib/sqlalchemy/connectors/mxodbc.py | 2 +- lib/sqlalchemy/dialects/mssql/pymssql.py | 1 - lib/sqlalchemy/dialects/mssql/pyodbc.py | 27 ++++++++---- lib/sqlalchemy/dialects/oracle/cx_oracle.py | 12 +++--- lib/sqlalchemy/dialects/postgresql/pg8000.py | 3 +- lib/sqlalchemy/dialects/postgresql/psycopg2.py | 2 +- lib/sqlalchemy/dialects/postgresql/pypostgresql.py | 1 - lib/sqlalchemy/dialects/sybase/pyodbc.py | 2 +- lib/sqlalchemy/types.py | 48 +++++++++++++++++----- lib/sqlalchemy/util/compat.py | 5 ++- 10 files changed, 71 insertions(+), 32 deletions(-) (limited to 'lib/sqlalchemy') diff --git a/lib/sqlalchemy/connectors/mxodbc.py b/lib/sqlalchemy/connectors/mxodbc.py index 4c4b0b070..1f1688a51 100644 --- a/lib/sqlalchemy/connectors/mxodbc.py +++ b/lib/sqlalchemy/connectors/mxodbc.py @@ -15,7 +15,7 @@ For more info on mxODBC, see http://www.egenix.com/ import sys import re import warnings -from decimal import Decimal +from sqlalchemy.util.compat import decimal from sqlalchemy.connectors import Connector from sqlalchemy import types as sqltypes diff --git a/lib/sqlalchemy/dialects/mssql/pymssql.py b/lib/sqlalchemy/dialects/mssql/pymssql.py index c5f471942..1368f6414 100644 --- a/lib/sqlalchemy/dialects/mssql/pymssql.py +++ b/lib/sqlalchemy/dialects/mssql/pymssql.py @@ -35,7 +35,6 @@ Please consult the pymssql documentation for further information. from sqlalchemy.dialects.mssql.base import MSDialect from sqlalchemy import types as sqltypes, util, processors import re -import decimal class _MSNumeric_pymssql(sqltypes.Numeric): def result_processor(self, dialect, type_): diff --git a/lib/sqlalchemy/dialects/mssql/pyodbc.py b/lib/sqlalchemy/dialects/mssql/pyodbc.py index 5bba24514..93a516706 100644 --- a/lib/sqlalchemy/dialects/mssql/pyodbc.py +++ b/lib/sqlalchemy/dialects/mssql/pyodbc.py @@ -88,7 +88,12 @@ class _MSNumeric_pyodbc(sqltypes.Numeric): """ def bind_processor(self, dialect): - super_process = super(_MSNumeric_pyodbc, self).bind_processor(dialect) + + super_process = super(_MSNumeric_pyodbc, self).\ + bind_processor(dialect) + + if not dialect._need_decimal_fix: + return super_process def process(value): if self.asdecimal and \ @@ -106,31 +111,35 @@ class _MSNumeric_pyodbc(sqltypes.Numeric): return value return process + # these routines needed for older versions of pyodbc. + # as of 2.1.8 this logic is integrated. + def _small_dec_to_string(self, value): return "%s0.%s%s" % ( (value < 0 and '-' or ''), '0' * (abs(value.adjusted()) - 1), - "".join([str(nint) for nint in value._int])) + "".join([str(nint) for nint in value.as_tuple()[1]])) def _large_dec_to_string(self, value): + _int = value.as_tuple()[1] if 'E' in str(value): result = "%s%s%s" % ( (value < 0 and '-' or ''), - "".join([str(s) for s in value._int]), - "0" * (value.adjusted() - (len(value._int)-1))) + "".join([str(s) for s in _int]), + "0" * (value.adjusted() - (len(_int)-1))) else: - if (len(value._int) - 1) > value.adjusted(): + if (len(_int) - 1) > value.adjusted(): result = "%s%s.%s" % ( (value < 0 and '-' or ''), "".join( - [str(s) for s in value._int][0:value.adjusted() + 1]), + [str(s) for s in _int][0:value.adjusted() + 1]), "".join( - [str(s) for s in value._int][value.adjusted() + 1:])) + [str(s) for s in _int][value.adjusted() + 1:])) else: result = "%s%s" % ( (value < 0 and '-' or ''), "".join( - [str(s) for s in value._int][0:value.adjusted() + 1])) + [str(s) for s in _int][0:value.adjusted() + 1])) return result @@ -200,5 +209,7 @@ class MSDialect_pyodbc(PyODBCConnector, MSDialect): self.description_encoding = description_encoding self.use_scope_identity = self.dbapi and \ hasattr(self.dbapi.Cursor, 'nextset') + self._need_decimal_fix = self.dbapi and \ + tuple(self.dbapi.version.split(".")) < (2, 1, 8) dialect = MSDialect_pyodbc diff --git a/lib/sqlalchemy/dialects/oracle/cx_oracle.py b/lib/sqlalchemy/dialects/oracle/cx_oracle.py index 87a84e514..b7d663138 100644 --- a/lib/sqlalchemy/dialects/oracle/cx_oracle.py +++ b/lib/sqlalchemy/dialects/oracle/cx_oracle.py @@ -121,7 +121,7 @@ from sqlalchemy.engine import base from sqlalchemy import types as sqltypes, util, exc, processors from datetime import datetime import random -from decimal import Decimal +from sqlalchemy.util.compat import decimal import re class _OracleNumeric(sqltypes.Numeric): @@ -148,10 +148,10 @@ class _OracleNumeric(sqltypes.Numeric): def to_decimal(value): if value is None: return None - elif isinstance(value, Decimal): + elif isinstance(value, decimal.Decimal): return value else: - return Decimal(fstring % value) + return decimal.Decimal(fstring % value) return to_decimal else: if self.precision is None and self.scale is None: @@ -569,15 +569,15 @@ class OracleDialect_cx_oracle(OracleDialect): self._detect_decimal = \ lambda value: _detect_decimal(value.replace(char, '.')) self._to_decimal = \ - lambda value: Decimal(value.replace(char, '.')) + lambda value: decimal.Decimal(value.replace(char, '.')) def _detect_decimal(self, value): if "." in value: - return Decimal(value) + return decimal.Decimal(value) else: return int(value) - _to_decimal = Decimal + _to_decimal = decimal.Decimal def on_connect(self): if self.cx_oracle_ver < (5,): diff --git a/lib/sqlalchemy/dialects/postgresql/pg8000.py b/lib/sqlalchemy/dialects/postgresql/pg8000.py index 7b1d8e6a7..3afa06eab 100644 --- a/lib/sqlalchemy/dialects/postgresql/pg8000.py +++ b/lib/sqlalchemy/dialects/postgresql/pg8000.py @@ -21,10 +21,9 @@ Passing data from/to the Interval type is not supported as of yet. """ -import decimal - from sqlalchemy.engine import default from sqlalchemy import util, exc +from sqlalchemy.util.compat import decimal from sqlalchemy import processors from sqlalchemy import types as sqltypes from sqlalchemy.dialects.postgresql.base import PGDialect, \ diff --git a/lib/sqlalchemy/dialects/postgresql/psycopg2.py b/lib/sqlalchemy/dialects/postgresql/psycopg2.py index 88e6ce670..b3f42c330 100644 --- a/lib/sqlalchemy/dialects/postgresql/psycopg2.py +++ b/lib/sqlalchemy/dialects/postgresql/psycopg2.py @@ -86,10 +86,10 @@ The following per-statement execution options are respected: import random import re -import decimal import logging from sqlalchemy import util, exc +from sqlalchemy.util.compat import decimal from sqlalchemy import processors from sqlalchemy.engine import base, default from sqlalchemy.sql import expression diff --git a/lib/sqlalchemy/dialects/postgresql/pypostgresql.py b/lib/sqlalchemy/dialects/postgresql/pypostgresql.py index b8f7991d5..9abdffb6e 100644 --- a/lib/sqlalchemy/dialects/postgresql/pypostgresql.py +++ b/lib/sqlalchemy/dialects/postgresql/pypostgresql.py @@ -8,7 +8,6 @@ URLs are of the form ``postgresql+pypostgresql://user@password@host:port/dbname[ """ from sqlalchemy.engine import default -import decimal from sqlalchemy import util from sqlalchemy import types as sqltypes from sqlalchemy.dialects.postgresql.base import PGDialect, PGExecutionContext diff --git a/lib/sqlalchemy/dialects/sybase/pyodbc.py b/lib/sqlalchemy/dialects/sybase/pyodbc.py index 1d955a7d9..68b16c051 100644 --- a/lib/sqlalchemy/dialects/sybase/pyodbc.py +++ b/lib/sqlalchemy/dialects/sybase/pyodbc.py @@ -31,8 +31,8 @@ Currently *not* supported are:: from sqlalchemy.dialects.sybase.base import SybaseDialect,\ SybaseExecutionContext from sqlalchemy.connectors.pyodbc import PyODBCConnector -import decimal from sqlalchemy import types as sqltypes, util, processors +from sqlalchemy.util.compat import decimal class _SybNumeric_pyodbc(sqltypes.Numeric): """Turns Decimals with adjusted() < -6 into floats. diff --git a/lib/sqlalchemy/types.py b/lib/sqlalchemy/types.py index 13dbc6a83..3e592ea51 100644 --- a/lib/sqlalchemy/types.py +++ b/lib/sqlalchemy/types.py @@ -22,14 +22,14 @@ __all__ = [ 'TypeEngine', 'TypeDecorator', 'AbstractType', 'UserDefinedType', import inspect import datetime as dt -from decimal import Decimal as _python_Decimal import codecs from sqlalchemy import exc, schema from sqlalchemy.sql import expression, operators import sys -schema.types = expression.sqltypes =sys.modules['sqlalchemy.types'] +schema.types = expression.sqltypes = sys.modules['sqlalchemy.types'] from sqlalchemy.util import pickle +from sqlalchemy.util.compat import decimal from sqlalchemy.sql.visitors import Visitable from sqlalchemy import util from sqlalchemy import processors @@ -1047,7 +1047,38 @@ class Numeric(_DateAffinity, TypeEngine): overhead, and is still subject to floating point data loss - in which case ``asdecimal=False`` will at least remove the extra conversion overhead. - + + Note that the "cdecimal" library is a high performing alternative + to Python's built-in "decimal" type, which performs very poorly + in high volume situations. SQLAlchemy 0.7 is tested against "cdecimal" + as well and supports it fully. The type is not necessarily supported by + DBAPI implementations however, most of which contain an import + for plain "decimal" in their source code, even though + some such as psycopg2 provide hooks for alternate adapters. + SQLAlchemy imports "decimal" globally as well. While the + alternate "Decimal" class can be patched into SQLA's "decimal" module, + overall the most straightforward and foolproof way to use + "cdecimal" given current support is to patch it directly + into sys.modules before anything else is imported:: + + import sys + import cdecimal + sys.modules["decimal"] = cdecimal + + While the global patch is a little ugly, it's particularly + important to use just one decimal library at a time since + Python Decimal and cdecimal Decimal objects + are not currently compatible *with each other*:: + + >>> import cdecimal + >>> import decimal + >>> decimal.Decimal("10") == cdecimal.Decimal("10") + False + + SQLAlchemy will provide more automatic support of + cdecimal if and when it becomes a standard part of Python + installations and is supported by all DBAPIs. + """ self.precision = precision self.scale = scale @@ -1085,10 +1116,10 @@ class Numeric(_DateAffinity, TypeEngine): # we're a "numeric", DBAPI returns floats, convert. if self.scale is not None: return processors.to_decimal_processor_factory( - _python_Decimal, self.scale) + decimal.Decimal, self.scale) else: return processors.to_decimal_processor_factory( - _python_Decimal) + decimal.Decimal) else: if dialect.supports_native_decimal: return processors.to_float @@ -1153,7 +1184,7 @@ class Float(Numeric): def result_processor(self, dialect, coltype): if self.asdecimal: - return processors.to_decimal_processor_factory(_python_Decimal) + return processors.to_decimal_processor_factory(decimal.Decimal) else: return None @@ -1928,9 +1959,6 @@ NULLTYPE = NullType() BOOLEANTYPE = Boolean() STRINGTYPE = String() -# using VARCHAR/NCHAR so that we dont get the genericized "String" -# type which usually resolves to TEXT/CLOB - _type_map = { str: String(), # Py3K @@ -1941,7 +1969,7 @@ _type_map = { int : Integer(), float : Numeric(), bool: BOOLEANTYPE, - _python_Decimal : Numeric(), + decimal.Decimal : Numeric(), dt.date : Date(), dt.datetime : DateTime(), dt.time : Time(), diff --git a/lib/sqlalchemy/util/compat.py b/lib/sqlalchemy/util/compat.py index 961aa1f8a..59dd9eaf0 100644 --- a/lib/sqlalchemy/util/compat.py +++ b/lib/sqlalchemy/util/compat.py @@ -171,7 +171,6 @@ if win32 or jython: else: time_func = time.time - if sys.version_info >= (2, 5): def decode_slice(slc): """decode a slice object as sent to __getitem__. @@ -188,3 +187,7 @@ if sys.version_info >= (2, 5): else: def decode_slice(slc): return (slc.start, slc.stop, slc.step) + + +import decimal + -- cgit v1.2.1