- pg8000 + postgresql dialects now check for float/numeric return

types to more intelligently determine float() vs. Decimal(),
[ticket:1567]
- since result processing is a hot issue of late, the DBAPI type
returned from cursor.description is certainly useful in cases like
these to determine an efficient result processor.   There's likely
other result processors that can make use of it.  But, backwards
incompat change to result_processor().  Happy major version number..

19 files changed, 109 insertions, 70 deletions

diff --git a/lib/sqlalchemy/dialects/access/base.py b/lib/sqlalchemy/dialects/access/base.py
index ed8297137..ee61190ff 100644
--- a/lib/sqlalchemy/dialects/access/base.py
+++ b/lib/sqlalchemy/dialects/access/base.py
@@ -18,7 +18,7 @@ from sqlalchemy.engine import default, base
 
 
 class AcNumeric(types.Numeric):
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         return None
 
     def bind_processor(self, dialect):
@@ -86,7 +86,7 @@ class AcUnicode(types.Unicode):
     def bind_processor(self, dialect):
         return None
 
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         return None
 
 class AcChar(types.CHAR):
@@ -101,7 +101,7 @@ class AcBoolean(types.Boolean):
     def get_col_spec(self):
         return "YESNO"
 
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         def process(value):
             if value is None:
                 return None
diff --git a/lib/sqlalchemy/dialects/firebird/base.py b/lib/sqlalchemy/dialects/firebird/base.py
index 921c70e98..21fec6b51 100644
--- a/lib/sqlalchemy/dialects/firebird/base.py
+++ b/lib/sqlalchemy/dialects/firebird/base.py
@@ -124,7 +124,7 @@ RESERVED_WORDS = set([
 
 
 class _FBBoolean(sqltypes.Boolean):
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         def process(value):
             if value is None:
                 return None
diff --git a/lib/sqlalchemy/dialects/informix/base.py b/lib/sqlalchemy/dialects/informix/base.py
index 5760488ae..6565a812f 100644
--- a/lib/sqlalchemy/dialects/informix/base.py
+++ b/lib/sqlalchemy/dialects/informix/base.py
@@ -41,7 +41,7 @@ class InfoTime(sqltypes.Time):
             return value
         return process
 
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         def process(value):
             if isinstance(value, datetime.datetime):
                 return value.time()
@@ -51,7 +51,7 @@ class InfoTime(sqltypes.Time):
 
 
 class InfoBoolean(sqltypes.Boolean):
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         def process(value):
             if value is None:
                 return None
diff --git a/lib/sqlalchemy/dialects/maxdb/base.py b/lib/sqlalchemy/dialects/maxdb/base.py
index d1c0191ed..d5f00dbdd 100644
--- a/lib/sqlalchemy/dialects/maxdb/base.py
+++ b/lib/sqlalchemy/dialects/maxdb/base.py
@@ -83,7 +83,7 @@ class _StringType(sqltypes.String):
                     return value
             return process
 
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         def process(value):
             while True:
                 if value is None:
@@ -169,7 +169,7 @@ class MaxTimestamp(sqltypes.DateTime):
                     dialect.datetimeformat,))
         return process
 
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         def process(value):
             if value is None:
                 return None
@@ -209,7 +209,7 @@ class MaxDate(sqltypes.Date):
                     dialect.datetimeformat,))
         return process
 
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         def process(value):
             if value is None:
                 return None
@@ -243,7 +243,7 @@ class MaxTime(sqltypes.Time):
                     dialect.datetimeformat,))
         return process
 
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         def process(value):
             if value is None:
                 return None
@@ -270,7 +270,7 @@ class MaxBlob(sqltypes.Binary):
                 return str(value)
         return process
 
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         def process(value):
             if value is None:
                 return None
diff --git a/lib/sqlalchemy/dialects/mssql/adodbapi.py b/lib/sqlalchemy/dialects/mssql/adodbapi.py
index 10b8b33b3..6ca1879d6 100644
--- a/lib/sqlalchemy/dialects/mssql/adodbapi.py
+++ b/lib/sqlalchemy/dialects/mssql/adodbapi.py
@@ -3,7 +3,7 @@ from sqlalchemy.dialects.mssql.base import MSDateTime, MSDialect
 import sys
 
 class MSDateTime_adodbapi(MSDateTime):
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         def process(value):
             # adodbapi will return datetimes with empty time values as datetime.date() objects.
             # Promote them back to full datetime.datetime()
diff --git a/lib/sqlalchemy/dialects/mssql/base.py b/lib/sqlalchemy/dialects/mssql/base.py
index 129125ca7..6c8937799 100644
--- a/lib/sqlalchemy/dialects/mssql/base.py
+++ b/lib/sqlalchemy/dialects/mssql/base.py
@@ -277,7 +277,7 @@ RESERVED_WORDS = set(
 
 
 class _MSNumeric(sqltypes.Numeric):
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         if self.asdecimal:
             def process(value):
                 if value is not None:
@@ -350,7 +350,7 @@ class _MSDate(sqltypes.Date):
         return process
 
     _reg = re.compile(r"(\d+)-(\d+)-(\d+)")
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         def process(value):
             if isinstance(value, datetime.datetime):
                 return value.date()
@@ -377,7 +377,7 @@ class TIME(sqltypes.TIME):
         return process
 
     _reg = re.compile(r"(\d+):(\d+):(\d+)(?:\.(\d+))?")
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         def process(value):
             if isinstance(value, datetime.datetime):
                 return value.time()
@@ -599,7 +599,7 @@ class BIT(sqltypes.TypeEngine):
     __visit_name__ = 'BIT'
     
 class _MSBoolean(sqltypes.Boolean):
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         def process(value):
             if value is None:
                 return None
diff --git a/lib/sqlalchemy/dialects/mysql/base.py b/lib/sqlalchemy/dialects/mysql/base.py
index 36a042589..a9acc2a01 100644
--- a/lib/sqlalchemy/dialects/mysql/base.py
+++ b/lib/sqlalchemy/dialects/mysql/base.py
@@ -546,7 +546,7 @@ class BIT(sqltypes.TypeEngine):
         """
         self.length = length
 
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         """Convert a MySQL's 64 bit, variable length binary string to a long."""
         def process(value):
             if value is not None:
@@ -562,7 +562,7 @@ class _MSTime(sqltypes.Time):
 
     __visit_name__ = 'TIME'
 
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         time = datetime.time
         def process(value):
             # convert from a timedelta value
@@ -1042,7 +1042,7 @@ class SET(_StringType):
         length = max([len(v) for v in strip_values] + [0])
         super(SET, self).__init__(length=length, **kw)
 
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         def process(value):
             # The good news:
             #   No ',' quoting issues- commas aren't allowed in SET values
@@ -1085,7 +1085,7 @@ class _MSBoolean(sqltypes.Boolean):
 
     __visit_name__ = 'BOOLEAN'
 
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         def process(value):
             if value is None:
                 return None
diff --git a/lib/sqlalchemy/dialects/mysql/mysqldb.py b/lib/sqlalchemy/dialects/mysql/mysqldb.py
index 49fa044a3..846de6580 100644
--- a/lib/sqlalchemy/dialects/mysql/mysqldb.py
+++ b/lib/sqlalchemy/dialects/mysql/mysqldb.py
@@ -48,7 +48,7 @@ class MySQL_mysqldbCompiler(MySQLCompiler):
 
 
 class _DecimalType(_NumericType):
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         if self.asdecimal:
             return
         def process(value):
diff --git a/lib/sqlalchemy/dialects/mysql/oursql.py b/lib/sqlalchemy/dialects/mysql/oursql.py
index 37537483d..5558c1a19 100644
--- a/lib/sqlalchemy/dialects/mysql/oursql.py
+++ b/lib/sqlalchemy/dialects/mysql/oursql.py
@@ -36,7 +36,7 @@ class _PlainQuery(unicode):
 
 
 class _oursqlNumeric(NUMERIC):
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         if self.asdecimal:
             return
         def process(value):
@@ -48,7 +48,7 @@ class _oursqlNumeric(NUMERIC):
 
 
 class _oursqlBIT(BIT):
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         """oursql already converts mysql bits, so."""
 
         return None
diff --git a/lib/sqlalchemy/dialects/mysql/zxjdbc.py b/lib/sqlalchemy/dialects/mysql/zxjdbc.py
index bf1267aed..dcb46789a 100644
--- a/lib/sqlalchemy/dialects/mysql/zxjdbc.py
+++ b/lib/sqlalchemy/dialects/mysql/zxjdbc.py
@@ -23,7 +23,7 @@ from sqlalchemy.connectors.zxJDBC import ZxJDBCConnector
 from sqlalchemy.dialects.mysql.base import BIT, MySQLDialect, MySQLExecutionContext
 
 class _ZxJDBCBit(BIT):
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         """Converts boolean or byte arrays from MySQL Connector/J to longs."""
         def process(value):
             if value is None:
diff --git a/lib/sqlalchemy/dialects/oracle/base.py b/lib/sqlalchemy/dialects/oracle/base.py
index 22ba2ce93..d13e37d60 100644
--- a/lib/sqlalchemy/dialects/oracle/base.py
+++ b/lib/sqlalchemy/dialects/oracle/base.py
@@ -159,7 +159,7 @@ class _OracleBoolean(sqltypes.Boolean):
     def get_dbapi_type(self, dbapi):
         return dbapi.NUMBER
     
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         def process(value):
             if value is None:
                 return None
diff --git a/lib/sqlalchemy/dialects/oracle/cx_oracle.py b/lib/sqlalchemy/dialects/oracle/cx_oracle.py
index eb5f2cb43..2db37a4fc 100644
--- a/lib/sqlalchemy/dialects/oracle/cx_oracle.py
+++ b/lib/sqlalchemy/dialects/oracle/cx_oracle.py
@@ -84,7 +84,7 @@ class _OracleDate(sqltypes.Date):
     def bind_processor(self, dialect):
         return None
 
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         def process(value):
             if not isinstance(value, datetime):
                 return value
@@ -93,7 +93,7 @@ class _OracleDate(sqltypes.Date):
         return process
 
 class _OracleDateTime(sqltypes.DateTime):
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         def process(value):
             if value is None or isinstance(value, datetime):
                 return value
@@ -110,7 +110,7 @@ class _OracleDateTime(sqltypes.DateTime):
 
 # only if cx_oracle contains TIMESTAMP
 class _OracleTimestamp(sqltypes.TIMESTAMP):
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         def process(value):
             if value is None or isinstance(value, datetime):
                 return value
@@ -121,13 +121,13 @@ class _OracleTimestamp(sqltypes.TIMESTAMP):
         return process
 
 class _LOBMixin(object):
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         if not dialect.auto_convert_lobs:
             # return the cx_oracle.LOB directly.
             # don't even call super.result_processor here.
             return None
             
-        super_process = super(_LOBMixin, self).result_processor(dialect)
+        super_process = super(_LOBMixin, self).result_processor(dialect, coltype)
         lob = dialect.dbapi.LOB
         if super_process:
             def process(value):
@@ -148,11 +148,11 @@ class _OracleChar(sqltypes.CHAR):
         return dbapi.FIXED_CHAR
 
 class _OracleNVarChar(sqltypes.NVARCHAR):
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         if dialect._cx_oracle_native_nvarchar:
             return None
         else:
-            return sqltypes.NVARCHAR.result_processor(self, dialect)
+            return sqltypes.NVARCHAR.result_processor(self, dialect, coltype)
         
 class _OracleText(_LOBMixin, sqltypes.Text):
     def get_dbapi_type(self, dbapi):
@@ -163,7 +163,7 @@ class _OracleUnicodeText(_LOBMixin, sqltypes.UnicodeText):
         return dbapi.NCLOB
 
 class _OracleInteger(sqltypes.Integer):
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         def to_int(val):
             if val is not None:
                 val = int(val)
@@ -259,11 +259,14 @@ class Oracle_cx_oracleExecutionContext(OracleExecutionContext):
                 for bind, name in self.compiled.bind_names.iteritems():
                     if name in self.out_parameters:
                         type = bind.type
-                        result_processor = type.dialect_impl(self.dialect).\
-                                                    result_processor(self.dialect)
+                        impl_type = type.dialect_impl(self.dialect)
+                        dbapi_type = impl_type.get_dbapi_type(self.dialect.dbapi)
+                        result_processor = impl_type.\
+                                                    result_processor(self.dialect, 
+                                                    dbapi_type)
                         if result_processor is not None:
                             out_parameters[name] = \
-                                    result_processor(self.out_parameters[name].getvalue())
+                                    result_processor(self.out_parameters[name].getvalue(), dbapi_type)
                         else:
                             out_parameters[name] = self.out_parameters[name].getvalue()
             else:
diff --git a/lib/sqlalchemy/dialects/oracle/zxjdbc.py b/lib/sqlalchemy/dialects/oracle/zxjdbc.py
index 6edef301c..42c43d369 100644
--- a/lib/sqlalchemy/dialects/oracle/zxjdbc.py
+++ b/lib/sqlalchemy/dialects/oracle/zxjdbc.py
@@ -20,7 +20,7 @@ SQLException = zxJDBC = None
 
 class _ZxJDBCDate(sqltypes.Date):
 
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         def process(value):
             if value is None:
                 return None
@@ -31,7 +31,7 @@ class _ZxJDBCDate(sqltypes.Date):
 
 class _ZxJDBCNumeric(sqltypes.Numeric):
 
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         if self.asdecimal:
             def process(value):
                 if isinstance(value, decimal.Decimal):
diff --git a/lib/sqlalchemy/dialects/postgresql/base.py b/lib/sqlalchemy/dialects/postgresql/base.py
index 97108b3cb..9c6de3623 100644
--- a/lib/sqlalchemy/dialects/postgresql/base.py
+++ b/lib/sqlalchemy/dialects/postgresql/base.py
@@ -174,8 +174,8 @@ class ARRAY(sqltypes.MutableType, sqltypes.Concatenable, sqltypes.TypeEngine):
             return [convert_item(item) for item in value]
         return process
 
-    def result_processor(self, dialect):
-        item_proc = self.item_type.result_processor(dialect)
+    def result_processor(self, dialect, coltype):
+        item_proc = self.item_type.result_processor(dialect, coltype)
         def process(value):
             if value is None:
                 return value
diff --git a/lib/sqlalchemy/dialects/postgresql/pg8000.py b/lib/sqlalchemy/dialects/postgresql/pg8000.py
index 17fe86be6..e90bebb6b 100644
--- a/lib/sqlalchemy/dialects/postgresql/pg8000.py
+++ b/lib/sqlalchemy/dialects/postgresql/pg8000.py
@@ -21,25 +21,47 @@ Passing data from/to the Interval type is not supported as of yet.
 """
 from sqlalchemy.engine import default
 import decimal
-from sqlalchemy import util
+from sqlalchemy import util, exc
 from sqlalchemy import types as sqltypes
-from sqlalchemy.dialects.postgresql.base import PGDialect, PGCompiler, PGIdentifierPreparer, PGExecutionContext
+from sqlalchemy.dialects.postgresql.base import PGDialect, \
+                PGCompiler, PGIdentifierPreparer, PGExecutionContext
 
 class _PGNumeric(sqltypes.Numeric):
     def bind_processor(self, dialect):
-        return None
-
-    def result_processor(self, dialect):
+        def process(value):
+            if value is not None:
+                return float(value)
+            else:
+                return value
+        return process
+    
+    def result_processor(self, dialect, coltype):
         if self.asdecimal:
-            return None
+            if coltype in (700, 701):
+                def process(value):
+                    if value is not None:
+                        return decimal.Decimal(str(value))
+                    else:
+                        return value
+                return process
+            elif coltype == 1700:
+                # pg8000 returns Decimal natively for 1700
+                return None
+            else:
+                raise exc.InvalidRequestError("Unknown PG numeric type: %d" % coltype)
         else:
-            def process(value):
-                if isinstance(value, decimal.Decimal):
-                    return float(value)
-                else:
-                    return value
-            return process
-
+            if coltype in (700, 701):
+                # pg8000 returns float natively for 701
+                return None
+            elif coltype == 1700:
+                def process(value):
+                    if value is not None:
+                        return float(value)
+                    else:
+                        return value
+                return process
+            else:
+                raise exc.InvalidRequestError("Unknown PG numeric type: %d" % coltype)
 
 class PostgreSQL_pg8000ExecutionContext(PGExecutionContext):
     pass
@@ -79,7 +101,6 @@ class PostgreSQL_pg8000(PGDialect):
         PGDialect.colspecs,
         {
             sqltypes.Numeric : _PGNumeric,
-            sqltypes.Float: sqltypes.Float,  # prevents _PGNumeric from being used
         }
     )
     
diff --git a/lib/sqlalchemy/dialects/postgresql/psycopg2.py b/lib/sqlalchemy/dialects/postgresql/psycopg2.py
index aa4e07bb3..a46fdbddb 100644
--- a/lib/sqlalchemy/dialects/postgresql/psycopg2.py
+++ b/lib/sqlalchemy/dialects/postgresql/psycopg2.py
@@ -51,17 +51,33 @@ class _PGNumeric(sqltypes.Numeric):
     def bind_processor(self, dialect):
         return None
 
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         if self.asdecimal:
-            return None
+            if coltype in (700, 701):
+                def process(value):
+                    if value is not None:
+                        return decimal.Decimal(str(value))
+                    else:
+                        return value
+                return process
+            elif coltype == 1700:
+                # pg8000 returns Decimal natively for 1700
+                return None
+            else:
+                raise exc.InvalidRequestError("Unknown PG numeric type: %d" % coltype)
         else:
-            def process(value):
-                if isinstance(value, decimal.Decimal):
-                    return float(value)
-                else:
-                    return value
-            return process
-
+            if coltype in (700, 701):
+                # pg8000 returns float natively for 701
+                return None
+            elif coltype == 1700:
+                def process(value):
+                    if value is not None:
+                        return float(value)
+                    else:
+                        return value
+                return process
+            else:
+                raise exc.InvalidRequestError("Unknown PG numeric type: %d" % coltype)
 
 class _PGEnum(ENUM):
     def __init__(self, *arg, **kw):
@@ -139,7 +155,6 @@ class PostgreSQL_psycopg2(PGDialect):
         PGDialect.colspecs,
         {
             sqltypes.Numeric : _PGNumeric,
-            sqltypes.Float: sqltypes.Float,  # prevents _PGNumeric from being used
             ENUM : _PGEnum, # needs force_unicode
             sqltypes.Enum : _PGEnum, # needs force_unicode
             ARRAY : _PGArray, # needs force_unicode
diff --git a/lib/sqlalchemy/dialects/postgresql/pypostgresql.py b/lib/sqlalchemy/dialects/postgresql/pypostgresql.py
index 517d41aaf..2c33b3eb5 100644
--- a/lib/sqlalchemy/dialects/postgresql/pypostgresql.py
+++ b/lib/sqlalchemy/dialects/postgresql/pypostgresql.py
@@ -17,7 +17,7 @@ class PGNumeric(sqltypes.Numeric):
     def bind_processor(self, dialect):
         return None
 
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         if self.asdecimal:
             return None
         else:
diff --git a/lib/sqlalchemy/dialects/sqlite/base.py b/lib/sqlalchemy/dialects/sqlite/base.py
index 47c797c21..33feaeaae 100644
--- a/lib/sqlalchemy/dialects/sqlite/base.py
+++ b/lib/sqlalchemy/dialects/sqlite/base.py
@@ -96,7 +96,7 @@ class _SLDateTime(_DateTimeMixin, sqltypes.DateTime):
                         )
 
     _reg = re.compile(r"(\d+)-(\d+)-(\d+)(?: (\d+):(\d+):(\d+)(?:\.(\d+))?)?")
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         return self._result_processor(datetime.datetime, self._reg)
 
 class _SLDate(_DateTimeMixin, sqltypes.Date):
@@ -107,7 +107,7 @@ class _SLDate(_DateTimeMixin, sqltypes.Date):
                 )
 
     _reg = re.compile(r"(\d+)-(\d+)-(\d+)")
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         return self._result_processor(datetime.date, self._reg)
 
 class _SLTime(_DateTimeMixin, sqltypes.Time):
@@ -126,7 +126,7 @@ class _SLTime(_DateTimeMixin, sqltypes.Time):
                     )
 
     _reg = re.compile(r"(\d+):(\d+):(\d+)(?:\.(\d+))?")
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         return self._result_processor(datetime.time, self._reg)
 
 
@@ -138,7 +138,7 @@ class _SLBoolean(sqltypes.Boolean):
             return value and 1 or 0
         return process
 
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         def process(value):
             if value is None:
                 return None
diff --git a/lib/sqlalchemy/dialects/sybase/base.py b/lib/sqlalchemy/dialects/sybase/base.py
index 6fc42c312..cfdbd321a 100644
--- a/lib/sqlalchemy/dialects/sybase/base.py
+++ b/lib/sqlalchemy/dialects/sybase/base.py
@@ -113,7 +113,7 @@ class SybaseUniqueIdentifier(sqltypes.TypeEngine):
     __visit_name__ = "UNIQUEIDENTIFIER"
     
 class SybaseBoolean(sqltypes.Boolean):
-    def result_processor(self, dialect):
+    def result_processor(self, dialect, coltype):
         def process(value):
             if value is None:
                 return None