- an executemany() now requires that all bound parameter

sets require that all keys are present which are
present in the first bound parameter set.  The structure
and behavior of an insert/update statement is very much
determined by the first parameter set, including which
defaults are going to fire off, and a minimum of
guesswork is performed with all the rest so that performance
is not impacted.  For this reason defaults would otherwise
silently "fail" for missing parameters, so this is now guarded
against. [ticket:1566]

4 files changed, 47 insertions, 18 deletions

diff --git a/lib/sqlalchemy/engine/default.py b/lib/sqlalchemy/engine/default.py
index 12ab605e4..ad728da9c 100644
--- a/lib/sqlalchemy/engine/default.py
+++ b/lib/sqlalchemy/engine/default.py
@@ -232,7 +232,7 @@ class DefaultExecutionContext(base.ExecutionContext):
                 self.compiled_parameters = [compiled.construct_params()]
                 self.executemany = False
             else:
-                self.compiled_parameters = [compiled.construct_params(m) for m in parameters]
+                self.compiled_parameters = [compiled.construct_params(m, _group_number=grp) for grp,m in enumerate(parameters)]
                 self.executemany = len(parameters) > 1
 
             self.cursor = self.create_cursor()
@@ -508,11 +508,22 @@ class DefaultExecutionContext(base.ExecutionContext):
 
         if self.executemany:
             if len(self.compiled.prefetch):
-                params = self.compiled_parameters
-                for param in params:
+                scalar_defaults = {}
+                
+                # pre-determine scalar Python-side defaults
+                # to avoid many calls of get_insert_default()/get_update_default()
+                for c in self.compiled.prefetch:
+                    if self.isinsert and c.default and c.default.is_scalar:
+                        scalar_defaults[c] = c.default.arg
+                    elif self.isupdate and c.onupdate and c.onupdate.is_scalar:
+                        scalar_defaults[c] = c.onupdate.arg
+                        
+                for param in self.compiled_parameters:
                     self.current_parameters = param
                     for c in self.compiled.prefetch:
-                        if self.isinsert:
+                        if c in scalar_defaults:
+                            val = scalar_defaults[c]
+                        elif self.isinsert:
                             val = self.get_insert_default(c)
                         else:
                             val = self.get_update_default(c)
diff --git a/lib/sqlalchemy/schema.py b/lib/sqlalchemy/schema.py
index 7965070d1..9798fc23a 100644
--- a/lib/sqlalchemy/schema.py
+++ b/lib/sqlalchemy/schema.py
@@ -1094,6 +1094,10 @@ class ColumnDefault(DefaultGenerator):
     @util.memoized_property
     def is_clause_element(self):
         return isinstance(self.arg, expression.ClauseElement)
+    
+    @util.memoized_property
+    def is_scalar(self):
+        return not self.is_callable and not self.is_clause_element and not self.is_sequence
         
     def _maybe_wrap_callable(self, fn):
         """Backward compat: Wrap callables that don't accept a context."""
diff --git a/lib/sqlalchemy/sql/compiler.py b/lib/sqlalchemy/sql/compiler.py
index 61c6c214f..b204f42b1 100644
--- a/lib/sqlalchemy/sql/compiler.py
+++ b/lib/sqlalchemy/sql/compiler.py
@@ -231,7 +231,7 @@ class SQLCompiler(engine.Compiled):
     def is_subquery(self):
         return len(self.stack) > 1
 
-    def construct_params(self, params=None):
+    def construct_params(self, params=None, _group_number=None):
         """return a dictionary of bind parameter keys and values"""
 
         if params:
@@ -242,7 +242,12 @@ class SQLCompiler(engine.Compiled):
                         pd[name] = params[paramname]
                         break
                 else:
-                    if util.callable(bindparam.value):
+                    if bindparam.required:
+                        if _group_number:
+                            raise exc.InvalidRequestError("A value is required for bind parameter %r, in parameter group %d" % (bindparam.key, _group_number))
+                        else:
+                            raise exc.InvalidRequestError("A value is required for bind parameter %r" % bindparam.key)
+                    elif util.callable(bindparam.value):
                         pd[name] = bindparam.value()
                     else:
                         pd[name] = bindparam.value
@@ -751,8 +756,8 @@ class SQLCompiler(engine.Compiled):
 
         return text
 
-    def _create_crud_bind_param(self, col, value):
-        bindparam = sql.bindparam(col.key, value, type_=col.type)
+    def _create_crud_bind_param(self, col, value, required=False):
+        bindparam = sql.bindparam(col.key, value, type_=col.type, required=required)
         self.binds[col.key] = bindparam
         return self.bindparam_string(self._truncate_bindparam(bindparam))
         
@@ -770,21 +775,23 @@ class SQLCompiler(engine.Compiled):
         self.postfetch = []
         self.prefetch = []
         self.returning = []
-        
+
         # no parameters in the statement, no parameters in the
         # compiled params - return binds for all columns
         if self.column_keys is None and stmt.parameters is None:
             return [
-                        (c, self._create_crud_bind_param(c, None)) 
+                        (c, self._create_crud_bind_param(c, None, required=True)) 
                         for c in stmt.table.columns
                     ]
 
+        required = object()
+        
         # if we have statement parameters - set defaults in the
         # compiled params
         if self.column_keys is None:
             parameters = {}
         else:
-            parameters = dict((sql._column_as_key(key), None)
+            parameters = dict((sql._column_as_key(key), required)
                               for key in self.column_keys)
 
         if stmt.parameters is not None:
@@ -808,7 +815,7 @@ class SQLCompiler(engine.Compiled):
             if c.key in parameters:
                 value = parameters[c.key]
                 if sql._is_literal(value):
-                    value = self._create_crud_bind_param(c, value)
+                    value = self._create_crud_bind_param(c, value, required=value is required)
                 else:
                     self.postfetch.append(c)
                     value = self.process(value.self_group())
diff --git a/lib/sqlalchemy/sql/expression.py b/lib/sqlalchemy/sql/expression.py
index 0ece67e20..0a703ad36 100644
--- a/lib/sqlalchemy/sql/expression.py
+++ b/lib/sqlalchemy/sql/expression.py
@@ -743,7 +743,7 @@ def table(name, *columns):
     """
     return TableClause(name, *columns)
 
-def bindparam(key, value=None, shortname=None, type_=None, unique=False):
+def bindparam(key, value=None, shortname=None, type_=None, unique=False, required=False):
     """Create a bind parameter clause with the given key.
 
     value
@@ -762,11 +762,14 @@ def bindparam(key, value=None, shortname=None, type_=None, unique=False):
       underlying ``key`` modified to a uniquely generated name.
       mostly useful with value-based bind params.
 
+    required
+      A value is required at execution time.
+      
     """
     if isinstance(key, ColumnClause):
-        return _BindParamClause(key.name, value, type_=key.type, unique=unique, shortname=shortname)
+        return _BindParamClause(key.name, value, type_=key.type, unique=unique, shortname=shortname, required=required)
     else:
-        return _BindParamClause(key, value, type_=type_, unique=unique, shortname=shortname)
+        return _BindParamClause(key, value, type_=type_, unique=unique, shortname=shortname, required=required)
 
 def outparam(key, type_=None):
     """Create an 'OUT' parameter for usage in functions (stored procedures), for
@@ -2071,7 +2074,7 @@ class _BindParamClause(ColumnElement):
     __visit_name__ = 'bindparam'
     quote = None
 
-    def __init__(self, key, value, type_=None, unique=False, isoutparam=False, shortname=None):
+    def __init__(self, key, value, type_=None, unique=False, isoutparam=False, shortname=None, required=False):
         """Construct a _BindParamClause.
 
         key
@@ -2100,7 +2103,10 @@ class _BindParamClause(ColumnElement):
           modified if another ``_BindParamClause`` of the same name
           already has been located within the containing
           ``ClauseElement``.
-
+        
+        required
+          a value is required at execution time.
+          
         isoutparam
           if True, the parameter should be treated like a stored procedure "OUT"
           parameter.
@@ -2115,7 +2121,8 @@ class _BindParamClause(ColumnElement):
         self.value = value
         self.isoutparam = isoutparam
         self.shortname = shortname
-
+        self.required = required
+        
         if type_ is None:
             self.type = sqltypes.type_map.get(type(value), sqltypes.NullType)()
         elif isinstance(type_, type):