- A large refactoring of the ``sqlalchemy.sql`` package has reorganized

the import structure of many core modules.
``sqlalchemy.schema`` and ``sqlalchemy.types``
remain in the top-level package, but are now just lists of names
that pull from within ``sqlalchemy.sql``.  Their implementations
are now broken out among ``sqlalchemy.sql.type_api``, ``sqlalchemy.sql.sqltypes``,
``sqlalchemy.sql.schema`` and ``sqlalchemy.sql.ddl``, the last of which was
moved from ``sqlalchemy.engine``.  ``sqlalchemy.sql.expression`` is also
a namespace now which pulls implementations mostly from ``sqlalchemy.sql.elements``,
``sqlalchemy.sql.selectable``, and ``sqlalchemy.sql.dml``.
Most of the "factory" functions
used to create SQL expression objects have been moved to classmethods
or constructors, which are exposed in ``sqlalchemy.sql.expression``
using a programmatic system.  Care has been taken such that all the
original import namespaces remain intact and there should be no impact
on any existing applications.   The rationale here was to break out these
very large modules into smaller ones, provide more manageable lists
of function names, to greatly reduce "import cycles" and clarify the
up-front importing of names, and to remove the need for redundant
functions and documentation throughout the expression package.

7 files changed, 21 insertions, 312 deletions

diff --git a/lib/sqlalchemy/engine/__init__.py b/lib/sqlalchemy/engine/__init__.py
index a00976d4c..16d214140 100644
--- a/lib/sqlalchemy/engine/__init__.py
+++ b/lib/sqlalchemy/engine/__init__.py
@@ -50,14 +50,13 @@ url.py
     within a URL.
 """
 
-# not sure what this was used for
-#import sqlalchemy.databases
-
 from .interfaces import (
-    Compiled,
     Connectable,
     Dialect,
     ExecutionContext,
+
+    # backwards compat
+    Compiled,
     TypeCompiler
 )
 
@@ -83,7 +82,11 @@ from .util import (
     connection_memoize
     )
 
-from . import util, strategies, ddl
+
+from . import util, strategies
+
+# backwards compat
+from ..sql import ddl
 
 default_strategy = 'plain'
 
diff --git a/lib/sqlalchemy/engine/base.py b/lib/sqlalchemy/engine/base.py
index f69bd3d4b..83fa34f2c 100644
--- a/lib/sqlalchemy/engine/base.py
+++ b/lib/sqlalchemy/engine/base.py
@@ -11,8 +11,8 @@ from __future__ import with_statement
 
 
 import sys
-from .. import exc, schema, util, log, interfaces
-from ..sql import expression, util as sql_util
+from .. import exc, util, log, interfaces
+from ..sql import expression, util as sql_util, schema, ddl
 from .interfaces import Connectable, Compiled
 from .util import _distill_params
 import contextlib
@@ -1039,7 +1039,7 @@ class Connection(Connectable):
         expression.ClauseElement: _execute_clauseelement,
         Compiled: _execute_compiled,
         schema.SchemaItem: _execute_default,
-        schema.DDLElement: _execute_ddl,
+        ddl.DDLElement: _execute_ddl,
         util.string_types[0]: _execute_text
     }
 
diff --git a/lib/sqlalchemy/engine/ddl.py b/lib/sqlalchemy/engine/ddl.py
deleted file mode 100644
index 6daa9be6b..000000000
--- a/lib/sqlalchemy/engine/ddl.py
+++ /dev/null
@@ -1,193 +0,0 @@
-# engine/ddl.py
-# Copyright (C) 2009-2013 the SQLAlchemy authors and contributors <see AUTHORS file>
-#
-# This module is part of SQLAlchemy and is released under
-# the MIT License: http://www.opensource.org/licenses/mit-license.php
-
-"""Routines to handle CREATE/DROP workflow."""
-
-from .. import schema
-from ..sql import util as sql_util
-
-
-class DDLBase(schema.SchemaVisitor):
-    def __init__(self, connection):
-        self.connection = connection
-
-
-class SchemaGenerator(DDLBase):
-
-    def __init__(self, dialect, connection, checkfirst=False,
-                 tables=None, **kwargs):
-        super(SchemaGenerator, self).__init__(connection, **kwargs)
-        self.checkfirst = checkfirst
-        self.tables = tables
-        self.preparer = dialect.identifier_preparer
-        self.dialect = dialect
-        self.memo = {}
-
-    def _can_create_table(self, table):
-        self.dialect.validate_identifier(table.name)
-        if table.schema:
-            self.dialect.validate_identifier(table.schema)
-        return not self.checkfirst or \
-                not self.dialect.has_table(self.connection,
-                                    table.name, schema=table.schema)
-
-    def _can_create_sequence(self, sequence):
-        return self.dialect.supports_sequences and \
-            (
-                (not self.dialect.sequences_optional or
-                 not sequence.optional) and
-                    (
-                        not self.checkfirst or
-                        not self.dialect.has_sequence(
-                                self.connection,
-                                sequence.name,
-                                schema=sequence.schema)
-                     )
-            )
-
-    def visit_metadata(self, metadata):
-        if self.tables is not None:
-            tables = self.tables
-        else:
-            tables = list(metadata.tables.values())
-        collection = [t for t in sql_util.sort_tables(tables)
-                        if self._can_create_table(t)]
-        seq_coll = [s for s in metadata._sequences.values()
-                        if s.column is None and self._can_create_sequence(s)]
-
-        metadata.dispatch.before_create(metadata, self.connection,
-                                    tables=collection,
-                                    checkfirst=self.checkfirst,
-                                            _ddl_runner=self)
-
-        for seq in seq_coll:
-            self.traverse_single(seq, create_ok=True)
-
-        for table in collection:
-            self.traverse_single(table, create_ok=True)
-
-        metadata.dispatch.after_create(metadata, self.connection,
-                                    tables=collection,
-                                    checkfirst=self.checkfirst,
-                                            _ddl_runner=self)
-
-    def visit_table(self, table, create_ok=False):
-        if not create_ok and not self._can_create_table(table):
-            return
-
-        table.dispatch.before_create(table, self.connection,
-                                        checkfirst=self.checkfirst,
-                                            _ddl_runner=self)
-
-        for column in table.columns:
-            if column.default is not None:
-                self.traverse_single(column.default)
-
-        self.connection.execute(schema.CreateTable(table))
-
-        if hasattr(table, 'indexes'):
-            for index in table.indexes:
-                self.traverse_single(index)
-
-        table.dispatch.after_create(table, self.connection,
-                                        checkfirst=self.checkfirst,
-                                            _ddl_runner=self)
-
-    def visit_sequence(self, sequence, create_ok=False):
-        if not create_ok and not self._can_create_sequence(sequence):
-            return
-        self.connection.execute(schema.CreateSequence(sequence))
-
-    def visit_index(self, index):
-        self.connection.execute(schema.CreateIndex(index))
-
-
-class SchemaDropper(DDLBase):
-
-    def __init__(self, dialect, connection, checkfirst=False,
-                 tables=None, **kwargs):
-        super(SchemaDropper, self).__init__(connection, **kwargs)
-        self.checkfirst = checkfirst
-        self.tables = tables
-        self.preparer = dialect.identifier_preparer
-        self.dialect = dialect
-        self.memo = {}
-
-    def visit_metadata(self, metadata):
-        if self.tables is not None:
-            tables = self.tables
-        else:
-            tables = list(metadata.tables.values())
-
-        collection = [
-            t
-            for t in reversed(sql_util.sort_tables(tables))
-            if self._can_drop_table(t)
-        ]
-
-        seq_coll = [
-            s
-            for s in metadata._sequences.values()
-            if s.column is None and self._can_drop_sequence(s)
-        ]
-
-        metadata.dispatch.before_drop(
-            metadata, self.connection, tables=collection,
-            checkfirst=self.checkfirst, _ddl_runner=self)
-
-        for table in collection:
-            self.traverse_single(table, drop_ok=True)
-
-        for seq in seq_coll:
-            self.traverse_single(seq, drop_ok=True)
-
-        metadata.dispatch.after_drop(
-            metadata, self.connection, tables=collection,
-            checkfirst=self.checkfirst, _ddl_runner=self)
-
-    def _can_drop_table(self, table):
-        self.dialect.validate_identifier(table.name)
-        if table.schema:
-            self.dialect.validate_identifier(table.schema)
-        return not self.checkfirst or self.dialect.has_table(self.connection,
-                                            table.name, schema=table.schema)
-
-    def _can_drop_sequence(self, sequence):
-        return self.dialect.supports_sequences and \
-            ((not self.dialect.sequences_optional or
-                 not sequence.optional) and
-                (not self.checkfirst or
-                self.dialect.has_sequence(
-                                self.connection,
-                                sequence.name,
-                                schema=sequence.schema))
-            )
-
-    def visit_index(self, index):
-        self.connection.execute(schema.DropIndex(index))
-
-    def visit_table(self, table, drop_ok=False):
-        if not drop_ok and not self._can_drop_table(table):
-            return
-
-        table.dispatch.before_drop(table, self.connection,
-                                    checkfirst=self.checkfirst,
-                                            _ddl_runner=self)
-
-        for column in table.columns:
-            if column.default is not None:
-                self.traverse_single(column.default)
-
-        self.connection.execute(schema.DropTable(table))
-
-        table.dispatch.after_drop(table, self.connection,
-                                        checkfirst=self.checkfirst,
-                                            _ddl_runner=self)
-
-    def visit_sequence(self, sequence, drop_ok=False):
-        if not drop_ok and not self._can_drop_sequence(sequence):
-            return
-        self.connection.execute(schema.DropSequence(sequence))
diff --git a/lib/sqlalchemy/engine/default.py b/lib/sqlalchemy/engine/default.py
index 7b5a22251..017dfa902 100644
--- a/lib/sqlalchemy/engine/default.py
+++ b/lib/sqlalchemy/engine/default.py
@@ -16,7 +16,8 @@ import re
 import random
 from . import reflection, interfaces, result
 from ..sql import compiler, expression
-from .. import exc, types as sqltypes, util, pool, processors
+from .. import types as sqltypes
+from .. import exc, util, pool, processors
 import codecs
 import weakref
 from .. import event
diff --git a/lib/sqlalchemy/engine/interfaces.py b/lib/sqlalchemy/engine/interfaces.py
index 750aa2fcd..8f4af6db5 100644
--- a/lib/sqlalchemy/engine/interfaces.py
+++ b/lib/sqlalchemy/engine/interfaces.py
@@ -8,6 +8,8 @@
 
 from .. import util, event, events
 
+# backwards compat
+from ..sql.compiler import Compiled, TypeCompiler
 
 class Dialect(object):
     """Define the behavior of a specific database and DB-API combination.
@@ -769,110 +771,6 @@ class ExecutionContext(object):
         raise NotImplementedError()
 
 
-class Compiled(object):
-    """Represent a compiled SQL or DDL expression.
-
-    The ``__str__`` method of the ``Compiled`` object should produce
-    the actual text of the statement.  ``Compiled`` objects are
-    specific to their underlying database dialect, and also may
-    or may not be specific to the columns referenced within a
-    particular set of bind parameters.  In no case should the
-    ``Compiled`` object be dependent on the actual values of those
-    bind parameters, even though it may reference those values as
-    defaults.
-    """
-
-    def __init__(self, dialect, statement, bind=None,
-                compile_kwargs=util.immutabledict()):
-        """Construct a new ``Compiled`` object.
-
-        :param dialect: ``Dialect`` to compile against.
-
-        :param statement: ``ClauseElement`` to be compiled.
-
-        :param bind: Optional Engine or Connection to compile this
-          statement against.
-
-        :param compile_kwargs: additional kwargs that will be
-         passed to the initial call to :meth:`.Compiled.process`.
-
-         .. versionadded:: 0.8
-
-        """
-
-        self.dialect = dialect
-        self.bind = bind
-        if statement is not None:
-            self.statement = statement
-            self.can_execute = statement.supports_execution
-            self.string = self.process(self.statement, **compile_kwargs)
-
-    @util.deprecated("0.7", ":class:`.Compiled` objects now compile "
-                        "within the constructor.")
-    def compile(self):
-        """Produce the internal string representation of this element."""
-        pass
-
-    @property
-    def sql_compiler(self):
-        """Return a Compiled that is capable of processing SQL expressions.
-
-        If this compiler is one, it would likely just return 'self'.
-
-        """
-
-        raise NotImplementedError()
-
-    def process(self, obj, **kwargs):
-        return obj._compiler_dispatch(self, **kwargs)
-
-    def __str__(self):
-        """Return the string text of the generated SQL or DDL."""
-
-        return self.string or ''
-
-    def construct_params(self, params=None):
-        """Return the bind params for this compiled object.
-
-        :param params: a dict of string/object pairs whose values will
-                       override bind values compiled in to the
-                       statement.
-        """
-
-        raise NotImplementedError()
-
-    @property
-    def params(self):
-        """Return the bind params for this compiled object."""
-        return self.construct_params()
-
-    def execute(self, *multiparams, **params):
-        """Execute this compiled object."""
-
-        e = self.bind
-        if e is None:
-            raise exc.UnboundExecutionError(
-                        "This Compiled object is not bound to any Engine "
-                        "or Connection.")
-        return e._execute_compiled(self, multiparams, params)
-
-    def scalar(self, *multiparams, **params):
-        """Execute this compiled object and return the result's
-        scalar value."""
-
-        return self.execute(*multiparams, **params).scalar()
-
-
-class TypeCompiler(object):
-    """Produces DDL specification for TypeEngine objects."""
-
-    def __init__(self, dialect):
-        self.dialect = dialect
-
-    def process(self, type_):
-        return type_._compiler_dispatch(self)
-
-
 class Connectable(object):
     """Interface for an object which supports execution of SQL constructs.
 
diff --git a/lib/sqlalchemy/engine/reflection.py b/lib/sqlalchemy/engine/reflection.py
index 50b3f774c..29ede9579 100644
--- a/lib/sqlalchemy/engine/reflection.py
+++ b/lib/sqlalchemy/engine/reflection.py
@@ -25,9 +25,9 @@ methods such as get_table_names, get_columns, etc.
 """
 
 from .. import exc, sql
-from .. import schema as sa_schema
+from ..sql import schema as sa_schema
 from .. import util
-from ..types import TypeEngine
+from ..sql.type_api import TypeEngine
 from ..util import deprecated
 from ..util import topological
 from .. import inspection
diff --git a/lib/sqlalchemy/engine/result.py b/lib/sqlalchemy/engine/result.py
index 65ce3b742..91f3c2275 100644
--- a/lib/sqlalchemy/engine/result.py
+++ b/lib/sqlalchemy/engine/result.py
@@ -9,8 +9,8 @@ and :class:`.RowProxy."""
 
 
 
-from .. import exc, types, util
-from ..sql import expression
+from .. import exc, util
+from ..sql import expression, sqltypes
 import collections
 
 # This reconstructor is necessary so that pickles with the C extension or
@@ -205,10 +205,10 @@ class ResultMetaData(object):
                                                     else colname.lower()]
                 except KeyError:
                     name, obj, type_ = \
-                        colname, None, typemap.get(coltype, types.NULLTYPE)
+                        colname, None, typemap.get(coltype, sqltypes.NULLTYPE)
             else:
                 name, obj, type_ = \
-                        colname, None, typemap.get(coltype, types.NULLTYPE)
+                        colname, None, typemap.get(coltype, sqltypes.NULLTYPE)
 
             processor = context.get_result_processor(type_, colname, coltype)