removals: all unicode encoding / decoding

Removed here includes:

* convert_unicode parameters
* encoding create_engine() parameter
* description encoding support
* "non-unicode fallback" modes under Python 2
* String symbols regarding Python 2 non-unicode fallbacks
* any concept of DBAPIs that don't accept unicode
  statements, unicode bound parameters, or that return bytes
  for strings anywhere except an explicit Binary / BLOB
  type
* unicode processors in Python / C

Risk factors:

* Whether all DBAPIs do in fact return Unicode objects for
  all entries in cursor.description now
* There was logic for mysql-connector trying to determine
  description encoding.   A quick test shows Unicode coming
  back but it's not clear if there are still edge cases where
  they return bytes.  if so, these are bugs in that driver,
  and at most we would only work around it in the mysql-connector
  DBAPI itself (but we won't do that either).
* It seems like Oracle 8 was not expecting unicode bound parameters.
  I'm assuming this was all Python 2 stuff and does not apply
  for modern cx_Oracle under Python 3.
* third party dialects relying upon built in unicode encoding/decoding
  but it's hard to imagine any non-SQLAlchemy database driver not
  dealing exclusively in Python unicode strings in Python 3

Change-Id: I97d762ef6d4dd836487b714d57d8136d0310f28a
References: #7257

9 files changed, 17 insertions, 264 deletions

diff --git a/test/aaa_profiling/test_memusage.py b/test/aaa_profiling/test_memusage.py
index 895cd9e0c..518b215dd 100644
--- a/test/aaa_profiling/test_memusage.py
+++ b/test/aaa_profiling/test_memusage.py
@@ -29,7 +29,6 @@ from sqlalchemy.orm import sessionmaker
 from sqlalchemy.orm import subqueryload
 from sqlalchemy.orm.session import _sessions
 from sqlalchemy.processors import to_decimal_processor_factory
-from sqlalchemy.processors import to_unicode_processor_factory
 from sqlalchemy.sql import column
 from sqlalchemy.sql import util as sql_util
 from sqlalchemy.sql.visitors import cloned_traverse
@@ -286,14 +285,6 @@ class MemUsageTest(EnsureZeroed):
         go()
 
     @testing.requires.cextensions
-    def test_UnicodeResultProcessor_init(self):
-        @profile_memory()
-        def go():
-            to_unicode_processor_factory("utf8")
-
-        go()
-
-    @testing.requires.cextensions
     def test_cycles_in_row(self):
 
         tup = result.result_tuple(["a", "b", "c"])
diff --git a/test/dialect/mssql/test_reflection.py b/test/dialect/mssql/test_reflection.py
index 01c5e845e..1789166ee 100644
--- a/test/dialect/mssql/test_reflection.py
+++ b/test/dialect/mssql/test_reflection.py
@@ -22,7 +22,6 @@ from sqlalchemy import types as sqltypes
 from sqlalchemy import util
 from sqlalchemy.dialects import mssql
 from sqlalchemy.dialects.mssql import base
-from sqlalchemy.dialects.mssql.information_schema import CoerceUnicode
 from sqlalchemy.dialects.mssql.information_schema import tables
 from sqlalchemy.schema import CreateIndex
 from sqlalchemy.testing import AssertsCompiledSQL
@@ -550,12 +549,6 @@ class ReflectionTest(fixtures.TestBase, ComparesTables, AssertsCompiledSQL):
 
 
 class InfoCoerceUnicodeTest(fixtures.TestBase, AssertsCompiledSQL):
-    def test_info_unicode_coercion(self):
-
-        dialect = mssql.dialect()
-        value = CoerceUnicode().bind_processor(dialect)("a string")
-        assert isinstance(value, util.text_type)
-
     def test_info_unicode_cast_no_2000(self):
         dialect = mssql.dialect()
         dialect.server_version_info = base.MS_2000_VERSION
diff --git a/test/dialect/oracle/test_dialect.py b/test/dialect/oracle/test_dialect.py
index f287a9a0b..ccf771f81 100644
--- a/test/dialect/oracle/test_dialect.py
+++ b/test/dialect/oracle/test_dialect.py
@@ -259,22 +259,15 @@ class EncodingErrorsTest(fixtures.TestBase):
     def test_older_cx_oracle_warning(self, cx_Oracle, cx_oracle_type):
         cx_Oracle.version = "6.3"
 
-        ignore_dialect = cx_oracle.dialect(
-            dbapi=cx_Oracle, encoding_errors="ignore"
-        )
-        ignore_outputhandler = (
-            ignore_dialect._generate_connection_outputtype_handler()
-        )
-
-        cursor = mock.Mock()
-
         with testing.expect_warnings(
             r"cx_oracle version \(6, 3\) does not support encodingErrors"
         ):
-            ignore_outputhandler(
-                cursor, "foo", cx_oracle_type, None, None, None
+            dialect = cx_oracle.dialect(
+                dbapi=cx_Oracle, encoding_errors="ignore"
             )
 
+            eq_(dialect._cursor_var_unicode_kwargs, {})
+
     @_oracle_char_combinations
     def test_encoding_errors_cx_oracle(
         self,
@@ -319,10 +312,18 @@ class EncodingErrorsTest(fixtures.TestBase):
         cursor = mock.Mock()
         plain_outputhandler(cursor, "foo", cx_oracle_type, None, None, None)
 
-        eq_(
-            cursor.mock_calls,
-            [mock.call.var(mock.ANY, None, cursor.arraysize)],
-        )
+        if cx_oracle_type in (cx_Oracle.FIXED_CHAR, cx_Oracle.STRING):
+            # no calls; without encodingErrors, use cx_Oracle's default unicode
+            # handling
+            eq_(
+                cursor.mock_calls,
+                [],
+            )
+        else:
+            eq_(
+                cursor.mock_calls,
+                [mock.call.var(mock.ANY, None, cursor.arraysize)],
+            )
 
 
 class ComputedReturningTest(fixtures.TablesTest):
diff --git a/test/dialect/oracle/test_types.py b/test/dialect/oracle/test_types.py
index 2b54f2b56..cbbb7be7c 100644
--- a/test/dialect/oracle/test_types.py
+++ b/test/dialect/oracle/test_types.py
@@ -714,18 +714,6 @@ class TypesTest(fixtures.TestBase):
 
             eq_(sqla_result, cx_oracle_result)
 
-    def test_coerce_to_unicode(self, connection):
-        engine = testing_engine(options=dict(coerce_to_unicode=False))
-        with engine.connect() as conn_no_coerce:
-            value = exec_sql(
-                conn_no_coerce, "SELECT 'hello' FROM DUAL"
-            ).scalar()
-            assert not isinstance(value, util.binary_type)
-            assert isinstance(value, util.text_type)
-
-        value = exec_sql(connection, "SELECT 'hello' FROM DUAL").scalar()
-        assert isinstance(value, util.text_type)
-
     def test_reflect_dates(self, metadata, connection):
         Table(
             "date_types",
diff --git a/test/dialect/postgresql/test_dialect.py b/test/dialect/postgresql/test_dialect.py
index fe3700bbb..c12f4a50a 100644
--- a/test/dialect/postgresql/test_dialect.py
+++ b/test/dialect/postgresql/test_dialect.py
@@ -161,14 +161,6 @@ $$ LANGUAGE plpgsql;"""
             future_connection.dialect.server_version_info,
         )
 
-    @testing.requires.psycopg2_compatibility
-    def test_pg_dialect_no_native_unicode_in(self, testing_engine):
-        with testing.expect_raises_message(
-            exc.ArgumentError,
-            "psycopg2 native_unicode mode is required under Python 3",
-        ):
-            testing_engine(options=dict(use_native_unicode=False))
-
     def test_psycopg2_empty_connection_string(self):
         dialect = psycopg2_dialect.dialect()
         u = url.make_url("postgresql+psycopg2://")
diff --git a/test/dialect/postgresql/test_types.py b/test/dialect/postgresql/test_types.py
index ebb4d4b12..96601d6ad 100644
--- a/test/dialect/postgresql/test_types.py
+++ b/test/dialect/postgresql/test_types.py
@@ -3826,10 +3826,7 @@ class JSONRoundTripTest(fixtures.TablesTest):
         result = connection.execute(
             select(data_table.c.data["k1"].astext)
         ).first()
-        if connection.dialect.returns_unicode_strings:
-            assert isinstance(result[0], util.text_type)
-        else:
-            assert isinstance(result[0], util.string_types)
+        assert isinstance(result[0], util.text_type)
 
     def test_query_returned_as_int(self, connection):
         self._fixture_data(connection)
diff --git a/test/sql/test_defaults.py b/test/sql/test_defaults.py
index 31cc15155..92e59d9a8 100644
--- a/test/sql/test_defaults.py
+++ b/test/sql/test_defaults.py
@@ -13,7 +13,6 @@ from sqlalchemy import MetaData
 from sqlalchemy import Sequence
 from sqlalchemy import String
 from sqlalchemy import testing
-from sqlalchemy import Unicode
 from sqlalchemy.schema import CreateTable
 from sqlalchemy.sql import literal_column
 from sqlalchemy.sql import select
@@ -29,8 +28,6 @@ from sqlalchemy.testing.schema import Column
 from sqlalchemy.testing.schema import Table
 from sqlalchemy.types import TypeDecorator
 from sqlalchemy.types import TypeEngine
-from sqlalchemy.util import b
-from sqlalchemy.util import u
 
 
 class DDLTest(fixtures.TestBase, AssertsCompiledSQL):
@@ -1428,29 +1425,6 @@ class ServerDefaultsOnPKTest(fixtures.TestBase):
         eq_(list(connection.execute(t.select())), [(5, "data")])
 
 
-class UnicodeDefaultsTest(fixtures.TestBase):
-    __backend__ = True
-
-    def test_no_default(self):
-        Column(Unicode(32))
-
-    def test_unicode_default(self):
-        default = u("foo")
-        Column(Unicode(32), default=default)
-
-    def test_nonunicode_default(self):
-        default = b("foo")
-        assert_raises_message(
-            sa.exc.SAWarning,
-            "Unicode column 'foobar' has non-unicode "
-            "default value b?'foo' specified.",
-            Column,
-            "foobar",
-            Unicode(32),
-            default=default,
-        )
-
-
 class InsertFromSelectTest(fixtures.TablesTest):
     __backend__ = True
 
diff --git a/test/sql/test_deprecations.py b/test/sql/test_deprecations.py
index 93e280d4e..2faae8b87 100644
--- a/test/sql/test_deprecations.py
+++ b/test/sql/test_deprecations.py
@@ -9,7 +9,6 @@ from sqlalchemy import bindparam
 from sqlalchemy import case
 from sqlalchemy import CHAR
 from sqlalchemy import column
-from sqlalchemy import create_engine
 from sqlalchemy import exc
 from sqlalchemy import exists
 from sqlalchemy import ForeignKey
@@ -29,7 +28,6 @@ from sqlalchemy import String
 from sqlalchemy import table
 from sqlalchemy import testing
 from sqlalchemy import text
-from sqlalchemy import util
 from sqlalchemy.engine import default
 from sqlalchemy.sql import coercions
 from sqlalchemy.sql import LABEL_STYLE_TABLENAME_PLUS_COL
@@ -159,29 +157,6 @@ class DeprecationWarningsTest(fixtures.TestBase, AssertsCompiledSQL):
         ):
             preparer.quote_schema("hi", True)
 
-    def test_string_convert_unicode(self):
-        with testing.expect_deprecated(
-            "The String.convert_unicode parameter is deprecated and "
-            "will be removed in a future release."
-        ):
-            String(convert_unicode=True)
-
-    def test_string_convert_unicode_force(self):
-        with testing.expect_deprecated(
-            "The String.convert_unicode parameter is deprecated and "
-            "will be removed in a future release."
-        ):
-            String(convert_unicode="force")
-
-    def test_engine_convert_unicode(self):
-        with testing.expect_deprecated(
-            "The create_engine.convert_unicode parameter and "
-            "corresponding dialect-level"
-        ):
-            create_engine(
-                "mysql+mysqldb://", convert_unicode=True, module=mock.Mock()
-            )
-
     def test_empty_and_or(self):
         with testing.expect_deprecated(
             r"Invoking and_\(\) without arguments is deprecated, and "
@@ -213,57 +188,6 @@ class DeprecationWarningsTest(fixtures.TestBase, AssertsCompiledSQL):
         )
 
 
-class ConvertUnicodeDeprecationTest(fixtures.TestBase):
-
-    __backend__ = True
-
-    data = util.u(
-        "Alors vous imaginez ma surprise, au lever du jour, quand "
-        "une drôle de petite voix m’a réveillé. "
-        "Elle disait: « S’il vous plaît… dessine-moi un mouton! »"
-    )
-
-    def test_unicode_warnings_dialectlevel(self):
-
-        unicodedata = self.data
-
-        with testing.expect_deprecated(
-            "The create_engine.convert_unicode parameter and "
-            "corresponding dialect-level"
-        ):
-            dialect = default.DefaultDialect(convert_unicode=True)
-        dialect.supports_unicode_binds = False
-
-        s = String()
-        uni = s.dialect_impl(dialect).bind_processor(dialect)
-
-        uni(util.b("x"))
-        assert isinstance(uni(unicodedata), util.binary_type)
-
-        eq_(uni(unicodedata), unicodedata.encode("utf-8"))
-
-    def test_ignoring_unicode_error(self):
-        """checks String(unicode_error='ignore') is passed to
-        underlying codec."""
-
-        unicodedata = self.data
-
-        with testing.expect_deprecated(
-            "The String.convert_unicode parameter is deprecated and "
-            "will be removed in a future release.",
-            "The String.unicode_errors parameter is deprecated and "
-            "will be removed in a future release.",
-        ):
-            type_ = String(
-                248, convert_unicode="force", unicode_error="ignore"
-            )
-        dialect = default.DefaultDialect(encoding="ascii")
-        proc = type_.result_processor(dialect, 10)
-
-        utfdata = unicodedata.encode("utf8")
-        eq_(proc(utfdata), unicodedata.encode("ascii", "ignore").decode())
-
-
 class SubqueryCoercionsTest(fixtures.TestBase, AssertsCompiledSQL):
     __dialect__ = "default"
 
diff --git a/test/sql/test_types.py b/test/sql/test_types.py
index 5acc5f076..a15d163e0 100644
--- a/test/sql/test_types.py
+++ b/test/sql/test_types.py
@@ -81,7 +81,6 @@ from sqlalchemy.testing import engines
 from sqlalchemy.testing import eq_
 from sqlalchemy.testing import expect_deprecated_20
 from sqlalchemy.testing import expect_raises
-from sqlalchemy.testing import expect_warnings
 from sqlalchemy.testing import fixtures
 from sqlalchemy.testing import is_
 from sqlalchemy.testing import is_not
@@ -92,7 +91,6 @@ from sqlalchemy.testing.schema import pep435_enum
 from sqlalchemy.testing.schema import Table
 from sqlalchemy.testing.util import picklers
 from sqlalchemy.testing.util import round_decimal
-from sqlalchemy.util import u
 
 
 def _all_dialect_modules():
@@ -338,12 +336,6 @@ class AdaptTest(fixtures.TestBase):
         t2 = t1.adapt(Text)
         eq_(t2.length, 50)
 
-    def test_convert_unicode_text_type(self):
-        with testing.expect_deprecated(
-            "The String.convert_unicode parameter is deprecated"
-        ):
-            eq_(types.String(convert_unicode=True).python_type, util.text_type)
-
 
 class TypeAffinityTest(fixtures.TestBase):
     @testing.combinations(
@@ -1026,50 +1018,6 @@ class UserDefinedTest(
         eq_(a.dialect_specific_args["bar"], "bar")
 
 
-class StringConvertUnicodeTest(fixtures.TestBase):
-    @testing.combinations((Unicode,), (String,), argnames="datatype")
-    @testing.combinations((True,), (False,), argnames="convert_unicode")
-    @testing.combinations(
-        (String.RETURNS_CONDITIONAL,),
-        (String.RETURNS_BYTES,),
-        (String.RETURNS_UNICODE),
-        argnames="returns_unicode_strings",
-    )
-    def test_convert_unicode(
-        self, datatype, convert_unicode, returns_unicode_strings
-    ):
-        s1 = datatype()
-        dialect = mock.Mock(
-            returns_unicode_strings=returns_unicode_strings,
-            encoding="utf-8",
-            convert_unicode=convert_unicode,
-        )
-
-        proc = s1.result_processor(dialect, None)
-
-        string = u("méil")
-        bytestring = string.encode("utf-8")
-
-        if (
-            datatype is Unicode or convert_unicode
-        ) and returns_unicode_strings in (
-            String.RETURNS_CONDITIONAL,
-            String.RETURNS_BYTES,
-        ):
-            eq_(proc(bytestring), string)
-
-            if returns_unicode_strings is String.RETURNS_CONDITIONAL:
-                eq_(proc(string), string)
-            else:
-                if util.py3k:
-                    # trying to decode a unicode
-                    assert_raises(TypeError, proc, string)
-                else:
-                    assert_raises(UnicodeEncodeError, proc, string)
-        else:
-            is_(proc, None)
-
-
 class TypeCoerceCastTest(fixtures.TablesTest):
     __backend__ = True
 
@@ -1668,59 +1616,6 @@ class VariantTest(fixtures.TestBase, AssertsCompiledSQL):
         )
 
 
-class UnicodeTest(fixtures.TestBase):
-
-    """Exercise the Unicode and related types.
-
-    Note:  unicode round trip tests are now in
-    sqlalchemy/testing/suite/test_types.py.
-
-    """
-
-    __backend__ = True
-
-    data = util.u(
-        "Alors vous imaginez ma surprise, au lever du jour, quand "
-        "une drôle de petite voix m’a réveillé. "
-        "Elle disait: « S’il vous plaît… dessine-moi un mouton! »"
-    )
-
-    def test_unicode_warnings_typelevel_native_unicode(self):
-
-        unicodedata = self.data
-        u = Unicode()
-        dialect = default.DefaultDialect()
-        dialect.supports_unicode_binds = True
-        uni = u.dialect_impl(dialect).bind_processor(dialect)
-        if util.py3k:
-            assert_raises(exc.SAWarning, uni, b"x")
-            assert isinstance(uni(unicodedata), str)
-        else:
-            assert_raises(exc.SAWarning, uni, "x")
-            assert isinstance(uni(unicodedata), unicode)  # noqa
-
-    def test_unicode_warnings_typelevel_sqla_unicode(self):
-        unicodedata = self.data
-        u = Unicode()
-        dialect = default.DefaultDialect()
-        dialect.supports_unicode_binds = False
-        uni = u.dialect_impl(dialect).bind_processor(dialect)
-        assert_raises(exc.SAWarning, uni, util.b("x"))
-        assert isinstance(uni(unicodedata), util.binary_type)
-
-        eq_(uni(unicodedata), unicodedata.encode("utf-8"))
-
-    def test_unicode_warnings_totally_wrong_type(self):
-        u = Unicode()
-        dialect = default.DefaultDialect()
-        dialect.supports_unicode_binds = False
-        uni = u.dialect_impl(dialect).bind_processor(dialect)
-        with expect_warnings(
-            "Unicode type received non-unicode bind param value 5."
-        ):
-            eq_(uni(5), 5)
-
-
 class EnumTest(AssertsCompiledSQL, fixtures.TablesTest):
     __backend__ = True
 
@@ -2479,13 +2374,11 @@ class EnumTest(AssertsCompiledSQL, fixtures.TablesTest):
         # depending on backend.
         assert "('x'," in e.print_sql()
 
-    @testing.uses_deprecated(".*convert_unicode")
     def test_repr(self):
         e = Enum(
             "x",
             "y",
             name="somename",
-            convert_unicode=True,
             quote=True,
             inherit_schema=True,
             native_enum=False,