diff options
| author | Mike Bayer <mike_mp@zzzcomputing.com> | 2016-01-06 12:40:34 -0500 |
|---|---|---|
| committer | Mike Bayer <mike_mp@zzzcomputing.com> | 2016-01-06 12:47:48 -0500 |
| commit | a80bb4e5aabc4850a202f3a4d114c543357e37d5 (patch) | |
| tree | 203dbaaa9002a2b9372dcee0976d7e1c3e0baf6b /test | |
| parent | 16746dd1a63198e3c27422517fa22ec76f441ceb (diff) | |
| download | sqlalchemy-a80bb4e5aabc4850a202f3a4d114c543357e37d5.tar.gz | |
- Added :class:`.mysql.JSON` for MySQL 5.7. The JSON type provides
persistence of JSON values in MySQL as well as basic operator support
of "getitem" and "getpath", making use of the ``JSON_EXTRACT``
function in order to refer to individual paths in a JSON structure.
fixes #3547
- Added a new type to core :class:`.types.JSON`. This is the
base of the PostgreSQL :class:`.postgresql.JSON` type as well as that
of the new :class:`.mysql.JSON` type, so that a PG/MySQL-agnostic
JSON column may be used. The type features basic index and path
searching support.
fixes #3619
- reorganization of migration docs etc. to try to refer both to
the fixes to JSON that helps Postgresql while at the same time
indicating these are new features of the new base JSON type.
- a rework of the Array/Indexable system some more, moving things
that are specific to Array out of Indexable.
- new operators for JSON indexing added to core so that these can
be compiled by the PG and MySQL dialects individually
- rename sqltypes.Array to sqltypes.ARRAY - as there is no generic
Array implementation, this is an uppercase type for now, consistent
with the new sqltypes.JSON type that is also not a generic implementation.
There may need to be some convention change to handle the case of
datatypes that aren't generic, rely upon DB-native implementations,
but aren't necessarily all named the same thing.
Diffstat (limited to 'test')
| -rw-r--r-- | test/dialect/postgresql/test_types.py | 116 | ||||
| -rw-r--r-- | test/requirements.py | 8 | ||||
| -rw-r--r-- | test/sql/test_functions.py | 8 | ||||
| -rw-r--r-- | test/sql/test_operators.py | 149 | ||||
| -rw-r--r-- | test/sql/test_types.py | 109 |
5 files changed, 262 insertions, 128 deletions
diff --git a/test/dialect/postgresql/test_types.py b/test/dialect/postgresql/test_types.py index 49a8cfabd..f8c5c1c48 100644 --- a/test/dialect/postgresql/test_types.py +++ b/test/dialect/postgresql/test_types.py @@ -7,7 +7,7 @@ from sqlalchemy import testing import datetime from sqlalchemy import Table, MetaData, Column, Integer, Enum, Float, select, \ func, DateTime, Numeric, exc, String, cast, REAL, TypeDecorator, Unicode, \ - Text, null, text, column, Array, any_, all_ + Text, null, text, column, ARRAY, any_, all_ from sqlalchemy.sql import operators from sqlalchemy import types import sqlalchemy as sa @@ -819,7 +819,7 @@ class ArrayTest(AssertsCompiledSQL, fixtures.TestBase): def test_array_index_map_dimensions(self): col = column('x', postgresql.ARRAY(Integer, dimensions=3)) is_( - col[5].type._type_affinity, Array + col[5].type._type_affinity, ARRAY ) assert isinstance( col[5].type, postgresql.ARRAY @@ -828,7 +828,7 @@ class ArrayTest(AssertsCompiledSQL, fixtures.TestBase): col[5].type.dimensions, 2 ) is_( - col[5][6].type._type_affinity, Array + col[5][6].type._type_affinity, ARRAY ) assert isinstance( col[5][6].type, postgresql.ARRAY @@ -859,8 +859,8 @@ class ArrayTest(AssertsCompiledSQL, fixtures.TestBase): ) # type affinity is Array... - is_(arrtable.c.intarr[1:3].type._type_affinity, Array) - is_(arrtable.c.strarr[1:3].type._type_affinity, Array) + is_(arrtable.c.intarr[1:3].type._type_affinity, ARRAY) + is_(arrtable.c.strarr[1:3].type._type_affinity, ARRAY) # but the slice returns the actual type assert isinstance(arrtable.c.intarr[1:3].type, postgresql.ARRAY) @@ -892,12 +892,12 @@ class ArrayTest(AssertsCompiledSQL, fixtures.TestBase): type_=postgresql.ARRAY(Integer) )[3], "(array_cat(ARRAY[%(param_1)s, %(param_2)s, %(param_3)s], " - "ARRAY[%(param_4)s, %(param_5)s, %(param_6)s]))[%(param_7)s]" + "ARRAY[%(param_4)s, %(param_5)s, %(param_6)s]))[%(array_cat_1)s]" ) def test_array_agg_generic(self): expr = func.array_agg(column('q', Integer)) - is_(expr.type.__class__, types.Array) + is_(expr.type.__class__, types.ARRAY) is_(expr.type.item_type.__class__, Integer) def test_array_agg_specific(self): @@ -1811,7 +1811,7 @@ class HStoreTest(AssertsCompiledSQL, fixtures.TestBase): def test_where_getitem(self): self._test_where( self.hashcol['bar'] == None, - "(test_table.hash -> %(hash_1)s) IS NULL" + "test_table.hash -> %(hash_1)s IS NULL" ) def test_cols_get(self): @@ -1894,7 +1894,7 @@ class HStoreTest(AssertsCompiledSQL, fixtures.TestBase): def test_cols_concat_get(self): self._test_cols( (self.hashcol + self.hashcol)['foo'], - "test_table.hash || test_table.hash -> %(param_1)s AS anon_1" + "(test_table.hash || test_table.hash) -> %(param_1)s AS anon_1" ) def test_cols_keys(self): @@ -1980,6 +1980,21 @@ class HStoreRoundTripTest(fixtures.TablesTest): cols = insp.get_columns('data_table') assert isinstance(cols[2]['type'], HSTORE) + def test_literal_round_trip(self): + # in particular, this tests that the array index + # operator against the function is handled by PG; with some + # array functions it requires outer parenthezisation on the left and + # we may not be doing that here + expr = hstore( + postgresql.array(['1', '2']), + postgresql.array(['3', None]))['1'] + eq_( + testing.db.scalar( + select([expr]) + ), + "3" + ) + @testing.requires.psycopg2_native_hstore def test_insert_native(self): engine = testing.db @@ -2411,100 +2426,33 @@ class JSONTest(AssertsCompiledSQL, fixtures.TestBase): ) % expected ) - def test_bind_serialize_default(self): - dialect = postgresql.dialect() - proc = self.test_table.c.test_column.type._cached_bind_processor( - dialect) - eq_( - proc({"A": [1, 2, 3, True, False]}), - '{"A": [1, 2, 3, true, false]}' - ) - - def test_bind_serialize_None(self): - dialect = postgresql.dialect() - proc = self.test_table.c.test_column.type._cached_bind_processor( - dialect) - eq_( - proc(None), - 'null' - ) - - def test_bind_serialize_none_as_null(self): - dialect = postgresql.dialect() - proc = JSON(none_as_null=True)._cached_bind_processor( - dialect) - eq_( - proc(None), - None - ) - eq_( - proc(null()), - None - ) - - def test_bind_serialize_null(self): - dialect = postgresql.dialect() - proc = self.test_table.c.test_column.type._cached_bind_processor( - dialect) - eq_( - proc(null()), - None - ) - - def test_result_deserialize_default(self): - dialect = postgresql.dialect() - proc = self.test_table.c.test_column.type._cached_result_processor( - dialect, None) - eq_( - proc('{"A": [1, 2, 3, true, false]}'), - {"A": [1, 2, 3, True, False]} - ) - - def test_result_deserialize_null(self): - dialect = postgresql.dialect() - proc = self.test_table.c.test_column.type._cached_result_processor( - dialect, None) - eq_( - proc('null'), - None - ) - - def test_result_deserialize_None(self): - dialect = postgresql.dialect() - proc = self.test_table.c.test_column.type._cached_result_processor( - dialect, None) - eq_( - proc(None), - None - ) - # This test is a bit misleading -- in real life you will need to cast to # do anything def test_where_getitem(self): self._test_where( self.jsoncol['bar'] == None, - "(test_table.test_column -> %(test_column_1)s) IS NULL" + "test_table.test_column -> %(test_column_1)s IS NULL" ) def test_where_path(self): self._test_where( self.jsoncol[("foo", 1)] == None, - "(test_table.test_column #> %(test_column_1)s) IS NULL" + "test_table.test_column #> %(test_column_1)s IS NULL" ) def test_path_typing(self): col = column('x', JSON()) is_( - col['q'].type._type_affinity, JSON + col['q'].type._type_affinity, types.JSON ) is_( - col[('q', )].type._type_affinity, JSON + col[('q', )].type._type_affinity, types.JSON ) is_( - col['q']['p'].type._type_affinity, JSON + col['q']['p'].type._type_affinity, types.JSON ) is_( - col[('q', 'p')].type._type_affinity, JSON + col[('q', 'p')].type._type_affinity, types.JSON ) def test_custom_astext_type(self): @@ -2528,7 +2476,7 @@ class JSONTest(AssertsCompiledSQL, fixtures.TestBase): def test_where_getitem_as_text(self): self._test_where( self.jsoncol['bar'].astext == None, - "(test_table.test_column ->> %(test_column_1)s) IS NULL" + "test_table.test_column ->> %(test_column_1)s IS NULL" ) def test_where_getitem_astext_cast(self): @@ -2548,7 +2496,7 @@ class JSONTest(AssertsCompiledSQL, fixtures.TestBase): def test_where_path_as_text(self): self._test_where( self.jsoncol[("foo", 1)].astext == None, - "(test_table.test_column #>> %(test_column_1)s) IS NULL" + "test_table.test_column #>> %(test_column_1)s IS NULL" ) def test_cols_get(self): diff --git a/test/requirements.py b/test/requirements.py index ff93a9c3d..7031a70dc 100644 --- a/test/requirements.py +++ b/test/requirements.py @@ -531,6 +531,14 @@ class DefaultRequirements(SuiteRequirements): 'sybase') @property + def json_type(self): + return only_on([ + lambda config: against(config, "mysql >= 5.7") and + not config.db.dialect._is_mariadb, + "postgresql >= 9.3" + ]) + + @property def datetime_literals(self): """target dialect supports rendering of a date, time, or datetime as a literal string, e.g. via the TypeEngine.literal_processor() method. diff --git a/test/sql/test_functions.py b/test/sql/test_functions.py index 51cfcb919..0074d789b 100644 --- a/test/sql/test_functions.py +++ b/test/sql/test_functions.py @@ -2,7 +2,7 @@ from sqlalchemy.testing import eq_, is_ import datetime from sqlalchemy import func, select, Integer, literal, DateTime, Table, \ Column, Sequence, MetaData, extract, Date, String, bindparam, \ - literal_column, Array, Numeric + literal_column, ARRAY, Numeric from sqlalchemy.sql import table, column from sqlalchemy import sql, util from sqlalchemy.sql.compiler import BIND_TEMPLATES @@ -558,7 +558,7 @@ class ReturnTypeTest(fixtures.TestBase): def test_array_agg(self): expr = func.array_agg(column('data', Integer)) - is_(expr.type._type_affinity, Array) + is_(expr.type._type_affinity, ARRAY) is_(expr.type.item_type._type_affinity, Integer) def test_mode(self): @@ -573,13 +573,13 @@ class ReturnTypeTest(fixtures.TestBase): def test_percentile_cont_array(self): expr = func.percentile_cont(0.5, 0.7).within_group( column('data', Integer)) - is_(expr.type._type_affinity, Array) + is_(expr.type._type_affinity, ARRAY) is_(expr.type.item_type._type_affinity, Integer) def test_percentile_cont_array_desc(self): expr = func.percentile_cont(0.5, 0.7).within_group( column('data', Integer).desc()) - is_(expr.type._type_affinity, Array) + is_(expr.type._type_affinity, ARRAY) is_(expr.type.item_type._type_affinity, Integer) def test_cume_dist(self): diff --git a/test/sql/test_operators.py b/test/sql/test_operators.py index 03c0f89be..6a6c749a4 100644 --- a/test/sql/test_operators.py +++ b/test/sql/test_operators.py @@ -15,7 +15,7 @@ from sqlalchemy.sql.elements import _literal_as_text from sqlalchemy.schema import Column, Table, MetaData from sqlalchemy.sql import compiler from sqlalchemy.types import TypeEngine, TypeDecorator, UserDefinedType, \ - Boolean, NullType, MatchType, Indexable, Concatenable, Array + Boolean, NullType, MatchType, Indexable, Concatenable, ARRAY, JSON from sqlalchemy.dialects import mysql, firebird, postgresql, oracle, \ sqlite, mssql from sqlalchemy import util @@ -632,7 +632,125 @@ class ExtensionOperatorTest(fixtures.TestBase, testing.AssertsCompiledSQL): ) -class IndexableTest(fixtures.TestBase, testing.AssertsCompiledSQL): +class JSONIndexOpTest(fixtures.TestBase, testing.AssertsCompiledSQL): + def setUp(self): + class MyTypeCompiler(compiler.GenericTypeCompiler): + def visit_mytype(self, type, **kw): + return "MYTYPE" + + def visit_myothertype(self, type, **kw): + return "MYOTHERTYPE" + + class MyCompiler(compiler.SQLCompiler): + + def visit_json_getitem_op_binary(self, binary, operator, **kw): + return self._generate_generic_binary( + binary, " -> ", **kw + ) + + def visit_json_path_getitem_op_binary( + self, binary, operator, **kw): + return self._generate_generic_binary( + binary, " #> ", **kw + ) + + def visit_getitem_binary(self, binary, operator, **kw): + raise NotImplementedError() + + class MyDialect(default.DefaultDialect): + statement_compiler = MyCompiler + type_compiler = MyTypeCompiler + + class MyType(JSON): + __visit_name__ = 'mytype' + + pass + + self.MyType = MyType + self.__dialect__ = MyDialect() + + def test_setup_getitem(self): + col = Column('x', self.MyType()) + + is_( + col[5].type._type_affinity, JSON + ) + is_( + col[5]['foo'].type._type_affinity, JSON + ) + is_( + col[('a', 'b', 'c')].type._type_affinity, JSON + ) + + def test_getindex_literal_integer(self): + + col = Column('x', self.MyType()) + + self.assert_compile( + col[5], + "x -> :x_1", + checkparams={'x_1': 5} + ) + + def test_getindex_literal_string(self): + + col = Column('x', self.MyType()) + + self.assert_compile( + col['foo'], + "x -> :x_1", + checkparams={'x_1': 'foo'} + ) + + def test_path_getindex_literal(self): + + col = Column('x', self.MyType()) + + self.assert_compile( + col[('a', 'b', 3, 4, 'd')], + "x #> :x_1", + checkparams={'x_1': ('a', 'b', 3, 4, 'd')} + ) + + def test_getindex_sqlexpr(self): + + col = Column('x', self.MyType()) + col2 = Column('y', Integer()) + + self.assert_compile( + col[col2], + "x -> y", + checkparams={} + ) + + self.assert_compile( + col[col2 + 8], + "x -> (y + :y_1)", + checkparams={'y_1': 8} + ) + + def test_override_operators(self): + special_index_op = operators.custom_op('$$>') + + class MyOtherType(JSON, TypeEngine): + __visit_name__ = 'myothertype' + + class Comparator(TypeEngine.Comparator): + + def _adapt_expression(self, op, other_comparator): + return special_index_op, MyOtherType() + + comparator_factory = Comparator + + col = Column('x', MyOtherType()) + self.assert_compile( + col[5], + "x $$> :x_1", + checkparams={'x_1': 5} + ) + + +class ArrayIndexOpTest(fixtures.TestBase, testing.AssertsCompiledSQL): def setUp(self): class MyTypeCompiler(compiler.GenericTypeCompiler): def visit_mytype(self, type, **kw): @@ -658,31 +776,14 @@ class IndexableTest(fixtures.TestBase, testing.AssertsCompiledSQL): statement_compiler = MyCompiler type_compiler = MyTypeCompiler - class MyType(Indexable, TypeEngine): + class MyType(ARRAY): __visit_name__ = 'mytype' def __init__(self, zero_indexes=False, dimensions=1): if zero_indexes: self.zero_indexes = zero_indexes self.dimensions = dimensions - - class Comparator(Indexable.Comparator): - def _setup_getitem(self, index): - if isinstance(index, slice): - return_type = self.type - elif self.type.dimensions is None or \ - self.type.dimensions == 1: - return_type = Integer() - else: - adapt_kw = {'dimensions': self.type.dimensions - 1} - # this is also testing the behavior of adapt() - # that we can pass kw that override constructor kws. - # required a small change to util.constructor_copy(). - return_type = self.type.adapt( - self.type.__class__, **adapt_kw) - - return operators.getitem, index, return_type - comparator_factory = Comparator + self.item_type = Integer() self.MyType = MyType self.__dialect__ = MyDialect() @@ -694,13 +795,13 @@ class IndexableTest(fixtures.TestBase, testing.AssertsCompiledSQL): col = Column('x', self.MyType(dimensions=3)) is_( - col[5].type._type_affinity, self.MyType + col[5].type._type_affinity, ARRAY ) eq_( col[5].type.dimensions, 2 ) is_( - col[5][6].type._type_affinity, self.MyType + col[5][6].type._type_affinity, ARRAY ) eq_( col[5][6].type.dimensions, 1 @@ -2273,7 +2374,7 @@ class AnyAllTest(fixtures.TestBase, testing.AssertsCompiledSQL): t = Table( 'tab1', m, - Column('arrval', Array(Integer)), + Column('arrval', ARRAY(Integer)), Column('data', Integer) ) return t diff --git a/test/sql/test_types.py b/test/sql/test_types.py index f1fb611fb..bb227bc5d 100644 --- a/test/sql/test_types.py +++ b/test/sql/test_types.py @@ -10,14 +10,14 @@ from sqlalchemy import ( and_, func, Date, LargeBinary, literal, cast, text, Enum, type_coerce, VARCHAR, Time, DateTime, BigInteger, SmallInteger, BOOLEAN, BLOB, NCHAR, NVARCHAR, CLOB, TIME, DATE, DATETIME, TIMESTAMP, SMALLINT, - INTEGER, DECIMAL, NUMERIC, FLOAT, REAL, Array) + INTEGER, DECIMAL, NUMERIC, FLOAT, REAL, ARRAY, JSON) from sqlalchemy.sql import ddl from sqlalchemy.sql import visitors from sqlalchemy import inspection from sqlalchemy import exc, types, util, dialects for name in dialects.__all__: __import__("sqlalchemy.dialects.%s" % name) -from sqlalchemy.sql import operators, column, table +from sqlalchemy.sql import operators, column, table, null from sqlalchemy.schema import CheckConstraint, AddConstraint from sqlalchemy.engine import default from sqlalchemy.testing.schema import Table, Column @@ -140,7 +140,7 @@ class AdaptTest(fixtures.TestBase): for is_down_adaption, typ, target_adaptions in adaptions(): if typ in (types.TypeDecorator, types.TypeEngine, types.Variant): continue - elif issubclass(typ, Array): + elif issubclass(typ, ARRAY): t1 = typ(String) else: t1 = typ() @@ -148,6 +148,8 @@ class AdaptTest(fixtures.TestBase): if not issubclass(typ, types.Enum) and \ issubclass(cls, types.Enum): continue + if cls.__module__.startswith("test"): + continue # print("ADAPT %s -> %s" % (t1.__class__, cls)) t2 = t1.adapt(cls) @@ -190,7 +192,7 @@ class AdaptTest(fixtures.TestBase): for typ in self._all_types(): if typ in (types.TypeDecorator, types.TypeEngine, types.Variant): continue - elif issubclass(typ, Array): + elif issubclass(typ, ARRAY): t1 = typ(String) else: t1 = typ() @@ -1406,23 +1408,98 @@ class BinaryTest(fixtures.TestBase, AssertsExecutionResults): return o.read() +class JSONTest(fixtures.TestBase): + + def setup(self): + metadata = MetaData() + self.test_table = Table('test_table', metadata, + Column('id', Integer, primary_key=True), + Column('test_column', JSON), + ) + self.jsoncol = self.test_table.c.test_column + + self.dialect = default.DefaultDialect() + self.dialect._json_serializer = None + self.dialect._json_deserializer = None + + def test_bind_serialize_default(self): + proc = self.test_table.c.test_column.type._cached_bind_processor( + self.dialect) + eq_( + proc({"A": [1, 2, 3, True, False]}), + '{"A": [1, 2, 3, true, false]}' + ) + + def test_bind_serialize_None(self): + proc = self.test_table.c.test_column.type._cached_bind_processor( + self.dialect) + eq_( + proc(None), + 'null' + ) + + def test_bind_serialize_none_as_null(self): + proc = JSON(none_as_null=True)._cached_bind_processor( + self.dialect) + eq_( + proc(None), + None + ) + eq_( + proc(null()), + None + ) + + def test_bind_serialize_null(self): + proc = self.test_table.c.test_column.type._cached_bind_processor( + self.dialect) + eq_( + proc(null()), + None + ) + + def test_result_deserialize_default(self): + proc = self.test_table.c.test_column.type._cached_result_processor( + self.dialect, None) + eq_( + proc('{"A": [1, 2, 3, true, false]}'), + {"A": [1, 2, 3, True, False]} + ) + + def test_result_deserialize_null(self): + proc = self.test_table.c.test_column.type._cached_result_processor( + self.dialect, None) + eq_( + proc('null'), + None + ) + + def test_result_deserialize_None(self): + proc = self.test_table.c.test_column.type._cached_result_processor( + self.dialect, None) + eq_( + proc(None), + None + ) + + class ArrayTest(fixtures.TestBase): def _myarray_fixture(self): - class MyArray(Array): + class MyArray(ARRAY): pass return MyArray def test_array_index_map_dimensions(self): - col = column('x', Array(Integer, dimensions=3)) + col = column('x', ARRAY(Integer, dimensions=3)) is_( - col[5].type._type_affinity, Array + col[5].type._type_affinity, ARRAY ) eq_( col[5].type.dimensions, 2 ) is_( - col[5][6].type._type_affinity, Array + col[5][6].type._type_affinity, ARRAY ) eq_( col[5][6].type.dimensions, 1 @@ -1435,8 +1512,8 @@ class ArrayTest(fixtures.TestBase): m = MetaData() arrtable = Table( 'arrtable', m, - Column('intarr', Array(Integer)), - Column('strarr', Array(String)), + Column('intarr', ARRAY(Integer)), + Column('strarr', ARRAY(String)), ) is_(arrtable.c.intarr[1].type._type_affinity, Integer) is_(arrtable.c.strarr[1].type._type_affinity, String) @@ -1445,11 +1522,11 @@ class ArrayTest(fixtures.TestBase): m = MetaData() arrtable = Table( 'arrtable', m, - Column('intarr', Array(Integer)), - Column('strarr', Array(String)), + Column('intarr', ARRAY(Integer)), + Column('strarr', ARRAY(String)), ) - is_(arrtable.c.intarr[1:3].type._type_affinity, Array) - is_(arrtable.c.strarr[1:3].type._type_affinity, Array) + is_(arrtable.c.intarr[1:3].type._type_affinity, ARRAY) + is_(arrtable.c.strarr[1:3].type._type_affinity, ARRAY) def test_array_getitem_slice_type_dialect_level(self): MyArray = self._myarray_fixture() @@ -1459,8 +1536,8 @@ class ArrayTest(fixtures.TestBase): Column('intarr', MyArray(Integer)), Column('strarr', MyArray(String)), ) - is_(arrtable.c.intarr[1:3].type._type_affinity, Array) - is_(arrtable.c.strarr[1:3].type._type_affinity, Array) + is_(arrtable.c.intarr[1:3].type._type_affinity, ARRAY) + is_(arrtable.c.strarr[1:3].type._type_affinity, ARRAY) # but the slice returns the actual type assert isinstance(arrtable.c.intarr[1:3].type, MyArray) |
