From 024f0dbada97fa5f9718f7d622dba232ac4ca65a Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 19 Sep 2012 04:26:35 +0100 Subject: Added json typecaster --- lib/_json.py | 187 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 lib/_json.py (limited to 'lib/_json.py') diff --git a/lib/_json.py b/lib/_json.py new file mode 100644 index 0000000..ec33a6a --- /dev/null +++ b/lib/_json.py @@ -0,0 +1,187 @@ +"""Implementation of the JSON adaptation objects + +This module exists to avoid a circular import problem: pyscopg2.extras depends +on psycopg2.extension, so I can't create the default JSON typecasters in +extensions importing register_json from extras. +""" + +# psycopg/extras.py - miscellaneous extra goodies for psycopg +# +# Copyright (C) 2012 Daniele Varrazzo +# +# psycopg2 is free software: you can redistribute it and/or modify it +# under the terms of the GNU Lesser General Public License as published +# by the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# In addition, as a special exception, the copyright holders give +# permission to link this program with the OpenSSL library (or with +# modified versions of OpenSSL that use the same license as OpenSSL), +# and distribute linked combinations including the two. +# +# You must obey the GNU Lesser General Public License in all respects for +# all of the code used other than OpenSSL. +# +# psycopg2 is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public +# License for more details. + +import sys + +from psycopg2._psycopg import ISQLQuote, QuotedString +from psycopg2._psycopg import new_type, new_array_type, register_type + + +# import the best json implementation available +if sys.version_info[:2] >= (2,6): + import json +else: + try: + import simplejson as json + except ImportError: + json = None + + +class Json(object): + """A wrapper to adapt a Python object to :sql:`json` data type. + + `!Json` can be used to wrap any object supported by the underlying + `!json` module. Raise `!ImportError` if no module is available. + + Any keyword argument will be passed to the underlying + :py:func:`json.dumps()` function, allowing extension and customization. :: + + curs.execute("insert into mytable (jsondata) values (%s)", + (Json({'a': 100}),)) + + .. note:: + + You can use `~psycopg2.extensions.register_adapter()` to adapt Python + dictionaries to JSON:: + + psycopg2.extensions.register_adapter(dict, + psycopg2.extras.Json) + + This setting is global though, so it is not compatible with the use of + `register_hstore()`. Any other object supported by the `!json` library + used by Psycopg can be registered the same way, but this will clobber + the default adaptation rule, so be careful to unwanted side effects. + + """ + def __init__(self, adapted, **kwargs): + self.adapted = adapted + self.kwargs = kwargs + + def __conform__(self, proto): + if proto is ISQLQuote: + return self + + def getquoted(self): + s = json.dumps(self.adapted, **self.kwargs) + return QuotedString(s).getquoted() + + +# clobber the above class if json is not available +if json is None: + class Json(Json): + def __init__(self, adapted): + raise ImportError("no json module available") + + +def register_json(conn_or_curs, globally=False, loads=None, + oid=None, array_oid=None): + """Create and register typecasters converting :sql:`json` type to Python objects. + + :param conn_or_curs: a connection or cursor used to find the :sql:`json` + and :sql:`json[]` oids; the typecasters are registered in a scope + limited to this object, unless *globally* is set to `!True`. It can be + `!None` if the oids are provided + :param globally: if `!False` register the typecasters only on + *conn_or_curs*, otherwise register them globally + :param loads: the function used to parse the data into a Python object. If + `!None` use `!json.loads()`, where `!json` is the module chosen + according to the Python version (see above) + :param oid: the OID of the :sql:`json` type if known; If not, it will be + queried on *conn_or_curs* + :param array_oid: the OID of the :sql:`json[]` array type if known; + if not, it will be queried on *conn_or_curs* + + Using the function is required to convert :sql:`json` data in PostgreSQL + versions before 9.2. Since 9.2 the oids are hardcoded so a default + typecaster is already registered. The :sql:`json` type is available as + `extension for PostgreSQL 9.1`__. + + .. __: http://people.planetpostgresql.org/andrew/index.php?/archives/255-JSON-for-PG-9.2-...-and-now-for-9.1!.html + + Another use of the function is to adapt :sql:`json` using a customized + load function. For example, if you want to convert the float values in the + :sql:`json` into :py:class:`~decimal.Decimal` you can use:: + + loads = lambda x: json.loads(x, parse_float=Decimal) + psycopg2.extras.register_json(conn, loads=loads) + + The connection or cursor passed to the function will be used to query the + database and look for the OID of the :sql:`json` type. No query is + performed if *oid* and *array_oid* are provided. Raise + `~psycopg2.ProgrammingError` if the type is not found. + + """ + if oid is None: + oid, array_oid = _get_json_oids(conn_or_curs) + + JSON, JSONARRAY = create_json_typecasters(oid, array_oid, loads) + + register_type(JSON, not globally and conn_or_curs or None) + + if JSONARRAY is not None: + register_type(JSONARRAY, not globally and conn_or_curs or None) + + return JSON, JSONARRAY + +def create_json_typecasters(oid, array_oid, loads=None): + """Create typecasters for json data type.""" + if loads is None: + if json is None: + raise ImportError("no json module available") + else: + loads = json.loads + + def typecast_json(s, cur): + return loads(s) + + JSON = new_type((oid, ), 'JSON', typecast_json) + JSONARRAY = new_array_type((array_oid, ), "JSONARRAY", JSON) + + return JSON, JSONARRAY + +def _get_json_oids(conn_or_curs): + # lazy imports + from psycopg2.extensions import STATUS_IN_TRANSACTION + from psycopg2.extras import _solve_conn_curs + + conn, curs = _solve_conn_curs(conn_or_curs) + + # Store the transaction status of the connection to revert it after use + conn_status = conn.status + + # column typarray not available before PG 8.3 + typarray = conn.server_version >= 80300 and "typarray" or "NULL" + + # get the oid for the hstore + curs.execute( + "SELECT t.oid, %s FROM pg_type t WHERE t.typname = 'json';" + % typarray) + r = curs.fetchone() + + # revert the status of the connection as before the command + if (conn_status != STATUS_IN_TRANSACTION and not conn.autocommit): + conn.rollback() + + if not r: + raise conn.ProgrammingError("json data type not found") + + return r + + + -- cgit v1.2.1 From 26d71b4cba31a62bc648bec353aab299adb391a6 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 19 Sep 2012 15:31:28 +0100 Subject: First parameter of register_json defaults to None --- lib/_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/_json.py') diff --git a/lib/_json.py b/lib/_json.py index ec33a6a..09d854b 100644 --- a/lib/_json.py +++ b/lib/_json.py @@ -89,7 +89,7 @@ if json is None: raise ImportError("no json module available") -def register_json(conn_or_curs, globally=False, loads=None, +def register_json(conn_or_curs=None, globally=False, loads=None, oid=None, array_oid=None): """Create and register typecasters converting :sql:`json` type to Python objects. -- cgit v1.2.1 From d963b478e27557dab74028b4fd7caf5ebe5f9f44 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 19 Sep 2012 15:49:00 +0100 Subject: Added register_default_json() function Register a typecaster for PostgreSQL 9.2 json. --- lib/_json.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) (limited to 'lib/_json.py') diff --git a/lib/_json.py b/lib/_json.py index 09d854b..3d21148 100644 --- a/lib/_json.py +++ b/lib/_json.py @@ -43,6 +43,10 @@ else: json = None +# oids from PostgreSQL 9.2 +JSON_OID = 114 +JSONARRAY_OID = 199 + class Json(object): """A wrapper to adapt a Python object to :sql:`json` data type. @@ -130,7 +134,7 @@ def register_json(conn_or_curs=None, globally=False, loads=None, if oid is None: oid, array_oid = _get_json_oids(conn_or_curs) - JSON, JSONARRAY = create_json_typecasters(oid, array_oid, loads) + JSON, JSONARRAY = _create_json_typecasters(oid, array_oid, loads) register_type(JSON, not globally and conn_or_curs or None) @@ -139,7 +143,20 @@ def register_json(conn_or_curs=None, globally=False, loads=None, return JSON, JSONARRAY -def create_json_typecasters(oid, array_oid, loads=None): +def register_default_json(conn_or_curs=None, globally=False, loads=None): + """ + Create and register :sql:`json` typecasters for PostgreSQL 9.2 and following. + + Since PostgreSQL 9.2 :sql:`json` is a builtin type, hence its oid is known + and fixed. This function allows specifying a customized *loads* function + for the default :sql:`json` type without querying the database. + All the parameters have the same meaning of `register_json()`. + """ + return register_json(conn_or_curs=conn_or_curs, globally=globally, + loads=loads, oid=JSON_OID, array_oid=JSONARRAY_OID) + + +def _create_json_typecasters(oid, array_oid, loads=None): """Create typecasters for json data type.""" if loads is None: if json is None: -- cgit v1.2.1 From 7386b8327cf4b77666a67e150ce660bf4ff489b6 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Wed, 19 Sep 2012 16:32:57 +0100 Subject: Dropped keywords passthrough in Json adapter Pass a dumps function instead. Allow customizing by either arg passing or subclassing. The basic Json class now raises ImportError on getquoted() if json is not available, thus allowing using a customized Json subclass even when the json module is not available. --- lib/_json.py | 78 ++++++++++++++++++++++++++++++++++++++++++------------------ 1 file changed, 55 insertions(+), 23 deletions(-) (limited to 'lib/_json.py') diff --git a/lib/_json.py b/lib/_json.py index 3d21148..c21d6c8 100644 --- a/lib/_json.py +++ b/lib/_json.py @@ -48,49 +48,81 @@ JSON_OID = 114 JSONARRAY_OID = 199 class Json(object): - """A wrapper to adapt a Python object to :sql:`json` data type. + """ + An `~psycopg2.extensions.ISQLQuote` wrapper to adapt a Python object to + :sql:`json` data type. `!Json` can be used to wrap any object supported by the underlying - `!json` module. Raise `!ImportError` if no module is available. + `!json` module. `~psycopg2.extensions.ISQLQuote.getquoted()` will raise + `!ImportError` if no module is available. + + The basic usage is to wrap `!Json` around the object to be adapted:: + + curs.execute("insert into mytable (jsondata) values (%s)", + [Json({'a': 100})]) + + If you want to customize the adaptation from Python to PostgreSQL you can + either provide a custom *dumps* function:: + + curs.execute("insert into mytable (jsondata) values (%s)", + [Json({'a': 100}, dumps=simplejson.dumps)]) + + or you can subclass `!Json` overriding the `dumps()` method:: - Any keyword argument will be passed to the underlying - :py:func:`json.dumps()` function, allowing extension and customization. :: + class MyJson(Json): + def dumps(self, obj): + return simplejson.dumps(obj) curs.execute("insert into mytable (jsondata) values (%s)", - (Json({'a': 100}),)) + [MyJson({'a': 100})]) .. note:: - You can use `~psycopg2.extensions.register_adapter()` to adapt Python - dictionaries to JSON:: + You can use `~psycopg2.extensions.register_adapter()` to adapt any + Python dictionary to JSON, either using `!Json` or any subclass or + factory creating a compatible adapter:: - psycopg2.extensions.register_adapter(dict, - psycopg2.extras.Json) + psycopg2.extensions.register_adapter(dict, psycopg2.extras.Json) - This setting is global though, so it is not compatible with the use of - `register_hstore()`. Any other object supported by the `!json` library - used by Psycopg can be registered the same way, but this will clobber - the default adaptation rule, so be careful to unwanted side effects. + This setting is global though, so it is not compatible with similar + adapters such as the one registered by `register_hstore()`. Any other + object supported by JSON can be registered the same way, but this will + clobber the default adaptation rule, so be careful to unwanted side + effects. """ - def __init__(self, adapted, **kwargs): + def __init__(self, adapted, dumps=None): self.adapted = adapted - self.kwargs = kwargs + + if dumps is not None: + self._dumps = dumps + elif json is not None: + self._dumps = json.dumps + else: + self._dumps = None def __conform__(self, proto): if proto is ISQLQuote: return self - def getquoted(self): - s = json.dumps(self.adapted, **self.kwargs) - return QuotedString(s).getquoted() + def dumps(self, obj): + """Serialize *obj* in JSON format. + The default is to call `!json.dumps()` or the *dumps* function + provided in the constructor. You can override this method to create a + customized JSON wrapper. + """ + dumps = self._dumps + if dumps is not None: + return dumps(obj) + else: + raise ImportError( + "json module not available: " + "you should provide a dumps function") -# clobber the above class if json is not available -if json is None: - class Json(Json): - def __init__(self, adapted): - raise ImportError("no json module available") + def getquoted(self): + s = self.dumps(self.adapted) + return QuotedString(s).getquoted() def register_json(conn_or_curs=None, globally=False, loads=None, -- cgit v1.2.1 From 9b2ad7abb5a67921cc2cc9e18e37eb6f9aeee4b4 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 20 Sep 2012 00:36:53 +0100 Subject: Fixed json typecaster with NULL input --- lib/_json.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'lib/_json.py') diff --git a/lib/_json.py b/lib/_json.py index c21d6c8..9142c57 100644 --- a/lib/_json.py +++ b/lib/_json.py @@ -197,6 +197,8 @@ def _create_json_typecasters(oid, array_oid, loads=None): loads = json.loads def typecast_json(s, cur): + if s is None: + return None return loads(s) JSON = new_type((oid, ), 'JSON', typecast_json) -- cgit v1.2.1 From 5645e7adef7644bd1acbe723dd5f0cc36c4815c1 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Thu, 20 Sep 2012 03:44:50 +0100 Subject: Pasto fixed --- lib/_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/_json.py') diff --git a/lib/_json.py b/lib/_json.py index 9142c57..e117252 100644 --- a/lib/_json.py +++ b/lib/_json.py @@ -5,7 +5,7 @@ on psycopg2.extension, so I can't create the default JSON typecasters in extensions importing register_json from extras. """ -# psycopg/extras.py - miscellaneous extra goodies for psycopg +# psycopg/_json.py - Implementation of the JSON adaptation objects # # Copyright (C) 2012 Daniele Varrazzo # -- cgit v1.2.1 From a3418052e9fdc1fc05cda4b632957decb0fadc7c Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 24 Sep 2012 11:23:09 +0100 Subject: Don't create/register a json array typecaster if no oid provided --- lib/_json.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'lib/_json.py') diff --git a/lib/_json.py b/lib/_json.py index e117252..f90bac5 100644 --- a/lib/_json.py +++ b/lib/_json.py @@ -202,7 +202,10 @@ def _create_json_typecasters(oid, array_oid, loads=None): return loads(s) JSON = new_type((oid, ), 'JSON', typecast_json) - JSONARRAY = new_array_type((array_oid, ), "JSONARRAY", JSON) + if array_oid is not None: + JSONARRAY = new_array_type((array_oid, ), "JSONARRAY", JSON) + else: + JSONARRAY = None return JSON, JSONARRAY -- cgit v1.2.1 From 71e2f190c2c7011dd5fd606d6e81dde0d9a56bf9 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Mon, 24 Sep 2012 11:51:35 +0100 Subject: Json documentation improved --- lib/_json.py | 57 +++++---------------------------------------------------- 1 file changed, 5 insertions(+), 52 deletions(-) (limited to 'lib/_json.py') diff --git a/lib/_json.py b/lib/_json.py index f90bac5..7757b6e 100644 --- a/lib/_json.py +++ b/lib/_json.py @@ -52,43 +52,11 @@ class Json(object): An `~psycopg2.extensions.ISQLQuote` wrapper to adapt a Python object to :sql:`json` data type. - `!Json` can be used to wrap any object supported by the underlying - `!json` module. `~psycopg2.extensions.ISQLQuote.getquoted()` will raise - `!ImportError` if no module is available. - - The basic usage is to wrap `!Json` around the object to be adapted:: - - curs.execute("insert into mytable (jsondata) values (%s)", - [Json({'a': 100})]) - - If you want to customize the adaptation from Python to PostgreSQL you can - either provide a custom *dumps* function:: - - curs.execute("insert into mytable (jsondata) values (%s)", - [Json({'a': 100}, dumps=simplejson.dumps)]) - - or you can subclass `!Json` overriding the `dumps()` method:: - - class MyJson(Json): - def dumps(self, obj): - return simplejson.dumps(obj) - - curs.execute("insert into mytable (jsondata) values (%s)", - [MyJson({'a': 100})]) - - .. note:: - - You can use `~psycopg2.extensions.register_adapter()` to adapt any - Python dictionary to JSON, either using `!Json` or any subclass or - factory creating a compatible adapter:: - - psycopg2.extensions.register_adapter(dict, psycopg2.extras.Json) - - This setting is global though, so it is not compatible with similar - adapters such as the one registered by `register_hstore()`. Any other - object supported by JSON can be registered the same way, but this will - clobber the default adaptation rule, so be careful to unwanted side - effects. + `!Json` can be used to wrap any object supported by the provided *dumps* + function. If none is provided, the standard :py:func:`json.dumps()` is + used (`!simplejson` for Python < 2.6; + `~psycopg2.extensions.ISQLQuote.getquoted()` will raise `!ImportError` if + the module is available). """ def __init__(self, adapted, dumps=None): @@ -143,20 +111,6 @@ def register_json(conn_or_curs=None, globally=False, loads=None, :param array_oid: the OID of the :sql:`json[]` array type if known; if not, it will be queried on *conn_or_curs* - Using the function is required to convert :sql:`json` data in PostgreSQL - versions before 9.2. Since 9.2 the oids are hardcoded so a default - typecaster is already registered. The :sql:`json` type is available as - `extension for PostgreSQL 9.1`__. - - .. __: http://people.planetpostgresql.org/andrew/index.php?/archives/255-JSON-for-PG-9.2-...-and-now-for-9.1!.html - - Another use of the function is to adapt :sql:`json` using a customized - load function. For example, if you want to convert the float values in the - :sql:`json` into :py:class:`~decimal.Decimal` you can use:: - - loads = lambda x: json.loads(x, parse_float=Decimal) - psycopg2.extras.register_json(conn, loads=loads) - The connection or cursor passed to the function will be used to query the database and look for the OID of the :sql:`json` type. No query is performed if *oid* and *array_oid* are provided. Raise @@ -187,7 +141,6 @@ def register_default_json(conn_or_curs=None, globally=False, loads=None): return register_json(conn_or_curs=conn_or_curs, globally=globally, loads=loads, oid=JSON_OID, array_oid=JSONARRAY_OID) - def _create_json_typecasters(oid, array_oid, loads=None): """Create typecasters for json data type.""" if loads is None: -- cgit v1.2.1 From b894539007ff0107b4a5295540e0fad112e11e92 Mon Sep 17 00:00:00 2001 From: Daniele Varrazzo Date: Tue, 25 Sep 2012 00:12:57 +0100 Subject: Fixed json doc error --- lib/_json.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'lib/_json.py') diff --git a/lib/_json.py b/lib/_json.py index 7757b6e..536dd58 100644 --- a/lib/_json.py +++ b/lib/_json.py @@ -56,7 +56,7 @@ class Json(object): function. If none is provided, the standard :py:func:`json.dumps()` is used (`!simplejson` for Python < 2.6; `~psycopg2.extensions.ISQLQuote.getquoted()` will raise `!ImportError` if - the module is available). + the module is not available). """ def __init__(self, adapted, dumps=None): -- cgit v1.2.1