Fix compilation on Python3

while still compiling on recent Python2:
  - change the handling of files, tweak the generator, get the fd
    instead of the FILE *, dup it and fdopen based on mode, add a
    Release function on Python3 and call to flush from the generated
    python stubs
  - switch to using Capsules instead of CObjects
  - fix PyString to PyBytes
  - fix PyInt to PyLong
  - tweak the module registration to compile on both versions
  - drop PyInstance check for passed xmlNodes and instead check
    attributes presence

Daniel

4 files changed, 340 insertions, 163 deletions

diff --git a/python/generator.py b/python/generator.py
index b864b16c..e4951437 100755
--- a/python/generator.py
+++ b/python/generator.py
@@ -133,8 +133,8 @@ class docParser(xml.sax.handler.ContentHandler):
                 str = str + c
             if self.in_function == 1:
                 self.function_cond = str
-                
-                
+
+
 def function(name, desc, ret, args, file, cond):
     functions[name] = (desc, ret, args, file, cond)
 
@@ -370,6 +370,7 @@ def print_function_wrapper(name, output, export, include):
     c_args=""
     c_return=""
     c_convert=""
+    c_release=""
     num_bufs=0
     for arg in args:
         # This should be correct
@@ -379,7 +380,7 @@ def print_function_wrapper(name, output, export, include):
         if arg[1] in py_types:
             (f, t, n, c) = py_types[arg[1]]
             if (f == 'z') and (name in foreign_encoding_args) and (num_bufs == 0):
-                f = 't#'
+                f = 's#'
             if f != None:
                 format = format + f
             if t != None:
@@ -390,13 +391,16 @@ def print_function_wrapper(name, output, export, include):
                    arg[1], t, arg[0])
             else:
                 format_args = format_args + ", &%s" % (arg[0])
-            if f == 't#':
+            if f == 's#':
                 format_args = format_args + ", &py_buffsize%d" % num_bufs
                 c_args = c_args + "    int py_buffsize%d;\n" % num_bufs
                 num_bufs = num_bufs + 1
             if c_call != "":
                 c_call = c_call + ", "
             c_call = c_call + "%s" % (arg[0])
+            if t == "File":
+                c_release = c_release + \
+		            "    PyFile_Release(%s);\n" % (arg[0])
         else:
             if arg[1] in skipped_types:
                 return 0
@@ -424,7 +428,7 @@ def print_function_wrapper(name, output, export, include):
         ret_convert = "    Py_INCREF(Py_None);\n    return(Py_None);\n"
     elif ret[0] in py_types:
         (f, t, n, c) = py_types[ret[0]]
-        c_return = "    %s c_retval;\n" % (ret[0])
+        c_return = c_return + "    %s c_retval;\n" % (ret[0])
         if file == "python_accessor" and ret[2] != None:
             c_call = "\n    c_retval = %s->%s;\n" % (args[0][0], ret[2])
         else:
@@ -433,7 +437,7 @@ def print_function_wrapper(name, output, export, include):
         ret_convert = ret_convert + "    return(py_retval);\n"
     elif ret[0] in py_return_types:
         (f, t, n, c) = py_return_types[ret[0]]
-        c_return = "    %s c_retval;\n" % (ret[0])
+        c_return = c_return + "    %s c_retval;\n" % (ret[0])
         c_call = "\n    c_retval = %s(%s);\n" % (name, c_call)
         ret_convert = "    py_retval = libxml_%sWrap((%s) c_retval);\n" % (n,c)
         ret_convert = ret_convert + "    return(py_retval);\n"
@@ -491,8 +495,10 @@ def print_function_wrapper(name, output, export, include):
         output.write("        return(NULL);\n")
     if c_convert != "":
         output.write(c_convert)
-                                                              
+
     output.write(c_call)
+    if c_release != "":
+        output.write(c_release)
     output.write(ret_convert)
     output.write("}\n\n")
     if cond != None and cond != "":
@@ -959,6 +965,12 @@ def buildWrappers():
                                   (arg[0], arg[0]))
                     classes.write("    else: %s__o = %s%s\n" %
                                   (arg[0], arg[0], classes_type[arg[1]][0]))
+                if arg[1] in py_types:
+                    (f, t, n, c) = py_types[arg[1]]
+                    if t == "File":
+                        classes.write("    if %s is not None: %s.flush()\n" % (
+                                      arg[0], arg[0]))
+
             if ret[0] != "void":
                 classes.write("    ret = ")
             else:
@@ -973,6 +985,16 @@ def buildWrappers():
                     classes.write("__o")
                 n = n + 1
             classes.write(")\n")
+
+# This may be needed to reposition the I/O, but likely to cause more harm
+# than good. Those changes in Python3 really break the model.
+#           for arg in args:
+#               if arg[1] in py_types:
+#                   (f, t, n, c) = py_types[arg[1]]
+#                   if t == "File":
+#                       classes.write("    if %s is not None: %s.seek(0,0)\n"%(
+#                                     arg[0], arg[0]))
+
             if ret[0] != "void":
                 if ret[0] in classes_type:
                     #
diff --git a/python/libxml.c b/python/libxml.c
index 8bb3db27..03cfb9fc 100644
--- a/python/libxml.c
+++ b/python/libxml.c
@@ -41,7 +41,17 @@
 /* #define DEBUG_FILES */
 /* #define DEBUG_LOADER */
 
+#if PY_MAJOR_VERSION >= 3
+PyObject *PyInit_libxml2mod(void);
+
+#define PY_IMPORT_STRING_SIZE PyUnicode_FromStringAndSize
+#define PY_IMPORT_STRING PyUnicode_FromString
+#else
 void initlibxml2mod(void);
+#define PY_IMPORT_STRING_SIZE PyString_FromStringAndSize
+#define PY_IMPORT_STRING PyString_FromString
+#endif
+
 
 /**
  * TODO:
@@ -280,18 +290,42 @@ xmlPythonFileReadRaw (void * context, char * buffer, int len) {
     if (ret == NULL) {
 	printf("xmlPythonFileReadRaw: result is NULL\n");
 	return(-1);
-    } else if (PyString_Check(ret)) {
-	lenread = PyString_Size(ret);
-	data = PyString_AsString(ret);
-	if (lenread > len)
-	    memcpy(buffer, data, len);
-	else
-	    memcpy(buffer, data, lenread);
-	Py_DECREF(ret);
+    } else if (PyBytes_Check(ret)) {
+	lenread = PyBytes_Size(ret);
+	data = PyBytes_AsString(ret);
+#ifdef PyUnicode_Check
+    } else if PyUnicode_Check (ret) {
+#if PY_VERSION_HEX >= 0x03030000
+        size_t size;
+	const char *tmp;
+
+	/* tmp doesn't need to be deallocated */
+        tmp = PyUnicode_AsUTF8AndSize(ret, &size);
+
+	lenread = (int) size;
+	data = (char *) tmp;
+#else
+        PyObject *b;
+	b = PyUnicode_AsUTF8String(ret);
+	if (b == NULL) {
+	    printf("xmlPythonFileReadRaw: failed to convert to UTF-8\n");
+	    return(-1);
+	}
+	lenread = PyBytes_Size(b);
+	data = PyBytes_AsString(b);
+	Py_DECREF(b);
+#endif
+#endif
     } else {
 	printf("xmlPythonFileReadRaw: result is not a String\n");
 	Py_DECREF(ret);
+	return(-1);
     }
+    if (lenread > len)
+	memcpy(buffer, data, len);
+    else
+	memcpy(buffer, data, lenread);
+    Py_DECREF(ret);
     return(lenread);
 }
 
@@ -321,18 +355,42 @@ xmlPythonFileRead (void * context, char * buffer, int len) {
     if (ret == NULL) {
 	printf("xmlPythonFileRead: result is NULL\n");
 	return(-1);
-    } else if (PyString_Check(ret)) {
-	lenread = PyString_Size(ret);
-	data = PyString_AsString(ret);
-	if (lenread > len)
-	    memcpy(buffer, data, len);
-	else
-	    memcpy(buffer, data, lenread);
-	Py_DECREF(ret);
+    } else if (PyBytes_Check(ret)) {
+	lenread = PyBytes_Size(ret);
+	data = PyBytes_AsString(ret);
+#ifdef PyUnicode_Check
+    } else if PyUnicode_Check (ret) {
+#if PY_VERSION_HEX >= 0x03030000
+        size_t size;
+	const char *tmp;
+
+	/* tmp doesn't need to be deallocated */
+        tmp = PyUnicode_AsUTF8AndSize(ret, &size);
+
+	lenread = (int) size;
+	data = (char *) tmp;
+#else
+        PyObject *b;
+	b = PyUnicode_AsUTF8String(ret);
+	if (b == NULL) {
+	    printf("xmlPythonFileRead: failed to convert to UTF-8\n");
+	    return(-1);
+	}
+	lenread = PyBytes_Size(b);
+	data = PyBytes_AsString(b);
+	Py_DECREF(b);
+#endif
+#endif
     } else {
 	printf("xmlPythonFileRead: result is not a String\n");
 	Py_DECREF(ret);
+	return(-1);
     }
+    if (lenread > len)
+	memcpy(buffer, data, len);
+    else
+	memcpy(buffer, data, lenread);
+    Py_DECREF(ret);
     return(lenread);
 }
 
@@ -358,7 +416,7 @@ xmlPythonFileWrite (void * context, const char * buffer, int len) {
 #endif
     file = (PyObject *) context;
     if (file == NULL) return(-1);
-    string = PyString_FromStringAndSize(buffer, len);
+    string = PY_IMPORT_STRING_SIZE(buffer, len);
     if (string == NULL) return(-1);
     if (PyObject_HasAttrString(file, (char *) "io_write")) {
         ret = PyEval_CallMethod(file, (char *) "io_write", (char *) "(O)",
@@ -371,8 +429,8 @@ xmlPythonFileWrite (void * context, const char * buffer, int len) {
     if (ret == NULL) {
 	printf("xmlPythonFileWrite: result is NULL\n");
 	return(-1);
-    } else if (PyInt_Check(ret)) {
-	written = (int) PyInt_AsLong(ret);
+    } else if (PyLong_Check(ret)) {
+	written = (int) PyLong_AsLong(ret);
 	Py_DECREF(ret);
     } else if (ret == Py_None) {
 	written = len;
@@ -727,7 +785,7 @@ libxml_xmlSetEntityLoader(ATTRIBUTE_UNUSED PyObject *self, PyObject *args) {
     Py_XINCREF(pythonExternalEntityLoaderObjext);
     xmlSetExternalEntityLoader(pythonExternalEntityLoader);
 
-    py_retval = PyInt_FromLong(0);
+    py_retval = PyLong_FromLong(0);
     return(py_retval);
 }
 
@@ -859,10 +917,10 @@ pythonStartElement(void *user_data, const xmlChar * name,
         } else {
             dict = PyDict_New();
             for (i = 0; attrs[i] != NULL; i++) {
-                attrname = PyString_FromString((char *) attrs[i]);
+                attrname = PY_IMPORT_STRING((char *) attrs[i]);
                 i++;
                 if (attrs[i] != NULL) {
-                    attrvalue = PyString_FromString((char *) attrs[i]);
+                    attrvalue = PY_IMPORT_STRING((char *) attrs[i]);
                 } else {
                     Py_XINCREF(Py_None);
                     attrvalue = Py_None;
@@ -1265,7 +1323,7 @@ pythonAttributeDecl(void *user_data,
         nameList = PyList_New(count);
         count = 0;
         for (node = tree; node != NULL; node = node->next) {
-            newName = PyString_FromString((char *) node->name);
+            newName = PY_IMPORT_STRING((char *) node->name);
             PyList_SetItem(nameList, count, newName);
 	    Py_DECREF(newName);
             count++;
@@ -2128,7 +2186,7 @@ libxml_xmlFreeTextReader(ATTRIBUTE_UNUSED PyObject *self, PyObject *args) {
 
     if (!PyArg_ParseTuple(args, (char *)"O:xmlFreeTextReader", &pyobj_reader))
         return(NULL);
-    if (!PyCObject_Check(pyobj_reader)) {
+    if (!PyCapsule_CheckExact(pyobj_reader)) {
 	Py_INCREF(Py_None);
 	return(Py_None);
     }
@@ -2687,6 +2745,10 @@ libxml_type(ATTRIBUTE_UNUSED PyObject * self, PyObject * args)
     if (!PyArg_ParseTuple(args, (char *) "O:last", &obj))
         return NULL;
     cur = PyxmlNode_Get(obj);
+    if (cur == NULL) {
+        Py_INCREF(Py_None);
+	return (Py_None);
+    }
 
 #ifdef DEBUG
     printf("libxml_type: cur = %p\n", cur);
@@ -2803,7 +2865,7 @@ libxml_xmlNodeRemoveNsDef(ATTRIBUTE_UNUSED PyObject * self, PyObject * args)
     PyObject *pyobj_node;
     xmlChar *href;
     xmlNsPtr c_retval;
-    
+
     if (!PyArg_ParseTuple
         (args, (char *) "Oz:xmlNodeRemoveNsDef", &pyobj_node, &href))
         return (NULL);
@@ -2965,16 +3027,12 @@ libxml_saveNodeTo(ATTRIBUTE_UNUSED PyObject * self, PyObject * args)
                           &py_file, &encoding, &format))
         return (NULL);
     node = (xmlNodePtr) PyxmlNode_Get(pyobj_node);
-
     if (node == NULL) {
-        return (PyInt_FromLong((long) -1));
+        return (PyLong_FromLong((long) -1));
     }
-    if ((py_file == NULL) || (!(PyFile_Check(py_file)))) {
-        return (PyInt_FromLong((long) -1));
-    }
-    output = PyFile_AsFile(py_file);
+    output = PyFile_Get(py_file);
     if (output == NULL) {
-        return (PyInt_FromLong((long) -1));
+        return (PyLong_FromLong((long) -1));
     }
 
     if (node->type == XML_DOCUMENT_NODE) {
@@ -2993,7 +3051,7 @@ libxml_saveNodeTo(ATTRIBUTE_UNUSED PyObject * self, PyObject * args)
     if (encoding != NULL) {
         handler = xmlFindCharEncodingHandler(encoding);
         if (handler == NULL) {
-            return (PyInt_FromLong((long) -1));
+            return (PyLong_FromLong((long) -1));
         }
     }
     if (doc->type == XML_HTML_DOCUMENT_NODE) {
@@ -3018,7 +3076,8 @@ libxml_saveNodeTo(ATTRIBUTE_UNUSED PyObject * self, PyObject * args)
         xmlNodeDumpOutput(buf, doc, node, 0, format, encoding);
         len = xmlOutputBufferClose(buf);
     }
-    return (PyInt_FromLong((long) len));
+    PyFile_Release(output);
+    return (PyLong_FromLong((long) len));
 }
 #endif /* LIBXML_OUTPUT_ENABLED */
 
@@ -3524,7 +3583,7 @@ PystringSet_Convert(PyObject *py_strings, xmlChar *** result)
     {
         int idx;
         for (idx=0; idx < count; ++idx) {
-            char* s = PyString_AsString
+            char* s = PyBytes_AsString
                 (is_tuple
                  ? PyTuple_GET_ITEM(py_strings, idx)
                  : PyList_GET_ITEM(py_strings, idx));
@@ -3613,8 +3672,8 @@ libxml_C14NDocDumpMemory(ATTRIBUTE_UNUSED PyObject * self,
         return NULL;
     }
     else {
-        py_retval = PyString_FromStringAndSize((const char *) doc_txt,
-                                               result);
+        py_retval = PY_IMPORT_STRING_SIZE((const char *) doc_txt,
+                                                result);
         xmlFree(doc_txt);
         return py_retval;
     }
@@ -3655,11 +3714,7 @@ libxml_C14NDocSaveTo(ATTRIBUTE_UNUSED PyObject * self,
         return NULL;
     }
 
-    if ((py_file == NULL) || (!(PyFile_Check(py_file)))) {
-        PyErr_SetString(PyExc_TypeError, "bad file.");
-        return NULL;
-    }
-    output = PyFile_AsFile(py_file);
+    output = PyFile_Get(py_file);
     if (output == NULL) {
         PyErr_SetString(PyExc_TypeError, "bad file.");
         return NULL;
@@ -3697,6 +3752,7 @@ libxml_C14NDocSaveTo(ATTRIBUTE_UNUSED PyObject * self,
         xmlFree(prefixes);
     }
 
+    PyFile_Release(output);
     len = xmlOutputBufferClose(buf);
 
     if (result < 0) {
@@ -3705,7 +3761,7 @@ libxml_C14NDocSaveTo(ATTRIBUTE_UNUSED PyObject * self,
         return NULL;
     }
     else
-        return PyInt_FromLong((long) len);
+        return PyLong_FromLong((long) len);
 }
 
 #endif
@@ -3719,7 +3775,7 @@ libxml_getObjDesc(PyObject *self ATTRIBUTE_UNUSED, PyObject *args) {
 
     if (!PyArg_ParseTuple(args, (char *)"O:getObjDesc", &obj))
         return NULL;
-    str = PyCObject_GetDesc(obj);
+    str = PyCapsule_GetPointer(obj, PyCapsule_GetName(obj));
     return Py_BuildValue((char *)"s", str);
 }
 
@@ -3819,28 +3875,59 @@ static PyMethodDef libxmlMethods[] = {
     {NULL, NULL, 0, NULL}
 };
 
+#if PY_MAJOR_VERSION >= 3
+#define INITERROR return NULL
+
+static struct PyModuleDef moduledef = {
+        PyModuleDef_HEAD_INIT,
+        "libxml2mod",
+        NULL,
+        -1,
+        libxmlMethods,
+        NULL,
+        NULL,
+        NULL,
+        NULL
+};
+
+#else
+#define INITERROR return
+
 #ifdef MERGED_MODULES
 extern void initlibxsltmod(void);
 #endif
 
-void
-initlibxml2mod(void)
-{
-    static int initialized = 0;
+#endif
 
-    if (initialized != 0)
-        return;
+#if PY_MAJOR_VERSION >= 3
+PyObject *PyInit_libxml2mod(void)
+#else
+void initlibxml2mod(void)
+#endif
+{
+    PyObject *module;
 
+#if PY_MAJOR_VERSION >= 3
+    module = PyModule_Create(&moduledef);
+#else
     /* intialize the python extension module */
-    Py_InitModule((char *) "libxml2mod", libxmlMethods);
+    module = Py_InitModule((char *) "libxml2mod", libxmlMethods);
+#endif
+    if (module == NULL)
+        INITERROR;
 
     /* initialize libxml2 */
     xmlInitParser();
+    /* TODO this probably need to be revamped for Python3 */
     libxml_xmlErrorInitialize();
 
-    initialized = 1;
-
+#if PY_MAJOR_VERSION < 3
 #ifdef MERGED_MODULES
     initlibxsltmod();
 #endif
+#endif
+
+#if PY_MAJOR_VERSION >= 3
+    return module;
+#endif
 }
diff --git a/python/libxml_wrap.h b/python/libxml_wrap.h
index ac5a6264..70248d76 100644
--- a/python/libxml_wrap.h
+++ b/python/libxml_wrap.h
@@ -150,8 +150,16 @@ typedef struct {
 } PyURI_Object;
 
 /* FILE * have their own internal representation */
+#if PY_MAJOR_VERSION >= 3
+FILE *libxml_PyFileGet(PyObject *f);
+void libxml_PyFileRelease(FILE *f);
+#define PyFile_Get(v) (((v) == Py_None) ? NULL : libxml_PyFileGet(v))
+#define PyFile_Release(f) libxml_PyFileRelease(f)
+#else
 #define PyFile_Get(v) (((v) == Py_None) ? NULL : \
 	(PyFile_Check(v) ? (PyFile_AsFile(v)) : stdout))
+#define PyFile_Release(f)
+#endif
 
 #ifdef LIBXML_SCHEMAS_ENABLED
 typedef struct {
@@ -249,3 +257,4 @@ PyObject * libxml_xmlErrorPtrWrap(xmlErrorPtr error);
 PyObject * libxml_xmlSchemaSetValidErrors(PyObject * self, PyObject * args);
 PyObject * libxml_xmlRegisterInputCallback(PyObject *self, PyObject *args);
 PyObject * libxml_xmlUnregisterInputCallback(PyObject *self, PyObject *args);
+PyObject * libxml_xmlNodeRemoveNsDef(PyObject * self, PyObject * args);
diff --git a/python/types.c b/python/types.c
index 2c9ab228..c865b65f 100644
--- a/python/types.c
+++ b/python/types.c
@@ -9,6 +9,83 @@
 #include "libxml_wrap.h"
 #include <libxml/xpathInternals.h>
 
+#if PY_MAJOR_VERSION >= 3
+#define PY_IMPORT_STRING_SIZE PyUnicode_FromStringAndSize
+#define PY_IMPORT_STRING PyUnicode_FromString
+#define PY_IMPORT_INT PyLong_FromLong
+#else
+#define PY_IMPORT_STRING_SIZE PyString_FromStringAndSize
+#define PY_IMPORT_STRING PyString_FromString
+#define PY_IMPORT_INT PyInt_FromLong
+#endif
+
+#if PY_MAJOR_VERSION >= 3
+#include <stdio.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+FILE *
+libxml_PyFileGet(PyObject *f) {
+    int fd, flags;
+    FILE *res;
+    const char *mode;
+
+    fd = PyObject_AsFileDescriptor(f);
+    if (!_PyVerify_fd(fd))
+        return(NULL);
+    /*
+     * Get the flags on the fd to understand how it was opened
+     */
+    flags = fcntl(fd, F_GETFL, 0);
+    switch (flags & O_ACCMODE) {
+        case O_RDWR:
+	    if (flags & O_APPEND)
+	        mode = "a+";
+	    else
+	        mode = "rw";
+	    break;
+        case O_RDONLY:
+	    if (flags & O_APPEND)
+	        mode = "r+";
+	    else
+	        mode = "r";
+	    break;
+	case O_WRONLY:
+	    if (flags & O_APPEND)
+	        mode = "a";
+	    else
+	        mode = "w";
+	    break;
+	default:
+	    return(NULL);
+    }
+
+    /*
+     * the FILE struct gets a new fd, so that it can be closed
+     * independently of the file descriptor given. The risk though is
+     * lack of sync. So at the python level sync must be implemented
+     * before and after a conversion took place. No way around it
+     * in the Python3 infrastructure !
+     * The duplicated fd and FILE * will be released in the subsequent
+     * call to libxml_PyFileRelease() which must be genrated accodingly
+     */
+    fd = dup(fd);
+    if (fd == -1)
+        return(NULL);
+    res = fdopen(fd, mode);
+    if (res == NULL) {
+        close(fd);
+	return(NULL);
+    }
+    return(res);
+}
+
+void libxml_PyFileRelease(FILE *f) {
+    if (f != NULL)
+        fclose(f);
+}
+#endif
+
 PyObject *
 libxml_intWrap(int val)
 {
@@ -17,7 +94,7 @@ libxml_intWrap(int val)
 #ifdef DEBUG
     printf("libxml_intWrap: val = %d\n", val);
 #endif
-    ret = PyInt_FromLong((long) val);
+    ret = PY_IMPORT_INT((long) val);
     return (ret);
 }
 
@@ -29,7 +106,7 @@ libxml_longWrap(long val)
 #ifdef DEBUG
     printf("libxml_longWrap: val = %ld\n", val);
 #endif
-    ret = PyInt_FromLong(val);
+    ret = PyLong_FromLong(val);
     return (ret);
 }
 
@@ -57,8 +134,7 @@ libxml_charPtrWrap(char *str)
         Py_INCREF(Py_None);
         return (Py_None);
     }
-    /* TODO: look at deallocation */
-    ret = PyString_FromString(str);
+    ret = PY_IMPORT_STRING(str);
     xmlFree(str);
     return (ret);
 }
@@ -75,8 +151,7 @@ libxml_charPtrConstWrap(const char *str)
         Py_INCREF(Py_None);
         return (Py_None);
     }
-    /* TODO: look at deallocation */
-    ret = PyString_FromString(str);
+    ret = PY_IMPORT_STRING(str);
     return (ret);
 }
 
@@ -92,8 +167,7 @@ libxml_xmlCharPtrWrap(xmlChar * str)
         Py_INCREF(Py_None);
         return (Py_None);
     }
-    /* TODO: look at deallocation */
-    ret = PyString_FromString((char *) str);
+    ret = PY_IMPORT_STRING((char *) str);
     xmlFree(str);
     return (ret);
 }
@@ -110,8 +184,7 @@ libxml_xmlCharPtrConstWrap(const xmlChar * str)
         Py_INCREF(Py_None);
         return (Py_None);
     }
-    /* TODO: look at deallocation */
-    ret = PyString_FromString((char *) str);
+    ret = PY_IMPORT_STRING((char *) str);
     return (ret);
 }
 
@@ -127,8 +200,7 @@ libxml_constcharPtrWrap(const char *str)
         Py_INCREF(Py_None);
         return (Py_None);
     }
-    /* TODO: look at deallocation */
-    ret = PyString_FromString(str);
+    ret = PY_IMPORT_STRING(str);
     return (ret);
 }
 
@@ -144,8 +216,7 @@ libxml_constxmlCharPtrWrap(const xmlChar * str)
         Py_INCREF(Py_None);
         return (Py_None);
     }
-    /* TODO: look at deallocation */
-    ret = PyString_FromString((char *) str);
+    ret = PY_IMPORT_STRING((char *) str);
     return (ret);
 }
 
@@ -162,9 +233,7 @@ libxml_xmlDocPtrWrap(xmlDocPtr doc)
         return (Py_None);
     }
     /* TODO: look at deallocation */
-    ret =
-        PyCObject_FromVoidPtrAndDesc((void *) doc, (char *) "xmlDocPtr",
-                                     NULL);
+    ret = PyCapsule_New((void *) doc, (char *) "xmlDocPtr", NULL);
     return (ret);
 }
 
@@ -180,9 +249,7 @@ libxml_xmlNodePtrWrap(xmlNodePtr node)
         Py_INCREF(Py_None);
         return (Py_None);
     }
-    ret =
-        PyCObject_FromVoidPtrAndDesc((void *) node, (char *) "xmlNodePtr",
-                                     NULL);
+    ret = PyCapsule_New((void *) node, (char *) "xmlNodePtr", NULL);
     return (ret);
 }
 
@@ -198,9 +265,7 @@ libxml_xmlURIPtrWrap(xmlURIPtr uri)
         Py_INCREF(Py_None);
         return (Py_None);
     }
-    ret =
-        PyCObject_FromVoidPtrAndDesc((void *) uri, (char *) "xmlURIPtr",
-                                     NULL);
+    ret = PyCapsule_New((void *) uri, (char *) "xmlURIPtr", NULL);
     return (ret);
 }
 
@@ -216,9 +281,7 @@ libxml_xmlNsPtrWrap(xmlNsPtr ns)
         Py_INCREF(Py_None);
         return (Py_None);
     }
-    ret =
-        PyCObject_FromVoidPtrAndDesc((void *) ns, (char *) "xmlNsPtr",
-                                     NULL);
+    ret = PyCapsule_New((void *) ns, (char *) "xmlNsPtr", NULL);
     return (ret);
 }
 
@@ -234,9 +297,7 @@ libxml_xmlAttrPtrWrap(xmlAttrPtr attr)
         Py_INCREF(Py_None);
         return (Py_None);
     }
-    ret =
-        PyCObject_FromVoidPtrAndDesc((void *) attr, (char *) "xmlAttrPtr",
-                                     NULL);
+    ret = PyCapsule_New((void *) attr, (char *) "xmlAttrPtr", NULL);
     return (ret);
 }
 
@@ -252,9 +313,7 @@ libxml_xmlAttributePtrWrap(xmlAttributePtr attr)
         Py_INCREF(Py_None);
         return (Py_None);
     }
-    ret =
-        PyCObject_FromVoidPtrAndDesc((void *) attr,
-                                     (char *) "xmlAttributePtr", NULL);
+    ret = PyCapsule_New((void *) attr, (char *) "xmlAttributePtr", NULL);
     return (ret);
 }
 
@@ -270,9 +329,7 @@ libxml_xmlElementPtrWrap(xmlElementPtr elem)
         Py_INCREF(Py_None);
         return (Py_None);
     }
-    ret =
-        PyCObject_FromVoidPtrAndDesc((void *) elem,
-                                     (char *) "xmlElementPtr", NULL);
+    ret = PyCapsule_New((void *) elem, (char *) "xmlElementPtr", NULL);
     return (ret);
 }
 
@@ -288,9 +345,7 @@ libxml_xmlXPathContextPtrWrap(xmlXPathContextPtr ctxt)
         Py_INCREF(Py_None);
         return (Py_None);
     }
-    ret =
-        PyCObject_FromVoidPtrAndDesc((void *) ctxt,
-                                     (char *) "xmlXPathContextPtr", NULL);
+    ret = PyCapsule_New((void *) ctxt, (char *) "xmlXPathContextPtr", NULL);
     return (ret);
 }
 
@@ -306,9 +361,7 @@ libxml_xmlXPathParserContextPtrWrap(xmlXPathParserContextPtr ctxt)
         Py_INCREF(Py_None);
         return (Py_None);
     }
-    ret = PyCObject_FromVoidPtrAndDesc((void *) ctxt,
-                                       (char *) "xmlXPathParserContextPtr",
-                                       NULL);
+    ret = PyCapsule_New((void *)ctxt, (char *)"xmlXPathParserContextPtr", NULL);
     return (ret);
 }
 
@@ -325,16 +378,13 @@ libxml_xmlParserCtxtPtrWrap(xmlParserCtxtPtr ctxt)
         return (Py_None);
     }
 
-    ret =
-        PyCObject_FromVoidPtrAndDesc((void *) ctxt,
-                                     (char *) "xmlParserCtxtPtr", NULL);
+    ret = PyCapsule_New((void *) ctxt, (char *) "xmlParserCtxtPtr", NULL);
     return (ret);
 }
 
 /**
  * libxml_xmlXPathDestructNsNode:
- * cobj: xmlNsPtr namespace node
- * desc: ignored string
+ * cobj: xmlNsPtr namespace node capsule object
  *
  * This function is called if and when a namespace node returned in
  * an XPath node set is to be destroyed. That's the only kind of
@@ -342,11 +392,11 @@ libxml_xmlParserCtxtPtrWrap(xmlParserCtxtPtr ctxt)
  * xmlDoc document, see xmlXPathNodeSetDupNs.
  */
 static void
-libxml_xmlXPathDestructNsNode(void *cobj, void *desc ATTRIBUTE_UNUSED) {
+libxml_xmlXPathDestructNsNode(PyObject *cap) {
 #ifdef DEBUG
     fprintf(stderr, "libxml_xmlXPathDestructNsNode called %p\n", cobj);
 #endif
-    xmlXPathNodeSetFreeNs((xmlNsPtr) cobj);
+    xmlXPathNodeSetFreeNs((xmlNsPtr) PyCapsule_GetPointer(cap, "xmlNsPtr"));
 }
 
 PyObject *
@@ -400,8 +450,7 @@ libxml_xmlXPathObjectPtrWrap(xmlXPathObjectPtr obj)
                 for (i = 0; i < obj->nodesetval->nodeNr; i++) {
                     node = obj->nodesetval->nodeTab[i];
                     if (node->type == XML_NAMESPACE_DECL) {
-		        PyObject *ns = 
-			    PyCObject_FromVoidPtrAndDesc((void *) node,
+		        PyObject *ns = PyCapsule_New((void *) node,
                                      (char *) "xmlNsPtr",
 				     libxml_xmlXPathDestructNsNode);
 			PyList_SetItem(ret, i, ns);
@@ -414,13 +463,13 @@ libxml_xmlXPathObjectPtrWrap(xmlXPathObjectPtr obj)
             }
             break;
         case XPATH_BOOLEAN:
-            ret = PyInt_FromLong((long) obj->boolval);
+            ret = PY_IMPORT_INT((long) obj->boolval);
             break;
         case XPATH_NUMBER:
             ret = PyFloat_FromDouble(obj->floatval);
             break;
         case XPATH_STRING:
-            ret = PyString_FromString((char *) obj->stringval);
+	    ret = PY_IMPORT_STRING((char *) obj->stringval);
             break;
         case XPATH_POINT:
         {
@@ -429,7 +478,7 @@ libxml_xmlXPathObjectPtrWrap(xmlXPathObjectPtr obj)
             PyObject *tuple;
 
             node = libxml_xmlNodePtrWrap(obj->user);
-            indexIntoNode = PyInt_FromLong((long) obj->index);
+            indexIntoNode = PY_IMPORT_INT((long) obj->index);
 
             tuple = PyTuple_New(2);
             PyTuple_SetItem(tuple, 0, node);
@@ -453,7 +502,7 @@ libxml_xmlXPathObjectPtrWrap(xmlXPathObjectPtr obj)
                 list = PyList_New(1);
 
                 node = libxml_xmlNodePtrWrap(obj->user);
-                indexIntoNode = PyInt_FromLong((long) obj->index);
+                indexIntoNode = PY_IMPORT_INT((long) obj->index);
 
                 tuple = PyTuple_New(2);
                 PyTuple_SetItem(tuple, 0, node);
@@ -471,7 +520,7 @@ libxml_xmlXPathObjectPtrWrap(xmlXPathObjectPtr obj)
                 list = PyList_New(2);
 
                 node = libxml_xmlNodePtrWrap(obj->user);
-                indexIntoNode = PyInt_FromLong((long) obj->index);
+                indexIntoNode = PY_IMPORT_INT((long) obj->index);
 
                 tuple = PyTuple_New(2);
                 PyTuple_SetItem(tuple, 0, node);
@@ -480,7 +529,7 @@ libxml_xmlXPathObjectPtrWrap(xmlXPathObjectPtr obj)
                 PyList_SetItem(list, 0, tuple);
 
                 node = libxml_xmlNodePtrWrap(obj->user2);
-                indexIntoNode = PyInt_FromLong((long) obj->index2);
+                indexIntoNode = PY_IMPORT_INT((long) obj->index2);
 
                 tuple = PyTuple_New(2);
                 PyTuple_SetItem(tuple, 0, node);
@@ -545,14 +594,14 @@ libxml_xmlXPathObjectPtrConvert(PyObject * obj)
     if (obj == NULL) {
         return (NULL);
     }
-    if PyFloat_Check
-        (obj) {
+    if PyFloat_Check (obj) {
         ret = xmlXPathNewFloat((double) PyFloat_AS_DOUBLE(obj));
-
-    } else if PyInt_Check(obj) {
-
+    } else if PyLong_Check(obj) {
+#ifdef PyLong_AS_LONG
+        ret = xmlXPathNewFloat((double) PyLong_AS_LONG(obj));
+#else
         ret = xmlXPathNewFloat((double) PyInt_AS_LONG(obj));
-
+#endif
 #ifdef PyBool_Check
     } else if PyBool_Check (obj) {
 
@@ -563,15 +612,37 @@ libxml_xmlXPathObjectPtrConvert(PyObject * obj)
           ret = xmlXPathNewBoolean(0);
         }
 #endif
-    } else if PyString_Check
-        (obj) {
+    } else if PyBytes_Check (obj) {
+        xmlChar *str;
+
+        str = xmlStrndup((const xmlChar *) PyBytes_AS_STRING(obj),
+                         PyBytes_GET_SIZE(obj));
+        ret = xmlXPathWrapString(str);
+#ifdef PyUnicode_Check
+    } else if PyUnicode_Check (obj) {
+#if PY_VERSION_HEX >= 0x03030000
         xmlChar *str;
+	const char *tmp;
+	size_t size;
 
-        str = xmlStrndup((const xmlChar *) PyString_AS_STRING(obj),
-                         PyString_GET_SIZE(obj));
+	/* tmp doesn't need to be deallocated */
+        tmp = PyUnicode_AsUTF8AndSize(obj, &size);
+        str = xmlStrndup(tmp, (int) size);
         ret = xmlXPathWrapString(str);
-    } else if PyList_Check
-        (obj) {
+#else
+        xmlChar *str = NULL;
+        PyObject *b;
+
+	b = PyUnicode_AsUTF8String(obj);
+	if (b != NULL) {
+	    str = xmlStrndup((const xmlChar *) PyBytes_AS_STRING(b),
+			     PyBytes_GET_SIZE(b));
+	    Py_DECREF(b);
+	}
+	ret = xmlXPathWrapString(str);
+#endif
+#endif
+    } else if PyList_Check (obj) {
         int i;
         PyObject *node;
         xmlNodePtr cur;
@@ -585,28 +656,18 @@ libxml_xmlXPathObjectPtrConvert(PyObject * obj)
                 continue;
 
             cur = NULL;
-            if (PyCObject_Check(node)) {
+            if (PyCapsule_CheckExact(node)) {
 #ifdef DEBUG
                 printf("Got a CObject\n");
 #endif
                 cur = PyxmlNode_Get(node);
-            } else if (PyInstance_Check(node)) {
-                PyInstanceObject *inst = (PyInstanceObject *) node;
-                PyObject *name = inst->in_class->cl_name;
-
-                if PyString_Check
-                    (name) {
-                    char *type = PyString_AS_STRING(name);
-                    PyObject *wrapper;
-
-                    if (!strcmp(type, "xmlNode")) {
-                        wrapper =
-                            PyObject_GetAttrString(node, (char *) "_o");
-                        if (wrapper != NULL) {
-                            cur = PyxmlNode_Get(wrapper);
-                        }
-                    }
-                    }
+            } else if ((PyObject_HasAttrString(node, (char *) "_o")) &&
+	               (PyObject_HasAttrString(node, (char *) "get_doc"))) {
+		PyObject *wrapper;
+
+		wrapper = PyObject_GetAttrString(node, (char *) "_o");
+		if (wrapper != NULL)
+		    cur = PyxmlNode_Get(wrapper);
             } else {
 #ifdef DEBUG
                 printf("Unknown object in Python return list\n");
@@ -629,7 +690,7 @@ PyObject *
 libxml_xmlValidCtxtPtrWrap(xmlValidCtxtPtr valid)
 {
 	PyObject *ret;
-	
+
 #ifdef DEBUG
 	printf("libxml_xmlValidCtxtPtrWrap: valid = %p\n", valid);
 #endif
@@ -639,7 +700,7 @@ libxml_xmlValidCtxtPtrWrap(xmlValidCtxtPtr valid)
 	}
 
 	ret = 
-		PyCObject_FromVoidPtrAndDesc((void *) valid,
+		PyCapsule_New((void *) valid,
 									 (char *) "xmlValidCtxtPtr", NULL);
 
 	return (ret);
@@ -658,7 +719,7 @@ libxml_xmlCatalogPtrWrap(xmlCatalogPtr catal)
         return (Py_None);
     }
     ret =
-        PyCObject_FromVoidPtrAndDesc((void *) catal,
+        PyCapsule_New((void *) catal,
                                      (char *) "xmlCatalogPtr", NULL);
     return (ret);
 }
@@ -676,7 +737,7 @@ libxml_xmlOutputBufferPtrWrap(xmlOutputBufferPtr buffer)
         return (Py_None);
     }
     ret =
-        PyCObject_FromVoidPtrAndDesc((void *) buffer,
+        PyCapsule_New((void *) buffer,
                                      (char *) "xmlOutputBufferPtr", NULL);
     return (ret);
 }
@@ -694,7 +755,7 @@ libxml_xmlParserInputBufferPtrWrap(xmlParserInputBufferPtr buffer)
         return (Py_None);
     }
     ret =
-        PyCObject_FromVoidPtrAndDesc((void *) buffer,
+        PyCapsule_New((void *) buffer,
                                      (char *) "xmlParserInputBufferPtr", NULL);
     return (ret);
 }
@@ -713,7 +774,7 @@ libxml_xmlRegexpPtrWrap(xmlRegexpPtr regexp)
         return (Py_None);
     }
     ret =
-        PyCObject_FromVoidPtrAndDesc((void *) regexp,
+        PyCapsule_New((void *) regexp,
                                      (char *) "xmlRegexpPtr", NULL);
     return (ret);
 }
@@ -733,7 +794,7 @@ libxml_xmlTextReaderPtrWrap(xmlTextReaderPtr reader)
         return (Py_None);
     }
     ret =
-        PyCObject_FromVoidPtrAndDesc((void *) reader,
+        PyCapsule_New((void *) reader,
                                      (char *) "xmlTextReaderPtr", NULL);
     return (ret);
 }
@@ -751,7 +812,7 @@ libxml_xmlTextReaderLocatorPtrWrap(xmlTextReaderLocatorPtr locator)
         return (Py_None);
     }
     ret =
-        PyCObject_FromVoidPtrAndDesc((void *) locator,
+        PyCapsule_New((void *) locator,
                                      (char *) "xmlTextReaderLocatorPtr", NULL);
     return (ret);
 }
@@ -771,7 +832,7 @@ libxml_xmlRelaxNGPtrWrap(xmlRelaxNGPtr ctxt)
         return (Py_None);
     }
     ret =
-        PyCObject_FromVoidPtrAndDesc((void *) ctxt,
+        PyCapsule_New((void *) ctxt,
                                      (char *) "xmlRelaxNGPtr", NULL);
     return (ret);
 }
@@ -789,7 +850,7 @@ libxml_xmlRelaxNGParserCtxtPtrWrap(xmlRelaxNGParserCtxtPtr ctxt)
         return (Py_None);
     }
     ret =
-        PyCObject_FromVoidPtrAndDesc((void *) ctxt,
+        PyCapsule_New((void *) ctxt,
                                      (char *) "xmlRelaxNGParserCtxtPtr", NULL);
     return (ret);
 }
@@ -806,7 +867,7 @@ libxml_xmlRelaxNGValidCtxtPtrWrap(xmlRelaxNGValidCtxtPtr valid)
         return (Py_None);
     }
     ret =
-        PyCObject_FromVoidPtrAndDesc((void *) valid,
+        PyCapsule_New((void *) valid,
                                      (char *) "xmlRelaxNGValidCtxtPtr", NULL);
     return (ret);
 }
@@ -824,7 +885,7 @@ libxml_xmlSchemaPtrWrap(xmlSchemaPtr ctxt)
 		return (Py_None);
 	}
 	ret =
-		PyCObject_FromVoidPtrAndDesc((void *) ctxt,
+		PyCapsule_New((void *) ctxt,
 									 (char *) "xmlSchemaPtr", NULL);
 	return (ret);
 }
@@ -842,7 +903,7 @@ libxml_xmlSchemaParserCtxtPtrWrap(xmlSchemaParserCtxtPtr ctxt)
 		return (Py_None);
 	}
 	ret = 
-		PyCObject_FromVoidPtrAndDesc((void *) ctxt,
+		PyCapsule_New((void *) ctxt,
 									 (char *) "xmlSchemaParserCtxtPtr", NULL);
 
 	return (ret);
@@ -862,7 +923,7 @@ libxml_xmlSchemaValidCtxtPtrWrap(xmlSchemaValidCtxtPtr valid)
 	}
 
 	ret = 
-		PyCObject_FromVoidPtrAndDesc((void *) valid,
+		PyCapsule_New((void *) valid,
 									 (char *) "xmlSchemaValidCtxtPtr", NULL);
 
 	return (ret);
@@ -881,8 +942,6 @@ libxml_xmlErrorPtrWrap(xmlErrorPtr error)
         Py_INCREF(Py_None);
         return (Py_None);
     }
-    ret =
-        PyCObject_FromVoidPtrAndDesc((void *) error,
-                                     (char *) "xmlErrorPtr", NULL);
+    ret = PyCapsule_New((void *) error, (char *) "xmlErrorPtr", NULL);
     return (ret);
 }