Properly integrate the compilation of _elementpath.py and some lxml.html modules into the build process and exclude it in PyPy.

Also finally rename lxml.etree.pyx and lxml.objectify.pyx to their correct module names and provide some legacy integration aid by keeping the original API header file names.

12 files changed, 83 insertions, 37 deletions

diff --git a/.gitignore b/.gitignore
index ea137ead..d10849a0 100644
--- a/.gitignore
+++ b/.gitignore
@@ -4,6 +4,9 @@
 build
 dist
 wheelhouse
+wheels
+venvs
+venv
 doc/html
 libs
 *.egg-info
@@ -15,7 +18,12 @@ MANIFEST
 
 src/lxml/includes/lxml-version.h
 src/lxml/*.html
+src/lxml/html/*.c
+src/lxml/etree.c
+src/lxml/etree.h
+src/lxml/etree_api.h
 src/lxml/lxml.etree.c
 src/lxml/lxml.etree.h
 src/lxml/lxml.etree_api.h
+src/lxml/objectify.c
 src/lxml/lxml.objectify.c
diff --git a/.hgignore b/.hgignore
index c30692ae..103fb6ed 100644
--- a/.hgignore
+++ b/.hgignore
@@ -6,14 +6,22 @@ __pycache__
 
 src/lxml/includes/lxml-version.h
 src/lxml/*.html
+src/lxml/html/*.c
+src/lxml/etree.c
+src/lxml/etree.h
+src/lxml/etree_api.h
 src/lxml/lxml.etree.c
 src/lxml/lxml.etree.h
 src/lxml/lxml.etree_api.h
+src/lxml/objectify.c
 src/lxml/lxml.objectify.c
 
 build/
 dist/
 wheelhouse/
+wheels/
+venvs/
+venv/
 doc/html/
 cython_debug/
 .idea/
diff --git a/MANIFEST.in b/MANIFEST.in
index 2ad2039e..d1a2965e 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -5,9 +5,9 @@ include update-error-constants.py
 include MANIFEST.in Makefile version.txt requirements.txt
 include CHANGES.txt CREDITS.txt INSTALL.txt LICENSES.txt README.rst TODO.txt
 include tools/*.py tools/manylinux/*.sh
+include src/lxml/*.c src/lxml/html/*.c
 recursive-include src *.pyx *.pxd *.pxi *.py
-recursive-include src/lxml lxml.etree.c lxml.objectify.c
-recursive-include src/lxml lxml.etree.h lxml.etree_api.h etree_defs.h lxml_endian.h
+recursive-include src/lxml lxml.etree.h lxml.etree_api.h etree.h etree_api.h etree_defs.h lxml_endian.h
 recursive-include src/lxml/isoschematron *.rng *.xsl *.txt
 recursive-include src/lxml/tests *.rng *.xslt *.xml *.dtd *.xsd *.sch *.html
 recursive-include src/lxml/html/tests *.data *.txt
diff --git a/setup.py b/setup.py
index 0118a547..942fcf2b 100644
--- a/setup.py
+++ b/setup.py
@@ -74,6 +74,8 @@ extra_options.update(setupinfo.extra_setup_args())
 
 extra_options['package_data'] = {
     'lxml': [
+        'etree.h',
+        'etree_api.h',
         'lxml.etree.h',
         'lxml.etree_api.h',
     ],
diff --git a/setupinfo.py b/setupinfo.py
index af785211..50d3e985 100644
--- a/setupinfo.py
+++ b/setupinfo.py
@@ -1,4 +1,7 @@
-import sys, os, os.path
+import sys
+import io
+import os
+import os.path
 from distutils.core import Extension
 from distutils.errors import CompileError, DistutilsOptionError
 from distutils.command.build_ext import build_ext as _build_ext
@@ -11,10 +14,16 @@ except ImportError:
     CYTHON_INSTALLED = False
 
 EXT_MODULES = ["lxml.etree", "lxml.objectify"]
-COMPILED_MODULES = ["_elementpath.py"]
+COMPILED_MODULES = ["lxml._elementpath", "lxml.html.diff", "lxml.html.clean"]
+HEADER_FILES = ['etree.h', 'etree_api.h']
 
-PACKAGE_PATH = "src%slxml%s" % (os.path.sep, os.path.sep)
-INCLUDE_PACKAGE_PATH = PACKAGE_PATH + 'includes'
+if hasattr(sys, 'pypy_version_info') or (
+        getattr(sys, 'implementation', None) and sys.implementation.name != 'cpython'):
+    # disable Cython compilation of Python modules in PyPy and other non-CPythons
+    del COMPILED_MODULES[:]
+
+SOURCE_PATH = "src"
+INCLUDE_PACKAGE_PATH = os.path.join(SOURCE_PATH, 'lxml', 'includes')
 
 if sys.version_info[0] >= 3:
     _system_encoding = sys.getdefaultencoding()
@@ -65,14 +74,14 @@ def ext_modules(static_include_dirs, static_library_dirs,
                 zlib_version=OPTION_ZLIB_VERSION,
                 multicore=OPTION_MULTICORE)
 
-    modules = EXT_MODULES
+    modules = EXT_MODULES + COMPILED_MODULES
     if OPTION_WITHOUT_OBJECTIFY:
         modules = [entry for entry in modules if 'objectify' not in entry]
 
-    c_files_exist = [os.path.exists('%s%s.c' % (PACKAGE_PATH, module))
-                     for module in modules]
+    module_files = list(os.path.join(SOURCE_PATH, *module.split('.')) for module in modules)
+    c_files_exist = [os.path.exists(module + '.c') for module in module_files]
 
-    source_extension = ".pyx"
+    use_cython = True
     if CYTHON_INSTALLED and (OPTION_WITH_CYTHON or not all(c_files_exist)):
         print("Building with Cython %s." % Cython.Compiler.Version.version)
         # generate module cleanup code
@@ -80,19 +89,18 @@ def ext_modules(static_include_dirs, static_library_dirs,
         Options.generate_cleanup_code = 3
         Options.clear_to_none = False
     elif not OPTION_WITHOUT_CYTHON and not all(c_files_exist):
-        for exists, module in zip(c_files_exist, modules):
+        for exists, module in zip(c_files_exist, module_files):
             if not exists:
                 raise RuntimeError(
-                    "ERROR: Trying to build without Cython, but pre-generated '%s%s.c' "
-                    "is not available (pass --without-cython to ignore this error)." % (
-                        PACKAGE_PATH, module))
+                    "ERROR: Trying to build without Cython, but pre-generated '%s.c' "
+                    "is not available (pass --without-cython to ignore this error)." % module)
     else:
         if not all(c_files_exist):
-            for exists, module in zip(c_files_exist, modules):
+            for exists, module in zip(c_files_exist, module_files):
                 if not exists:
                     print("WARNING: Trying to build without Cython, but pre-generated "
-                          "'%s%s.c' is not available." % (PACKAGE_PATH, module))
-        source_extension = ".c"
+                          "'%s.c' is not available." % module)
+        use_cython = False
         print("Building without Cython.")
 
     lib_versions = get_library_versions()
@@ -110,7 +118,10 @@ def ext_modules(static_include_dirs, static_library_dirs,
 
     base_dir = get_base_dir()
     _include_dirs = _prefer_reldirs(
-        base_dir, include_dirs(static_include_dirs) + [INCLUDE_PACKAGE_PATH])
+        base_dir, include_dirs(static_include_dirs) + [
+            SOURCE_PATH,
+            INCLUDE_PACKAGE_PATH,
+        ])
     _library_dirs = _prefer_reldirs(base_dir, library_dirs(static_library_dirs))
     _cflags = cflags(static_cflags)
     _define_macros = define_macros()
@@ -135,13 +146,16 @@ def ext_modules(static_include_dirs, static_library_dirs,
         from Cython.Compiler import Errors
         Errors.LEVEL = 0
 
-    cythonize_options = {}
+    cythonize_directives = {
+        'binding': True,
+    }
     if OPTION_WITH_COVERAGE:
-        cythonize_options['compiler_directives'] = {'linetrace': True}
+        cythonize_directives['linetrace'] = True
 
     result = []
-    for module in modules:
-        main_module_source = PACKAGE_PATH + module + source_extension
+    for module, src_file in zip(modules, module_files):
+        main_module_source = src_file + (
+            '.c' if not use_cython else '.py' if module in COMPILED_MODULES else '.pyx')
         result.append(
             Extension(
                 module,
@@ -159,21 +173,35 @@ def ext_modules(static_include_dirs, static_library_dirs,
         for ext in result:
             ext.cython_gdb = True
 
-    if CYTHON_INSTALLED and source_extension == '.pyx':
+    if CYTHON_INSTALLED and use_cython:
         # build .c files right now and convert Extension() objects
         from Cython.Build import cythonize
-        result = cythonize(
-            result + [PACKAGE_PATH + module for module in COMPILED_MODULES],
-            **cythonize_options)
+        result = cythonize(result, compiler_directives=cythonize_directives)
+
+    # for backwards compatibility reasons, provide "etree[_api].h" also as "lxml.etree[_api].h"
+    for header_filename in HEADER_FILES:
+        src_file = os.path.join(SOURCE_PATH, 'lxml', header_filename)
+        dst_file = os.path.join(SOURCE_PATH, 'lxml', 'lxml.' + header_filename)
+        if not os.path.exists(src_file):
+            continue
+        if os.path.exists(dst_file) and os.path.getmtime(dst_file) >= os.path.getmtime(src_file):
+            continue
+
+        with io.open(src_file, 'r', encoding='iso8859-1') as f:
+            content = f.read()
+        for filename in HEADER_FILES:
+            content = content.replace('"%s"' % filename, '"lxml.%s"' % filename)
+        with io.open(dst_file, 'w', encoding='iso8859-1') as f:
+            f.write(content)
 
     return result
 
 
 def find_dependencies(module):
-    if not CYTHON_INSTALLED:
+    if not CYTHON_INSTALLED or 'lxml.html' in module:
         return []
     base_dir = get_base_dir()
-    package_dir = os.path.join(base_dir, PACKAGE_PATH)
+    package_dir = os.path.join(base_dir, SOURCE_PATH, 'lxml')
     includes_dir = os.path.join(base_dir, INCLUDE_PACKAGE_PATH)
 
     pxd_files = [
@@ -182,9 +210,9 @@ def find_dependencies(module):
         if filename.endswith('.pxd')
     ]
 
-    if 'etree' in module:
+    if module == 'lxml.etree':
         pxi_files = [
-            os.path.join(PACKAGE_PATH, filename)
+            os.path.join(SOURCE_PATH, 'lxml', filename)
             for filename in os.listdir(package_dir)
             if filename.endswith('.pxi') and 'objectpath' not in filename
         ]
@@ -192,10 +220,10 @@ def find_dependencies(module):
             filename for filename in pxd_files
             if 'etreepublic' not in filename
         ]
-    elif 'objectify' in module:
-        pxi_files = [os.path.join(PACKAGE_PATH, 'objectpath.pxi')]
+    elif module == 'lxml.objectify':
+        pxi_files = [os.path.join(SOURCE_PATH, 'lxml', 'objectpath.pxi')]
     else:
-        pxi_files = []
+        pxi_files = pxd_files = []
 
     return pxd_files + pxi_files
 
diff --git a/src/lxml/__init__.pxd b/src/lxml/__init__.pxd
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/src/lxml/__init__.pxd
diff --git a/src/lxml/cvarargs.pxd b/src/lxml/cvarargs.pxd
index 824c1f0c..5fe9b89c 100644
--- a/src/lxml/cvarargs.pxd
+++ b/src/lxml/cvarargs.pxd
@@ -3,6 +3,6 @@ cdef extern from "stdarg.h":
     void va_start(va_list ap, void *last) nogil
     void va_end(va_list ap) nogil
 
-cdef extern from "etree_defs.h":
+cdef extern from "includes/etree_defs.h":
     cdef int va_int(va_list ap) nogil
     cdef char *va_charptr(va_list ap) nogil
diff --git a/src/lxml/lxml.etree.pyx b/src/lxml/etree.pyx
index 9fd555dd..49f7b857 100644
--- a/src/lxml/lxml.etree.pyx
+++ b/src/lxml/etree.pyx
@@ -1600,7 +1600,7 @@ cdef public class _Element [ type LxmlElementType, object LxmlElement ]:
         return CSSSelector(expr, translator=translator)(self)
 
 
-cdef extern from "etree_defs.h":
+cdef extern from "includes/etree_defs.h":
     # macro call to 't->tp_new()' for fast instantiation
     cdef object NEW_ELEMENT "PY_NEW" (object t)
 
diff --git a/src/lxml/includes/__init__.pxd b/src/lxml/includes/__init__.pxd
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/src/lxml/includes/__init__.pxd
diff --git a/src/lxml/includes/etreepublic.pxd b/src/lxml/includes/etreepublic.pxd
index 665cf788..94fe2e8d 100644
--- a/src/lxml/includes/etreepublic.pxd
+++ b/src/lxml/includes/etreepublic.pxd
@@ -19,7 +19,7 @@ cdef extern from "etree_defs.h":
                                           int start_node_inclusive) nogil
     cdef void END_FOR_EACH_ELEMENT_FROM(tree.xmlNode* start_node) nogil
 
-cdef extern from "lxml.etree_api.h":
+cdef extern from "etree_api.h":
 
     # first function to call!
     cdef int import_lxml__etree() except -1
diff --git a/src/lxml/lxml.objectify.pyx b/src/lxml/objectify.pyx
index a4cf19d0..a4cf19d0 100644
--- a/src/lxml/lxml.objectify.pyx
+++ b/src/lxml/objectify.pyx
diff --git a/src/lxml/python.pxd b/src/lxml/python.pxd
index 6a4d3ec6..5eb9271c 100644
--- a/src/lxml/python.pxd
+++ b/src/lxml/python.pxd
@@ -115,7 +115,7 @@ cdef extern from "pythread.h":
         WAIT_LOCK
         NOWAIT_LOCK
 
-cdef extern from "etree_defs.h": # redefines some functions as macros
+cdef extern from "includes/etree_defs.h": # redefines some functions as macros
     cdef void* lxml_malloc(size_t count, size_t item_size)
     cdef void* lxml_realloc(void* mem, size_t count, size_t item_size)
     cdef void lxml_free(void* mem)