diff options
Diffstat (limited to 'Lib/packaging/command/build_ext.py')
| -rw-r--r-- | Lib/packaging/command/build_ext.py | 641 |
1 files changed, 641 insertions, 0 deletions
diff --git a/Lib/packaging/command/build_ext.py b/Lib/packaging/command/build_ext.py new file mode 100644 index 0000000000..4c85822a92 --- /dev/null +++ b/Lib/packaging/command/build_ext.py @@ -0,0 +1,641 @@ +"""Build extension modules.""" + +import os +import re +import sys +import site +import sysconfig + +from packaging.util import get_platform +from packaging.command.cmd import Command +from packaging.errors import (CCompilerError, CompileError, PackagingError, + PackagingPlatformError, PackagingSetupError) +from packaging.compiler import customize_compiler, show_compilers +from packaging.util import newer_group +from packaging.compiler.extension import Extension +from packaging import logger + +if os.name == 'nt': + from packaging.compiler.msvccompiler import get_build_version + MSVC_VERSION = int(get_build_version()) + +# An extension name is just a dot-separated list of Python NAMEs (ie. +# the same as a fully-qualified module name). +extension_name_re = re.compile \ + (r'^[a-zA-Z_][a-zA-Z_0-9]*(\.[a-zA-Z_][a-zA-Z_0-9]*)*$') + + +class build_ext(Command): + + description = "build C/C++ extension modules (compile/link to build directory)" + + # XXX thoughts on how to deal with complex command-line options like + # these, i.e. how to make it so fancy_getopt can suck them off the + # command line and turn them into the appropriate + # lists of tuples of what-have-you. + # - each command needs a callback to process its command-line options + # - Command.__init__() needs access to its share of the whole + # command line (must ultimately come from + # Distribution.parse_command_line()) + # - it then calls the current command class' option-parsing + # callback to deal with weird options like -D, which have to + # parse the option text and churn out some custom data + # structure + # - that data structure (in this case, a list of 2-tuples) + # will then be present in the command object by the time + # we get to finalize_options() (i.e. the constructor + # takes care of both command-line and client options + # in between initialize_options() and finalize_options()) + + sep_by = " (separated by '%s')" % os.pathsep + user_options = [ + ('build-lib=', 'b', + "directory for compiled extension modules"), + ('build-temp=', 't', + "directory for temporary files (build by-products)"), + ('plat-name=', 'p', + "platform name to cross-compile for, if supported " + "(default: %s)" % get_platform()), + ('inplace', 'i', + "ignore build-lib and put compiled extensions into the source " + + "directory alongside your pure Python modules"), + ('user', None, + "add user include, library and rpath"), + ('include-dirs=', 'I', + "list of directories to search for header files" + sep_by), + ('define=', 'D', + "C preprocessor macros to define"), + ('undef=', 'U', + "C preprocessor macros to undefine"), + ('libraries=', 'l', + "external C libraries to link with"), + ('library-dirs=', 'L', + "directories to search for external C libraries" + sep_by), + ('rpath=', 'R', + "directories to search for shared C libraries at runtime"), + ('link-objects=', 'O', + "extra explicit link objects to include in the link"), + ('debug', 'g', + "compile/link with debugging information"), + ('force', 'f', + "forcibly build everything (ignore file timestamps)"), + ('compiler=', 'c', + "specify the compiler type"), + ('swig-opts=', None, + "list of SWIG command-line options"), + ('swig=', None, + "path to the SWIG executable"), + ] + + boolean_options = ['inplace', 'debug', 'force', 'user'] + + + help_options = [ + ('help-compiler', None, + "list available compilers", show_compilers), + ] + + def initialize_options(self): + self.extensions = None + self.build_lib = None + self.plat_name = None + self.build_temp = None + self.inplace = False + self.package = None + + self.include_dirs = None + self.define = None + self.undef = None + self.libraries = None + self.library_dirs = None + self.rpath = None + self.link_objects = None + self.debug = None + self.force = None + self.compiler = None + self.swig = None + self.swig_opts = None + self.user = None + + def finalize_options(self): + self.set_undefined_options('build', + 'build_lib', 'build_temp', 'compiler', + 'debug', 'force', 'plat_name') + + if self.package is None: + self.package = self.distribution.ext_package + + # Ensure that the list of extensions is valid, i.e. it is a list of + # Extension objects. + self.extensions = self.distribution.ext_modules + if self.extensions: + if not isinstance(self.extensions, (list, tuple)): + type_name = (self.extensions is None and 'None' + or type(self.extensions).__name__) + raise PackagingSetupError( + "'ext_modules' must be a sequence of Extension instances," + " not %s" % (type_name,)) + for i, ext in enumerate(self.extensions): + if isinstance(ext, Extension): + continue # OK! (assume type-checking done + # by Extension constructor) + type_name = (ext is None and 'None' or type(ext).__name__) + raise PackagingSetupError( + "'ext_modules' item %d must be an Extension instance," + " not %s" % (i, type_name)) + + # Make sure Python's include directories (for Python.h, pyconfig.h, + # etc.) are in the include search path. + py_include = sysconfig.get_path('include') + plat_py_include = sysconfig.get_path('platinclude') + if self.include_dirs is None: + self.include_dirs = self.distribution.include_dirs or [] + if isinstance(self.include_dirs, str): + self.include_dirs = self.include_dirs.split(os.pathsep) + + # Put the Python "system" include dir at the end, so that + # any local include dirs take precedence. + self.include_dirs.append(py_include) + if plat_py_include != py_include: + self.include_dirs.append(plat_py_include) + + self.ensure_string_list('libraries') + + # Life is easier if we're not forever checking for None, so + # simplify these options to empty lists if unset + if self.libraries is None: + self.libraries = [] + if self.library_dirs is None: + self.library_dirs = [] + elif isinstance(self.library_dirs, str): + self.library_dirs = self.library_dirs.split(os.pathsep) + + if self.rpath is None: + self.rpath = [] + elif isinstance(self.rpath, str): + self.rpath = self.rpath.split(os.pathsep) + + # for extensions under windows use different directories + # for Release and Debug builds. + # also Python's library directory must be appended to library_dirs + if os.name == 'nt': + # the 'libs' directory is for binary installs - we assume that + # must be the *native* platform. But we don't really support + # cross-compiling via a binary install anyway, so we let it go. + self.library_dirs.append(os.path.join(sys.exec_prefix, 'libs')) + if self.debug: + self.build_temp = os.path.join(self.build_temp, "Debug") + else: + self.build_temp = os.path.join(self.build_temp, "Release") + + # Append the source distribution include and library directories, + # this allows distutils on windows to work in the source tree + self.include_dirs.append(os.path.join(sys.exec_prefix, 'PC')) + if MSVC_VERSION == 9: + # Use the .lib files for the correct architecture + if self.plat_name == 'win32': + suffix = '' + else: + # win-amd64 or win-ia64 + suffix = self.plat_name[4:] + new_lib = os.path.join(sys.exec_prefix, 'PCbuild') + if suffix: + new_lib = os.path.join(new_lib, suffix) + self.library_dirs.append(new_lib) + + elif MSVC_VERSION == 8: + self.library_dirs.append(os.path.join(sys.exec_prefix, + 'PC', 'VS8.0')) + elif MSVC_VERSION == 7: + self.library_dirs.append(os.path.join(sys.exec_prefix, + 'PC', 'VS7.1')) + else: + self.library_dirs.append(os.path.join(sys.exec_prefix, + 'PC', 'VC6')) + + # OS/2 (EMX) doesn't support Debug vs Release builds, but has the + # import libraries in its "Config" subdirectory + if os.name == 'os2': + self.library_dirs.append(os.path.join(sys.exec_prefix, 'Config')) + + # for extensions under Cygwin and AtheOS Python's library directory must be + # appended to library_dirs + if sys.platform[:6] == 'cygwin' or sys.platform[:6] == 'atheos': + if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): + # building third party extensions + self.library_dirs.append(os.path.join(sys.prefix, "lib", + "python" + sysconfig.get_python_version(), + "config")) + else: + # building python standard extensions + self.library_dirs.append(os.curdir) + + # for extensions under Linux or Solaris with a shared Python library, + # Python's library directory must be appended to library_dirs + sysconfig.get_config_var('Py_ENABLE_SHARED') + if (sys.platform.startswith(('linux', 'gnu', 'sunos')) + and sysconfig.get_config_var('Py_ENABLE_SHARED')): + if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): + # building third party extensions + self.library_dirs.append(sysconfig.get_config_var('LIBDIR')) + else: + # building python standard extensions + self.library_dirs.append(os.curdir) + + # The argument parsing will result in self.define being a string, but + # it has to be a list of 2-tuples. All the preprocessor symbols + # specified by the 'define' option will be set to '1'. Multiple + # symbols can be separated with commas. + + if self.define: + defines = self.define.split(',') + self.define = [(symbol, '1') for symbol in defines] + + # The option for macros to undefine is also a string from the + # option parsing, but has to be a list. Multiple symbols can also + # be separated with commas here. + if self.undef: + self.undef = self.undef.split(',') + + if self.swig_opts is None: + self.swig_opts = [] + else: + self.swig_opts = self.swig_opts.split(' ') + + # Finally add the user include and library directories if requested + if self.user: + user_include = os.path.join(site.USER_BASE, "include") + user_lib = os.path.join(site.USER_BASE, "lib") + if os.path.isdir(user_include): + self.include_dirs.append(user_include) + if os.path.isdir(user_lib): + self.library_dirs.append(user_lib) + self.rpath.append(user_lib) + + def run(self): + from packaging.compiler import new_compiler + + if not self.extensions: + return + + # If we were asked to build any C/C++ libraries, make sure that the + # directory where we put them is in the library search path for + # linking extensions. + if self.distribution.has_c_libraries(): + build_clib = self.get_finalized_command('build_clib') + self.libraries.extend(build_clib.get_library_names() or []) + self.library_dirs.append(build_clib.build_clib) + + # Setup the CCompiler object that we'll use to do all the + # compiling and linking + self.compiler_obj = new_compiler(compiler=self.compiler, + dry_run=self.dry_run, + force=self.force) + + customize_compiler(self.compiler_obj) + # If we are cross-compiling, init the compiler now (if we are not + # cross-compiling, init would not hurt, but people may rely on + # late initialization of compiler even if they shouldn't...) + if os.name == 'nt' and self.plat_name != get_platform(): + self.compiler_obj.initialize(self.plat_name) + + # And make sure that any compile/link-related options (which might + # come from the command line or from the setup script) are set in + # that CCompiler object -- that way, they automatically apply to + # all compiling and linking done here. + if self.include_dirs is not None: + self.compiler_obj.set_include_dirs(self.include_dirs) + if self.define is not None: + # 'define' option is a list of (name,value) tuples + for name, value in self.define: + self.compiler_obj.define_macro(name, value) + if self.undef is not None: + for macro in self.undef: + self.compiler_obj.undefine_macro(macro) + if self.libraries is not None: + self.compiler_obj.set_libraries(self.libraries) + if self.library_dirs is not None: + self.compiler_obj.set_library_dirs(self.library_dirs) + if self.rpath is not None: + self.compiler_obj.set_runtime_library_dirs(self.rpath) + if self.link_objects is not None: + self.compiler_obj.set_link_objects(self.link_objects) + + # Now actually compile and link everything. + self.build_extensions() + + def get_source_files(self): + filenames = [] + + # Wouldn't it be neat if we knew the names of header files too... + for ext in self.extensions: + filenames.extend(ext.sources) + + return filenames + + def get_outputs(self): + # And build the list of output (built) filenames. Note that this + # ignores the 'inplace' flag, and assumes everything goes in the + # "build" tree. + outputs = [] + for ext in self.extensions: + outputs.append(self.get_ext_fullpath(ext.name)) + return outputs + + def build_extensions(self): + for ext in self.extensions: + try: + self.build_extension(ext) + except (CCompilerError, PackagingError, CompileError) as e: + if not ext.optional: + raise + logger.warning('%s: building extension %r failed: %s', + self.get_command_name(), ext.name, e) + + def build_extension(self, ext): + sources = ext.sources + if sources is None or not isinstance(sources, (list, tuple)): + raise PackagingSetupError(("in 'ext_modules' option (extension '%s'), " + + "'sources' must be present and must be " + + "a list of source filenames") % ext.name) + sources = list(sources) + + ext_path = self.get_ext_fullpath(ext.name) + depends = sources + ext.depends + if not (self.force or newer_group(depends, ext_path, 'newer')): + logger.debug("skipping '%s' extension (up-to-date)", ext.name) + return + else: + logger.info("building '%s' extension", ext.name) + + # First, scan the sources for SWIG definition files (.i), run + # SWIG on 'em to create .c files, and modify the sources list + # accordingly. + sources = self.swig_sources(sources, ext) + + # Next, compile the source code to object files. + + # XXX not honouring 'define_macros' or 'undef_macros' -- the + # CCompiler API needs to change to accommodate this, and I + # want to do one thing at a time! + + # Two possible sources for extra compiler arguments: + # - 'extra_compile_args' in Extension object + # - CFLAGS environment variable (not particularly + # elegant, but people seem to expect it and I + # guess it's useful) + # The environment variable should take precedence, and + # any sensible compiler will give precedence to later + # command-line args. Hence we combine them in order: + extra_args = ext.extra_compile_args or [] + + macros = ext.define_macros[:] + for undef in ext.undef_macros: + macros.append((undef,)) + + objects = self.compiler_obj.compile(sources, + output_dir=self.build_temp, + macros=macros, + include_dirs=ext.include_dirs, + debug=self.debug, + extra_postargs=extra_args, + depends=ext.depends) + + # XXX -- this is a Vile HACK! + # + # The setup.py script for Python on Unix needs to be able to + # get this list so it can perform all the clean up needed to + # avoid keeping object files around when cleaning out a failed + # build of an extension module. Since Packaging does not + # track dependencies, we have to get rid of intermediates to + # ensure all the intermediates will be properly re-built. + # + self._built_objects = objects[:] + + # Now link the object files together into a "shared object" -- + # of course, first we have to figure out all the other things + # that go into the mix. + if ext.extra_objects: + objects.extend(ext.extra_objects) + extra_args = ext.extra_link_args or [] + + # Detect target language, if not provided + language = ext.language or self.compiler_obj.detect_language(sources) + + self.compiler_obj.link_shared_object( + objects, ext_path, + libraries=self.get_libraries(ext), + library_dirs=ext.library_dirs, + runtime_library_dirs=ext.runtime_library_dirs, + extra_postargs=extra_args, + export_symbols=self.get_export_symbols(ext), + debug=self.debug, + build_temp=self.build_temp, + target_lang=language) + + + def swig_sources(self, sources, extension): + """Walk the list of source files in 'sources', looking for SWIG + interface (.i) files. Run SWIG on all that are found, and + return a modified 'sources' list with SWIG source files replaced + by the generated C (or C++) files. + """ + new_sources = [] + swig_sources = [] + swig_targets = {} + + # XXX this drops generated C/C++ files into the source tree, which + # is fine for developers who want to distribute the generated + # source -- but there should be an option to put SWIG output in + # the temp dir. + + if ('-c++' in self.swig_opts or '-c++' in extension.swig_opts): + target_ext = '.cpp' + else: + target_ext = '.c' + + for source in sources: + base, ext = os.path.splitext(source) + if ext == ".i": # SWIG interface file + new_sources.append(base + '_wrap' + target_ext) + swig_sources.append(source) + swig_targets[source] = new_sources[-1] + else: + new_sources.append(source) + + if not swig_sources: + return new_sources + + swig = self.swig or self.find_swig() + swig_cmd = [swig, "-python"] + swig_cmd.extend(self.swig_opts) + + # Do not override commandline arguments + if not self.swig_opts: + for o in extension.swig_opts: + swig_cmd.append(o) + + for source in swig_sources: + target = swig_targets[source] + logger.info("swigging %s to %s", source, target) + self.spawn(swig_cmd + ["-o", target, source]) + + return new_sources + + def find_swig(self): + """Return the name of the SWIG executable. On Unix, this is + just "swig" -- it should be in the PATH. Tries a bit harder on + Windows. + """ + + if os.name == "posix": + return "swig" + elif os.name == "nt": + + # Look for SWIG in its standard installation directory on + # Windows (or so I presume!). If we find it there, great; + # if not, act like Unix and assume it's in the PATH. + for vers in ("1.3", "1.2", "1.1"): + fn = os.path.join("c:\\swig%s" % vers, "swig.exe") + if os.path.isfile(fn): + return fn + else: + return "swig.exe" + + elif os.name == "os2": + # assume swig available in the PATH. + return "swig.exe" + + else: + raise PackagingPlatformError(("I don't know how to find (much less run) SWIG " + "on platform '%s'") % os.name) + + # -- Name generators ----------------------------------------------- + # (extension names, filenames, whatever) + def get_ext_fullpath(self, ext_name): + """Returns the path of the filename for a given extension. + + The file is located in `build_lib` or directly in the package + (inplace option). + """ + fullname = self.get_ext_fullname(ext_name) + modpath = fullname.split('.') + filename = self.get_ext_filename(modpath[-1]) + + if not self.inplace: + # no further work needed + # returning : + # build_dir/package/path/filename + filename = os.path.join(*modpath[:-1]+[filename]) + return os.path.join(self.build_lib, filename) + + # the inplace option requires to find the package directory + # using the build_py command for that + package = '.'.join(modpath[0:-1]) + build_py = self.get_finalized_command('build_py') + package_dir = os.path.abspath(build_py.get_package_dir(package)) + + # returning + # package_dir/filename + return os.path.join(package_dir, filename) + + def get_ext_fullname(self, ext_name): + """Returns the fullname of a given extension name. + + Adds the `package.` prefix""" + if self.package is None: + return ext_name + else: + return self.package + '.' + ext_name + + def get_ext_filename(self, ext_name): + r"""Convert the name of an extension (eg. "foo.bar") into the name + of the file from which it will be loaded (eg. "foo/bar.so", or + "foo\bar.pyd"). + """ + ext_path = ext_name.split('.') + # OS/2 has an 8 character module (extension) limit :-( + if os.name == "os2": + ext_path[len(ext_path) - 1] = ext_path[len(ext_path) - 1][:8] + # extensions in debug_mode are named 'module_d.pyd' under windows + so_ext = sysconfig.get_config_var('SO') + if os.name == 'nt' and self.debug: + return os.path.join(*ext_path) + '_d' + so_ext + return os.path.join(*ext_path) + so_ext + + def get_export_symbols(self, ext): + """Return the list of symbols that a shared extension has to + export. This either uses 'ext.export_symbols' or, if it's not + provided, "init" + module_name. Only relevant on Windows, where + the .pyd file (DLL) must export the module "init" function. + """ + initfunc_name = "PyInit_" + ext.name.split('.')[-1] + if initfunc_name not in ext.export_symbols: + ext.export_symbols.append(initfunc_name) + return ext.export_symbols + + def get_libraries(self, ext): + """Return the list of libraries to link against when building a + shared extension. On most platforms, this is just 'ext.libraries'; + on Windows and OS/2, we add the Python library (eg. python20.dll). + """ + # The python library is always needed on Windows. For MSVC, this + # is redundant, since the library is mentioned in a pragma in + # pyconfig.h that MSVC groks. The other Windows compilers all seem + # to need it mentioned explicitly, though, so that's what we do. + # Append '_d' to the python import library on debug builds. + if sys.platform == "win32": + from packaging.compiler.msvccompiler import MSVCCompiler + if not isinstance(self.compiler_obj, MSVCCompiler): + template = "python%d%d" + if self.debug: + template = template + '_d' + pythonlib = template % sys.version_info[:2] + # don't extend ext.libraries, it may be shared with other + # extensions, it is a reference to the original list + return ext.libraries + [pythonlib] + else: + return ext.libraries + elif sys.platform == "os2emx": + # EMX/GCC requires the python library explicitly, and I + # believe VACPP does as well (though not confirmed) - AIM Apr01 + template = "python%d%d" + # debug versions of the main DLL aren't supported, at least + # not at this time - AIM Apr01 + #if self.debug: + # template = template + '_d' + pythonlib = template % sys.version_info[:2] + # don't extend ext.libraries, it may be shared with other + # extensions, it is a reference to the original list + return ext.libraries + [pythonlib] + elif sys.platform[:6] == "cygwin": + template = "python%d.%d" + pythonlib = template % sys.version_info[:2] + # don't extend ext.libraries, it may be shared with other + # extensions, it is a reference to the original list + return ext.libraries + [pythonlib] + elif sys.platform[:6] == "atheos": + template = "python%d.%d" + pythonlib = template % sys.version_info[:2] + # Get SHLIBS from Makefile + extra = [] + for lib in sysconfig.get_config_var('SHLIBS').split(): + if lib.startswith('-l'): + extra.append(lib[2:]) + else: + extra.append(lib) + # don't extend ext.libraries, it may be shared with other + # extensions, it is a reference to the original list + return ext.libraries + [pythonlib, "m"] + extra + + elif sys.platform == 'darwin': + # Don't use the default code below + return ext.libraries + + else: + if sysconfig.get_config_var('Py_ENABLE_SHARED'): + template = 'python%d.%d' + sys.abiflags + pythonlib = template % sys.version_info[:2] + return ext.libraries + [pythonlib] + else: + return ext.libraries |