From c656624a4fc638e4c0cc098dab0f59a6f8331b1c Mon Sep 17 00:00:00 2001 From: Eli Schwartz Date: Fri, 5 Aug 2022 05:49:39 -0400 Subject: [0.29] implement the --depfile command-line option for the "cython" tool (GH-4949) Backports https://github.com/cython/cython/pull/4916 --- Cython/Build/Dependencies.py | 19 +-------- Cython/Compiler/CmdLine.py | 5 ++- Cython/Compiler/Main.py | 5 +++ Cython/Utils.py | 21 +++++++++ tests/build/depfile_package.srctree | 57 ------------------------- tests/build/depfile_package_cython.srctree | 61 +++++++++++++++++++++++++++ tests/build/depfile_package_cythonize.srctree | 60 ++++++++++++++++++++++++++ 7 files changed, 152 insertions(+), 76 deletions(-) delete mode 100644 tests/build/depfile_package.srctree create mode 100644 tests/build/depfile_package_cython.srctree create mode 100644 tests/build/depfile_package_cythonize.srctree diff --git a/Cython/Build/Dependencies.py b/Cython/Build/Dependencies.py index 7eb55e260..1ba574d52 100644 --- a/Cython/Build/Dependencies.py +++ b/Cython/Build/Dependencies.py @@ -43,7 +43,7 @@ except: pythran = None from .. import Utils -from ..Utils import (cached_function, cached_method, path_exists, +from ..Utils import (cached_function, cached_method, path_exists, write_depfile, safe_makedirs, copy_file_to_dir_if_newer, is_package_dir, replace_suffix) from ..Compiler.Main import Context, CompilationOptions, default_options @@ -1030,22 +1030,7 @@ def cythonize(module_list, exclude=None, nthreads=0, aliases=None, quiet=False, # write out the depfile, if requested if depfile: dependencies = deps.all_dependencies(source) - src_base_dir, _ = os.path.split(source) - if not src_base_dir.endswith(os.sep): - src_base_dir += os.sep - # paths below the base_dir are relative, otherwise absolute - paths = [] - for fname in dependencies: - if (fname.startswith(src_base_dir) or - fname.startswith('.' + os.path.sep)): - paths.append(os.path.relpath(fname, src_base_dir)) - else: - paths.append(os.path.abspath(fname)) - - depline = os.path.split(c_file)[1] + ": \\\n " - depline += " \\\n ".join(paths) + "\n" - with open(c_file+'.dep', 'w') as outfile: - outfile.write(depline) + write_depfile(c_file, source, dependencies) if os.path.exists(c_file): c_timestamp = os.path.getmtime(c_file) diff --git a/Cython/Compiler/CmdLine.py b/Cython/Compiler/CmdLine.py index 9e2f8beb0..470fe6bd4 100644 --- a/Cython/Compiler/CmdLine.py +++ b/Cython/Compiler/CmdLine.py @@ -53,6 +53,7 @@ Options: --module-name Fully qualified module name. If not given, it is deduced from the import path if source file is in a package, or equals the filename otherwise. + -M, --depfile Produce depfiles for the sources """ @@ -66,7 +67,6 @@ def bad_usage(): sys.stderr.write(usage) sys.exit(1) - def parse_command_line(args): from .Main import CompilationOptions, default_options @@ -195,6 +195,8 @@ def parse_command_line(args): sys.exit(1) elif option == "--module-name": options.module_name = pop_value() + elif option in ('-M', '--depfile'): + options.depfile = True elif option.startswith('--debug'): option = option[2:].replace('-', '_') from . import DebugFlags @@ -236,4 +238,3 @@ def parse_command_line(args): "cython: Only one source file allowed when using --module-name\n") sys.exit(1) return options, sources - diff --git a/Cython/Compiler/Main.py b/Cython/Compiler/Main.py index 128441da6..9c57452ba 100644 --- a/Cython/Compiler/Main.py +++ b/Cython/Compiler/Main.py @@ -514,6 +514,10 @@ def run_pipeline(source, options, full_module_name=None, context=None): context.setup_errors(options, result) err, enddata = Pipeline.run_pipeline(pipeline, source) context.teardown_errors(err, options, result) + if options.depfile: + from ..Build.Dependencies import create_dependency_tree + dependencies = create_dependency_tree(context).all_dependencies(result.main_source_file) + Utils.write_depfile(result.c_file, result.main_source_file, dependencies) return result @@ -881,6 +885,7 @@ default_options = dict( errors_to_stderr = 1, cplus = 0, output_file = None, + depfile = None, annotate = None, annotate_coverage_xml = None, generate_pxi = 0, diff --git a/Cython/Utils.py b/Cython/Utils.py index d59d67d78..69563794c 100644 --- a/Cython/Utils.py +++ b/Cython/Utils.py @@ -447,3 +447,24 @@ def build_hex_version(version_string): hexversion = (hexversion << 8) + digit return '0x%08X' % hexversion + + +def write_depfile(target, source, dependencies): + src_base_dir = os.path.dirname(source) + cwd = os.getcwd() + if not src_base_dir.endswith(os.sep): + src_base_dir += os.sep + # paths below the base_dir are relative, otherwise absolute + paths = [] + for fname in dependencies: + fname = os.path.abspath(fname) + if fname.startswith(src_base_dir): + paths.append(os.path.relpath(fname, cwd)) + else: + paths.append(fname) + + depline = os.path.relpath(target, cwd) + ": \\\n " + depline += " \\\n ".join(paths) + "\n" + + with open(target+'.dep', 'w') as outfile: + outfile.write(depline) diff --git a/tests/build/depfile_package.srctree b/tests/build/depfile_package.srctree deleted file mode 100644 index c1de7b868..000000000 --- a/tests/build/depfile_package.srctree +++ /dev/null @@ -1,57 +0,0 @@ -""" -PYTHON -m Cython.Build.Cythonize -i pkg --depfile -PYTHON package_test.py -""" - -######## package_test.py ######## - -import os.path - -with open(os.path.join("pkg", "test.c.dep"), "r") as f: - contents = f.read().replace("\\\n", " ").replace("\n", " ") - -assert sorted(contents.split()) == sorted(['test.c:', os.path.join('sub', 'incl.pxi'), 'test.pxd', 'test.pyx']), contents - - -with open(os.path.join("pkg", "sub", "test.c.dep"), "r") as f: - contents = f.read().replace("\\\n", " ").replace("\n", " ") - -contents = [os.path.relpath(entry, '.') - if os.path.isabs(entry) else entry for entry in contents.split()] -assert sorted(contents) == sorted(['test.c:', 'incl.pxi', 'test.pyx', os.path.join('..', 'test.pxd')]), contents - - -######## pkg/__init__.py ######## - - -######## pkg/test.pyx ######## - -TEST = "pkg.test" - -include "sub/incl.pxi" - -cdef object get_str(): - return TEST - - -######## pkg/test.pxd ######## - -cdef object get_str() - - -######## pkg/sub/__init__.py ######## - - -######## pkg/sub/test.pyx ######## -# cython: language_level=3 - -from ..test cimport get_str - -include 'incl.pxi' - -TEST = 'pkg.sub.test' - - -######## pkg/sub/incl.pxi ######## - -pass diff --git a/tests/build/depfile_package_cython.srctree b/tests/build/depfile_package_cython.srctree new file mode 100644 index 000000000..5a6674a3d --- /dev/null +++ b/tests/build/depfile_package_cython.srctree @@ -0,0 +1,61 @@ +""" +PYTHON -c 'import os; os.makedirs("builddir/pkg/sub")' +CYTHON -M pkg/test.pyx -o builddir/pkg/test.c +CYTHON --depfile pkg/sub/test.pyx -o builddir/pkg/sub/test.c +PYTHON check.py +""" + +######## check.py ######## + +import os.path + +def pkgpath(*args): + return os.path.join('pkg', *args) + +with open(os.path.join("builddir", "pkg", "test.c.dep"), "r") as f: + contents = f.read().replace("\\\n", " ").replace("\n", " ") + +assert sorted(contents.split()) == sorted([os.path.join('builddir', 'pkg', 'test.c:'), pkgpath('sub', 'incl.pxi'), pkgpath('test.pxd'), pkgpath('test.pyx')]), contents + + +with open(os.path.join("builddir", "pkg", "sub", "test.c.dep"), "r") as f: + contents = f.read().replace("\\\n", " ").replace("\n", " ") + +contents = [os.path.relpath(entry, '.') + if os.path.isabs(entry) else entry for entry in contents.split()] +assert sorted(contents) == sorted([os.path.join('builddir', 'pkg', 'sub', 'test.c:'), pkgpath('sub', 'incl.pxi'), pkgpath('sub', 'test.pyx')]), contents # last is really one level up + +######## pkg/__init__.py ######## + + +######## pkg/test.pyx ######## + +TEST = "pkg.test" + +include "sub/incl.pxi" + +cdef object get_str(): + return TEST + + +######## pkg/test.pxd ######## + +cdef object get_str() + + +######## pkg/sub/__init__.py ######## + + +######## pkg/sub/test.pyx ######## +# cython: language_level=3 + +from ..test cimport get_str + +include 'incl.pxi' + +TEST = 'pkg.sub.test' + + +######## pkg/sub/incl.pxi ######## + +pass diff --git a/tests/build/depfile_package_cythonize.srctree b/tests/build/depfile_package_cythonize.srctree new file mode 100644 index 000000000..0ad4cab78 --- /dev/null +++ b/tests/build/depfile_package_cythonize.srctree @@ -0,0 +1,60 @@ +""" +PYTHON -m Cython.Build.Cythonize -i pkg --depfile +PYTHON package_test.py +""" + +######## package_test.py ######## + +import os.path + +def pkgpath(*args): + return os.path.join('pkg', *args) + +with open(os.path.join("pkg", "test.c.dep"), "r") as f: + contents = f.read().replace("\\\n", " ").replace("\n", " ") + +assert sorted(contents.split()) == sorted([pkgpath('test.c:'), pkgpath('sub', 'incl.pxi'), pkgpath('test.pxd'), pkgpath('test.pyx')]), contents + + +with open(os.path.join("pkg", "sub", "test.c.dep"), "r") as f: + contents = f.read().replace("\\\n", " ").replace("\n", " ") + +contents = [os.path.relpath(entry, '.') + if os.path.isabs(entry) else entry for entry in contents.split()] +assert sorted(contents) == sorted([pkgpath('sub', 'test.c:'), pkgpath('sub', 'incl.pxi'), pkgpath('sub', 'test.pyx'), pkgpath('test.pxd')]), contents # last is really one level up + + +######## pkg/__init__.py ######## + + +######## pkg/test.pyx ######## + +TEST = "pkg.test" + +include "sub/incl.pxi" + +cdef object get_str(): + return TEST + + +######## pkg/test.pxd ######## + +cdef object get_str() + + +######## pkg/sub/__init__.py ######## + + +######## pkg/sub/test.pyx ######## +# cython: language_level=3 + +from ..test cimport get_str + +include 'incl.pxi' + +TEST = 'pkg.sub.test' + + +######## pkg/sub/incl.pxi ######## + +pass -- cgit v1.2.1