summaryrefslogtreecommitdiff
path: root/chromium/tools/clang
diff options
context:
space:
mode:
authorAllan Sandfeld Jensen <allan.jensen@qt.io>2020-10-29 10:46:47 +0100
committerAllan Sandfeld Jensen <allan.jensen@qt.io>2020-11-02 12:02:10 +0000
commit99677208ff3b216fdfec551fbe548da5520cd6fb (patch)
tree476a4865c10320249360e859d8fdd3e01833b03a /chromium/tools/clang
parentc30a6232df03e1efbd9f3b226777b07e087a1122 (diff)
downloadqtwebengine-chromium-99677208ff3b216fdfec551fbe548da5520cd6fb.tar.gz
BASELINE: Update Chromium to 86.0.4240.124
Change-Id: Ide0ff151e94cd665ae6521a446995d34a9d1d644 Reviewed-by: Allan Sandfeld Jensen <allan.jensen@qt.io>
Diffstat (limited to 'chromium/tools/clang')
-rwxr-xr-xchromium/tools/clang/blink_gc_plugin/process-graph.py6
-rw-r--r--chromium/tools/clang/empty_string/EmptyStringConverter.cpp8
-rwxr-xr-xchromium/tools/clang/pylib/clang/compile_db.py13
-rw-r--r--chromium/tools/clang/rewrite_raw_ptr_fields/RewriteRawPtrFields.cpp153
-rw-r--r--chromium/tools/clang/rewrite_raw_ptr_fields/manual-fields-to-ignore.txt6
-rw-r--r--chromium/tools/clang/rewrite_raw_ptr_fields/manual-paths-to-ignore.txt118
-rwxr-xr-xchromium/tools/clang/rewrite_raw_ptr_fields/rewrite.sh11
-rwxr-xr-xchromium/tools/clang/scripts/build.py120
-rwxr-xr-xchromium/tools/clang/scripts/goma_ld.py13
-rwxr-xr-xchromium/tools/clang/scripts/goma_link.py164
-rwxr-xr-xchromium/tools/clang/scripts/goma_link_integration_tests.py159
-rwxr-xr-xchromium/tools/clang/scripts/package.py16
-rwxr-xr-xchromium/tools/clang/scripts/test_tool.py28
-rwxr-xr-xchromium/tools/clang/scripts/update.py11
-rwxr-xr-xchromium/tools/clang/scripts/upload_revision.py51
15 files changed, 638 insertions, 239 deletions
diff --git a/chromium/tools/clang/blink_gc_plugin/process-graph.py b/chromium/tools/clang/blink_gc_plugin/process-graph.py
index 39e53f18e12..06031838528 100755
--- a/chromium/tools/clang/blink_gc_plugin/process-graph.py
+++ b/chromium/tools/clang/blink_gc_plugin/process-graph.py
@@ -4,9 +4,13 @@
# found in the LICENSE file.
from __future__ import print_function
-from StringIO import StringIO
import argparse, os, sys, json, subprocess, pickle
+try:
+ from StringIO import StringIO # Python 2
+except:
+ from io import StringIO
+
parser = argparse.ArgumentParser(
description =
"Process the Blink points-to graph generated by the Blink GC plugin.")
diff --git a/chromium/tools/clang/empty_string/EmptyStringConverter.cpp b/chromium/tools/clang/empty_string/EmptyStringConverter.cpp
index db82a7782e6..f6f68009864 100644
--- a/chromium/tools/clang/empty_string/EmptyStringConverter.cpp
+++ b/chromium/tools/clang/empty_string/EmptyStringConverter.cpp
@@ -80,12 +80,12 @@ class EmptyStringConverter {
};
void EmptyStringConverter::SetupMatchers(MatchFinder* match_finder) {
- const clang::ast_matchers::StatementMatcher& constructor_call = id(
- "call",
+ const clang::ast_matchers::StatementMatcher& constructor_call =
cxxConstructExpr(
hasDeclaration(cxxMethodDecl(ofClass(hasName("std::basic_string")))),
- argumentCountIs(2), hasArgument(0, id("literal", stringLiteral())),
- hasArgument(1, cxxDefaultArgExpr())));
+ argumentCountIs(2), hasArgument(0, stringLiteral().bind("literal")),
+ hasArgument(1, cxxDefaultArgExpr()))
+ .bind("call");
// Note that expr(has()) in the matcher is significant; the Clang AST wraps
// calls to the std::string constructor with exprWithCleanups nodes. Without
diff --git a/chromium/tools/clang/pylib/clang/compile_db.py b/chromium/tools/clang/pylib/clang/compile_db.py
index 50decf71907..6fe04981baf 100755
--- a/chromium/tools/clang/pylib/clang/compile_db.py
+++ b/chromium/tools/clang/pylib/clang/compile_db.py
@@ -14,12 +14,13 @@ import subprocess
_RSP_RE = re.compile(r' (@(.+?\.rsp)) ')
_CMD_LINE_RE = re.compile(
- r'^(?P<gomacc>.*gomacc\.exe"?\s+)?(?P<clang>\S*clang\S*)\s+(?P<args>.*)$')
+ r'^(?P<gomacc>.*gomacc(\.exe)?"?\s+)?(?P<clang>\S*clang\S*)\s+(?P<args>.*)$'
+)
_debugging = False
def _ProcessCommand(command):
- """Removes gomacc.exe and inserts --driver-mode=cl as the first argument.
+ """Removes gomacc(.exe). On Windows inserts --driver-mode=cl as the first arg.
Note that we deliberately don't use shlex.split here, because it doesn't work
predictably for Windows commands (specifically, it doesn't parse args the same
@@ -31,10 +32,9 @@ def _ProcessCommand(command):
# If the driver mode is not already set then define it. Driver mode is
# automatically included in the compile db by clang starting with release
# 9.0.0.
- if "--driver_mode" in command:
- driver_mode = ""
+ driver_mode = ''
# Only specify for Windows. Other platforms do fine without it.
- elif sys.platform == 'win32':
+ if sys.platform == 'win32' and '--driver_mode' not in command:
driver_mode = '--driver-mode=cl'
match = _CMD_LINE_RE.search(command)
@@ -88,6 +88,8 @@ def ProcessCompileDatabaseIfNeeded(compile_db):
Returns:
A postprocessed compile db that clang tooling can use.
"""
+ compile_db = [_ProcessEntry(e) for e in compile_db]
+
# TODO(dcheng): Ideally this would check target_os... but not sure there's an
# easy way to do that, and (for now) cross-compiles don't work without custom
# patches anyway.
@@ -96,7 +98,6 @@ def ProcessCompileDatabaseIfNeeded(compile_db):
if _debugging:
print('Read in %d entries from the compile db' % len(compile_db))
- compile_db = [_ProcessEntry(e) for e in compile_db]
original_length = len(compile_db)
# Filter out NaCl stuff. The clang tooling chokes on them.
diff --git a/chromium/tools/clang/rewrite_raw_ptr_fields/RewriteRawPtrFields.cpp b/chromium/tools/clang/rewrite_raw_ptr_fields/RewriteRawPtrFields.cpp
index f94ce4e03f3..27b3fbb196e 100644
--- a/chromium/tools/clang/rewrite_raw_ptr_fields/RewriteRawPtrFields.cpp
+++ b/chromium/tools/clang/rewrite_raw_ptr_fields/RewriteRawPtrFields.cpp
@@ -70,9 +70,17 @@ const char kIncludePath[] = "base/memory/checked_ptr.h";
//
// See also:
// - OutputSectionHelper
-// - FieldDeclFilterFile
+// - FilterFile
const char kExcludeFieldsParamName[] = "exclude-fields";
+// Name of a cmdline parameter that can be used to specify a file listing
+// regular expressions describing paths that should be excluded from the
+// rewrite.
+//
+// See also:
+// - PathFilterFile
+const char kExcludePathsParamName[] = "exclude-paths";
+
// OutputSectionHelper helps gather and emit a section of output.
//
// The section of output is delimited in a way that makes it easy to extract it
@@ -93,13 +101,16 @@ const char kExcludeFieldsParamName[] = "exclude-fields";
// changes).
//
// See also:
-// - FieldDeclFilterFile
+// - FilterFile
// - OutputHelper
class OutputSectionHelper {
public:
explicit OutputSectionHelper(llvm::StringRef output_delimiter)
: output_delimiter_(output_delimiter.str()) {}
+ OutputSectionHelper(const OutputSectionHelper&) = delete;
+ OutputSectionHelper& operator=(const OutputSectionHelper&) = delete;
+
void Add(llvm::StringRef output_line, llvm::StringRef tag = "") {
// Look up |tags| associated with |output_line|. As a side effect of the
// lookup, |output_line| will be inserted if it wasn't already present in
@@ -152,6 +163,9 @@ class OutputHelper : public clang::tooling::SourceFileCallbacks {
: edits_helper_("EDITS"), field_decl_filter_helper_("FIELD FILTERS") {}
~OutputHelper() = default;
+ OutputHelper(const OutputHelper&) = delete;
+ OutputHelper& operator=(const OutputHelper&) = delete;
+
void AddReplacement(const clang::SourceManager& source_manager,
const clang::SourceRange& replacement_range,
std::string replacement_text,
@@ -264,14 +278,9 @@ AST_MATCHER(clang::FieldDecl, isInThirdPartyLocation) {
if (file_path.contains("third_party/blink/"))
return false;
- // V8 needs to be considered "third party", even though its paths do not
- // contain the "third_party" substring. In particular, the rewriter should
- // not append |.get()| to references to |v8::RegisterState::pc|, because
- // //v8/include/v8.h will *not* get rewritten.
- if (file_path.contains("v8/include/"))
- return true;
-
// Otherwise, just check if the paths contains the "third_party" substring.
+ // We don't want to rewrite content of such paths even if they are in the main
+ // Chromium git repository.
return file_path.contains("third_party");
}
@@ -282,23 +291,37 @@ AST_MATCHER(clang::FieldDecl, isInGeneratedLocation) {
return file_path.startswith("gen/") || file_path.contains("/gen/");
}
-// Represents a filter file specified via cmdline, that can be used to filter
-// out specific FieldDecls.
-//
-// See also:
-// - kExcludeFieldsParamName
-// - OutputSectionHelper
-class FieldDeclFilterFile {
+// Represents a filter file specified via cmdline.
+class FilterFile {
public:
- explicit FieldDeclFilterFile(const std::string& filepath) {
- if (!filepath.empty())
- ParseInputFile(filepath);
+ explicit FilterFile(const llvm::cl::opt<std::string>& cmdline_param) {
+ ParseInputFile(cmdline_param);
}
- bool Contains(const clang::FieldDecl& field_decl) const {
- std::string qualified_name = field_decl.getQualifiedNameAsString();
- auto it = fields_to_filter_.find(qualified_name);
- return it != fields_to_filter_.end();
+ FilterFile(const FilterFile&) = delete;
+ FilterFile& operator=(const FilterFile&) = delete;
+
+ // Returns true if any of the filter file lines is exactly equal to |line|.
+ bool ContainsLine(llvm::StringRef line) const {
+ auto it = file_lines_.find(line);
+ return it != file_lines_.end();
+ }
+
+ // Returns true if any of the filter file lines is a substring of
+ // |string_to_match|.
+ bool ContainsSubstringOf(llvm::StringRef string_to_match) const {
+ if (!substring_regex_.hasValue()) {
+ std::vector<std::string> regex_escaped_file_lines;
+ regex_escaped_file_lines.reserve(file_lines_.size());
+ for (const llvm::StringRef& file_line : file_lines_.keys())
+ regex_escaped_file_lines.push_back(llvm::Regex::escape(file_line));
+ std::string substring_regex_pattern =
+ llvm::join(regex_escaped_file_lines.begin(),
+ regex_escaped_file_lines.end(), "|");
+ substring_regex_.emplace(substring_regex_pattern);
+ }
+
+ return substring_regex_->match(string_to_match);
}
private:
@@ -310,13 +333,17 @@ class FieldDeclFilterFile {
// autofill::AddressField::address1_ # some comment
// - Templates are represented without template arguments, like:
// WTF::HashTable::table_ # some comment
- void ParseInputFile(const std::string& filepath) {
+ void ParseInputFile(const llvm::cl::opt<std::string>& cmdline_param) {
+ std::string filepath = cmdline_param;
+ if (filepath.empty())
+ return;
+
llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> file_or_err =
llvm::MemoryBuffer::getFile(filepath);
if (std::error_code err = file_or_err.getError()) {
llvm::errs() << "ERROR: Cannot open the file specified in --"
- << kExcludeFieldsParamName << " argument: " << filepath
- << ": " << err.message() << "\n";
+ << cmdline_param.ArgStr << " argument: " << filepath << ": "
+ << err.message() << "\n";
assert(false);
return;
}
@@ -334,19 +361,32 @@ class FieldDeclFilterFile {
if (line.empty())
continue;
- fields_to_filter_.insert(line);
+ file_lines_.insert(line);
}
}
- // Stores fully-namespace-qualified names of fields matched by the filter.
- llvm::StringSet<> fields_to_filter_;
+ // Stores all file lines (after stripping comments and blank lines).
+ llvm::StringSet<> file_lines_;
+
+ // Lazily-constructed regex that matches strings that contain any of the
+ // |file_lines_|.
+ mutable llvm::Optional<llvm::Regex> substring_regex_;
};
AST_MATCHER_P(clang::FieldDecl,
- isListedInFilterFile,
- FieldDeclFilterFile,
+ isFieldDeclListedInFilterFile,
+ const FilterFile*,
+ Filter) {
+ return Filter->ContainsLine(Node.getQualifiedNameAsString());
+}
+
+AST_MATCHER_P(clang::FieldDecl,
+ isInLocationListedInFilterFile,
+ const FilterFile*,
Filter) {
- return Filter.Contains(Node);
+ llvm::StringRef file_path =
+ GetFilePath(Finder->getASTContext().getSourceManager(), Node);
+ return Filter->ContainsSubstringOf(file_path);
}
AST_MATCHER(clang::Decl, isInExternCContext) {
@@ -641,6 +681,9 @@ class FieldDeclRewriter : public MatchFinder::MatchCallback {
explicit FieldDeclRewriter(OutputHelper* output_helper)
: output_helper_(output_helper) {}
+ FieldDeclRewriter(const FieldDeclRewriter&) = delete;
+ FieldDeclRewriter& operator=(const FieldDeclRewriter&) = delete;
+
void run(const MatchFinder::MatchResult& result) override {
const clang::ASTContext& ast_context = *result.Context;
const clang::SourceManager& source_manager = *result.SourceManager;
@@ -720,6 +763,9 @@ class AffectedExprRewriter : public MatchFinder::MatchCallback {
explicit AffectedExprRewriter(OutputHelper* output_helper)
: output_helper_(output_helper) {}
+ AffectedExprRewriter(const AffectedExprRewriter&) = delete;
+ AffectedExprRewriter& operator=(const AffectedExprRewriter&) = delete;
+
void run(const MatchFinder::MatchResult& result) override {
const clang::SourceManager& source_manager = *result.SourceManager;
@@ -747,6 +793,9 @@ class FilteredExprWriter : public MatchFinder::MatchCallback {
FilteredExprWriter(OutputHelper* output_helper, llvm::StringRef filter_tag)
: output_helper_(output_helper), filter_tag_(filter_tag) {}
+ FilteredExprWriter(const FilteredExprWriter&) = delete;
+ FilteredExprWriter& operator=(const FilteredExprWriter&) = delete;
+
void run(const MatchFinder::MatchResult& result) override {
const clang::FieldDecl* field_decl =
result.Nodes.getNodeAs<clang::FieldDecl>("affectedFieldDecl");
@@ -772,6 +821,9 @@ int main(int argc, const char* argv[]) {
llvm::cl::opt<std::string> exclude_fields_param(
kExcludeFieldsParamName, llvm::cl::value_desc("filepath"),
llvm::cl::desc("file listing fields to be blocked (not rewritten)"));
+ llvm::cl::opt<std::string> exclude_paths_param(
+ kExcludePathsParamName, llvm::cl::value_desc("filepath"),
+ llvm::cl::desc("file listing paths to be blocked (not rewritten)"));
clang::tooling::CommonOptionsParser options(argc, argv, category);
clang::tooling::ClangTool tool(options.getCompilations(),
options.getSourcePathList());
@@ -822,16 +874,19 @@ int main(int argc, const char* argv[]) {
// matches |int* y|. Doesn't match:
// - non-pointer types
// - fields of lambda-supporting classes
- // - fields listed in the --exclude-fields cmdline param
+ // - fields listed in the --exclude-fields cmdline param or located in paths
+ // matched by --exclude-paths cmdline param
// - "implicit" fields (i.e. field decls that are not explicitly present in
// the source code)
- FieldDeclFilterFile fields_to_exclude(exclude_fields_param);
+ FilterFile fields_to_exclude(exclude_fields_param);
+ FilterFile paths_to_exclude(exclude_paths_param);
auto field_decl_matcher =
fieldDecl(
allOf(hasType(supported_pointer_types_matcher),
- unless(anyOf(isInThirdPartyLocation(), isInGeneratedLocation(),
- isExpansionInSystemHeader(), isInExternCContext(),
- isListedInFilterFile(fields_to_exclude),
+ unless(anyOf(isExpansionInSystemHeader(), isInExternCContext(),
+ isInThirdPartyLocation(), isInGeneratedLocation(),
+ isInLocationListedInFilterFile(&paths_to_exclude),
+ isFieldDeclListedInFilterFile(&fields_to_exclude),
implicit_field_decl_matcher))))
.bind("affectedFieldDecl");
FieldDeclRewriter field_decl_rewriter(&output_helper);
@@ -849,15 +904,7 @@ int main(int argc, const char* argv[]) {
auto affected_member_expr_matcher =
memberExpr(member(fieldDecl(hasExplicitFieldDecl(field_decl_matcher))))
.bind("affectedMemberExpr");
- auto affected_implicit_expr_matcher = implicitCastExpr(has(expr(anyOf(
- // Only single implicitCastExpr is present in case of:
- // |auto* v = s.ptr_field;|
- expr(affected_member_expr_matcher),
- // 2nd nested implicitCastExpr is present in case of:
- // |const auto* v = s.ptr_field;|
- expr(implicitCastExpr(has(affected_member_expr_matcher)))))));
- auto affected_expr_matcher =
- expr(anyOf(affected_member_expr_matcher, affected_implicit_expr_matcher));
+ auto affected_expr_matcher = ignoringImplicit(affected_member_expr_matcher);
// Places where |.get()| needs to be appended =========
// Given
@@ -960,10 +1007,9 @@ int main(int argc, const char* argv[]) {
//
// See also the testcases in tests/gen-in-out-arg-test.cc.
auto affected_in_out_ref_arg_matcher = callExpr(forEachArgumentWithParam(
- affected_expr_matcher.bind("expr"),
- hasExplicitParmVarDecl(
- hasType(qualType(allOf(referenceType(pointee(pointerType())),
- unless(rValueReferenceType())))))));
+ affected_expr_matcher, hasExplicitParmVarDecl(hasType(qualType(
+ allOf(referenceType(pointee(pointerType())),
+ unless(rValueReferenceType())))))));
FilteredExprWriter filtered_in_out_ref_arg_writer(&output_helper,
"in-out-param-ref");
match_finder.addMatcher(affected_in_out_ref_arg_matcher,
@@ -988,9 +1034,10 @@ int main(int argc, const char* argv[]) {
// See the doc comment for the anyCharType matcher
// and the testcases in tests/gen-char-test.cc.
auto char_ptr_field_decl_matcher = fieldDecl(allOf(
- field_decl_matcher, hasType(pointerType(pointee(
- hasUnqualifiedDesugaredType(anyCharType()))))));
- FilteredExprWriter char_ptr_field_decl_writer(&output_helper, "char");
+ field_decl_matcher,
+ hasType(pointerType(pointee(qualType(allOf(
+ isConstQualified(), hasUnqualifiedDesugaredType(anyCharType()))))))));
+ FilteredExprWriter char_ptr_field_decl_writer(&output_helper, "const-char");
match_finder.addMatcher(char_ptr_field_decl_matcher,
&char_ptr_field_decl_writer);
diff --git a/chromium/tools/clang/rewrite_raw_ptr_fields/manual-fields-to-ignore.txt b/chromium/tools/clang/rewrite_raw_ptr_fields/manual-fields-to-ignore.txt
index b8cc64f3bb3..9f18a75980d 100644
--- a/chromium/tools/clang/rewrite_raw_ptr_fields/manual-fields-to-ignore.txt
+++ b/chromium/tools/clang/rewrite_raw_ptr_fields/manual-fields-to-ignore.txt
@@ -14,10 +14,11 @@ base::CheckedContiguousIterator::current_
base::CheckedContiguousIterator::end_
base::CheckedContiguousIterator::start_
base::CheckedContiguousRange::container_
+base::FeatureParam<base::TimeDelta, false>::feature
base::FeatureParam<bool, false>::feature
base::FeatureParam<double, false>::feature
base::FeatureParam<int, false>::feature
-base::FeatureParam<std::__Cr::basic_string<char, std::__Cr::char_traits<char>, std::__Cr::allocator<char>>, false>::feature
+base::FeatureParam<std::__1::basic_string<char>, false>::feature
base::FeatureParam<type-parameter-0-0, true>::feature
base::FeatureParam<type-parameter-0-0, true>::options
base::span::data_
@@ -154,6 +155,9 @@ device::BluetoothDevice::adapter_
vr::LocationBarState::vector_icon
vr::OmniboxSuggestion::icon
+# Populated manually - other compile-time reasons
+logging::CheckOpResult::message_ # cyclic #include
+
# Populated manually - these pointers are assigned invalid address (with top
# bits sets), which CheckedPtr is unable to handle, leading to run-time crashes.
(anonymous namespace)::TlsVectorEntry::data
diff --git a/chromium/tools/clang/rewrite_raw_ptr_fields/manual-paths-to-ignore.txt b/chromium/tools/clang/rewrite_raw_ptr_fields/manual-paths-to-ignore.txt
new file mode 100644
index 00000000000..62e16dc46c9
--- /dev/null
+++ b/chromium/tools/clang/rewrite_raw_ptr_fields/manual-paths-to-ignore.txt
@@ -0,0 +1,118 @@
+# File that lists regular expressions of paths that should be ignored when
+# running the rewrite_raw_ptr_fields tool on Chromium sources.
+#
+# If a source file path contains any of the lines in the filter file below,
+# then such source file will not be rewritten.
+#
+# Note that the rewriter has a hardcoded logic for a handful of path-based
+# exclusions that cannot be expressed as substring matches:
+# - Excluding paths containing "third_party/", but still covering
+# "third_party/blink/"
+# (see the isInThirdPartyLocation AST matcher in RewriteRawPtrFields.cpp).
+# - Excluding paths _starting_ with "gen/" or containing "/gen/"
+# (i.e. hopefully just the paths under out/.../gen/... directory)
+# via the isInGeneratedLocation AST matcher in RewriteRawPtrFields.cpp.
+
+# Exclude to prevent PartitionAlloc<->CheckedPtr cyclical dependency.
+base/allocator/
+
+# Exclude - deprecated and contains legacy C++ and pre-C++11 code.
+ppapi/
+
+# Exclude tools that do not ship in the Chrome binary.
+base/android/linker/
+tools/
+net/tools/
+
+# Exclude paths in separate repositories - i.e. in directories that
+# 1. Contain a ".git" subdirectory
+# 2. And hasn't been excluded via "third_party/" substring in their path
+# (see the isInThirdPartyLocation AST matcher in RewriteRawPtrFields.cpp).
+#
+# The list below has been generated with:
+#
+# $ find . -type d -name .git | \
+# sed -e 's/\.git$//g' | \
+# sed -e 's/\.\///g' | \
+# grep -v third_party | \
+# grep -v '^$' | \
+# sort | uniq > ~/scratch/git-paths
+buildtools/
+buildtools/clang_format/script/
+chrome/app/theme/default_100_percent/google_chrome/
+chrome/app/theme/default_200_percent/google_chrome/
+chrome/app/theme/google_chrome/
+chrome/app/vector_icons/google_chrome/
+chrome/browser/chromeos/arc/voice_interaction/internal/
+chrome/browser/google/linkdoctor_internal/
+chrome/browser/internal/
+chrome/browser/media/engagement_internal/
+chrome/browser/media/kaleidoscope/internal/
+chrome/browser/resources/chromeos/quickoffice/
+chrome/browser/resources/media_router/extension/src/
+chrome/browser/resources/media_router_internal/
+chrome/browser/resources/settings_internal/
+chrome/browser/spellchecker/internal/
+chrome/browser/ui/media_router/internal/
+chrome/installer/mac/internal/
+chromeos/assistant/internal/
+chromeos/assistant/libassistant/deps/
+chromeos/assistant/libassistant/src/
+chromeos/assistant/libassistant/src/buildtools/
+chromeos/assistant/libassistant/src/buildtools/clang_format/script/
+chromeos/components/media_app_ui/resources/app/
+chrome/services/soda/internal/
+chrome/test/data/firefox3_profile/searchplugins/
+chrome/test/data/firefox3_searchplugins/
+chrome/test/data/gpu/vt/
+chrome/test/data/osdd/
+chrome/test/data/pdf_private/
+chrome/test/data/perf/canvas_bench/
+chrome/test/data/perf/frame_rate/content/
+chrome/test/data/perf/frame_rate/private/
+chrome/test/data/perf/private/
+chrome/test/data/xr/webvr_info/
+chrome/test/media_router/internal/
+chrome/test/python_tests/
+chrome/tools/test/reference_build/chrome_linux/
+clank/
+components/ntp_tiles/resources/internal/
+components/resources/default_100_percent/google_chrome/
+components/resources/default_200_percent/google_chrome/
+components/resources/default_300_percent/google_chrome/
+components/site_isolation/internal/
+content/test/data/plugin/
+data/autodiscovery/
+data/dom_perf/
+data/mach_ports/
+data/memory_test/
+data/mozilla_js_tests/
+data/page_cycler/
+data/selenium_core/
+data/tab_switching/
+google_apis/internal/
+libassistant/communication/
+libassistant/contrib/
+libassistant/internal/
+libassistant/resources/
+libassistant/shared/
+media/cdm/api/
+mojo/internal/
+native_client/
+remoting/android/internal/
+remoting/host/installer/linux/internal/
+remoting/internal/
+remoting/test/internal/
+remoting/tools/internal/
+remoting/webapp/app_remoting/internal/
+skia/tools/clusterfuzz-data/
+tools/histograms/
+tools/page_cycler/acid3/
+tools/perf/data/
+tools/swarming_client/
+ui/file_manager/internal/
+ui/webui/internal/
+v8/
+webkit/data/bmp_decoder/
+webkit/data/ico_decoder/
+webkit/data/test_shell/plugins/
diff --git a/chromium/tools/clang/rewrite_raw_ptr_fields/rewrite.sh b/chromium/tools/clang/rewrite_raw_ptr_fields/rewrite.sh
index e932c3248d5..30eebc984ba 100755
--- a/chromium/tools/clang/rewrite_raw_ptr_fields/rewrite.sh
+++ b/chromium/tools/clang/rewrite_raw_ptr_fields/rewrite.sh
@@ -24,6 +24,9 @@ then
OUT_DIR="$1"
fi
+SCRIPT_PATH=$(realpath $0)
+REWRITER_SRC_DIR=$(dirname $SCRIPT_PATH)
+
COMPILE_DIRS=.
EDIT_DIRS=.
@@ -49,6 +52,7 @@ time ninja -C $OUT_DIR $GEN_H_TARGETS
echo "*** Generating the ignore list ***"
time tools/clang/scripts/run_tool.py \
--tool rewrite_raw_ptr_fields \
+ --tool-arg=--exclude-paths=$REWRITER_SRC_DIR/manual-paths-to-ignore.txt \
--generate-compdb \
-p $OUT_DIR \
$COMPILE_DIRS > ~/scratch/rewriter.out
@@ -64,6 +68,7 @@ echo "*** Running the main rewrite phase ***"
time tools/clang/scripts/run_tool.py \
--tool rewrite_raw_ptr_fields \
--tool-arg=--exclude-fields=$HOME/scratch/combined-fields-to-ignore.txt \
+ --tool-arg=--exclude-paths=$REWRITER_SRC_DIR/manual-paths-to-ignore.txt \
-p $OUT_DIR \
$COMPILE_DIRS > ~/scratch/rewriter.main.out
@@ -73,12 +78,6 @@ cat ~/scratch/rewriter.main.out | \
tools/clang/scripts/extract_edits.py | \
tools/clang/scripts/apply_edits.py -p $OUT_DIR $EDIT_DIRS
-# Revert directories that are known to be troublesome and/or not needed.
-git checkout -- base/allocator/ # prevent cycles; CheckedPtr uses allocator
-git checkout -- ppapi/ # lots of legacy C and pre-C++11 code
-git checkout -- tools/ # not built into Chrome
-git checkout -- net/tools/ # not built into Chrome
-
# Format sources, as many lines are likely over 80 chars now.
echo "*** Formatting ***"
time git cl format
diff --git a/chromium/tools/clang/scripts/build.py b/chromium/tools/clang/scripts/build.py
index 3f2af8b3cf9..78ab2b8ddbd 100755
--- a/chromium/tools/clang/scripts/build.py
+++ b/chromium/tools/clang/scripts/build.py
@@ -50,8 +50,6 @@ BUG_REPORT_URL = ('https://crbug.com and run'
' tools/clang/scripts/process_crashreports.py'
' (only works inside Google) which will upload a report')
-FIRST_LLVM_COMMIT = '97724f18c79c7cc81ced24239eb5e883bf1398ef'
-
win_sdk_dir = None
dia_dll = None
@@ -158,8 +156,7 @@ def CheckoutLLVM(commit, dir):
# Also check that the first commit is reachable.
if (RunCommand(['git', 'diff-index', '--quiet', 'HEAD'], fail_hard=False)
and RunCommand(['git', 'fetch'], fail_hard=False)
- and RunCommand(['git', 'checkout', commit], fail_hard=False)
- and RunCommand(['git', 'show', FIRST_LLVM_COMMIT], fail_hard=False)):
+ and RunCommand(['git', 'checkout', commit], fail_hard=False)):
return
# If we can't use the current repo, delete it.
@@ -191,12 +188,12 @@ def GetLatestLLVMCommit():
return ref['object']['sha']
-def GetCommitCount(commit):
- """Get the number of commits from FIRST_LLVM_COMMIT to commit.
+def GetCommitDescription(commit):
+ """Get the output of `git describe`.
Needs to be called from inside the git repository dir."""
- return subprocess.check_output(['git', 'rev-list', '--count',
- FIRST_LLVM_COMMIT + '..' + commit]).rstrip()
+ return subprocess.check_output(
+ ['git', 'describe', '--long', '--abbrev=8', commit]).rstrip()
def DeleteChromeToolsShim():
@@ -256,7 +253,7 @@ def AddGnuWinToPath():
return
gnuwin_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'gnuwin')
- GNUWIN_VERSION = '12'
+ GNUWIN_VERSION = '13'
GNUWIN_STAMP = os.path.join(gnuwin_dir, 'stamp')
if ReadStampFile(GNUWIN_STAMP) == GNUWIN_VERSION:
print('GNU Win tools already up to date.')
@@ -337,6 +334,26 @@ def VerifyVersionOfBuiltClangMatchesVERSION():
sys.exit(1)
+def VerifyZlibSupport():
+ """Check that clang was built with zlib support enabled."""
+ clang = os.path.join(LLVM_BUILD_DIR, 'bin', 'clang')
+ test_file = '/dev/null'
+ if sys.platform == 'win32':
+ clang += '-cl.exe'
+ test_file = 'nul'
+
+ print('Checking for zlib support')
+ clang_out = subprocess.check_output([
+ clang, '--driver-mode=gcc', '-target', 'x86_64-unknown-linux-gnu', '-gz',
+ '-c', '-###', '-x', 'c', test_file ],
+ stderr=subprocess.STDOUT, universal_newlines=True)
+ if (re.search(r'--compress-debug-sections', clang_out)):
+ print('OK')
+ else:
+ print(('Failed to detect zlib support!\n\n(driver output: %s)') % clang_out)
+ sys.exit(1)
+
+
def CopyLibstdcpp(args, build_dir):
if not args.gcc_toolchain:
return
@@ -383,6 +400,9 @@ def main():
'building; --gcc-toolchain=/opt/foo picks '
'/opt/foo/bin/gcc')
parser.add_argument('--pgo', action='store_true', help='build with PGO')
+ parser.add_argument('--thinlto',
+ action='store_true',
+ help='build with ThinLTO')
parser.add_argument('--llvm-force-head-revision', action='store_true',
help='build the latest revision')
parser.add_argument('--run-tests', action='store_true',
@@ -408,8 +428,8 @@ def main():
default=sys.platform in ('linux2', 'darwin'))
args = parser.parse_args()
- if args.pgo and not args.bootstrap:
- print('--pgo requires --bootstrap')
+ if (args.pgo or args.thinlto) and not args.bootstrap:
+ print('--pgo/--thinlto requires --bootstrap')
return 1
if args.with_android and not os.path.exists(ANDROID_NDK_DIR):
print('Android NDK not found at ' + ANDROID_NDK_DIR)
@@ -456,16 +476,21 @@ def main():
# enough GCC to build Clang.
MaybeDownloadHostGcc(args)
+ if sys.platform == 'darwin':
+ isysroot = subprocess.check_output(['xcrun', '--show-sdk-path']).rstrip()
+
global CLANG_REVISION, PACKAGE_VERSION
if args.llvm_force_head_revision:
- CLANG_REVISION = GetLatestLLVMCommit()
+ checkout_revision = GetLatestLLVMCommit()
+ else:
+ checkout_revision = CLANG_REVISION
if not args.skip_checkout:
- CheckoutLLVM(CLANG_REVISION, LLVM_DIR);
+ CheckoutLLVM(checkout_revision, LLVM_DIR)
if args.llvm_force_head_revision:
- PACKAGE_VERSION = 'n%s-%s-0' % (GetCommitCount(CLANG_REVISION),
- CLANG_REVISION[:8])
+ CLANG_REVISION = GetCommitDescription(checkout_revision)
+ PACKAGE_VERSION = '%s-0' % CLANG_REVISION
print('Locally building clang %s...' % PACKAGE_VERSION)
WriteStampFile('', STAMP_FILE)
@@ -516,6 +541,8 @@ def main():
'-DCOMPILER_RT_USE_LIBCXX=NO',
# Don't run Go bindings tests; PGO makes them confused.
'-DLLVM_INCLUDE_GO_TESTS=OFF',
+ # TODO(crbug.com/1113475): Update binutils.
+ '-DENABLE_X86_RELAX_RELOCATIONS=NO',
]
if args.gcc_toolchain:
@@ -533,8 +560,10 @@ def main():
if sys.platform == 'darwin':
# For libc++, we only want the headers.
base_cmake_args.extend([
- '-DLIBCXX_ENABLE_SHARED=OFF', '-DLIBCXX_ENABLE_STATIC=OFF',
- '-DLIBCXX_INCLUDE_TESTS=OFF'
+ '-DLIBCXX_ENABLE_SHARED=OFF',
+ '-DLIBCXX_ENABLE_STATIC=OFF',
+ '-DLIBCXX_INCLUDE_TESTS=OFF',
+ '-DLIBCXX_ENABLE_EXPERIMENTAL_LIBRARY=OFF',
])
# Prefer Python 2. TODO(crbug.com/1076834): Remove this.
base_cmake_args.append('-DPython3_EXECUTABLE=/nonexistent')
@@ -685,6 +714,8 @@ def main():
if cc is not None: instrument_args.append('-DCMAKE_C_COMPILER=' + cc)
if cxx is not None: instrument_args.append('-DCMAKE_CXX_COMPILER=' + cxx)
if lld is not None: instrument_args.append('-DCMAKE_LINKER=' + lld)
+ if args.thinlto:
+ instrument_args.append('-DLLVM_ENABLE_LTO=Thin')
RunCommand(['cmake'] + instrument_args + [os.path.join(LLVM_DIR, 'llvm')],
msvc_arch='x64')
@@ -715,15 +746,13 @@ def main():
# from PGO as well. Perhaps the training could be done asynchronously by
# dedicated buildbots that upload profiles to the cloud.
training_source = 'pgo_training-1.ii'
- with open(training_source, 'w') as f:
+ with open(training_source, 'wb') as f:
DownloadUrl(CDS_URL + '/' + training_source, f)
train_cmd = [os.path.join(LLVM_INSTRUMENTED_DIR, 'bin', 'clang++'),
'-target', 'x86_64-unknown-unknown', '-O2', '-g', '-std=c++14',
'-fno-exceptions', '-fno-rtti', '-w', '-c', training_source]
if sys.platform == 'darwin':
- train_cmd.extend(['-stdlib=libc++', '-isysroot',
- subprocess.check_output(['xcrun',
- '--show-sdk-path']).rstrip()])
+ train_cmd.extend(['-stdlib=libc++', '-isysroot', isysroot])
RunCommand(train_cmd, msvc_arch='x64')
# Merge profiles.
@@ -750,9 +779,48 @@ def main():
# iPhones). armv7k is Apple Watch, which we don't need.
'-DDARWIN_ios_ARCHS=armv7;armv7s;arm64',
'-DDARWIN_iossim_ARCHS=i386;x86_64',
- # We don't need 32-bit intel support for macOS, we only ship 64-bit.
- '-DDARWIN_osx_ARCHS=x86_64',
])
+ if args.bootstrap:
+ # mac/arm64 needs MacOSX11.0.sdk. System Xcode (+ SDK) on the chrome bots
+ # is something much older.
+ # Options:
+ # - temporarily set system Xcode to Xcode 12 beta while running this
+ # script, (cf build/swarming_xcode_install.py, but it looks unused)
+ # - use Xcode 12 beta for everything on tot bots, only need to fuzz with
+ # scripts/slave/recipes/chromium_upload_clang.py then (but now the
+ # chrome/ios build will use the 11.0 SDK too and we'd be on the hook for
+ # keeping it green -- if it's currently green, who knows)
+ # - pass flags to cmake to try to coax it into using Xcode 12 beta for the
+ # LLVM build without it being system Xcode.
+ #
+ # The last option seems best, so let's go with that. We need to pass
+ # -isysroot to the 11.0 SDK and -B to the /usr/bin so that the new ld64 is
+ # used.
+ # The compiler-rt build overrides -isysroot flags set via cflags, and we
+ # only need to use the 11 SDK for the compiler-rt build. So set only
+ # DARWIN_macosx_CACHED_SYSROOT to the 11.0 SDK and use the regular SDK
+ # for the rest of the build. (The new ld is used for all links.)
+ sys.path.insert(1, os.path.join(CHROMIUM_DIR, 'build'))
+ import mac_toolchain
+ LLVM_XCODE = os.path.join(THIRD_PARTY_DIR, 'llvm-xcode')
+ mac_toolchain.InstallXcodeBinaries('xcode_12_beta', LLVM_XCODE)
+ isysroot_11 = os.path.join(LLVM_XCODE, 'Contents', 'Developer',
+ 'Platforms', 'MacOSX.platform', 'Developer',
+ 'SDKs', 'MacOSX11.0.sdk')
+ xcode_bin = os.path.join(LLVM_XCODE, 'Contents', 'Developer',
+ 'Toolchains', 'XcodeDefault.xctoolchain', 'usr',
+ 'bin')
+ # Include an arm64 slice for libclang_rt.osx.a. This requires using
+ # MacOSX11.0.sdk (via -isysroot, via DARWIN_macosx_CACHED_SYSROOT) and
+ # the new ld, via -B
+ compiler_rt_args.extend([
+ # We don't need 32-bit intel support for macOS, we only ship 64-bit.
+ '-DDARWIN_osx_ARCHS=arm64;x86_64',
+ '-DDARWIN_macosx_CACHED_SYSROOT=' + isysroot_11,
+ ])
+ ldflags += ['-B', xcode_bin]
+ else:
+ compiler_rt_args.extend(['-DDARWIN_osx_ARCHS=x86_64'])
else:
compiler_rt_args.append('-DCOMPILER_RT_BUILD_BUILTINS=OFF')
@@ -765,8 +833,7 @@ def main():
if sys.platform == 'darwin' and args.bootstrap:
# When building on 10.9, /usr/include usually doesn't exist, and while
# Xcode's clang automatically sets a sysroot, self-built clangs don't.
- cflags = ['-isysroot', subprocess.check_output(
- ['xcrun', '--show-sdk-path']).rstrip()]
+ cflags = ['-isysroot', isysroot]
cxxflags = ['-stdlib=libc++'] + cflags
ldflags += ['-stdlib=libc++']
deployment_target = '10.7'
@@ -808,6 +875,8 @@ def main():
'-DCHROMIUM_TOOLS=%s' % ';'.join(chrome_tools)]
if args.pgo:
cmake_args.append('-DLLVM_PROFDATA_FILE=' + LLVM_PROFDATA_FILE)
+ if args.thinlto:
+ cmake_args.append('-DLLVM_ENABLE_LTO=Thin')
if sys.platform == 'win32':
cmake_args.append('-DLLVM_ENABLE_ZLIB=FORCE_ON')
if sys.platform == 'darwin':
@@ -831,6 +900,7 @@ def main():
RunCommand(['ninja', 'cr-install'], msvc_arch='x64')
VerifyVersionOfBuiltClangMatchesVERSION()
+ VerifyZlibSupport()
if sys.platform == 'win32':
platform = 'windows'
diff --git a/chromium/tools/clang/scripts/goma_ld.py b/chromium/tools/clang/scripts/goma_ld.py
index 576dfa9f9d2..21a7dd2286c 100755
--- a/chromium/tools/clang/scripts/goma_ld.py
+++ b/chromium/tools/clang/scripts/goma_ld.py
@@ -35,20 +35,21 @@ class GomaLinkUnix(goma_link.GomaLinkBase):
PREFIX_REPLACE = TLTO + '-prefix-replace' + SEP
XIR = '-x ir '
- WHITELISTED_TARGETS = {
+ ALLOWLIST = {
'chrome',
}
def analyze_args(self, args, *posargs, **kwargs):
# TODO(crbug.com/1040196): Builds are unreliable when all targets use
- # distributed ThinLTO, so we only enable it for whitelisted targets.
+ # distributed ThinLTO, so we only enable it for some targets.
# For other targets, we fall back to local ThinLTO. We must use ThinLTO
# because we build with -fsplit-lto-unit, which requires code generation
# be done for each object and target.
- if args.output is None or os.path.basename(
- args.output) not in self.WHITELISTED_TARGETS:
- # Returning None causes the original, non-distributed linker command to be
- # invoked.
+ # Returning None from this function causes the original, non-distributed
+ # linker command to be invoked.
+ if args.output is None:
+ return None
+ if not (args.allowlist or os.path.basename(args.output) in self.ALLOWLIST):
return None
return super(GomaLinkUnix, self).analyze_args(args, *posargs, **kwargs)
diff --git a/chromium/tools/clang/scripts/goma_link.py b/chromium/tools/clang/scripts/goma_link.py
index 8bf194e4bdb..15fb617c9b3 100755
--- a/chromium/tools/clang/scripts/goma_link.py
+++ b/chromium/tools/clang/scripts/goma_link.py
@@ -36,8 +36,8 @@ except NameError:
# Type returned by analyze_args.
AnalyzeArgsResult = namedtuple('AnalyzeArgsResult', [
- 'output', 'linker', 'compiler', 'splitfile', 'index_params', 'codegen',
- 'codegen_params', 'final_params'
+ 'output', 'linker', 'compiler', 'splitfile', 'index_inputs', 'index_params',
+ 'codegen', 'codegen_params', 'final_inputs', 'final_params'
])
@@ -181,9 +181,15 @@ def parse_args(args):
# that set some values for this script. If these optional options are
# present, they must be followed by '--'.
ap = argparse.ArgumentParser()
+ ap.add_argument('--generate',
+ action='store_true',
+ help='generate ninja file, but do not invoke it.')
ap.add_argument('--gomacc', help='path to gomacc.')
ap.add_argument('--jobs', '-j', help='maximum number of concurrent jobs.')
ap.add_argument('--no-gomacc', action='store_true', help='do not use gomacc.')
+ ap.add_argument('--allowlist',
+ action='store_true',
+ help='act as if the target is on the allow list.')
try:
splitpos = args.index('--')
except:
@@ -234,6 +240,19 @@ class GomaLinkBase(object):
MLLVM_RE = re.compile('(?:-Wl,)?([-/]mllvm)[:=]?(.*)', re.IGNORECASE)
OBJ_RE = re.compile('(.*)\\.(o(?:bj)?)', re.IGNORECASE)
+ def _no_codegen(self, args):
+ """
+ Helper function for the case where no distributed code generation
+ is necessary. It invokes the original command, unless --generate
+ was passed, in which case it informs the user that no code generation
+ is necessary.
+ """
+ if args.generate:
+ sys.stderr.write(
+ 'No code generation required; no ninja file generated.\n')
+ return 5 # Indicates no code generation required.
+ return subprocess.call([args.linker] + args.linker_args)
+
def transform_codegen_param(self, param):
return self.transform_codegen_param_common(param)
@@ -378,12 +397,14 @@ class GomaLinkBase(object):
obj_dir = gen_dir
common_index = common_dir + '/empty.thinlto.bc'
+ index_inputs = set()
index_params = []
codegen = []
codegen_params = [
'-Wno-unused-command-line-argument',
'-Wno-override-module',
]
+ final_inputs = set()
final_params = []
in_mllvm = [False]
@@ -455,6 +476,7 @@ class GomaLinkBase(object):
return
index_params.append(param)
if os.path.exists(param):
+ index_inputs.add(param)
match = self.OBJ_RE.match(param)
if match and is_bitcode_file(param):
native = obj_dir + '/' + match.group(1) + '.' + match.group(2)
@@ -465,6 +487,7 @@ class GomaLinkBase(object):
ensure_file(index)
codegen.append((os.path.normpath(native), param, index))
else:
+ final_inputs.add(param)
final_params.append(param)
elif not self.LTO_RE.match(param):
final_params.append(param)
@@ -497,11 +520,12 @@ class GomaLinkBase(object):
splitfile = None
for tup in codegen:
final_params.append(tup[0])
+ index_inputs = []
else:
splitfile = gen_dir + '/' + output + '.split' + self.OBJ_SUFFIX
final_params.append(splitfile)
index_params.append(self.WL + self.OBJ_PATH + splitfile)
- used_obj_file = gen_dir + '/' + output + '.objs'
+ used_obj_file = gen_dir + '/' + os.path.basename(output) + '.objs'
final_params.append('@' + used_obj_file)
return AnalyzeArgsResult(
@@ -509,13 +533,15 @@ class GomaLinkBase(object):
linker=linker,
compiler=compiler,
splitfile=splitfile,
+ index_inputs=index_inputs,
index_params=index_params,
codegen=codegen,
codegen_params=codegen_params,
+ final_inputs=final_inputs,
final_params=final_params,
)
- def gen_ninja(self, ninjaname, params, objs):
+ def gen_ninja(self, ninjaname, params, gen_dir):
"""
Generates a ninja build file at path ninjaname, using original command line
params and with objs being a list of bitcode files for which to generate
@@ -525,70 +551,72 @@ class GomaLinkBase(object):
gomacc_prefix = ninjaenc(self.gomacc) + ' '
else:
gomacc_prefix = ''
+ base = gen_dir + '/' + os.path.basename(params.output)
+ ensure_dir(gen_dir)
ensure_dir(os.path.dirname(ninjaname))
+ codegen_cmd = ('%s%s -c %s -fthinlto-index=$index %s$bitcode -o $native' %
+ (gomacc_prefix, ninjaenc(params.compiler),
+ ninjajoin(params.codegen_params), self.XIR))
+ if params.index_inputs:
+ used_obj_file = base + '.objs'
+ index_rsp = base + '.index.rsp'
+ ensure_dir(os.path.dirname(used_obj_file))
+ if params.splitfile:
+ ensure_dir(os.path.dirname(params.splitfile))
+ # We use grep here to only codegen native objects which are actually
+ # used by the native link step. Ninja 1.10 introduced a dyndep feature
+ # which allows for a more elegant implementation, but Chromium still
+ # uses an older ninja version which doesn't have this feature.
+ codegen_cmd = '( ! grep -qF $native %s || %s)' % (
+ ninjaenc(used_obj_file), codegen_cmd)
+
with open(ninjaname, 'w') as f:
+ if params.index_inputs:
+ self.write_rsp(index_rsp, params.index_params)
+ f.write('\nrule index\n command = %s %s %s @$rspfile\n' %
+ (ninjaenc(params.linker),
+ ninjaenc(self.WL + self.TLTO + '-index-only' + self.SEP) +
+ '$out', self.WL + self.TLTO + '-emit-imports-files'))
+
f.write(('\nrule native-link\n command = %s @$rspname'
'\n rspfile = $rspname\n rspfile_content = $params\n') %
(ninjaenc(params.linker), ))
- f.write(('\nrule codegen\n command = %s%s -c %s'
- ' -fthinlto-index=$index %s$bitcode -o $out\n') %
- (gomacc_prefix, ninjaenc(params.compiler),
- ninjajoin(params.codegen_params), self.XIR))
+ f.write('\nrule codegen\n command = %s && touch $out\n' %
+ (codegen_cmd, ))
+
+ native_link_deps = []
+ if params.index_inputs:
+ f.write(
+ ('\nbuild %s | %s : index %s\n'
+ ' rspfile = %s\n'
+ ' rspfile_content = %s\n') %
+ (ninjaenc(used_obj_file), ninjajoin(
+ [x[2] for x in params.codegen]), ninjajoin(params.index_inputs),
+ ninjaenc(index_rsp), ninjajoin(params.index_params)))
+ native_link_deps.append(used_obj_file)
for tup in params.codegen:
obj, bitcode, index = tup
- f.write('\nbuild %s : codegen %s %s\n bitcode = %s\n index = %s\n' %
- tuple(map(ninjaenc, (obj, bitcode, index, bitcode, index))))
-
- f.write('\nbuild %s : native-link %s\n rspname = %s\n params = %s\n' %
- (ninjaenc(params.output), ninjajoin(objs),
- ninjaenc(params.output + '.final.rsp'),
- ninjajoin(params.final_params)))
+ stamp = obj + '.stamp'
+ native_link_deps.append(obj)
+ f.write(
+ ('\nbuild %s : codegen %s %s\n'
+ ' bitcode = %s\n'
+ ' index = %s\n'
+ ' native = %s\n'
+ '\nbuild %s : phony %s\n') % tuple(
+ map(ninjaenc,
+ (stamp, bitcode, index, bitcode, index, obj, obj, stamp))))
+
+ f.write(('\nbuild %s : native-link %s\n'
+ ' rspname = %s\n params = %s\n') %
+ (ninjaenc(params.output),
+ ninjajoin(list(params.final_inputs) + native_link_deps),
+ ninjaenc(base + '.final.rsp'), ninjajoin(params.final_params)))
f.write('\ndefault %s\n' % (ninjaenc(params.output), ))
- def thin_link(self, params, gen_dir):
- """
- Performs the thin link step.
- This generates the index files, imports files, split LTO object,
- and used object file.
- Returns a list of native objects we need to generate from bitcode
- files for the final link step.
- """
- used_obj_file = gen_dir + '/' + params.output + '.objs'
- index_rsp = gen_dir + '/' + params.output + '.index.rsp'
- ensure_dir(gen_dir)
- ensure_dir(os.path.dirname(used_obj_file))
- if params.splitfile:
- ensure_dir(os.path.dirname(params.splitfile))
- self.write_rsp(index_rsp, params.index_params)
- index_cmd = [
- params.linker,
- self.WL + self.TLTO + '-index-only' + self.SEP + used_obj_file,
- self.WL + self.TLTO + '-emit-imports-files', '@' + index_rsp
- ]
- report_run(index_cmd)
- with open(used_obj_file) as f:
- codegen_objs = [
- os.path.normpath(x) for x in f.read().split('\n') if len(x) > 0
- ]
- return codegen_objs
-
- def codegen_and_link(self, params, gen_dir, objs):
- """
- Performs code generation for selected bitcode files and
- the final link step.
- objs should be the list of native object files expected to be generated
- (as returned by thin_link()).
- """
- ninjaname = gen_dir + '/build.ninja'
- self.gen_ninja(ninjaname, params, objs)
- cmd = [autoninja(), '-f', ninjaname]
- if self.jobs:
- cmd.extend(['-j', str(self.jobs)])
- report_run(cmd)
-
def do_main(self, argv):
"""
This function contains the main code to run. Not intended to be called
@@ -598,7 +626,7 @@ class GomaLinkBase(object):
args = parse_args(argv)
args.output = self.output_path(argv[1:])
if args.output is None:
- return subprocess.call([args.linker] + args.linker_args)
+ return self._no_codegen(args)
if args.gomacc:
self.gomacc = args.gomacc
if args.no_gomacc:
@@ -607,7 +635,7 @@ class GomaLinkBase(object):
self.jobs = int(args.jobs)
basename = os.path.basename(args.output)
- # Only generate tailored native object files for whitelisted targets.
+ # Only generate tailored native object files for targets on the allow list.
# TODO: Find a better way to structure this. There are three different
# ways we can perform linking: Local ThinLTO, distributed ThinLTO,
# and distributed ThinLTO with common object files.
@@ -616,20 +644,26 @@ class GomaLinkBase(object):
# Currently, we don't detect this situation. We could, but it might
# be better to instead move this logic out of this script and into
# the build system.
- use_common_objects = basename not in self.WHITELISTED_TARGETS
+ use_common_objects = not (args.allowlist or basename in self.ALLOWLIST)
common_dir = 'common_objs'
gen_dir = 'lto.' + basename
params = self.analyze_args(args, gen_dir, common_dir, use_common_objects)
# If we determined that no distributed code generation need be done, just
# invoke the original command.
if params is None:
- return subprocess.call([args.linker] + args.linker_args)
+ return self._no_codegen(args)
if use_common_objects:
objs = [x[0] for x in params.codegen]
ensure_file(common_dir + '/empty.thinlto.bc')
+ ninjaname = gen_dir + '/build.ninja'
+ self.gen_ninja(ninjaname, params, gen_dir)
+ if args.generate:
+ sys.stderr.write('Generated ninja file %s\n' % (shquote(ninjaname), ))
else:
- objs = self.thin_link(params, gen_dir)
- self.codegen_and_link(params, gen_dir, objs)
+ cmd = [autoninja(), '-f', ninjaname]
+ if self.jobs:
+ cmd.extend(['-j', str(self.jobs)])
+ report_run(cmd)
return 0
def main(self, argv):
@@ -654,13 +688,13 @@ class GomaLinkWindows(GomaLinkBase):
PREFIX_REPLACE = TLTO + '-prefix-replace' + SEP
XIR = ''
- WHITELISTED_TARGETS = {
+ ALLOWLIST = {
'chrome.exe',
'chrome.dll',
'chrome_child.dll',
- # TODO: The following targets have been whitelisted because the
+ # TODO: The following targets are on the allow list because the
# common objects flow does not link them successfully. This should
- # be fixed, after which they can be removed from the whitelist.
+ # be fixed, after which they can be removed from the list.
'tls_edit.exe',
}
diff --git a/chromium/tools/clang/scripts/goma_link_integration_tests.py b/chromium/tools/clang/scripts/goma_link_integration_tests.py
index 3a2b4ecc699..00a01043f75 100755
--- a/chromium/tools/clang/scripts/goma_link_integration_tests.py
+++ b/chromium/tools/clang/scripts/goma_link_integration_tests.py
@@ -18,10 +18,13 @@
import goma_ld
import goma_link
+from io import StringIO
import os
import re
+import shlex
import subprocess
import unittest
+from unittest import mock
from goma_link_test_utils import named_directory, working_directory
@@ -45,24 +48,24 @@ def _create_inputs(path):
f.write('int bar() {\n return 9;\n}\n')
-class GomaLinkUnixWhitelistMain(goma_ld.GomaLinkUnix):
+class GomaLinkUnixAllowMain(goma_ld.GomaLinkUnix):
"""
- Same as goma_ld.GomaLinkUnix, but whitelists "main".
+ Same as goma_ld.GomaLinkUnix, but has "main" on the allow list.
"""
def __init__(self, *args, **kwargs):
- super(GomaLinkUnixWhitelistMain, self).__init__(*args, **kwargs)
- self.WHITELISTED_TARGETS = {'main'}
+ super(GomaLinkUnixAllowMain, self).__init__(*args, **kwargs)
+ self.ALLOWLIST = {'main'}
-class GomaLinkWindowsWhitelistMain(goma_link.GomaLinkWindows):
+class GomaLinkWindowsAllowMain(goma_link.GomaLinkWindows):
"""
- Same as goma_ld.GomaLinkWindows, but whitelists "main".
+ Same as goma_ld.GomaLinkWindows, but has "main" on the allow list.
"""
def __init__(self, *args, **kwargs):
- super(GomaLinkWindowsWhitelistMain, self).__init__(*args, **kwargs)
- self.WHITELISTED_TARGETS = {'main.exe'}
+ super(GomaLinkWindowsAllowMain, self).__init__(*args, **kwargs)
+ self.ALLOWLIST = {'main.exe'}
class GomaLinkIntegrationTest(unittest.TestCase):
@@ -109,8 +112,10 @@ class GomaLinkIntegrationTest(unittest.TestCase):
codegen_text = codegen_match.group(0)
self.assertIn('my_goma.sh', codegen_text)
self.assertNotIn('-flto', codegen_text)
- self.assertIn('build common_objs/obj/main.obj : codegen ', buildrules)
- self.assertIn('build common_objs/obj/foo.obj : codegen ', buildrules)
+ self.assertIn('build common_objs/obj/main.obj.stamp : codegen ',
+ buildrules)
+ self.assertIn('build common_objs/obj/foo.obj.stamp : codegen ',
+ buildrules)
self.assertIn(' index = common_objs/empty.thinlto.bc', buildrules)
link_match = re.search('^build main.exe : native-link\\b.*?^[^ ]',
buildrules, re.MULTILINE | re.DOTALL)
@@ -123,7 +128,7 @@ class GomaLinkIntegrationTest(unittest.TestCase):
# functions, one of which calls the other.
self.assertTrue(b'call' in disasm or b'jmp' in disasm)
- def test_distributed_lto_whitelisted(self):
+ def test_distributed_lto_allowlist(self):
with named_directory() as d, working_directory(d):
_create_inputs(d)
os.makedirs('obj')
@@ -143,7 +148,7 @@ class GomaLinkIntegrationTest(unittest.TestCase):
['llvm-ar', 'crsT', 'obj/foobar.lib', 'obj/bar.obj', 'obj/foo.obj'])
with open('main.rsp', 'w') as f:
f.write('obj/main.obj\n' 'obj/foobar.lib\n')
- rc = GomaLinkWindowsWhitelistMain().main([
+ rc = GomaLinkWindowsAllowMain().main([
'goma_link.py', '--gomacc', 'gomacc', '--',
self.lld_link(), '-nodefaultlib', '-entry:main', '-machine:X86',
'-opt:lldlto=2', '-mllvm:-import-instr-limit=10', '-out:main.exe',
@@ -162,8 +167,10 @@ class GomaLinkIntegrationTest(unittest.TestCase):
self.assertIn('-m32', codegen_text)
self.assertIn('-mllvm -import-instr-limit=10', codegen_text)
self.assertNotIn('-flto', codegen_text)
- self.assertIn('build lto.main.exe/obj/main.obj : codegen ', buildrules)
- self.assertIn('build lto.main.exe/obj/foo.obj : codegen ', buildrules)
+ self.assertIn('build lto.main.exe/obj/main.obj.stamp : codegen ',
+ buildrules)
+ self.assertIn('build lto.main.exe/obj/foo.obj.stamp : codegen ',
+ buildrules)
link_match = re.search('^build main.exe : native-link\\b.*?^[^ ]',
buildrules, re.MULTILINE | re.DOTALL)
self.assertIsNotNone(link_match)
@@ -176,6 +183,41 @@ class GomaLinkIntegrationTest(unittest.TestCase):
self.assertNotIn(b'jmp', disasm)
self.assertNotIn(b'call', disasm)
+ def test_override_allowlist(self):
+ with named_directory() as d, working_directory(d):
+ _create_inputs(d)
+ os.makedirs('obj')
+ subprocess.check_call([
+ self.clangcl(), '-c', '-O2', '-flto=thin', 'main.cpp',
+ '-Foobj/main.obj'
+ ])
+ subprocess.check_call([
+ self.clangcl(), '-c', '-O2', '-flto=thin', 'foo.cpp', '-Foobj/foo.obj'
+ ])
+ rc = goma_link.GomaLinkWindows().main([
+ 'goma_link.py', '--generate', '--allowlist', '--',
+ self.lld_link(), '-nodefaultlib', '-entry:main', '-opt:lldlto=2',
+ '-out:main.exe', 'obj/main.obj', 'obj/foo.obj'
+ ])
+ # Should succeed.
+ self.assertEqual(rc, 0)
+ # Check that we have rules for main and foo, and that they are
+ # not common objects.
+ with open(os.path.join(d, 'lto.main.exe', 'build.ninja')) as f:
+ buildrules = f.read()
+ codegen_match = re.search(r'^rule codegen\b.*?^[^ ]', buildrules,
+ re.MULTILINE | re.DOTALL)
+ self.assertIsNotNone(codegen_match)
+ codegen_text = codegen_match.group(0)
+ self.assertNotIn('-flto', codegen_text)
+ self.assertIn('build lto.main.exe/obj/main.obj.stamp : codegen ',
+ buildrules)
+ self.assertIn('build lto.main.exe/obj/foo.obj.stamp : codegen ',
+ buildrules)
+ link_match = re.search(r'^build main.exe : native-link\b.*?^[^ ]',
+ buildrules, re.MULTILINE | re.DOTALL)
+ self.assertIsNotNone(link_match)
+
class GomaLdIntegrationTest(unittest.TestCase):
def clangxx(self):
@@ -188,7 +230,7 @@ class GomaLdIntegrationTest(unittest.TestCase):
[self.clangxx(), '-c', '-Os', 'main.cpp', '-o', 'main.o'])
subprocess.check_call(
[self.clangxx(), '-c', '-Os', 'foo.cpp', '-o', 'foo.o'])
- rc = GomaLinkUnixWhitelistMain().main([
+ rc = GomaLinkUnixAllowMain().main([
'goma_ld.py', '--gomacc', 'gomacc', '--',
self.clangxx(), '-fuse-ld=lld', 'main.o', 'foo.o', '-o', 'main'
])
@@ -235,7 +277,7 @@ class GomaLdIntegrationTest(unittest.TestCase):
])
subprocess.check_call(
[self.clangxx(), '-c', '-Os', '-flto=thin', 'foo.cpp', '-o', 'foo.o'])
- rc = GomaLinkUnixWhitelistMain().main([
+ rc = GomaLinkUnixAllowMain().main([
'goma_ld.py', '-j', '16', '--',
self.clangxx(), '-fuse-ld=lld', '-flto=thin', 'main.o', 'foo.o', '-o',
'main'
@@ -246,8 +288,8 @@ class GomaLdIntegrationTest(unittest.TestCase):
with open(os.path.join(d, 'lto.main', 'build.ninja')) as f:
buildrules = f.read()
self.assertIn('gomacc ', buildrules)
- self.assertIn('build lto.main/main.o : codegen ', buildrules)
- self.assertIn('build lto.main/foo.o : codegen ', buildrules)
+ self.assertIn('build lto.main/main.o.stamp : codegen ', buildrules)
+ self.assertIn('build lto.main/foo.o.stamp : codegen ', buildrules)
# Check that main does not call foo.
disasm = subprocess.check_output(['llvm-objdump', '-d', 'main'])
main_idx = disasm.index(b' <main>:\n')
@@ -267,7 +309,7 @@ class GomaLdIntegrationTest(unittest.TestCase):
[self.clangxx(), '-c', '-Os', '-flto=thin', 'bar.cpp', '-o', 'bar.o'])
subprocess.check_call(
['llvm-ar', 'crsT', 'libfoobar.a', 'bar.o', 'foo.o'])
- rc = GomaLinkUnixWhitelistMain().main([
+ rc = GomaLinkUnixAllowMain().main([
'goma_ld.py',
self.clangxx(), '-fuse-ld=lld', '-flto=thin', 'main.o', 'libfoobar.a',
'-o', 'main'
@@ -278,8 +320,8 @@ class GomaLdIntegrationTest(unittest.TestCase):
with open(os.path.join(d, 'lto.main', 'build.ninja')) as f:
buildrules = f.read()
self.assertIn('gomacc ', buildrules)
- self.assertIn('build lto.main/main.o : codegen ', buildrules)
- self.assertIn('build lto.main/foo.o : codegen ', buildrules)
+ self.assertIn('build lto.main/main.o.stamp : codegen ', buildrules)
+ self.assertIn('build lto.main/foo.o.stamp : codegen ', buildrules)
# Check that main does not call foo.
disasm = subprocess.check_output(['llvm-objdump', '-d', 'main'])
main_idx = disasm.index(b' <main>:\n')
@@ -305,7 +347,7 @@ class GomaLdIntegrationTest(unittest.TestCase):
])
subprocess.check_call(
['llvm-ar', 'crsT', 'obj/libfoobar.a', 'obj/bar.o', 'obj/foo.o'])
- rc = GomaLinkUnixWhitelistMain().main([
+ rc = GomaLinkUnixAllowMain().main([
'goma_ld.py',
self.clangxx(), '-fuse-ld=lld', '-flto=thin', 'obj/main.o',
'obj/libfoobar.a', '-o', 'main'
@@ -316,8 +358,8 @@ class GomaLdIntegrationTest(unittest.TestCase):
with open(os.path.join(d, 'lto.main', 'build.ninja')) as f:
buildrules = f.read()
self.assertIn('gomacc ', buildrules)
- self.assertIn('build lto.main/obj/main.o : codegen ', buildrules)
- self.assertIn('build lto.main/obj/foo.o : codegen ', buildrules)
+ self.assertIn('build lto.main/obj/main.o.stamp : codegen ', buildrules)
+ self.assertIn('build lto.main/obj/foo.o.stamp : codegen ', buildrules)
# Check that main does not call foo.
disasm = subprocess.check_output(['llvm-objdump', '-d', 'main'])
main_idx = disasm.index(b' <main>:\n')
@@ -339,7 +381,7 @@ class GomaLdIntegrationTest(unittest.TestCase):
])
with open('main.rsp', 'w') as f:
f.write('obj/main.o\n' 'obj/foo.o\n')
- rc = GomaLinkUnixWhitelistMain().main([
+ rc = GomaLinkUnixAllowMain().main([
'goma_ld.py',
self.clangxx(), '-fuse-ld=lld', '-flto=thin', '-g', '-gsplit-dwarf',
'-Wl,--lto-O2', '-o', 'main', '@main.rsp'
@@ -377,7 +419,7 @@ class GomaLdIntegrationTest(unittest.TestCase):
'-fwhole-program-vtables\n'
'obj/main.o\n'
'obj/libfoobar.a\n')
- rc = GomaLinkUnixWhitelistMain().main([
+ rc = GomaLinkUnixAllowMain().main([
'goma_ld.py',
self.clangxx(), '-fuse-ld=lld', '-flto=thin', '-m32', '-Wl,-mllvm',
'-Wl,-generate-type-units', '-Wl,--lto-O2', '-o', 'main',
@@ -396,8 +438,8 @@ class GomaLdIntegrationTest(unittest.TestCase):
self.assertIn('-m32', codegen_text)
self.assertIn('-mllvm -generate-type-units', codegen_text)
self.assertNotIn('-flto', codegen_text)
- self.assertIn('build lto.main/obj/main.o : codegen ', buildrules)
- self.assertIn('build lto.main/obj/foo.o : codegen ', buildrules)
+ self.assertIn('build lto.main/obj/main.o.stamp : codegen ', buildrules)
+ self.assertIn('build lto.main/obj/foo.o.stamp : codegen ', buildrules)
link_match = re.search('^build main : native-link\\b.*?^[^ ]',
buildrules, re.MULTILINE | re.DOTALL)
self.assertIsNotNone(link_match)
@@ -418,7 +460,7 @@ class GomaLdIntegrationTest(unittest.TestCase):
])
subprocess.check_call(
[self.clangxx(), '-c', '-Os', '-flto=thin', 'foo.cpp', '-o', 'foo.o'])
- rc = GomaLinkUnixWhitelistMain().main([
+ rc = GomaLinkUnixAllowMain().main([
'goma_ld.py', '--no-gomacc', '-j', '16', '--',
self.clangxx(), '-fuse-ld=lld', '-flto=thin', 'main.o', 'foo.o', '-o',
'main'
@@ -429,8 +471,8 @@ class GomaLdIntegrationTest(unittest.TestCase):
with open(os.path.join(d, 'lto.main', 'build.ninja')) as f:
buildrules = f.read()
self.assertNotIn('gomacc ', buildrules)
- self.assertIn('build lto.main/main.o : codegen ', buildrules)
- self.assertIn('build lto.main/foo.o : codegen ', buildrules)
+ self.assertIn('build lto.main/main.o.stamp : codegen ', buildrules)
+ self.assertIn('build lto.main/foo.o.stamp : codegen ', buildrules)
# Check that main does not call foo.
disasm = subprocess.check_output(['llvm-objdump', '-d', 'main'])
main_idx = disasm.index(b' <main>:\n')
@@ -438,6 +480,61 @@ class GomaLdIntegrationTest(unittest.TestCase):
main_disasm = disasm[main_idx:after_main_idx]
self.assertNotIn(b'foo', main_disasm)
+ def test_generate_no_codegen(self):
+ with named_directory() as d, working_directory(d):
+ with open('main.o', 'wb') as f:
+ f.write(b'\7fELF')
+ with mock.patch('sys.stderr', new_callable=StringIO) as stderr:
+ rc = GomaLinkUnixAllowMain().main([
+ 'goma_ld.py', '--generate', '--',
+ self.clangxx(), 'main.o', '-o', 'main'
+ ])
+ self.assertEqual(rc, 5)
+ self.assertIn('no ninja file generated.\n', stderr.getvalue())
+
+ def test_generate(self):
+ with named_directory() as d, working_directory(d):
+ with open('main.o', 'wb') as f:
+ f.write(b'BC\xc0\xde')
+ with mock.patch('sys.stderr', new_callable=StringIO) as stderr:
+ rc = GomaLinkUnixAllowMain().main([
+ 'goma_ld.py', '--generate', '--',
+ self.clangxx(), 'main.o', '-o', 'main'
+ ])
+ self.assertEqual(rc, 0)
+ m = re.search('ninja file (.*)', stderr.getvalue())
+ self.assertIsNotNone(m)
+ path = shlex.split(m.group(1))[0]
+ self.assertTrue(os.path.exists(path))
+ content = open(path).read()
+ self.assertRegex(
+ content,
+ re.compile('^build [^:]+/main\\.o\\.stamp : codegen ',
+ re.MULTILINE))
+
+ def test_override_allowlist(self):
+ with named_directory() as d, working_directory(d):
+ _create_inputs(d)
+ subprocess.check_call([
+ self.clangxx(), '-c', '-Os', '-flto=thin', 'main.cpp', '-o', 'main.o'
+ ])
+ subprocess.check_call(
+ [self.clangxx(), '-c', '-Os', '-flto=thin', 'foo.cpp', '-o', 'foo.o'])
+ rc = goma_ld.GomaLinkUnix().main([
+ 'goma_ld.py', '--generate', '--allowlist', '--',
+ self.clangxx(), '-fuse-ld=lld', '-flto=thin', 'main.o', 'foo.o', '-o',
+ 'main'
+ ])
+ # Should succeed.
+ self.assertEqual(rc, 0)
+ # build.ninja file should have rules for main and foo.
+ ninjafile = os.path.join(d, 'lto.main', 'build.ninja')
+ self.assertTrue(os.path.exists(ninjafile))
+ with open(ninjafile) as f:
+ buildrules = f.read()
+ self.assertIn('build lto.main/main.o.stamp : codegen ', buildrules)
+ self.assertIn('build lto.main/foo.o.stamp : codegen ', buildrules)
+
if __name__ == '__main__':
unittest.main()
diff --git a/chromium/tools/clang/scripts/package.py b/chromium/tools/clang/scripts/package.py
index 4e4574a6e65..cfed526d3b5 100755
--- a/chromium/tools/clang/scripts/package.py
+++ b/chromium/tools/clang/scripts/package.py
@@ -184,9 +184,14 @@ def main():
shutil.rmtree(LLVM_BOOTSTRAP_INSTALL_DIR, ignore_errors=True)
shutil.rmtree(LLVM_BUILD_DIR, ignore_errors=True)
- build_cmd = [sys.executable, os.path.join(THIS_DIR, 'build.py'),
- '--bootstrap', '--disable-asserts',
- '--run-tests', '--pgo']
+ build_cmd = [
+ sys.executable,
+ os.path.join(THIS_DIR, 'build.py'), '--bootstrap', '--disable-asserts',
+ '--run-tests', '--pgo'
+ ]
+ if sys.platform.startswith('linux'):
+ build_cmd.append('--thinlto')
+
TeeCmd(build_cmd, log)
stamp = open(STAMP_FILE).read().rstrip()
@@ -228,9 +233,10 @@ def main():
'lib/clang/$V/lib/darwin/libclang_rt.asan_iossim_dynamic.dylib',
'lib/clang/$V/lib/darwin/libclang_rt.asan_osx_dynamic.dylib',
- # OS X and iOS builtin libraries (iossim is lipo'd into ios) for the
- # _IsOSVersionAtLeast runtime function.
+ # OS X and iOS builtin libraries for the _IsOSVersionAtLeast runtime
+ # function.
'lib/clang/$V/lib/darwin/libclang_rt.ios.a',
+ 'lib/clang/$V/lib/darwin/libclang_rt.iossim.a',
'lib/clang/$V/lib/darwin/libclang_rt.osx.a',
# Profile runtime (used by profiler and code coverage).
diff --git a/chromium/tools/clang/scripts/test_tool.py b/chromium/tools/clang/scripts/test_tool.py
index 171e33eb608..00190937018 100755
--- a/chromium/tools/clang/scripts/test_tool.py
+++ b/chromium/tools/clang/scripts/test_tool.py
@@ -13,6 +13,7 @@ import glob
import json
import os
import os.path
+import re
import shutil
import subprocess
import sys
@@ -126,6 +127,28 @@ def _ApplyTool(tools_clang_scripts_directory,
_RunGit(args)
+def _NormalizePathInRawOutput(path, test_dir):
+ if not os.path.isabs(path):
+ path = os.path.join(test_dir, path)
+
+ return os.path.relpath(path, test_dir)
+
+
+def _NormalizeSingleRawOutputLine(output_line, test_dir):
+ if not re.match('^[^:]+(:::.*){4,4}$', output_line):
+ return output_line
+
+ edit_type, path, offset, length, replacement = output_line.split(':::', 4)
+ path = _NormalizePathInRawOutput(path, test_dir)
+ return "%s:::%s:::%s:::%s:::%s" % (edit_type, path, offset, length,
+ replacement)
+
+
+def _NormalizeRawOutput(output_lines, test_dir):
+ return map(lambda line: _NormalizeSingleRawOutputLine(line, test_dir),
+ output_lines)
+
+
def main(argv):
parser = argparse.ArgumentParser()
parser.add_argument(
@@ -209,6 +232,11 @@ def main(argv):
expected_output = f.readlines()
with open(actual, 'r') as f:
actual_output = f.readlines()
+ if not args.apply_edits:
+ actual_output = _NormalizeRawOutput(actual_output,
+ test_directory_for_tool)
+ expected_output = _NormalizeRawOutput(expected_output,
+ test_directory_for_tool)
if actual_output != expected_output:
failed += 1
lines = difflib.unified_diff(expected_output, actual_output,
diff --git a/chromium/tools/clang/scripts/update.py b/chromium/tools/clang/scripts/update.py
index c9bdcf3ae20..b1018321ec3 100755
--- a/chromium/tools/clang/scripts/update.py
+++ b/chromium/tools/clang/scripts/update.py
@@ -37,13 +37,12 @@ import zipfile
# Do NOT CHANGE this if you don't know what you're doing -- see
# https://chromium.googlesource.com/chromium/src/+/master/docs/updating_clang.md
# Reverting problematic clang rolls is safe, though.
-CLANG_REVISION = '4e813bbdf'
-CLANG_SVN_REVISION = 'n356902'
+# This is the output of `git describe` and is usable as a commit-ish.
+CLANG_REVISION = 'llvmorg-12-init-3492-ga1caa302'
CLANG_SUB_REVISION = 1
-PACKAGE_VERSION = '%s-%s-%s' % (CLANG_SVN_REVISION, CLANG_REVISION[:8],
- CLANG_SUB_REVISION)
-RELEASE_VERSION = '11.0.0'
+PACKAGE_VERSION = '%s-%s' % (CLANG_REVISION, CLANG_SUB_REVISION)
+RELEASE_VERSION = '12.0.0'
CDS_URL = os.environ.get('CDS_CLANG_BUCKET_OVERRIDE',
@@ -235,7 +234,7 @@ def UpdatePackage(package_name, host_os):
try:
GCLIENT_CONFIG = os.path.join(os.path.dirname(CHROMIUM_DIR), '.gclient')
env = {}
- execfile(GCLIENT_CONFIG, env, env)
+ exec (open(GCLIENT_CONFIG).read(), env, env)
target_os = env.get('target_os', target_os)
except:
pass
diff --git a/chromium/tools/clang/scripts/upload_revision.py b/chromium/tools/clang/scripts/upload_revision.py
index 9cd4484cd2b..8eb81b3a368 100755
--- a/chromium/tools/clang/scripts/upload_revision.py
+++ b/chromium/tools/clang/scripts/upload_revision.py
@@ -18,7 +18,7 @@ import shutil
import subprocess
import sys
-from build import GetCommitCount, CheckoutLLVM, LLVM_DIR
+from build import CheckoutLLVM, GetCommitDescription, LLVM_DIR
from update import CHROMIUM_DIR
# Path constants.
@@ -29,8 +29,7 @@ CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..'))
# Keep lines in here at <= 72 columns, else they wrap in gerrit.
COMMIT_FOOTER = \
'''
-TODO: Add bug number.
-
+Bug: TODO
Cq-Include-Trybots: chromium/try:mac_chromium_asan_rel_ng
Cq-Include-Trybots: chromium/try:linux_chromium_cfi_rel_ng
Cq-Include-Trybots: chromium/try:linux_chromium_chromeos_asan_rel_ng
@@ -40,7 +39,8 @@ Cq-Include-Trybots: chromium/try:linux-chromeos-dbg,win-asan
Cq-Include-Trybots: chromium/try:chromeos-amd64-generic-cfi-thin-lto-rel
Cq-Include-Trybots: chromium/try:linux_chromium_compile_dbg_32_ng
Cq-Include-Trybots: chromium/try:win7-rel,win-angle-deqp-rel-32
-Cq-Include-Trybots: chromium/try:win-angle-deqp-rel-64,linux_angle_deqp_rel_ng
+Cq-Include-Trybots: chromium/try:linux_angle_deqp_rel_ng
+Cq-Include-Trybots: chromium/try:win-angle-deqp-rel-64
Cq-Include-Trybots: chromium/try:dawn-win10-x86-deps-rel
Cq-Include-Trybots: chrome/try:iphone-device,ipad-device
Cq-Include-Trybots: chrome/try:linux-chromeos-chrome
@@ -49,28 +49,25 @@ Cq-Include-Trybots: chrome/try:win-chrome,win64-chrome,mac-chrome
is_win = sys.platform.startswith('win32')
-def PatchRevision(clang_git_revision, clang_svn_revision, clang_sub_revision):
+
+def PatchRevision(clang_git_revision, clang_sub_revision):
with open(UPDATE_PY_PATH, 'rb') as f:
content = f.read()
- m = re.search("CLANG_REVISION = '([0-9a-f]+)'", content)
+ m = re.search("CLANG_REVISION = '([0-9a-z-]+)'", content)
clang_old_git_revision = m.group(1)
- m = re.search("CLANG_SVN_REVISION = '(n[0-9]+)'", content)
- clang_old_svn_revision = m.group(1)
m = re.search("CLANG_SUB_REVISION = ([0-9]+)", content)
clang_old_sub_revision = m.group(1)
- content = re.sub("CLANG_REVISION = '[0-9a-f]+'",
+ content = re.sub("CLANG_REVISION = '[0-9a-z-]+'",
"CLANG_REVISION = '{}'".format(clang_git_revision),
- content, count=1)
- content = re.sub("CLANG_SVN_REVISION = 'n[0-9]+'",
- "CLANG_SVN_REVISION = '{}'".format(clang_svn_revision),
- content, count=1)
+ content,
+ count=1)
content = re.sub("CLANG_SUB_REVISION = [0-9]+",
"CLANG_SUB_REVISION = {}".format(clang_sub_revision),
content, count=1)
with open(UPDATE_PY_PATH, 'wb') as f:
f.write(content)
- return clang_old_git_revision, clang_old_svn_revision, clang_old_sub_revision
+ return "{}-{}".format(clang_old_git_revision, clang_old_sub_revision)
def Git(args):
@@ -87,30 +84,24 @@ def main():
args = parser.parse_args()
- clang_git_revision = args.clang_git_revision[0]
+ clang_raw_git_revision = args.clang_git_revision[0]
- # To get the commit count, we need a checkout.
- CheckoutLLVM(clang_git_revision, LLVM_DIR);
- clang_svn_revision = 'n' + GetCommitCount(clang_git_revision)
+ # To `git describe`, we need a checkout.
+ CheckoutLLVM(clang_raw_git_revision, LLVM_DIR)
+ clang_git_revision = GetCommitDescription(clang_raw_git_revision)
clang_sub_revision = args.clang_sub_revision
os.chdir(CHROMIUM_DIR)
- print("Making a patch for Clang {}-{}-{}".format(
- clang_svn_revision, clang_git_revision[:8], clang_sub_revision))
+ print("Making a patch for Clang {}-{}".format(clang_git_revision,
+ clang_sub_revision))
- clang_old_git_revision, clang_old_svn_revision, clang_old_sub_revision = \
- PatchRevision(clang_git_revision, clang_svn_revision, clang_sub_revision)
-
- rev_string = "{}-{}-{}".format(clang_svn_revision,
- clang_git_revision[:8],
- clang_sub_revision)
+ rev_string = "{}-{}".format(clang_git_revision, clang_sub_revision)
Git(["checkout", "origin/master", "-b", "clang-{}".format(rev_string)])
- Git(["add", UPDATE_PY_PATH])
- old_rev_string = "{}-{}-{}".format(clang_old_svn_revision,
- clang_old_git_revision[:8],
- clang_old_sub_revision)
+ old_rev_string = PatchRevision(clang_git_revision, clang_sub_revision)
+
+ Git(["add", UPDATE_PY_PATH])
commit_message = 'Ran `{}`.'.format(' '.join(sys.argv)) + COMMIT_FOOTER
Git(["commit", "-m", "Roll clang {} : {}.\n\n{}".format(