diff options
author | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2020-10-29 10:46:47 +0100 |
---|---|---|
committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2020-11-02 12:02:10 +0000 |
commit | 99677208ff3b216fdfec551fbe548da5520cd6fb (patch) | |
tree | 476a4865c10320249360e859d8fdd3e01833b03a /chromium/tools/clang | |
parent | c30a6232df03e1efbd9f3b226777b07e087a1122 (diff) | |
download | qtwebengine-chromium-99677208ff3b216fdfec551fbe548da5520cd6fb.tar.gz |
BASELINE: Update Chromium to 86.0.4240.124
Change-Id: Ide0ff151e94cd665ae6521a446995d34a9d1d644
Reviewed-by: Allan Sandfeld Jensen <allan.jensen@qt.io>
Diffstat (limited to 'chromium/tools/clang')
-rwxr-xr-x | chromium/tools/clang/blink_gc_plugin/process-graph.py | 6 | ||||
-rw-r--r-- | chromium/tools/clang/empty_string/EmptyStringConverter.cpp | 8 | ||||
-rwxr-xr-x | chromium/tools/clang/pylib/clang/compile_db.py | 13 | ||||
-rw-r--r-- | chromium/tools/clang/rewrite_raw_ptr_fields/RewriteRawPtrFields.cpp | 153 | ||||
-rw-r--r-- | chromium/tools/clang/rewrite_raw_ptr_fields/manual-fields-to-ignore.txt | 6 | ||||
-rw-r--r-- | chromium/tools/clang/rewrite_raw_ptr_fields/manual-paths-to-ignore.txt | 118 | ||||
-rwxr-xr-x | chromium/tools/clang/rewrite_raw_ptr_fields/rewrite.sh | 11 | ||||
-rwxr-xr-x | chromium/tools/clang/scripts/build.py | 120 | ||||
-rwxr-xr-x | chromium/tools/clang/scripts/goma_ld.py | 13 | ||||
-rwxr-xr-x | chromium/tools/clang/scripts/goma_link.py | 164 | ||||
-rwxr-xr-x | chromium/tools/clang/scripts/goma_link_integration_tests.py | 159 | ||||
-rwxr-xr-x | chromium/tools/clang/scripts/package.py | 16 | ||||
-rwxr-xr-x | chromium/tools/clang/scripts/test_tool.py | 28 | ||||
-rwxr-xr-x | chromium/tools/clang/scripts/update.py | 11 | ||||
-rwxr-xr-x | chromium/tools/clang/scripts/upload_revision.py | 51 |
15 files changed, 638 insertions, 239 deletions
diff --git a/chromium/tools/clang/blink_gc_plugin/process-graph.py b/chromium/tools/clang/blink_gc_plugin/process-graph.py index 39e53f18e12..06031838528 100755 --- a/chromium/tools/clang/blink_gc_plugin/process-graph.py +++ b/chromium/tools/clang/blink_gc_plugin/process-graph.py @@ -4,9 +4,13 @@ # found in the LICENSE file. from __future__ import print_function -from StringIO import StringIO import argparse, os, sys, json, subprocess, pickle +try: + from StringIO import StringIO # Python 2 +except: + from io import StringIO + parser = argparse.ArgumentParser( description = "Process the Blink points-to graph generated by the Blink GC plugin.") diff --git a/chromium/tools/clang/empty_string/EmptyStringConverter.cpp b/chromium/tools/clang/empty_string/EmptyStringConverter.cpp index db82a7782e6..f6f68009864 100644 --- a/chromium/tools/clang/empty_string/EmptyStringConverter.cpp +++ b/chromium/tools/clang/empty_string/EmptyStringConverter.cpp @@ -80,12 +80,12 @@ class EmptyStringConverter { }; void EmptyStringConverter::SetupMatchers(MatchFinder* match_finder) { - const clang::ast_matchers::StatementMatcher& constructor_call = id( - "call", + const clang::ast_matchers::StatementMatcher& constructor_call = cxxConstructExpr( hasDeclaration(cxxMethodDecl(ofClass(hasName("std::basic_string")))), - argumentCountIs(2), hasArgument(0, id("literal", stringLiteral())), - hasArgument(1, cxxDefaultArgExpr()))); + argumentCountIs(2), hasArgument(0, stringLiteral().bind("literal")), + hasArgument(1, cxxDefaultArgExpr())) + .bind("call"); // Note that expr(has()) in the matcher is significant; the Clang AST wraps // calls to the std::string constructor with exprWithCleanups nodes. Without diff --git a/chromium/tools/clang/pylib/clang/compile_db.py b/chromium/tools/clang/pylib/clang/compile_db.py index 50decf71907..6fe04981baf 100755 --- a/chromium/tools/clang/pylib/clang/compile_db.py +++ b/chromium/tools/clang/pylib/clang/compile_db.py @@ -14,12 +14,13 @@ import subprocess _RSP_RE = re.compile(r' (@(.+?\.rsp)) ') _CMD_LINE_RE = re.compile( - r'^(?P<gomacc>.*gomacc\.exe"?\s+)?(?P<clang>\S*clang\S*)\s+(?P<args>.*)$') + r'^(?P<gomacc>.*gomacc(\.exe)?"?\s+)?(?P<clang>\S*clang\S*)\s+(?P<args>.*)$' +) _debugging = False def _ProcessCommand(command): - """Removes gomacc.exe and inserts --driver-mode=cl as the first argument. + """Removes gomacc(.exe). On Windows inserts --driver-mode=cl as the first arg. Note that we deliberately don't use shlex.split here, because it doesn't work predictably for Windows commands (specifically, it doesn't parse args the same @@ -31,10 +32,9 @@ def _ProcessCommand(command): # If the driver mode is not already set then define it. Driver mode is # automatically included in the compile db by clang starting with release # 9.0.0. - if "--driver_mode" in command: - driver_mode = "" + driver_mode = '' # Only specify for Windows. Other platforms do fine without it. - elif sys.platform == 'win32': + if sys.platform == 'win32' and '--driver_mode' not in command: driver_mode = '--driver-mode=cl' match = _CMD_LINE_RE.search(command) @@ -88,6 +88,8 @@ def ProcessCompileDatabaseIfNeeded(compile_db): Returns: A postprocessed compile db that clang tooling can use. """ + compile_db = [_ProcessEntry(e) for e in compile_db] + # TODO(dcheng): Ideally this would check target_os... but not sure there's an # easy way to do that, and (for now) cross-compiles don't work without custom # patches anyway. @@ -96,7 +98,6 @@ def ProcessCompileDatabaseIfNeeded(compile_db): if _debugging: print('Read in %d entries from the compile db' % len(compile_db)) - compile_db = [_ProcessEntry(e) for e in compile_db] original_length = len(compile_db) # Filter out NaCl stuff. The clang tooling chokes on them. diff --git a/chromium/tools/clang/rewrite_raw_ptr_fields/RewriteRawPtrFields.cpp b/chromium/tools/clang/rewrite_raw_ptr_fields/RewriteRawPtrFields.cpp index f94ce4e03f3..27b3fbb196e 100644 --- a/chromium/tools/clang/rewrite_raw_ptr_fields/RewriteRawPtrFields.cpp +++ b/chromium/tools/clang/rewrite_raw_ptr_fields/RewriteRawPtrFields.cpp @@ -70,9 +70,17 @@ const char kIncludePath[] = "base/memory/checked_ptr.h"; // // See also: // - OutputSectionHelper -// - FieldDeclFilterFile +// - FilterFile const char kExcludeFieldsParamName[] = "exclude-fields"; +// Name of a cmdline parameter that can be used to specify a file listing +// regular expressions describing paths that should be excluded from the +// rewrite. +// +// See also: +// - PathFilterFile +const char kExcludePathsParamName[] = "exclude-paths"; + // OutputSectionHelper helps gather and emit a section of output. // // The section of output is delimited in a way that makes it easy to extract it @@ -93,13 +101,16 @@ const char kExcludeFieldsParamName[] = "exclude-fields"; // changes). // // See also: -// - FieldDeclFilterFile +// - FilterFile // - OutputHelper class OutputSectionHelper { public: explicit OutputSectionHelper(llvm::StringRef output_delimiter) : output_delimiter_(output_delimiter.str()) {} + OutputSectionHelper(const OutputSectionHelper&) = delete; + OutputSectionHelper& operator=(const OutputSectionHelper&) = delete; + void Add(llvm::StringRef output_line, llvm::StringRef tag = "") { // Look up |tags| associated with |output_line|. As a side effect of the // lookup, |output_line| will be inserted if it wasn't already present in @@ -152,6 +163,9 @@ class OutputHelper : public clang::tooling::SourceFileCallbacks { : edits_helper_("EDITS"), field_decl_filter_helper_("FIELD FILTERS") {} ~OutputHelper() = default; + OutputHelper(const OutputHelper&) = delete; + OutputHelper& operator=(const OutputHelper&) = delete; + void AddReplacement(const clang::SourceManager& source_manager, const clang::SourceRange& replacement_range, std::string replacement_text, @@ -264,14 +278,9 @@ AST_MATCHER(clang::FieldDecl, isInThirdPartyLocation) { if (file_path.contains("third_party/blink/")) return false; - // V8 needs to be considered "third party", even though its paths do not - // contain the "third_party" substring. In particular, the rewriter should - // not append |.get()| to references to |v8::RegisterState::pc|, because - // //v8/include/v8.h will *not* get rewritten. - if (file_path.contains("v8/include/")) - return true; - // Otherwise, just check if the paths contains the "third_party" substring. + // We don't want to rewrite content of such paths even if they are in the main + // Chromium git repository. return file_path.contains("third_party"); } @@ -282,23 +291,37 @@ AST_MATCHER(clang::FieldDecl, isInGeneratedLocation) { return file_path.startswith("gen/") || file_path.contains("/gen/"); } -// Represents a filter file specified via cmdline, that can be used to filter -// out specific FieldDecls. -// -// See also: -// - kExcludeFieldsParamName -// - OutputSectionHelper -class FieldDeclFilterFile { +// Represents a filter file specified via cmdline. +class FilterFile { public: - explicit FieldDeclFilterFile(const std::string& filepath) { - if (!filepath.empty()) - ParseInputFile(filepath); + explicit FilterFile(const llvm::cl::opt<std::string>& cmdline_param) { + ParseInputFile(cmdline_param); } - bool Contains(const clang::FieldDecl& field_decl) const { - std::string qualified_name = field_decl.getQualifiedNameAsString(); - auto it = fields_to_filter_.find(qualified_name); - return it != fields_to_filter_.end(); + FilterFile(const FilterFile&) = delete; + FilterFile& operator=(const FilterFile&) = delete; + + // Returns true if any of the filter file lines is exactly equal to |line|. + bool ContainsLine(llvm::StringRef line) const { + auto it = file_lines_.find(line); + return it != file_lines_.end(); + } + + // Returns true if any of the filter file lines is a substring of + // |string_to_match|. + bool ContainsSubstringOf(llvm::StringRef string_to_match) const { + if (!substring_regex_.hasValue()) { + std::vector<std::string> regex_escaped_file_lines; + regex_escaped_file_lines.reserve(file_lines_.size()); + for (const llvm::StringRef& file_line : file_lines_.keys()) + regex_escaped_file_lines.push_back(llvm::Regex::escape(file_line)); + std::string substring_regex_pattern = + llvm::join(regex_escaped_file_lines.begin(), + regex_escaped_file_lines.end(), "|"); + substring_regex_.emplace(substring_regex_pattern); + } + + return substring_regex_->match(string_to_match); } private: @@ -310,13 +333,17 @@ class FieldDeclFilterFile { // autofill::AddressField::address1_ # some comment // - Templates are represented without template arguments, like: // WTF::HashTable::table_ # some comment - void ParseInputFile(const std::string& filepath) { + void ParseInputFile(const llvm::cl::opt<std::string>& cmdline_param) { + std::string filepath = cmdline_param; + if (filepath.empty()) + return; + llvm::ErrorOr<std::unique_ptr<llvm::MemoryBuffer>> file_or_err = llvm::MemoryBuffer::getFile(filepath); if (std::error_code err = file_or_err.getError()) { llvm::errs() << "ERROR: Cannot open the file specified in --" - << kExcludeFieldsParamName << " argument: " << filepath - << ": " << err.message() << "\n"; + << cmdline_param.ArgStr << " argument: " << filepath << ": " + << err.message() << "\n"; assert(false); return; } @@ -334,19 +361,32 @@ class FieldDeclFilterFile { if (line.empty()) continue; - fields_to_filter_.insert(line); + file_lines_.insert(line); } } - // Stores fully-namespace-qualified names of fields matched by the filter. - llvm::StringSet<> fields_to_filter_; + // Stores all file lines (after stripping comments and blank lines). + llvm::StringSet<> file_lines_; + + // Lazily-constructed regex that matches strings that contain any of the + // |file_lines_|. + mutable llvm::Optional<llvm::Regex> substring_regex_; }; AST_MATCHER_P(clang::FieldDecl, - isListedInFilterFile, - FieldDeclFilterFile, + isFieldDeclListedInFilterFile, + const FilterFile*, + Filter) { + return Filter->ContainsLine(Node.getQualifiedNameAsString()); +} + +AST_MATCHER_P(clang::FieldDecl, + isInLocationListedInFilterFile, + const FilterFile*, Filter) { - return Filter.Contains(Node); + llvm::StringRef file_path = + GetFilePath(Finder->getASTContext().getSourceManager(), Node); + return Filter->ContainsSubstringOf(file_path); } AST_MATCHER(clang::Decl, isInExternCContext) { @@ -641,6 +681,9 @@ class FieldDeclRewriter : public MatchFinder::MatchCallback { explicit FieldDeclRewriter(OutputHelper* output_helper) : output_helper_(output_helper) {} + FieldDeclRewriter(const FieldDeclRewriter&) = delete; + FieldDeclRewriter& operator=(const FieldDeclRewriter&) = delete; + void run(const MatchFinder::MatchResult& result) override { const clang::ASTContext& ast_context = *result.Context; const clang::SourceManager& source_manager = *result.SourceManager; @@ -720,6 +763,9 @@ class AffectedExprRewriter : public MatchFinder::MatchCallback { explicit AffectedExprRewriter(OutputHelper* output_helper) : output_helper_(output_helper) {} + AffectedExprRewriter(const AffectedExprRewriter&) = delete; + AffectedExprRewriter& operator=(const AffectedExprRewriter&) = delete; + void run(const MatchFinder::MatchResult& result) override { const clang::SourceManager& source_manager = *result.SourceManager; @@ -747,6 +793,9 @@ class FilteredExprWriter : public MatchFinder::MatchCallback { FilteredExprWriter(OutputHelper* output_helper, llvm::StringRef filter_tag) : output_helper_(output_helper), filter_tag_(filter_tag) {} + FilteredExprWriter(const FilteredExprWriter&) = delete; + FilteredExprWriter& operator=(const FilteredExprWriter&) = delete; + void run(const MatchFinder::MatchResult& result) override { const clang::FieldDecl* field_decl = result.Nodes.getNodeAs<clang::FieldDecl>("affectedFieldDecl"); @@ -772,6 +821,9 @@ int main(int argc, const char* argv[]) { llvm::cl::opt<std::string> exclude_fields_param( kExcludeFieldsParamName, llvm::cl::value_desc("filepath"), llvm::cl::desc("file listing fields to be blocked (not rewritten)")); + llvm::cl::opt<std::string> exclude_paths_param( + kExcludePathsParamName, llvm::cl::value_desc("filepath"), + llvm::cl::desc("file listing paths to be blocked (not rewritten)")); clang::tooling::CommonOptionsParser options(argc, argv, category); clang::tooling::ClangTool tool(options.getCompilations(), options.getSourcePathList()); @@ -822,16 +874,19 @@ int main(int argc, const char* argv[]) { // matches |int* y|. Doesn't match: // - non-pointer types // - fields of lambda-supporting classes - // - fields listed in the --exclude-fields cmdline param + // - fields listed in the --exclude-fields cmdline param or located in paths + // matched by --exclude-paths cmdline param // - "implicit" fields (i.e. field decls that are not explicitly present in // the source code) - FieldDeclFilterFile fields_to_exclude(exclude_fields_param); + FilterFile fields_to_exclude(exclude_fields_param); + FilterFile paths_to_exclude(exclude_paths_param); auto field_decl_matcher = fieldDecl( allOf(hasType(supported_pointer_types_matcher), - unless(anyOf(isInThirdPartyLocation(), isInGeneratedLocation(), - isExpansionInSystemHeader(), isInExternCContext(), - isListedInFilterFile(fields_to_exclude), + unless(anyOf(isExpansionInSystemHeader(), isInExternCContext(), + isInThirdPartyLocation(), isInGeneratedLocation(), + isInLocationListedInFilterFile(&paths_to_exclude), + isFieldDeclListedInFilterFile(&fields_to_exclude), implicit_field_decl_matcher)))) .bind("affectedFieldDecl"); FieldDeclRewriter field_decl_rewriter(&output_helper); @@ -849,15 +904,7 @@ int main(int argc, const char* argv[]) { auto affected_member_expr_matcher = memberExpr(member(fieldDecl(hasExplicitFieldDecl(field_decl_matcher)))) .bind("affectedMemberExpr"); - auto affected_implicit_expr_matcher = implicitCastExpr(has(expr(anyOf( - // Only single implicitCastExpr is present in case of: - // |auto* v = s.ptr_field;| - expr(affected_member_expr_matcher), - // 2nd nested implicitCastExpr is present in case of: - // |const auto* v = s.ptr_field;| - expr(implicitCastExpr(has(affected_member_expr_matcher))))))); - auto affected_expr_matcher = - expr(anyOf(affected_member_expr_matcher, affected_implicit_expr_matcher)); + auto affected_expr_matcher = ignoringImplicit(affected_member_expr_matcher); // Places where |.get()| needs to be appended ========= // Given @@ -960,10 +1007,9 @@ int main(int argc, const char* argv[]) { // // See also the testcases in tests/gen-in-out-arg-test.cc. auto affected_in_out_ref_arg_matcher = callExpr(forEachArgumentWithParam( - affected_expr_matcher.bind("expr"), - hasExplicitParmVarDecl( - hasType(qualType(allOf(referenceType(pointee(pointerType())), - unless(rValueReferenceType()))))))); + affected_expr_matcher, hasExplicitParmVarDecl(hasType(qualType( + allOf(referenceType(pointee(pointerType())), + unless(rValueReferenceType()))))))); FilteredExprWriter filtered_in_out_ref_arg_writer(&output_helper, "in-out-param-ref"); match_finder.addMatcher(affected_in_out_ref_arg_matcher, @@ -988,9 +1034,10 @@ int main(int argc, const char* argv[]) { // See the doc comment for the anyCharType matcher // and the testcases in tests/gen-char-test.cc. auto char_ptr_field_decl_matcher = fieldDecl(allOf( - field_decl_matcher, hasType(pointerType(pointee( - hasUnqualifiedDesugaredType(anyCharType())))))); - FilteredExprWriter char_ptr_field_decl_writer(&output_helper, "char"); + field_decl_matcher, + hasType(pointerType(pointee(qualType(allOf( + isConstQualified(), hasUnqualifiedDesugaredType(anyCharType())))))))); + FilteredExprWriter char_ptr_field_decl_writer(&output_helper, "const-char"); match_finder.addMatcher(char_ptr_field_decl_matcher, &char_ptr_field_decl_writer); diff --git a/chromium/tools/clang/rewrite_raw_ptr_fields/manual-fields-to-ignore.txt b/chromium/tools/clang/rewrite_raw_ptr_fields/manual-fields-to-ignore.txt index b8cc64f3bb3..9f18a75980d 100644 --- a/chromium/tools/clang/rewrite_raw_ptr_fields/manual-fields-to-ignore.txt +++ b/chromium/tools/clang/rewrite_raw_ptr_fields/manual-fields-to-ignore.txt @@ -14,10 +14,11 @@ base::CheckedContiguousIterator::current_ base::CheckedContiguousIterator::end_ base::CheckedContiguousIterator::start_ base::CheckedContiguousRange::container_ +base::FeatureParam<base::TimeDelta, false>::feature base::FeatureParam<bool, false>::feature base::FeatureParam<double, false>::feature base::FeatureParam<int, false>::feature -base::FeatureParam<std::__Cr::basic_string<char, std::__Cr::char_traits<char>, std::__Cr::allocator<char>>, false>::feature +base::FeatureParam<std::__1::basic_string<char>, false>::feature base::FeatureParam<type-parameter-0-0, true>::feature base::FeatureParam<type-parameter-0-0, true>::options base::span::data_ @@ -154,6 +155,9 @@ device::BluetoothDevice::adapter_ vr::LocationBarState::vector_icon vr::OmniboxSuggestion::icon +# Populated manually - other compile-time reasons +logging::CheckOpResult::message_ # cyclic #include + # Populated manually - these pointers are assigned invalid address (with top # bits sets), which CheckedPtr is unable to handle, leading to run-time crashes. (anonymous namespace)::TlsVectorEntry::data diff --git a/chromium/tools/clang/rewrite_raw_ptr_fields/manual-paths-to-ignore.txt b/chromium/tools/clang/rewrite_raw_ptr_fields/manual-paths-to-ignore.txt new file mode 100644 index 00000000000..62e16dc46c9 --- /dev/null +++ b/chromium/tools/clang/rewrite_raw_ptr_fields/manual-paths-to-ignore.txt @@ -0,0 +1,118 @@ +# File that lists regular expressions of paths that should be ignored when +# running the rewrite_raw_ptr_fields tool on Chromium sources. +# +# If a source file path contains any of the lines in the filter file below, +# then such source file will not be rewritten. +# +# Note that the rewriter has a hardcoded logic for a handful of path-based +# exclusions that cannot be expressed as substring matches: +# - Excluding paths containing "third_party/", but still covering +# "third_party/blink/" +# (see the isInThirdPartyLocation AST matcher in RewriteRawPtrFields.cpp). +# - Excluding paths _starting_ with "gen/" or containing "/gen/" +# (i.e. hopefully just the paths under out/.../gen/... directory) +# via the isInGeneratedLocation AST matcher in RewriteRawPtrFields.cpp. + +# Exclude to prevent PartitionAlloc<->CheckedPtr cyclical dependency. +base/allocator/ + +# Exclude - deprecated and contains legacy C++ and pre-C++11 code. +ppapi/ + +# Exclude tools that do not ship in the Chrome binary. +base/android/linker/ +tools/ +net/tools/ + +# Exclude paths in separate repositories - i.e. in directories that +# 1. Contain a ".git" subdirectory +# 2. And hasn't been excluded via "third_party/" substring in their path +# (see the isInThirdPartyLocation AST matcher in RewriteRawPtrFields.cpp). +# +# The list below has been generated with: +# +# $ find . -type d -name .git | \ +# sed -e 's/\.git$//g' | \ +# sed -e 's/\.\///g' | \ +# grep -v third_party | \ +# grep -v '^$' | \ +# sort | uniq > ~/scratch/git-paths +buildtools/ +buildtools/clang_format/script/ +chrome/app/theme/default_100_percent/google_chrome/ +chrome/app/theme/default_200_percent/google_chrome/ +chrome/app/theme/google_chrome/ +chrome/app/vector_icons/google_chrome/ +chrome/browser/chromeos/arc/voice_interaction/internal/ +chrome/browser/google/linkdoctor_internal/ +chrome/browser/internal/ +chrome/browser/media/engagement_internal/ +chrome/browser/media/kaleidoscope/internal/ +chrome/browser/resources/chromeos/quickoffice/ +chrome/browser/resources/media_router/extension/src/ +chrome/browser/resources/media_router_internal/ +chrome/browser/resources/settings_internal/ +chrome/browser/spellchecker/internal/ +chrome/browser/ui/media_router/internal/ +chrome/installer/mac/internal/ +chromeos/assistant/internal/ +chromeos/assistant/libassistant/deps/ +chromeos/assistant/libassistant/src/ +chromeos/assistant/libassistant/src/buildtools/ +chromeos/assistant/libassistant/src/buildtools/clang_format/script/ +chromeos/components/media_app_ui/resources/app/ +chrome/services/soda/internal/ +chrome/test/data/firefox3_profile/searchplugins/ +chrome/test/data/firefox3_searchplugins/ +chrome/test/data/gpu/vt/ +chrome/test/data/osdd/ +chrome/test/data/pdf_private/ +chrome/test/data/perf/canvas_bench/ +chrome/test/data/perf/frame_rate/content/ +chrome/test/data/perf/frame_rate/private/ +chrome/test/data/perf/private/ +chrome/test/data/xr/webvr_info/ +chrome/test/media_router/internal/ +chrome/test/python_tests/ +chrome/tools/test/reference_build/chrome_linux/ +clank/ +components/ntp_tiles/resources/internal/ +components/resources/default_100_percent/google_chrome/ +components/resources/default_200_percent/google_chrome/ +components/resources/default_300_percent/google_chrome/ +components/site_isolation/internal/ +content/test/data/plugin/ +data/autodiscovery/ +data/dom_perf/ +data/mach_ports/ +data/memory_test/ +data/mozilla_js_tests/ +data/page_cycler/ +data/selenium_core/ +data/tab_switching/ +google_apis/internal/ +libassistant/communication/ +libassistant/contrib/ +libassistant/internal/ +libassistant/resources/ +libassistant/shared/ +media/cdm/api/ +mojo/internal/ +native_client/ +remoting/android/internal/ +remoting/host/installer/linux/internal/ +remoting/internal/ +remoting/test/internal/ +remoting/tools/internal/ +remoting/webapp/app_remoting/internal/ +skia/tools/clusterfuzz-data/ +tools/histograms/ +tools/page_cycler/acid3/ +tools/perf/data/ +tools/swarming_client/ +ui/file_manager/internal/ +ui/webui/internal/ +v8/ +webkit/data/bmp_decoder/ +webkit/data/ico_decoder/ +webkit/data/test_shell/plugins/ diff --git a/chromium/tools/clang/rewrite_raw_ptr_fields/rewrite.sh b/chromium/tools/clang/rewrite_raw_ptr_fields/rewrite.sh index e932c3248d5..30eebc984ba 100755 --- a/chromium/tools/clang/rewrite_raw_ptr_fields/rewrite.sh +++ b/chromium/tools/clang/rewrite_raw_ptr_fields/rewrite.sh @@ -24,6 +24,9 @@ then OUT_DIR="$1" fi +SCRIPT_PATH=$(realpath $0) +REWRITER_SRC_DIR=$(dirname $SCRIPT_PATH) + COMPILE_DIRS=. EDIT_DIRS=. @@ -49,6 +52,7 @@ time ninja -C $OUT_DIR $GEN_H_TARGETS echo "*** Generating the ignore list ***" time tools/clang/scripts/run_tool.py \ --tool rewrite_raw_ptr_fields \ + --tool-arg=--exclude-paths=$REWRITER_SRC_DIR/manual-paths-to-ignore.txt \ --generate-compdb \ -p $OUT_DIR \ $COMPILE_DIRS > ~/scratch/rewriter.out @@ -64,6 +68,7 @@ echo "*** Running the main rewrite phase ***" time tools/clang/scripts/run_tool.py \ --tool rewrite_raw_ptr_fields \ --tool-arg=--exclude-fields=$HOME/scratch/combined-fields-to-ignore.txt \ + --tool-arg=--exclude-paths=$REWRITER_SRC_DIR/manual-paths-to-ignore.txt \ -p $OUT_DIR \ $COMPILE_DIRS > ~/scratch/rewriter.main.out @@ -73,12 +78,6 @@ cat ~/scratch/rewriter.main.out | \ tools/clang/scripts/extract_edits.py | \ tools/clang/scripts/apply_edits.py -p $OUT_DIR $EDIT_DIRS -# Revert directories that are known to be troublesome and/or not needed. -git checkout -- base/allocator/ # prevent cycles; CheckedPtr uses allocator -git checkout -- ppapi/ # lots of legacy C and pre-C++11 code -git checkout -- tools/ # not built into Chrome -git checkout -- net/tools/ # not built into Chrome - # Format sources, as many lines are likely over 80 chars now. echo "*** Formatting ***" time git cl format diff --git a/chromium/tools/clang/scripts/build.py b/chromium/tools/clang/scripts/build.py index 3f2af8b3cf9..78ab2b8ddbd 100755 --- a/chromium/tools/clang/scripts/build.py +++ b/chromium/tools/clang/scripts/build.py @@ -50,8 +50,6 @@ BUG_REPORT_URL = ('https://crbug.com and run' ' tools/clang/scripts/process_crashreports.py' ' (only works inside Google) which will upload a report') -FIRST_LLVM_COMMIT = '97724f18c79c7cc81ced24239eb5e883bf1398ef' - win_sdk_dir = None dia_dll = None @@ -158,8 +156,7 @@ def CheckoutLLVM(commit, dir): # Also check that the first commit is reachable. if (RunCommand(['git', 'diff-index', '--quiet', 'HEAD'], fail_hard=False) and RunCommand(['git', 'fetch'], fail_hard=False) - and RunCommand(['git', 'checkout', commit], fail_hard=False) - and RunCommand(['git', 'show', FIRST_LLVM_COMMIT], fail_hard=False)): + and RunCommand(['git', 'checkout', commit], fail_hard=False)): return # If we can't use the current repo, delete it. @@ -191,12 +188,12 @@ def GetLatestLLVMCommit(): return ref['object']['sha'] -def GetCommitCount(commit): - """Get the number of commits from FIRST_LLVM_COMMIT to commit. +def GetCommitDescription(commit): + """Get the output of `git describe`. Needs to be called from inside the git repository dir.""" - return subprocess.check_output(['git', 'rev-list', '--count', - FIRST_LLVM_COMMIT + '..' + commit]).rstrip() + return subprocess.check_output( + ['git', 'describe', '--long', '--abbrev=8', commit]).rstrip() def DeleteChromeToolsShim(): @@ -256,7 +253,7 @@ def AddGnuWinToPath(): return gnuwin_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'gnuwin') - GNUWIN_VERSION = '12' + GNUWIN_VERSION = '13' GNUWIN_STAMP = os.path.join(gnuwin_dir, 'stamp') if ReadStampFile(GNUWIN_STAMP) == GNUWIN_VERSION: print('GNU Win tools already up to date.') @@ -337,6 +334,26 @@ def VerifyVersionOfBuiltClangMatchesVERSION(): sys.exit(1) +def VerifyZlibSupport(): + """Check that clang was built with zlib support enabled.""" + clang = os.path.join(LLVM_BUILD_DIR, 'bin', 'clang') + test_file = '/dev/null' + if sys.platform == 'win32': + clang += '-cl.exe' + test_file = 'nul' + + print('Checking for zlib support') + clang_out = subprocess.check_output([ + clang, '--driver-mode=gcc', '-target', 'x86_64-unknown-linux-gnu', '-gz', + '-c', '-###', '-x', 'c', test_file ], + stderr=subprocess.STDOUT, universal_newlines=True) + if (re.search(r'--compress-debug-sections', clang_out)): + print('OK') + else: + print(('Failed to detect zlib support!\n\n(driver output: %s)') % clang_out) + sys.exit(1) + + def CopyLibstdcpp(args, build_dir): if not args.gcc_toolchain: return @@ -383,6 +400,9 @@ def main(): 'building; --gcc-toolchain=/opt/foo picks ' '/opt/foo/bin/gcc') parser.add_argument('--pgo', action='store_true', help='build with PGO') + parser.add_argument('--thinlto', + action='store_true', + help='build with ThinLTO') parser.add_argument('--llvm-force-head-revision', action='store_true', help='build the latest revision') parser.add_argument('--run-tests', action='store_true', @@ -408,8 +428,8 @@ def main(): default=sys.platform in ('linux2', 'darwin')) args = parser.parse_args() - if args.pgo and not args.bootstrap: - print('--pgo requires --bootstrap') + if (args.pgo or args.thinlto) and not args.bootstrap: + print('--pgo/--thinlto requires --bootstrap') return 1 if args.with_android and not os.path.exists(ANDROID_NDK_DIR): print('Android NDK not found at ' + ANDROID_NDK_DIR) @@ -456,16 +476,21 @@ def main(): # enough GCC to build Clang. MaybeDownloadHostGcc(args) + if sys.platform == 'darwin': + isysroot = subprocess.check_output(['xcrun', '--show-sdk-path']).rstrip() + global CLANG_REVISION, PACKAGE_VERSION if args.llvm_force_head_revision: - CLANG_REVISION = GetLatestLLVMCommit() + checkout_revision = GetLatestLLVMCommit() + else: + checkout_revision = CLANG_REVISION if not args.skip_checkout: - CheckoutLLVM(CLANG_REVISION, LLVM_DIR); + CheckoutLLVM(checkout_revision, LLVM_DIR) if args.llvm_force_head_revision: - PACKAGE_VERSION = 'n%s-%s-0' % (GetCommitCount(CLANG_REVISION), - CLANG_REVISION[:8]) + CLANG_REVISION = GetCommitDescription(checkout_revision) + PACKAGE_VERSION = '%s-0' % CLANG_REVISION print('Locally building clang %s...' % PACKAGE_VERSION) WriteStampFile('', STAMP_FILE) @@ -516,6 +541,8 @@ def main(): '-DCOMPILER_RT_USE_LIBCXX=NO', # Don't run Go bindings tests; PGO makes them confused. '-DLLVM_INCLUDE_GO_TESTS=OFF', + # TODO(crbug.com/1113475): Update binutils. + '-DENABLE_X86_RELAX_RELOCATIONS=NO', ] if args.gcc_toolchain: @@ -533,8 +560,10 @@ def main(): if sys.platform == 'darwin': # For libc++, we only want the headers. base_cmake_args.extend([ - '-DLIBCXX_ENABLE_SHARED=OFF', '-DLIBCXX_ENABLE_STATIC=OFF', - '-DLIBCXX_INCLUDE_TESTS=OFF' + '-DLIBCXX_ENABLE_SHARED=OFF', + '-DLIBCXX_ENABLE_STATIC=OFF', + '-DLIBCXX_INCLUDE_TESTS=OFF', + '-DLIBCXX_ENABLE_EXPERIMENTAL_LIBRARY=OFF', ]) # Prefer Python 2. TODO(crbug.com/1076834): Remove this. base_cmake_args.append('-DPython3_EXECUTABLE=/nonexistent') @@ -685,6 +714,8 @@ def main(): if cc is not None: instrument_args.append('-DCMAKE_C_COMPILER=' + cc) if cxx is not None: instrument_args.append('-DCMAKE_CXX_COMPILER=' + cxx) if lld is not None: instrument_args.append('-DCMAKE_LINKER=' + lld) + if args.thinlto: + instrument_args.append('-DLLVM_ENABLE_LTO=Thin') RunCommand(['cmake'] + instrument_args + [os.path.join(LLVM_DIR, 'llvm')], msvc_arch='x64') @@ -715,15 +746,13 @@ def main(): # from PGO as well. Perhaps the training could be done asynchronously by # dedicated buildbots that upload profiles to the cloud. training_source = 'pgo_training-1.ii' - with open(training_source, 'w') as f: + with open(training_source, 'wb') as f: DownloadUrl(CDS_URL + '/' + training_source, f) train_cmd = [os.path.join(LLVM_INSTRUMENTED_DIR, 'bin', 'clang++'), '-target', 'x86_64-unknown-unknown', '-O2', '-g', '-std=c++14', '-fno-exceptions', '-fno-rtti', '-w', '-c', training_source] if sys.platform == 'darwin': - train_cmd.extend(['-stdlib=libc++', '-isysroot', - subprocess.check_output(['xcrun', - '--show-sdk-path']).rstrip()]) + train_cmd.extend(['-stdlib=libc++', '-isysroot', isysroot]) RunCommand(train_cmd, msvc_arch='x64') # Merge profiles. @@ -750,9 +779,48 @@ def main(): # iPhones). armv7k is Apple Watch, which we don't need. '-DDARWIN_ios_ARCHS=armv7;armv7s;arm64', '-DDARWIN_iossim_ARCHS=i386;x86_64', - # We don't need 32-bit intel support for macOS, we only ship 64-bit. - '-DDARWIN_osx_ARCHS=x86_64', ]) + if args.bootstrap: + # mac/arm64 needs MacOSX11.0.sdk. System Xcode (+ SDK) on the chrome bots + # is something much older. + # Options: + # - temporarily set system Xcode to Xcode 12 beta while running this + # script, (cf build/swarming_xcode_install.py, but it looks unused) + # - use Xcode 12 beta for everything on tot bots, only need to fuzz with + # scripts/slave/recipes/chromium_upload_clang.py then (but now the + # chrome/ios build will use the 11.0 SDK too and we'd be on the hook for + # keeping it green -- if it's currently green, who knows) + # - pass flags to cmake to try to coax it into using Xcode 12 beta for the + # LLVM build without it being system Xcode. + # + # The last option seems best, so let's go with that. We need to pass + # -isysroot to the 11.0 SDK and -B to the /usr/bin so that the new ld64 is + # used. + # The compiler-rt build overrides -isysroot flags set via cflags, and we + # only need to use the 11 SDK for the compiler-rt build. So set only + # DARWIN_macosx_CACHED_SYSROOT to the 11.0 SDK and use the regular SDK + # for the rest of the build. (The new ld is used for all links.) + sys.path.insert(1, os.path.join(CHROMIUM_DIR, 'build')) + import mac_toolchain + LLVM_XCODE = os.path.join(THIRD_PARTY_DIR, 'llvm-xcode') + mac_toolchain.InstallXcodeBinaries('xcode_12_beta', LLVM_XCODE) + isysroot_11 = os.path.join(LLVM_XCODE, 'Contents', 'Developer', + 'Platforms', 'MacOSX.platform', 'Developer', + 'SDKs', 'MacOSX11.0.sdk') + xcode_bin = os.path.join(LLVM_XCODE, 'Contents', 'Developer', + 'Toolchains', 'XcodeDefault.xctoolchain', 'usr', + 'bin') + # Include an arm64 slice for libclang_rt.osx.a. This requires using + # MacOSX11.0.sdk (via -isysroot, via DARWIN_macosx_CACHED_SYSROOT) and + # the new ld, via -B + compiler_rt_args.extend([ + # We don't need 32-bit intel support for macOS, we only ship 64-bit. + '-DDARWIN_osx_ARCHS=arm64;x86_64', + '-DDARWIN_macosx_CACHED_SYSROOT=' + isysroot_11, + ]) + ldflags += ['-B', xcode_bin] + else: + compiler_rt_args.extend(['-DDARWIN_osx_ARCHS=x86_64']) else: compiler_rt_args.append('-DCOMPILER_RT_BUILD_BUILTINS=OFF') @@ -765,8 +833,7 @@ def main(): if sys.platform == 'darwin' and args.bootstrap: # When building on 10.9, /usr/include usually doesn't exist, and while # Xcode's clang automatically sets a sysroot, self-built clangs don't. - cflags = ['-isysroot', subprocess.check_output( - ['xcrun', '--show-sdk-path']).rstrip()] + cflags = ['-isysroot', isysroot] cxxflags = ['-stdlib=libc++'] + cflags ldflags += ['-stdlib=libc++'] deployment_target = '10.7' @@ -808,6 +875,8 @@ def main(): '-DCHROMIUM_TOOLS=%s' % ';'.join(chrome_tools)] if args.pgo: cmake_args.append('-DLLVM_PROFDATA_FILE=' + LLVM_PROFDATA_FILE) + if args.thinlto: + cmake_args.append('-DLLVM_ENABLE_LTO=Thin') if sys.platform == 'win32': cmake_args.append('-DLLVM_ENABLE_ZLIB=FORCE_ON') if sys.platform == 'darwin': @@ -831,6 +900,7 @@ def main(): RunCommand(['ninja', 'cr-install'], msvc_arch='x64') VerifyVersionOfBuiltClangMatchesVERSION() + VerifyZlibSupport() if sys.platform == 'win32': platform = 'windows' diff --git a/chromium/tools/clang/scripts/goma_ld.py b/chromium/tools/clang/scripts/goma_ld.py index 576dfa9f9d2..21a7dd2286c 100755 --- a/chromium/tools/clang/scripts/goma_ld.py +++ b/chromium/tools/clang/scripts/goma_ld.py @@ -35,20 +35,21 @@ class GomaLinkUnix(goma_link.GomaLinkBase): PREFIX_REPLACE = TLTO + '-prefix-replace' + SEP XIR = '-x ir ' - WHITELISTED_TARGETS = { + ALLOWLIST = { 'chrome', } def analyze_args(self, args, *posargs, **kwargs): # TODO(crbug.com/1040196): Builds are unreliable when all targets use - # distributed ThinLTO, so we only enable it for whitelisted targets. + # distributed ThinLTO, so we only enable it for some targets. # For other targets, we fall back to local ThinLTO. We must use ThinLTO # because we build with -fsplit-lto-unit, which requires code generation # be done for each object and target. - if args.output is None or os.path.basename( - args.output) not in self.WHITELISTED_TARGETS: - # Returning None causes the original, non-distributed linker command to be - # invoked. + # Returning None from this function causes the original, non-distributed + # linker command to be invoked. + if args.output is None: + return None + if not (args.allowlist or os.path.basename(args.output) in self.ALLOWLIST): return None return super(GomaLinkUnix, self).analyze_args(args, *posargs, **kwargs) diff --git a/chromium/tools/clang/scripts/goma_link.py b/chromium/tools/clang/scripts/goma_link.py index 8bf194e4bdb..15fb617c9b3 100755 --- a/chromium/tools/clang/scripts/goma_link.py +++ b/chromium/tools/clang/scripts/goma_link.py @@ -36,8 +36,8 @@ except NameError: # Type returned by analyze_args. AnalyzeArgsResult = namedtuple('AnalyzeArgsResult', [ - 'output', 'linker', 'compiler', 'splitfile', 'index_params', 'codegen', - 'codegen_params', 'final_params' + 'output', 'linker', 'compiler', 'splitfile', 'index_inputs', 'index_params', + 'codegen', 'codegen_params', 'final_inputs', 'final_params' ]) @@ -181,9 +181,15 @@ def parse_args(args): # that set some values for this script. If these optional options are # present, they must be followed by '--'. ap = argparse.ArgumentParser() + ap.add_argument('--generate', + action='store_true', + help='generate ninja file, but do not invoke it.') ap.add_argument('--gomacc', help='path to gomacc.') ap.add_argument('--jobs', '-j', help='maximum number of concurrent jobs.') ap.add_argument('--no-gomacc', action='store_true', help='do not use gomacc.') + ap.add_argument('--allowlist', + action='store_true', + help='act as if the target is on the allow list.') try: splitpos = args.index('--') except: @@ -234,6 +240,19 @@ class GomaLinkBase(object): MLLVM_RE = re.compile('(?:-Wl,)?([-/]mllvm)[:=]?(.*)', re.IGNORECASE) OBJ_RE = re.compile('(.*)\\.(o(?:bj)?)', re.IGNORECASE) + def _no_codegen(self, args): + """ + Helper function for the case where no distributed code generation + is necessary. It invokes the original command, unless --generate + was passed, in which case it informs the user that no code generation + is necessary. + """ + if args.generate: + sys.stderr.write( + 'No code generation required; no ninja file generated.\n') + return 5 # Indicates no code generation required. + return subprocess.call([args.linker] + args.linker_args) + def transform_codegen_param(self, param): return self.transform_codegen_param_common(param) @@ -378,12 +397,14 @@ class GomaLinkBase(object): obj_dir = gen_dir common_index = common_dir + '/empty.thinlto.bc' + index_inputs = set() index_params = [] codegen = [] codegen_params = [ '-Wno-unused-command-line-argument', '-Wno-override-module', ] + final_inputs = set() final_params = [] in_mllvm = [False] @@ -455,6 +476,7 @@ class GomaLinkBase(object): return index_params.append(param) if os.path.exists(param): + index_inputs.add(param) match = self.OBJ_RE.match(param) if match and is_bitcode_file(param): native = obj_dir + '/' + match.group(1) + '.' + match.group(2) @@ -465,6 +487,7 @@ class GomaLinkBase(object): ensure_file(index) codegen.append((os.path.normpath(native), param, index)) else: + final_inputs.add(param) final_params.append(param) elif not self.LTO_RE.match(param): final_params.append(param) @@ -497,11 +520,12 @@ class GomaLinkBase(object): splitfile = None for tup in codegen: final_params.append(tup[0]) + index_inputs = [] else: splitfile = gen_dir + '/' + output + '.split' + self.OBJ_SUFFIX final_params.append(splitfile) index_params.append(self.WL + self.OBJ_PATH + splitfile) - used_obj_file = gen_dir + '/' + output + '.objs' + used_obj_file = gen_dir + '/' + os.path.basename(output) + '.objs' final_params.append('@' + used_obj_file) return AnalyzeArgsResult( @@ -509,13 +533,15 @@ class GomaLinkBase(object): linker=linker, compiler=compiler, splitfile=splitfile, + index_inputs=index_inputs, index_params=index_params, codegen=codegen, codegen_params=codegen_params, + final_inputs=final_inputs, final_params=final_params, ) - def gen_ninja(self, ninjaname, params, objs): + def gen_ninja(self, ninjaname, params, gen_dir): """ Generates a ninja build file at path ninjaname, using original command line params and with objs being a list of bitcode files for which to generate @@ -525,70 +551,72 @@ class GomaLinkBase(object): gomacc_prefix = ninjaenc(self.gomacc) + ' ' else: gomacc_prefix = '' + base = gen_dir + '/' + os.path.basename(params.output) + ensure_dir(gen_dir) ensure_dir(os.path.dirname(ninjaname)) + codegen_cmd = ('%s%s -c %s -fthinlto-index=$index %s$bitcode -o $native' % + (gomacc_prefix, ninjaenc(params.compiler), + ninjajoin(params.codegen_params), self.XIR)) + if params.index_inputs: + used_obj_file = base + '.objs' + index_rsp = base + '.index.rsp' + ensure_dir(os.path.dirname(used_obj_file)) + if params.splitfile: + ensure_dir(os.path.dirname(params.splitfile)) + # We use grep here to only codegen native objects which are actually + # used by the native link step. Ninja 1.10 introduced a dyndep feature + # which allows for a more elegant implementation, but Chromium still + # uses an older ninja version which doesn't have this feature. + codegen_cmd = '( ! grep -qF $native %s || %s)' % ( + ninjaenc(used_obj_file), codegen_cmd) + with open(ninjaname, 'w') as f: + if params.index_inputs: + self.write_rsp(index_rsp, params.index_params) + f.write('\nrule index\n command = %s %s %s @$rspfile\n' % + (ninjaenc(params.linker), + ninjaenc(self.WL + self.TLTO + '-index-only' + self.SEP) + + '$out', self.WL + self.TLTO + '-emit-imports-files')) + f.write(('\nrule native-link\n command = %s @$rspname' '\n rspfile = $rspname\n rspfile_content = $params\n') % (ninjaenc(params.linker), )) - f.write(('\nrule codegen\n command = %s%s -c %s' - ' -fthinlto-index=$index %s$bitcode -o $out\n') % - (gomacc_prefix, ninjaenc(params.compiler), - ninjajoin(params.codegen_params), self.XIR)) + f.write('\nrule codegen\n command = %s && touch $out\n' % + (codegen_cmd, )) + + native_link_deps = [] + if params.index_inputs: + f.write( + ('\nbuild %s | %s : index %s\n' + ' rspfile = %s\n' + ' rspfile_content = %s\n') % + (ninjaenc(used_obj_file), ninjajoin( + [x[2] for x in params.codegen]), ninjajoin(params.index_inputs), + ninjaenc(index_rsp), ninjajoin(params.index_params))) + native_link_deps.append(used_obj_file) for tup in params.codegen: obj, bitcode, index = tup - f.write('\nbuild %s : codegen %s %s\n bitcode = %s\n index = %s\n' % - tuple(map(ninjaenc, (obj, bitcode, index, bitcode, index)))) - - f.write('\nbuild %s : native-link %s\n rspname = %s\n params = %s\n' % - (ninjaenc(params.output), ninjajoin(objs), - ninjaenc(params.output + '.final.rsp'), - ninjajoin(params.final_params))) + stamp = obj + '.stamp' + native_link_deps.append(obj) + f.write( + ('\nbuild %s : codegen %s %s\n' + ' bitcode = %s\n' + ' index = %s\n' + ' native = %s\n' + '\nbuild %s : phony %s\n') % tuple( + map(ninjaenc, + (stamp, bitcode, index, bitcode, index, obj, obj, stamp)))) + + f.write(('\nbuild %s : native-link %s\n' + ' rspname = %s\n params = %s\n') % + (ninjaenc(params.output), + ninjajoin(list(params.final_inputs) + native_link_deps), + ninjaenc(base + '.final.rsp'), ninjajoin(params.final_params))) f.write('\ndefault %s\n' % (ninjaenc(params.output), )) - def thin_link(self, params, gen_dir): - """ - Performs the thin link step. - This generates the index files, imports files, split LTO object, - and used object file. - Returns a list of native objects we need to generate from bitcode - files for the final link step. - """ - used_obj_file = gen_dir + '/' + params.output + '.objs' - index_rsp = gen_dir + '/' + params.output + '.index.rsp' - ensure_dir(gen_dir) - ensure_dir(os.path.dirname(used_obj_file)) - if params.splitfile: - ensure_dir(os.path.dirname(params.splitfile)) - self.write_rsp(index_rsp, params.index_params) - index_cmd = [ - params.linker, - self.WL + self.TLTO + '-index-only' + self.SEP + used_obj_file, - self.WL + self.TLTO + '-emit-imports-files', '@' + index_rsp - ] - report_run(index_cmd) - with open(used_obj_file) as f: - codegen_objs = [ - os.path.normpath(x) for x in f.read().split('\n') if len(x) > 0 - ] - return codegen_objs - - def codegen_and_link(self, params, gen_dir, objs): - """ - Performs code generation for selected bitcode files and - the final link step. - objs should be the list of native object files expected to be generated - (as returned by thin_link()). - """ - ninjaname = gen_dir + '/build.ninja' - self.gen_ninja(ninjaname, params, objs) - cmd = [autoninja(), '-f', ninjaname] - if self.jobs: - cmd.extend(['-j', str(self.jobs)]) - report_run(cmd) - def do_main(self, argv): """ This function contains the main code to run. Not intended to be called @@ -598,7 +626,7 @@ class GomaLinkBase(object): args = parse_args(argv) args.output = self.output_path(argv[1:]) if args.output is None: - return subprocess.call([args.linker] + args.linker_args) + return self._no_codegen(args) if args.gomacc: self.gomacc = args.gomacc if args.no_gomacc: @@ -607,7 +635,7 @@ class GomaLinkBase(object): self.jobs = int(args.jobs) basename = os.path.basename(args.output) - # Only generate tailored native object files for whitelisted targets. + # Only generate tailored native object files for targets on the allow list. # TODO: Find a better way to structure this. There are three different # ways we can perform linking: Local ThinLTO, distributed ThinLTO, # and distributed ThinLTO with common object files. @@ -616,20 +644,26 @@ class GomaLinkBase(object): # Currently, we don't detect this situation. We could, but it might # be better to instead move this logic out of this script and into # the build system. - use_common_objects = basename not in self.WHITELISTED_TARGETS + use_common_objects = not (args.allowlist or basename in self.ALLOWLIST) common_dir = 'common_objs' gen_dir = 'lto.' + basename params = self.analyze_args(args, gen_dir, common_dir, use_common_objects) # If we determined that no distributed code generation need be done, just # invoke the original command. if params is None: - return subprocess.call([args.linker] + args.linker_args) + return self._no_codegen(args) if use_common_objects: objs = [x[0] for x in params.codegen] ensure_file(common_dir + '/empty.thinlto.bc') + ninjaname = gen_dir + '/build.ninja' + self.gen_ninja(ninjaname, params, gen_dir) + if args.generate: + sys.stderr.write('Generated ninja file %s\n' % (shquote(ninjaname), )) else: - objs = self.thin_link(params, gen_dir) - self.codegen_and_link(params, gen_dir, objs) + cmd = [autoninja(), '-f', ninjaname] + if self.jobs: + cmd.extend(['-j', str(self.jobs)]) + report_run(cmd) return 0 def main(self, argv): @@ -654,13 +688,13 @@ class GomaLinkWindows(GomaLinkBase): PREFIX_REPLACE = TLTO + '-prefix-replace' + SEP XIR = '' - WHITELISTED_TARGETS = { + ALLOWLIST = { 'chrome.exe', 'chrome.dll', 'chrome_child.dll', - # TODO: The following targets have been whitelisted because the + # TODO: The following targets are on the allow list because the # common objects flow does not link them successfully. This should - # be fixed, after which they can be removed from the whitelist. + # be fixed, after which they can be removed from the list. 'tls_edit.exe', } diff --git a/chromium/tools/clang/scripts/goma_link_integration_tests.py b/chromium/tools/clang/scripts/goma_link_integration_tests.py index 3a2b4ecc699..00a01043f75 100755 --- a/chromium/tools/clang/scripts/goma_link_integration_tests.py +++ b/chromium/tools/clang/scripts/goma_link_integration_tests.py @@ -18,10 +18,13 @@ import goma_ld import goma_link +from io import StringIO import os import re +import shlex import subprocess import unittest +from unittest import mock from goma_link_test_utils import named_directory, working_directory @@ -45,24 +48,24 @@ def _create_inputs(path): f.write('int bar() {\n return 9;\n}\n') -class GomaLinkUnixWhitelistMain(goma_ld.GomaLinkUnix): +class GomaLinkUnixAllowMain(goma_ld.GomaLinkUnix): """ - Same as goma_ld.GomaLinkUnix, but whitelists "main". + Same as goma_ld.GomaLinkUnix, but has "main" on the allow list. """ def __init__(self, *args, **kwargs): - super(GomaLinkUnixWhitelistMain, self).__init__(*args, **kwargs) - self.WHITELISTED_TARGETS = {'main'} + super(GomaLinkUnixAllowMain, self).__init__(*args, **kwargs) + self.ALLOWLIST = {'main'} -class GomaLinkWindowsWhitelistMain(goma_link.GomaLinkWindows): +class GomaLinkWindowsAllowMain(goma_link.GomaLinkWindows): """ - Same as goma_ld.GomaLinkWindows, but whitelists "main". + Same as goma_ld.GomaLinkWindows, but has "main" on the allow list. """ def __init__(self, *args, **kwargs): - super(GomaLinkWindowsWhitelistMain, self).__init__(*args, **kwargs) - self.WHITELISTED_TARGETS = {'main.exe'} + super(GomaLinkWindowsAllowMain, self).__init__(*args, **kwargs) + self.ALLOWLIST = {'main.exe'} class GomaLinkIntegrationTest(unittest.TestCase): @@ -109,8 +112,10 @@ class GomaLinkIntegrationTest(unittest.TestCase): codegen_text = codegen_match.group(0) self.assertIn('my_goma.sh', codegen_text) self.assertNotIn('-flto', codegen_text) - self.assertIn('build common_objs/obj/main.obj : codegen ', buildrules) - self.assertIn('build common_objs/obj/foo.obj : codegen ', buildrules) + self.assertIn('build common_objs/obj/main.obj.stamp : codegen ', + buildrules) + self.assertIn('build common_objs/obj/foo.obj.stamp : codegen ', + buildrules) self.assertIn(' index = common_objs/empty.thinlto.bc', buildrules) link_match = re.search('^build main.exe : native-link\\b.*?^[^ ]', buildrules, re.MULTILINE | re.DOTALL) @@ -123,7 +128,7 @@ class GomaLinkIntegrationTest(unittest.TestCase): # functions, one of which calls the other. self.assertTrue(b'call' in disasm or b'jmp' in disasm) - def test_distributed_lto_whitelisted(self): + def test_distributed_lto_allowlist(self): with named_directory() as d, working_directory(d): _create_inputs(d) os.makedirs('obj') @@ -143,7 +148,7 @@ class GomaLinkIntegrationTest(unittest.TestCase): ['llvm-ar', 'crsT', 'obj/foobar.lib', 'obj/bar.obj', 'obj/foo.obj']) with open('main.rsp', 'w') as f: f.write('obj/main.obj\n' 'obj/foobar.lib\n') - rc = GomaLinkWindowsWhitelistMain().main([ + rc = GomaLinkWindowsAllowMain().main([ 'goma_link.py', '--gomacc', 'gomacc', '--', self.lld_link(), '-nodefaultlib', '-entry:main', '-machine:X86', '-opt:lldlto=2', '-mllvm:-import-instr-limit=10', '-out:main.exe', @@ -162,8 +167,10 @@ class GomaLinkIntegrationTest(unittest.TestCase): self.assertIn('-m32', codegen_text) self.assertIn('-mllvm -import-instr-limit=10', codegen_text) self.assertNotIn('-flto', codegen_text) - self.assertIn('build lto.main.exe/obj/main.obj : codegen ', buildrules) - self.assertIn('build lto.main.exe/obj/foo.obj : codegen ', buildrules) + self.assertIn('build lto.main.exe/obj/main.obj.stamp : codegen ', + buildrules) + self.assertIn('build lto.main.exe/obj/foo.obj.stamp : codegen ', + buildrules) link_match = re.search('^build main.exe : native-link\\b.*?^[^ ]', buildrules, re.MULTILINE | re.DOTALL) self.assertIsNotNone(link_match) @@ -176,6 +183,41 @@ class GomaLinkIntegrationTest(unittest.TestCase): self.assertNotIn(b'jmp', disasm) self.assertNotIn(b'call', disasm) + def test_override_allowlist(self): + with named_directory() as d, working_directory(d): + _create_inputs(d) + os.makedirs('obj') + subprocess.check_call([ + self.clangcl(), '-c', '-O2', '-flto=thin', 'main.cpp', + '-Foobj/main.obj' + ]) + subprocess.check_call([ + self.clangcl(), '-c', '-O2', '-flto=thin', 'foo.cpp', '-Foobj/foo.obj' + ]) + rc = goma_link.GomaLinkWindows().main([ + 'goma_link.py', '--generate', '--allowlist', '--', + self.lld_link(), '-nodefaultlib', '-entry:main', '-opt:lldlto=2', + '-out:main.exe', 'obj/main.obj', 'obj/foo.obj' + ]) + # Should succeed. + self.assertEqual(rc, 0) + # Check that we have rules for main and foo, and that they are + # not common objects. + with open(os.path.join(d, 'lto.main.exe', 'build.ninja')) as f: + buildrules = f.read() + codegen_match = re.search(r'^rule codegen\b.*?^[^ ]', buildrules, + re.MULTILINE | re.DOTALL) + self.assertIsNotNone(codegen_match) + codegen_text = codegen_match.group(0) + self.assertNotIn('-flto', codegen_text) + self.assertIn('build lto.main.exe/obj/main.obj.stamp : codegen ', + buildrules) + self.assertIn('build lto.main.exe/obj/foo.obj.stamp : codegen ', + buildrules) + link_match = re.search(r'^build main.exe : native-link\b.*?^[^ ]', + buildrules, re.MULTILINE | re.DOTALL) + self.assertIsNotNone(link_match) + class GomaLdIntegrationTest(unittest.TestCase): def clangxx(self): @@ -188,7 +230,7 @@ class GomaLdIntegrationTest(unittest.TestCase): [self.clangxx(), '-c', '-Os', 'main.cpp', '-o', 'main.o']) subprocess.check_call( [self.clangxx(), '-c', '-Os', 'foo.cpp', '-o', 'foo.o']) - rc = GomaLinkUnixWhitelistMain().main([ + rc = GomaLinkUnixAllowMain().main([ 'goma_ld.py', '--gomacc', 'gomacc', '--', self.clangxx(), '-fuse-ld=lld', 'main.o', 'foo.o', '-o', 'main' ]) @@ -235,7 +277,7 @@ class GomaLdIntegrationTest(unittest.TestCase): ]) subprocess.check_call( [self.clangxx(), '-c', '-Os', '-flto=thin', 'foo.cpp', '-o', 'foo.o']) - rc = GomaLinkUnixWhitelistMain().main([ + rc = GomaLinkUnixAllowMain().main([ 'goma_ld.py', '-j', '16', '--', self.clangxx(), '-fuse-ld=lld', '-flto=thin', 'main.o', 'foo.o', '-o', 'main' @@ -246,8 +288,8 @@ class GomaLdIntegrationTest(unittest.TestCase): with open(os.path.join(d, 'lto.main', 'build.ninja')) as f: buildrules = f.read() self.assertIn('gomacc ', buildrules) - self.assertIn('build lto.main/main.o : codegen ', buildrules) - self.assertIn('build lto.main/foo.o : codegen ', buildrules) + self.assertIn('build lto.main/main.o.stamp : codegen ', buildrules) + self.assertIn('build lto.main/foo.o.stamp : codegen ', buildrules) # Check that main does not call foo. disasm = subprocess.check_output(['llvm-objdump', '-d', 'main']) main_idx = disasm.index(b' <main>:\n') @@ -267,7 +309,7 @@ class GomaLdIntegrationTest(unittest.TestCase): [self.clangxx(), '-c', '-Os', '-flto=thin', 'bar.cpp', '-o', 'bar.o']) subprocess.check_call( ['llvm-ar', 'crsT', 'libfoobar.a', 'bar.o', 'foo.o']) - rc = GomaLinkUnixWhitelistMain().main([ + rc = GomaLinkUnixAllowMain().main([ 'goma_ld.py', self.clangxx(), '-fuse-ld=lld', '-flto=thin', 'main.o', 'libfoobar.a', '-o', 'main' @@ -278,8 +320,8 @@ class GomaLdIntegrationTest(unittest.TestCase): with open(os.path.join(d, 'lto.main', 'build.ninja')) as f: buildrules = f.read() self.assertIn('gomacc ', buildrules) - self.assertIn('build lto.main/main.o : codegen ', buildrules) - self.assertIn('build lto.main/foo.o : codegen ', buildrules) + self.assertIn('build lto.main/main.o.stamp : codegen ', buildrules) + self.assertIn('build lto.main/foo.o.stamp : codegen ', buildrules) # Check that main does not call foo. disasm = subprocess.check_output(['llvm-objdump', '-d', 'main']) main_idx = disasm.index(b' <main>:\n') @@ -305,7 +347,7 @@ class GomaLdIntegrationTest(unittest.TestCase): ]) subprocess.check_call( ['llvm-ar', 'crsT', 'obj/libfoobar.a', 'obj/bar.o', 'obj/foo.o']) - rc = GomaLinkUnixWhitelistMain().main([ + rc = GomaLinkUnixAllowMain().main([ 'goma_ld.py', self.clangxx(), '-fuse-ld=lld', '-flto=thin', 'obj/main.o', 'obj/libfoobar.a', '-o', 'main' @@ -316,8 +358,8 @@ class GomaLdIntegrationTest(unittest.TestCase): with open(os.path.join(d, 'lto.main', 'build.ninja')) as f: buildrules = f.read() self.assertIn('gomacc ', buildrules) - self.assertIn('build lto.main/obj/main.o : codegen ', buildrules) - self.assertIn('build lto.main/obj/foo.o : codegen ', buildrules) + self.assertIn('build lto.main/obj/main.o.stamp : codegen ', buildrules) + self.assertIn('build lto.main/obj/foo.o.stamp : codegen ', buildrules) # Check that main does not call foo. disasm = subprocess.check_output(['llvm-objdump', '-d', 'main']) main_idx = disasm.index(b' <main>:\n') @@ -339,7 +381,7 @@ class GomaLdIntegrationTest(unittest.TestCase): ]) with open('main.rsp', 'w') as f: f.write('obj/main.o\n' 'obj/foo.o\n') - rc = GomaLinkUnixWhitelistMain().main([ + rc = GomaLinkUnixAllowMain().main([ 'goma_ld.py', self.clangxx(), '-fuse-ld=lld', '-flto=thin', '-g', '-gsplit-dwarf', '-Wl,--lto-O2', '-o', 'main', '@main.rsp' @@ -377,7 +419,7 @@ class GomaLdIntegrationTest(unittest.TestCase): '-fwhole-program-vtables\n' 'obj/main.o\n' 'obj/libfoobar.a\n') - rc = GomaLinkUnixWhitelistMain().main([ + rc = GomaLinkUnixAllowMain().main([ 'goma_ld.py', self.clangxx(), '-fuse-ld=lld', '-flto=thin', '-m32', '-Wl,-mllvm', '-Wl,-generate-type-units', '-Wl,--lto-O2', '-o', 'main', @@ -396,8 +438,8 @@ class GomaLdIntegrationTest(unittest.TestCase): self.assertIn('-m32', codegen_text) self.assertIn('-mllvm -generate-type-units', codegen_text) self.assertNotIn('-flto', codegen_text) - self.assertIn('build lto.main/obj/main.o : codegen ', buildrules) - self.assertIn('build lto.main/obj/foo.o : codegen ', buildrules) + self.assertIn('build lto.main/obj/main.o.stamp : codegen ', buildrules) + self.assertIn('build lto.main/obj/foo.o.stamp : codegen ', buildrules) link_match = re.search('^build main : native-link\\b.*?^[^ ]', buildrules, re.MULTILINE | re.DOTALL) self.assertIsNotNone(link_match) @@ -418,7 +460,7 @@ class GomaLdIntegrationTest(unittest.TestCase): ]) subprocess.check_call( [self.clangxx(), '-c', '-Os', '-flto=thin', 'foo.cpp', '-o', 'foo.o']) - rc = GomaLinkUnixWhitelistMain().main([ + rc = GomaLinkUnixAllowMain().main([ 'goma_ld.py', '--no-gomacc', '-j', '16', '--', self.clangxx(), '-fuse-ld=lld', '-flto=thin', 'main.o', 'foo.o', '-o', 'main' @@ -429,8 +471,8 @@ class GomaLdIntegrationTest(unittest.TestCase): with open(os.path.join(d, 'lto.main', 'build.ninja')) as f: buildrules = f.read() self.assertNotIn('gomacc ', buildrules) - self.assertIn('build lto.main/main.o : codegen ', buildrules) - self.assertIn('build lto.main/foo.o : codegen ', buildrules) + self.assertIn('build lto.main/main.o.stamp : codegen ', buildrules) + self.assertIn('build lto.main/foo.o.stamp : codegen ', buildrules) # Check that main does not call foo. disasm = subprocess.check_output(['llvm-objdump', '-d', 'main']) main_idx = disasm.index(b' <main>:\n') @@ -438,6 +480,61 @@ class GomaLdIntegrationTest(unittest.TestCase): main_disasm = disasm[main_idx:after_main_idx] self.assertNotIn(b'foo', main_disasm) + def test_generate_no_codegen(self): + with named_directory() as d, working_directory(d): + with open('main.o', 'wb') as f: + f.write(b'\7fELF') + with mock.patch('sys.stderr', new_callable=StringIO) as stderr: + rc = GomaLinkUnixAllowMain().main([ + 'goma_ld.py', '--generate', '--', + self.clangxx(), 'main.o', '-o', 'main' + ]) + self.assertEqual(rc, 5) + self.assertIn('no ninja file generated.\n', stderr.getvalue()) + + def test_generate(self): + with named_directory() as d, working_directory(d): + with open('main.o', 'wb') as f: + f.write(b'BC\xc0\xde') + with mock.patch('sys.stderr', new_callable=StringIO) as stderr: + rc = GomaLinkUnixAllowMain().main([ + 'goma_ld.py', '--generate', '--', + self.clangxx(), 'main.o', '-o', 'main' + ]) + self.assertEqual(rc, 0) + m = re.search('ninja file (.*)', stderr.getvalue()) + self.assertIsNotNone(m) + path = shlex.split(m.group(1))[0] + self.assertTrue(os.path.exists(path)) + content = open(path).read() + self.assertRegex( + content, + re.compile('^build [^:]+/main\\.o\\.stamp : codegen ', + re.MULTILINE)) + + def test_override_allowlist(self): + with named_directory() as d, working_directory(d): + _create_inputs(d) + subprocess.check_call([ + self.clangxx(), '-c', '-Os', '-flto=thin', 'main.cpp', '-o', 'main.o' + ]) + subprocess.check_call( + [self.clangxx(), '-c', '-Os', '-flto=thin', 'foo.cpp', '-o', 'foo.o']) + rc = goma_ld.GomaLinkUnix().main([ + 'goma_ld.py', '--generate', '--allowlist', '--', + self.clangxx(), '-fuse-ld=lld', '-flto=thin', 'main.o', 'foo.o', '-o', + 'main' + ]) + # Should succeed. + self.assertEqual(rc, 0) + # build.ninja file should have rules for main and foo. + ninjafile = os.path.join(d, 'lto.main', 'build.ninja') + self.assertTrue(os.path.exists(ninjafile)) + with open(ninjafile) as f: + buildrules = f.read() + self.assertIn('build lto.main/main.o.stamp : codegen ', buildrules) + self.assertIn('build lto.main/foo.o.stamp : codegen ', buildrules) + if __name__ == '__main__': unittest.main() diff --git a/chromium/tools/clang/scripts/package.py b/chromium/tools/clang/scripts/package.py index 4e4574a6e65..cfed526d3b5 100755 --- a/chromium/tools/clang/scripts/package.py +++ b/chromium/tools/clang/scripts/package.py @@ -184,9 +184,14 @@ def main(): shutil.rmtree(LLVM_BOOTSTRAP_INSTALL_DIR, ignore_errors=True) shutil.rmtree(LLVM_BUILD_DIR, ignore_errors=True) - build_cmd = [sys.executable, os.path.join(THIS_DIR, 'build.py'), - '--bootstrap', '--disable-asserts', - '--run-tests', '--pgo'] + build_cmd = [ + sys.executable, + os.path.join(THIS_DIR, 'build.py'), '--bootstrap', '--disable-asserts', + '--run-tests', '--pgo' + ] + if sys.platform.startswith('linux'): + build_cmd.append('--thinlto') + TeeCmd(build_cmd, log) stamp = open(STAMP_FILE).read().rstrip() @@ -228,9 +233,10 @@ def main(): 'lib/clang/$V/lib/darwin/libclang_rt.asan_iossim_dynamic.dylib', 'lib/clang/$V/lib/darwin/libclang_rt.asan_osx_dynamic.dylib', - # OS X and iOS builtin libraries (iossim is lipo'd into ios) for the - # _IsOSVersionAtLeast runtime function. + # OS X and iOS builtin libraries for the _IsOSVersionAtLeast runtime + # function. 'lib/clang/$V/lib/darwin/libclang_rt.ios.a', + 'lib/clang/$V/lib/darwin/libclang_rt.iossim.a', 'lib/clang/$V/lib/darwin/libclang_rt.osx.a', # Profile runtime (used by profiler and code coverage). diff --git a/chromium/tools/clang/scripts/test_tool.py b/chromium/tools/clang/scripts/test_tool.py index 171e33eb608..00190937018 100755 --- a/chromium/tools/clang/scripts/test_tool.py +++ b/chromium/tools/clang/scripts/test_tool.py @@ -13,6 +13,7 @@ import glob import json import os import os.path +import re import shutil import subprocess import sys @@ -126,6 +127,28 @@ def _ApplyTool(tools_clang_scripts_directory, _RunGit(args) +def _NormalizePathInRawOutput(path, test_dir): + if not os.path.isabs(path): + path = os.path.join(test_dir, path) + + return os.path.relpath(path, test_dir) + + +def _NormalizeSingleRawOutputLine(output_line, test_dir): + if not re.match('^[^:]+(:::.*){4,4}$', output_line): + return output_line + + edit_type, path, offset, length, replacement = output_line.split(':::', 4) + path = _NormalizePathInRawOutput(path, test_dir) + return "%s:::%s:::%s:::%s:::%s" % (edit_type, path, offset, length, + replacement) + + +def _NormalizeRawOutput(output_lines, test_dir): + return map(lambda line: _NormalizeSingleRawOutputLine(line, test_dir), + output_lines) + + def main(argv): parser = argparse.ArgumentParser() parser.add_argument( @@ -209,6 +232,11 @@ def main(argv): expected_output = f.readlines() with open(actual, 'r') as f: actual_output = f.readlines() + if not args.apply_edits: + actual_output = _NormalizeRawOutput(actual_output, + test_directory_for_tool) + expected_output = _NormalizeRawOutput(expected_output, + test_directory_for_tool) if actual_output != expected_output: failed += 1 lines = difflib.unified_diff(expected_output, actual_output, diff --git a/chromium/tools/clang/scripts/update.py b/chromium/tools/clang/scripts/update.py index c9bdcf3ae20..b1018321ec3 100755 --- a/chromium/tools/clang/scripts/update.py +++ b/chromium/tools/clang/scripts/update.py @@ -37,13 +37,12 @@ import zipfile # Do NOT CHANGE this if you don't know what you're doing -- see # https://chromium.googlesource.com/chromium/src/+/master/docs/updating_clang.md # Reverting problematic clang rolls is safe, though. -CLANG_REVISION = '4e813bbdf' -CLANG_SVN_REVISION = 'n356902' +# This is the output of `git describe` and is usable as a commit-ish. +CLANG_REVISION = 'llvmorg-12-init-3492-ga1caa302' CLANG_SUB_REVISION = 1 -PACKAGE_VERSION = '%s-%s-%s' % (CLANG_SVN_REVISION, CLANG_REVISION[:8], - CLANG_SUB_REVISION) -RELEASE_VERSION = '11.0.0' +PACKAGE_VERSION = '%s-%s' % (CLANG_REVISION, CLANG_SUB_REVISION) +RELEASE_VERSION = '12.0.0' CDS_URL = os.environ.get('CDS_CLANG_BUCKET_OVERRIDE', @@ -235,7 +234,7 @@ def UpdatePackage(package_name, host_os): try: GCLIENT_CONFIG = os.path.join(os.path.dirname(CHROMIUM_DIR), '.gclient') env = {} - execfile(GCLIENT_CONFIG, env, env) + exec (open(GCLIENT_CONFIG).read(), env, env) target_os = env.get('target_os', target_os) except: pass diff --git a/chromium/tools/clang/scripts/upload_revision.py b/chromium/tools/clang/scripts/upload_revision.py index 9cd4484cd2b..8eb81b3a368 100755 --- a/chromium/tools/clang/scripts/upload_revision.py +++ b/chromium/tools/clang/scripts/upload_revision.py @@ -18,7 +18,7 @@ import shutil import subprocess import sys -from build import GetCommitCount, CheckoutLLVM, LLVM_DIR +from build import CheckoutLLVM, GetCommitDescription, LLVM_DIR from update import CHROMIUM_DIR # Path constants. @@ -29,8 +29,7 @@ CHROMIUM_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..', '..')) # Keep lines in here at <= 72 columns, else they wrap in gerrit. COMMIT_FOOTER = \ ''' -TODO: Add bug number. - +Bug: TODO Cq-Include-Trybots: chromium/try:mac_chromium_asan_rel_ng Cq-Include-Trybots: chromium/try:linux_chromium_cfi_rel_ng Cq-Include-Trybots: chromium/try:linux_chromium_chromeos_asan_rel_ng @@ -40,7 +39,8 @@ Cq-Include-Trybots: chromium/try:linux-chromeos-dbg,win-asan Cq-Include-Trybots: chromium/try:chromeos-amd64-generic-cfi-thin-lto-rel Cq-Include-Trybots: chromium/try:linux_chromium_compile_dbg_32_ng Cq-Include-Trybots: chromium/try:win7-rel,win-angle-deqp-rel-32 -Cq-Include-Trybots: chromium/try:win-angle-deqp-rel-64,linux_angle_deqp_rel_ng +Cq-Include-Trybots: chromium/try:linux_angle_deqp_rel_ng +Cq-Include-Trybots: chromium/try:win-angle-deqp-rel-64 Cq-Include-Trybots: chromium/try:dawn-win10-x86-deps-rel Cq-Include-Trybots: chrome/try:iphone-device,ipad-device Cq-Include-Trybots: chrome/try:linux-chromeos-chrome @@ -49,28 +49,25 @@ Cq-Include-Trybots: chrome/try:win-chrome,win64-chrome,mac-chrome is_win = sys.platform.startswith('win32') -def PatchRevision(clang_git_revision, clang_svn_revision, clang_sub_revision): + +def PatchRevision(clang_git_revision, clang_sub_revision): with open(UPDATE_PY_PATH, 'rb') as f: content = f.read() - m = re.search("CLANG_REVISION = '([0-9a-f]+)'", content) + m = re.search("CLANG_REVISION = '([0-9a-z-]+)'", content) clang_old_git_revision = m.group(1) - m = re.search("CLANG_SVN_REVISION = '(n[0-9]+)'", content) - clang_old_svn_revision = m.group(1) m = re.search("CLANG_SUB_REVISION = ([0-9]+)", content) clang_old_sub_revision = m.group(1) - content = re.sub("CLANG_REVISION = '[0-9a-f]+'", + content = re.sub("CLANG_REVISION = '[0-9a-z-]+'", "CLANG_REVISION = '{}'".format(clang_git_revision), - content, count=1) - content = re.sub("CLANG_SVN_REVISION = 'n[0-9]+'", - "CLANG_SVN_REVISION = '{}'".format(clang_svn_revision), - content, count=1) + content, + count=1) content = re.sub("CLANG_SUB_REVISION = [0-9]+", "CLANG_SUB_REVISION = {}".format(clang_sub_revision), content, count=1) with open(UPDATE_PY_PATH, 'wb') as f: f.write(content) - return clang_old_git_revision, clang_old_svn_revision, clang_old_sub_revision + return "{}-{}".format(clang_old_git_revision, clang_old_sub_revision) def Git(args): @@ -87,30 +84,24 @@ def main(): args = parser.parse_args() - clang_git_revision = args.clang_git_revision[0] + clang_raw_git_revision = args.clang_git_revision[0] - # To get the commit count, we need a checkout. - CheckoutLLVM(clang_git_revision, LLVM_DIR); - clang_svn_revision = 'n' + GetCommitCount(clang_git_revision) + # To `git describe`, we need a checkout. + CheckoutLLVM(clang_raw_git_revision, LLVM_DIR) + clang_git_revision = GetCommitDescription(clang_raw_git_revision) clang_sub_revision = args.clang_sub_revision os.chdir(CHROMIUM_DIR) - print("Making a patch for Clang {}-{}-{}".format( - clang_svn_revision, clang_git_revision[:8], clang_sub_revision)) + print("Making a patch for Clang {}-{}".format(clang_git_revision, + clang_sub_revision)) - clang_old_git_revision, clang_old_svn_revision, clang_old_sub_revision = \ - PatchRevision(clang_git_revision, clang_svn_revision, clang_sub_revision) - - rev_string = "{}-{}-{}".format(clang_svn_revision, - clang_git_revision[:8], - clang_sub_revision) + rev_string = "{}-{}".format(clang_git_revision, clang_sub_revision) Git(["checkout", "origin/master", "-b", "clang-{}".format(rev_string)]) - Git(["add", UPDATE_PY_PATH]) - old_rev_string = "{}-{}-{}".format(clang_old_svn_revision, - clang_old_git_revision[:8], - clang_old_sub_revision) + old_rev_string = PatchRevision(clang_git_revision, clang_sub_revision) + + Git(["add", UPDATE_PY_PATH]) commit_message = 'Ran `{}`.'.format(' '.join(sys.argv)) + COMMIT_FOOTER Git(["commit", "-m", "Roll clang {} : {}.\n\n{}".format( |