diff options
author | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2021-05-20 09:47:09 +0200 |
---|---|---|
committer | Allan Sandfeld Jensen <allan.jensen@qt.io> | 2021-06-07 11:15:42 +0000 |
commit | 189d4fd8fad9e3c776873be51938cd31a42b6177 (patch) | |
tree | 6497caeff5e383937996768766ab3bb2081a40b2 /chromium/tools/clang | |
parent | 8bc75099d364490b22f43a7ce366b366c08f4164 (diff) | |
download | qtwebengine-chromium-189d4fd8fad9e3c776873be51938cd31a42b6177.tar.gz |
BASELINE: Update Chromium to 90.0.4430.221
Change-Id: Iff4d9d18d2fcf1a576f3b1f453010f744a232920
Reviewed-by: Allan Sandfeld Jensen <allan.jensen@qt.io>
Diffstat (limited to 'chromium/tools/clang')
26 files changed, 984 insertions, 181 deletions
diff --git a/chromium/tools/clang/blink_gc_plugin/BadPatternFinder.cpp b/chromium/tools/clang/blink_gc_plugin/BadPatternFinder.cpp index e081cf3cdc6..36d5d8cef1e 100644 --- a/chromium/tools/clang/blink_gc_plugin/BadPatternFinder.cpp +++ b/chromium/tools/clang/blink_gc_plugin/BadPatternFinder.cpp @@ -3,6 +3,7 @@ // found in the LICENSE file. #include "BadPatternFinder.h" +#include <clang/AST/Decl.h> #include "DiagnosticsReporter.h" #include <algorithm> @@ -18,7 +19,9 @@ namespace { TypeMatcher GarbageCollectedType() { auto has_gc_base = hasCanonicalType(hasDeclaration( cxxRecordDecl(isDerivedFrom(hasAnyName("::blink::GarbageCollected", - "::blink::GarbageCollectedMixin"))) + "::blink::GarbageCollectedMixin", + "::cppgc::GarbageCollected", + "::cppgc::GarbageCollectedMixin"))) .bind("gctype"))); return anyOf(has_gc_base, hasCanonicalType(arrayType(hasElementType(has_gc_base)))); @@ -60,24 +63,30 @@ class OptionalGarbageCollectedMatcher : public MatchFinder::MatchCallback { : diagnostics_(diagnostics) {} void Register(MatchFinder& match_finder) { - // Matches any application of make_unique where the template argument is - // known to refer to a garbage-collected type. - auto optional_construction = - cxxConstructExpr(hasDeclaration(cxxConstructorDecl(ofClass( - classTemplateSpecializationDecl( - hasName("::base::Optional"), - hasTemplateArgument( - 0, refersToType(GarbageCollectedType()))) - .bind("optional"))))) - .bind("bad"); - match_finder.addDynamicMatcher(optional_construction, this); + // Matches fields and new-expressions of type base::Optional where the + // template argument is known to refer to a garbage-collected type. + auto optional_type = hasType( + classTemplateSpecializationDecl( + hasName("::base::Optional"), + hasTemplateArgument(0, refersToType(GarbageCollectedType()))) + .bind("optional")); + auto optional_field = fieldDecl(optional_type).bind("bad_field"); + auto optional_new_expression = + cxxNewExpr(has(cxxConstructExpr(optional_type))).bind("bad_new"); + match_finder.addDynamicMatcher(optional_field, this); + match_finder.addDynamicMatcher(optional_new_expression, this); } void run(const MatchFinder::MatchResult& result) override { - auto* bad_use = result.Nodes.getNodeAs<clang::Expr>("bad"); auto* optional = result.Nodes.getNodeAs<clang::CXXRecordDecl>("optional"); auto* gc_type = result.Nodes.getNodeAs<clang::CXXRecordDecl>("gctype"); - diagnostics_.OptionalUsedWithGC(bad_use, optional, gc_type); + if (auto* bad_field = + result.Nodes.getNodeAs<clang::FieldDecl>("bad_field")) { + diagnostics_.OptionalFieldUsedWithGC(bad_field, optional, gc_type); + } else { + auto* bad_new = result.Nodes.getNodeAs<clang::Expr>("bad_new"); + diagnostics_.OptionalNewExprUsedWithGC(bad_new, optional, gc_type); + } } private: diff --git a/chromium/tools/clang/blink_gc_plugin/BlinkGCPluginConsumer.cpp b/chromium/tools/clang/blink_gc_plugin/BlinkGCPluginConsumer.cpp index 826b94c1140..f829ab22201 100644 --- a/chromium/tools/clang/blink_gc_plugin/BlinkGCPluginConsumer.cpp +++ b/chromium/tools/clang/blink_gc_plugin/BlinkGCPluginConsumer.cpp @@ -86,10 +86,14 @@ BlinkGCPluginConsumer::BlinkGCPluginConsumer( json_(0) { // Only check structures in the blink and WebKit namespaces. options_.checked_namespaces.insert("blink"); + options_.checked_namespaces.insert("cppgc"); // Ignore GC implementation files. options_.ignored_directories.push_back( "third_party/blink/renderer/platform/heap/"); + options_.ignored_directories.push_back("v8/src/heap/cppgc/"); + options_.ignored_directories.push_back("v8/src/heap/cppgc-js/"); + options_.allowed_directories.push_back( "third_party/blink/renderer/platform/heap/test/"); } @@ -106,18 +110,14 @@ void BlinkGCPluginConsumer::HandleTranslationUnit(ASTContext& context) { if (options_.dump_graph) { std::error_code err; - // TODO: Make createDefaultOutputFile or a shorter createOutputFile work. + SmallString<128> OutputFile(instance_.getFrontendOpts().OutputFile); + llvm::sys::path::replace_extension(OutputFile, "graph.json"); json_ = JsonWriter::from(instance_.createOutputFile( - "", // OutputPath - err, // Errors + OutputFile, // OutputPath true, // Binary true, // RemoveFileOnSignal - instance_.getFrontendOpts().OutputFile, // BaseInput - "graph.json", // Extension false, // UseTemporary - false, // CreateMissingDirectories - 0, // ResultPathName - 0)); // TempPathName + false)); // CreateMissingDirectories if (!err && json_) { json_->OpenList(); } else { @@ -371,7 +371,9 @@ void BlinkGCPluginConsumer::CheckPolymorphicClass( CXXRecordDecl* BlinkGCPluginConsumer::GetLeftMostBase( CXXRecordDecl* left_most) { CXXRecordDecl::base_class_iterator it = left_most->bases_begin(); + CXXRecordDecl* previous_left_most = left_most; while (it != left_most->bases_end()) { + previous_left_most = left_most; if (it->getType()->isDependentType()) left_most = RecordInfo::GetDependentTemplatedDecl(*it->getType()); else @@ -380,6 +382,12 @@ CXXRecordDecl* BlinkGCPluginConsumer::GetLeftMostBase( return 0; it = left_most->bases_begin(); } + if (Config::IsCppgcGCBase(left_most->getName())) { + // In the cppgc library, the GC base classes share a common parent. The + // common parent should be ignored for the purposes of getting the left + // most base. + return previous_left_most; + } return left_most; } diff --git a/chromium/tools/clang/blink_gc_plugin/Config.h b/chromium/tools/clang/blink_gc_plugin/Config.h index 3f768b4d5d0..7e6e4fe54aa 100644 --- a/chromium/tools/clang/blink_gc_plugin/Config.h +++ b/chromium/tools/clang/blink_gc_plugin/Config.h @@ -118,6 +118,10 @@ class Config { name == "ThreadSafeRefCounted"; } + static bool IsCppgcGCBase(llvm::StringRef name) { + return name == "GarbageCollectedBase"; + } + static bool IsGCSimpleBase(llvm::StringRef name) { return name == "GarbageCollected"; } diff --git a/chromium/tools/clang/blink_gc_plugin/DiagnosticsReporter.cpp b/chromium/tools/clang/blink_gc_plugin/DiagnosticsReporter.cpp index 4a6d9ddeb8d..de7853c41ef 100644 --- a/chromium/tools/clang/blink_gc_plugin/DiagnosticsReporter.cpp +++ b/chromium/tools/clang/blink_gc_plugin/DiagnosticsReporter.cpp @@ -141,9 +141,15 @@ const char kUniquePtrUsedWithGC[] = "[blink-gc] Disallowed use of %0 found; %1 is a garbage-collected type. " "std::unique_ptr cannot hold garbage-collected objects."; -const char kOptionalUsedWithGC[] = - "[blink-gc] Disallowed construction of %0 found; %1 is a garbage-collected " - "type. optional cannot hold garbage-collected objects."; +const char kOptionalFieldUsedWithGC[] = + "[blink-gc] Disallowed optional field of %0 found; %1 is a " + "garbage-collected " + "type. Optional fields cannot hold garbage-collected objects."; + +const char kOptionalNewExprUsedWithGC[] = + "[blink-gc] Disallowed new-expression of %0 found; %1 is a " + "garbage-collected " + "type. GCed types cannot be created with new."; const char kVariantUsedWithGC[] = "[blink-gc] Disallowed construction of %0 found; %1 is a garbage-collected " @@ -248,8 +254,10 @@ DiagnosticsReporter::DiagnosticsReporter( diag_unique_ptr_used_with_gc_ = diagnostic_.getCustomDiagID(getErrorLevel(), kUniquePtrUsedWithGC); - diag_optional_used_with_gc_ = - diagnostic_.getCustomDiagID(getErrorLevel(), kOptionalUsedWithGC); + diag_optional_field_used_with_gc_ = + diagnostic_.getCustomDiagID(getErrorLevel(), kOptionalFieldUsedWithGC); + diag_optional_new_expr_used_with_gc_ = + diagnostic_.getCustomDiagID(getErrorLevel(), kOptionalNewExprUsedWithGC); diag_variant_used_with_gc_ = diagnostic_.getCustomDiagID(getErrorLevel(), kVariantUsedWithGC); } @@ -540,11 +548,19 @@ void DiagnosticsReporter::UniquePtrUsedWithGC( << bad_function << gc_type << expr->getSourceRange(); } -void DiagnosticsReporter::OptionalUsedWithGC( +void DiagnosticsReporter::OptionalFieldUsedWithGC( + const clang::FieldDecl* field, + const clang::CXXRecordDecl* optional, + const clang::CXXRecordDecl* gc_type) { + ReportDiagnostic(field->getBeginLoc(), diag_optional_field_used_with_gc_) + << optional << gc_type << field->getSourceRange(); +} + +void DiagnosticsReporter::OptionalNewExprUsedWithGC( const clang::Expr* expr, const clang::CXXRecordDecl* optional, const clang::CXXRecordDecl* gc_type) { - ReportDiagnostic(expr->getBeginLoc(), diag_optional_used_with_gc_) + ReportDiagnostic(expr->getBeginLoc(), diag_optional_new_expr_used_with_gc_) << optional << gc_type << expr->getSourceRange(); } diff --git a/chromium/tools/clang/blink_gc_plugin/DiagnosticsReporter.h b/chromium/tools/clang/blink_gc_plugin/DiagnosticsReporter.h index 24cf7a1b2ab..73ddcd0e4c9 100644 --- a/chromium/tools/clang/blink_gc_plugin/DiagnosticsReporter.h +++ b/chromium/tools/clang/blink_gc_plugin/DiagnosticsReporter.h @@ -79,9 +79,12 @@ class DiagnosticsReporter { void UniquePtrUsedWithGC(const clang::Expr* expr, const clang::FunctionDecl* bad_function, const clang::CXXRecordDecl* gc_type); - void OptionalUsedWithGC(const clang::Expr* expr, - const clang::CXXRecordDecl* optional, - const clang::CXXRecordDecl* gc_type); + void OptionalFieldUsedWithGC(const clang::FieldDecl* decl, + const clang::CXXRecordDecl* optional, + const clang::CXXRecordDecl* gc_type); + void OptionalNewExprUsedWithGC(const clang::Expr* expr, + const clang::CXXRecordDecl* optional, + const clang::CXXRecordDecl* gc_type); void VariantUsedWithGC(const clang::Expr* expr, const clang::CXXRecordDecl* variant, const clang::CXXRecordDecl* gc_type); @@ -142,7 +145,8 @@ class DiagnosticsReporter { unsigned diag_member_in_stack_allocated_class_; unsigned diag_unique_ptr_used_with_gc_; - unsigned diag_optional_used_with_gc_; + unsigned diag_optional_field_used_with_gc_; + unsigned diag_optional_new_expr_used_with_gc_; unsigned diag_variant_used_with_gc_; }; diff --git a/chromium/tools/clang/blink_gc_plugin/RecordInfo.cpp b/chromium/tools/clang/blink_gc_plugin/RecordInfo.cpp index d00cd0ec2be..2758a5a9e71 100644 --- a/chromium/tools/clang/blink_gc_plugin/RecordInfo.cpp +++ b/chromium/tools/clang/blink_gc_plugin/RecordInfo.cpp @@ -159,7 +159,18 @@ CXXRecordDecl* RecordInfo::GetDependentTemplatedDecl(const Type& type) { if (!tmpl_decl) return 0; - return dyn_cast_or_null<CXXRecordDecl>(tmpl_decl->getTemplatedDecl()); + if (CXXRecordDecl* record_decl = + dyn_cast_or_null<CXXRecordDecl>(tmpl_decl->getTemplatedDecl())) + return record_decl; + + // Type is an alias. + TypeAliasDecl* alias_decl = + dyn_cast<TypeAliasDecl>(tmpl_decl->getTemplatedDecl()); + assert(alias_decl); + const Type* alias_type = alias_decl->getUnderlyingType().getTypePtr(); + if (CXXRecordDecl* record_decl = alias_type->getAsCXXRecordDecl()) + return record_decl; + return GetDependentTemplatedDecl(*alias_type); } void RecordInfo::walkBases() { @@ -693,7 +704,12 @@ Edge* RecordInfo::CreateEdge(const Type* type) { // TODO: Consider using a more canonical identification than names. NamespaceDecl* ns = dyn_cast<NamespaceDecl>(info->record()->getDeclContext()); - if (!ns || ns->getName() != "blink") + // Find outer-most namespace. + while (NamespaceDecl* outer_ns = + dyn_cast<NamespaceDecl>(ns->getDeclContext())) { + ns = outer_ns; + } + if (!ns || (ns->getName() != "blink") && (ns->getName() != "cppgc")) return 0; if (!info->GetTemplateArgs(1, &args)) return 0; diff --git a/chromium/tools/clang/blink_gc_plugin/RecordInfo.h b/chromium/tools/clang/blink_gc_plugin/RecordInfo.h index de44d8652e6..20ef058a6e2 100644 --- a/chromium/tools/clang/blink_gc_plugin/RecordInfo.h +++ b/chromium/tools/clang/blink_gc_plugin/RecordInfo.h @@ -40,7 +40,7 @@ class BasePoint : public GraphPoint { RecordInfo* info, const TracingStatus& status) : spec_(spec), info_(info), status_(status) {} - const TracingStatus NeedsTracing() { return status_; } + const TracingStatus NeedsTracing() override { return status_; } const clang::CXXBaseSpecifier& spec() { return spec_; } RecordInfo* info() { return info_; } @@ -54,7 +54,7 @@ class FieldPoint : public GraphPoint { public: FieldPoint(clang::FieldDecl* field, Edge* edge) : field_(field), edge_(edge) {} - const TracingStatus NeedsTracing() { + const TracingStatus NeedsTracing() override { return edge_->NeedsTracing(Edge::kRecursive); } clang::FieldDecl* field() { return field_; } diff --git a/chromium/tools/clang/blink_gc_plugin/process-graph.py b/chromium/tools/clang/blink_gc_plugin/process-graph.py index 06031838528..eac8f363c64 100755 --- a/chromium/tools/clang/blink_gc_plugin/process-graph.py +++ b/chromium/tools/clang/blink_gc_plugin/process-graph.py @@ -364,6 +364,8 @@ def read_ignored_cycles(): gc_bases = ( 'blink::GarbageCollected', 'blink::GarbageCollectedMixin', + 'cppgc::GarbageCollected', + 'cppgc::GarbageCollectedMixin', ) ref_bases = ( 'WTF::RefCounted', @@ -418,7 +420,8 @@ def print_stats(): % ( stats['ref'] == 0 and stats['ref-mixins'] == 0 and "*" or " ", total == 0 and 100 or stats['mem'] * 100 / total, - node.name.replace('blink::', ''), + node.name.replace('blink::', '').replace( + 'cppgc::subtle::', '').replace('cppgc::', ''), stats['classes'], stats['mem'], stats['ref'], diff --git a/chromium/tools/clang/plugins/FindBadConstructsAction.h b/chromium/tools/clang/plugins/FindBadConstructsAction.h index 383db84206d..04cfc811928 100644 --- a/chromium/tools/clang/plugins/FindBadConstructsAction.h +++ b/chromium/tools/clang/plugins/FindBadConstructsAction.h @@ -16,12 +16,11 @@ class FindBadConstructsAction : public clang::PluginASTAction { FindBadConstructsAction(); protected: - // Overridden from PluginASTAction: - virtual std::unique_ptr<clang::ASTConsumer> CreateASTConsumer( + std::unique_ptr<clang::ASTConsumer> CreateASTConsumer( clang::CompilerInstance& instance, - llvm::StringRef ref); - virtual bool ParseArgs(const clang::CompilerInstance& instance, - const std::vector<std::string>& args); + llvm::StringRef ref) override; + bool ParseArgs(const clang::CompilerInstance& instance, + const std::vector<std::string>& args) override; private: Options options_; diff --git a/chromium/tools/clang/pylib/clang/compile_db.py b/chromium/tools/clang/pylib/clang/compile_db.py index 2502810cfbb..2966ea92f07 100755 --- a/chromium/tools/clang/pylib/clang/compile_db.py +++ b/chromium/tools/clang/pylib/clang/compile_db.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2016 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. @@ -10,6 +10,7 @@ import os import re import sys import subprocess +import shutil _RSP_RE = re.compile(r' (@(.+?\.rsp)) ') @@ -140,8 +141,11 @@ def GenerateWithNinja(path, targets=[]): # TODO(dcheng): Ensure that clang is enabled somehow. # First, generate the compile database. + ninja_path = GetNinjaPath() + if not os.path.exists(ninja_path): + ninja_path = shutil.which("ninja") json_compile_db = subprocess.check_output( - [GetNinjaPath(), '-C', path] + targets + + [ninja_path, '-C', path] + targets + ['-t', 'compdb', 'cc', 'cxx', 'objc', 'objcxx']) return json.loads(json_compile_db) diff --git a/chromium/tools/clang/rewrite_raw_ptr_fields/OWNERS b/chromium/tools/clang/rewrite_raw_ptr_fields/OWNERS new file mode 100644 index 00000000000..b9574f85df5 --- /dev/null +++ b/chromium/tools/clang/rewrite_raw_ptr_fields/OWNERS @@ -0,0 +1,9 @@ +dcheng@chromium.org +lukasza@chromium.org + +per-file manual-*-to-ignore.txt=bartekn@chromium.org +per-file manual-*-to-ignore.txt=glazunov@google.com +per-file manual-*-to-ignore.txt=keishi@chromium.org +per-file rewrite*.sh=bartekn@chromium.org +per-file rewrite*.sh=keishi@chromium.org +per-file rewrite*.sh=yukishiino@chromium.org diff --git a/chromium/tools/clang/rewrite_raw_ptr_fields/RewriteRawPtrFields.cpp b/chromium/tools/clang/rewrite_raw_ptr_fields/RewriteRawPtrFields.cpp index 41766cc99a8..b8a00c7c6c2 100644 --- a/chromium/tools/clang/rewrite_raw_ptr_fields/RewriteRawPtrFields.cpp +++ b/chromium/tools/clang/rewrite_raw_ptr_fields/RewriteRawPtrFields.cpp @@ -307,21 +307,35 @@ class FilterFile { return it != file_lines_.end(); } - // Returns true if any of the filter file lines is a substring of - // |string_to_match|. + // Returns true if |string_to_match| matches based on the filter file lines. + // Filter file lines can contain both inclusions and exclusions in the filter. + // Only returns true if |string_to_match| both matches an inclusion filter and + // is *not* matched by an exclusion filter. bool ContainsSubstringOf(llvm::StringRef string_to_match) const { - if (!substring_regex_.hasValue()) { - std::vector<std::string> regex_escaped_file_lines; - regex_escaped_file_lines.reserve(file_lines_.size()); - for (const llvm::StringRef& file_line : file_lines_.keys()) - regex_escaped_file_lines.push_back(llvm::Regex::escape(file_line)); - std::string substring_regex_pattern = - llvm::join(regex_escaped_file_lines.begin(), - regex_escaped_file_lines.end(), "|"); - substring_regex_.emplace(substring_regex_pattern); + if (!inclusion_substring_regex_.hasValue()) { + std::vector<std::string> regex_escaped_inclusion_file_lines; + std::vector<std::string> regex_escaped_exclusion_file_lines; + regex_escaped_inclusion_file_lines.reserve(file_lines_.size()); + for (const llvm::StringRef& file_line : file_lines_.keys()) { + if (file_line.startswith("!")) { + regex_escaped_exclusion_file_lines.push_back( + llvm::Regex::escape(file_line.substr(1))); + } else { + regex_escaped_inclusion_file_lines.push_back( + llvm::Regex::escape(file_line)); + } + } + std::string inclusion_substring_regex_pattern = + llvm::join(regex_escaped_inclusion_file_lines.begin(), + regex_escaped_inclusion_file_lines.end(), "|"); + inclusion_substring_regex_.emplace(inclusion_substring_regex_pattern); + std::string exclusion_substring_regex_pattern = + llvm::join(regex_escaped_exclusion_file_lines.begin(), + regex_escaped_exclusion_file_lines.end(), "|"); + exclusion_substring_regex_.emplace(exclusion_substring_regex_pattern); } - - return substring_regex_->match(string_to_match); + return inclusion_substring_regex_->match(string_to_match) && + !exclusion_substring_regex_->match(string_to_match); } private: @@ -368,9 +382,16 @@ class FilterFile { // Stores all file lines (after stripping comments and blank lines). llvm::StringSet<> file_lines_; + // |file_lines_| is partitioned based on whether the line starts with a ! + // (exclusion line) or not (inclusion line). Inclusion lines specify things to + // be matched by the filter. The exclusion lines specify what to force exclude + // from the filter. Lazily-constructed regex that matches strings that contain + // any of the inclusion lines in |file_lines_|. + mutable llvm::Optional<llvm::Regex> inclusion_substring_regex_; + // Lazily-constructed regex that matches strings that contain any of the - // |file_lines_|. - mutable llvm::Optional<llvm::Regex> substring_regex_; + // exclusion lines in |file_lines_|. + mutable llvm::Optional<llvm::Regex> exclusion_substring_regex_; }; AST_MATCHER_P(clang::FieldDecl, @@ -445,6 +466,10 @@ AST_MATCHER(clang::FunctionDecl, isImplicitFunctionTemplateSpecialization) { } } +AST_MATCHER(clang::Type, anyCharType) { + return Node.isAnyCharacterType(); +} + AST_POLYMORPHIC_MATCHER(isInMacroLocation, AST_POLYMORPHIC_SUPPORTED_TYPES(clang::Decl, clang::Stmt, @@ -937,9 +962,11 @@ int main(int argc, const char* argv[]) { llvm::cl::opt<std::string> exclude_paths_param( kExcludePathsParamName, llvm::cl::value_desc("filepath"), llvm::cl::desc("file listing paths to be blocked (not rewritten)")); - clang::tooling::CommonOptionsParser options(argc, argv, category); - clang::tooling::ClangTool tool(options.getCompilations(), - options.getSourcePathList()); + llvm::Expected<clang::tooling::CommonOptionsParser> options = + clang::tooling::CommonOptionsParser::create(argc, argv, category); + assert(static_cast<bool>(options)); // Should not return an error. + clang::tooling::ClangTool tool(options->getCompilations(), + options->getSourcePathList()); MatchFinder match_finder; OutputHelper output_helper; @@ -1046,6 +1073,26 @@ int main(int argc, const char* argv[]) { match_finder.addMatcher(affected_ternary_operator_arg_matcher, &affected_expr_rewriter); + // Affected string binary operator ========= + // Given + // struct S { const char* y; } + // void foo(const S& s) { + // std::string other; + // bool v1 = s.y == other; + // std::string v2 = s.y + other; + // } + // binds the |s.y| expr if it matches the |affected_expr_matcher| above. + // + // See also testcases in tests/affected-expr-original.cc + auto std_string_expr_matcher = + expr(hasType(cxxRecordDecl(hasName("::std::basic_string")))); + auto affected_string_binary_operator_arg_matcher = cxxOperatorCallExpr( + hasAnyOverloadedOperatorName("+", "==", "!=", "<", "<=", ">", ">="), + hasAnyArgument(std_string_expr_matcher), + forEachArgumentWithParam(affected_expr_matcher, parmVarDecl())); + match_finder.addMatcher(affected_string_binary_operator_arg_matcher, + &affected_expr_rewriter); + // Calls to templated functions ========= // Given // struct S { int* y; }; @@ -1066,7 +1113,7 @@ int main(int argc, const char* argv[]) { // TODO(lukasza): It is unclear why |traverse| below is needed. Maybe it can // be removed if https://bugs.llvm.org/show_bug.cgi?id=46287 is fixed. match_finder.addMatcher( - traverse(clang::ast_type_traits::TK_AsIs, + traverse(clang::TraversalKind::TK_AsIs, cxxConstructExpr(templated_function_arg_matcher)), &affected_expr_rewriter); @@ -1081,10 +1128,12 @@ int main(int argc, const char* argv[]) { // binds the |s.y| expr if it matches the |affected_expr_matcher| above. // // See also testcases in tests/affected-expr-original.cc - auto implicit_ctor_expr_matcher = implicitCastExpr(has(cxxConstructExpr(allOf( + auto implicit_ctor_expr_matcher = cxxConstructExpr(allOf( + anyOf(hasParent(materializeTemporaryExpr()), + hasParent(implicitCastExpr())), hasDeclaration( cxxConstructorDecl(allOf(parameterCountIs(1), unless(isExplicit())))), - forEachArgumentWithParam(affected_expr_matcher, parmVarDecl()))))); + forEachArgumentWithParam(affected_expr_matcher, parmVarDecl()))); match_finder.addMatcher(implicit_ctor_expr_matcher, &affected_expr_rewriter); // |auto| type declarations ========= @@ -1182,6 +1231,16 @@ int main(int argc, const char* argv[]) { FilteredExprWriter macro_field_decl_writer(&output_helper, "macro"); match_finder.addMatcher(macro_field_decl_matcher, ¯o_field_decl_writer); + // See the doc comment for the anyCharType matcher + // and the testcases in tests/gen-char-test.cc. + auto char_ptr_field_decl_matcher = fieldDecl(allOf( + field_decl_matcher, + hasType(pointerType(pointee(qualType(allOf( + isConstQualified(), hasUnqualifiedDesugaredType(anyCharType())))))))); + FilteredExprWriter char_ptr_field_decl_writer(&output_helper, "const-char"); + match_finder.addMatcher(char_ptr_field_decl_matcher, + &char_ptr_field_decl_writer); + // See the testcases in tests/gen-global-destructor-test.cc. auto global_destructor_matcher = varDecl(allOf(hasGlobalStorage(), @@ -1230,6 +1289,17 @@ int main(int argc, const char* argv[]) { FilteredExprWriter union_field_decl_writer(&output_helper, "union"); match_finder.addMatcher(union_field_decl_matcher, &union_field_decl_writer); + // Matches rewritable fields of struct `SomeStruct` if that struct happens to + // be a destination type of a `reinterpret_cast<SomeStruct*>` cast. + auto reinterpret_cast_struct_matcher = + cxxReinterpretCastExpr(hasDestinationType( + pointerType(pointee(hasUnqualifiedDesugaredType(recordType( + hasDeclaration(recordDecl(forEach(field_decl_matcher))))))))); + FilteredExprWriter reinterpret_cast_struct_writer(&output_helper, + "reinterpret-cast-struct"); + match_finder.addMatcher(reinterpret_cast_struct_matcher, + &reinterpret_cast_struct_writer); + // Prepare and run the tool. std::unique_ptr<clang::tooling::FrontendActionFactory> factory = clang::tooling::newFrontendActionFactory(&match_finder, &output_helper); diff --git a/chromium/tools/clang/rewrite_raw_ptr_fields/manual-fields-to-ignore.txt b/chromium/tools/clang/rewrite_raw_ptr_fields/manual-fields-to-ignore.txt index 7b3dbd2252c..6ac34e65192 100644 --- a/chromium/tools/clang/rewrite_raw_ptr_fields/manual-fields-to-ignore.txt +++ b/chromium/tools/clang/rewrite_raw_ptr_fields/manual-fields-to-ignore.txt @@ -77,10 +77,6 @@ remoting::ContinueWindowGtk::continue_window_ ui::AXPlatformNodeAuraLinux::atk_hyperlink_ PrintDialogGtk::dialog_ -# Populated manually - using nmap or base::AllocPages directly -blink::GCInfoTable::table_ -disk_cache::MappedFile::buffer_ - # Populated manually, because of in-out-arg usage. blink::PaintController::IdAsHashKey::client ui::AXPlatformNodeAuraLinux::atk_object_ @@ -112,6 +108,32 @@ cc::GpuImageDecodeCache::dark_mode_filter_ blink::scheduler::MainThreadSchedulerImpl::current_agent_group_scheduler_ performance_manager::v8_memory::(anonymous namespace)::V8ContextTrackerTest::tracker content::(anonymous namespace)::BackForwardCacheMessageFilter::interface_name_ +base::trace_event::MemoryDumpProviderInfo::name + +# Populated manually - requires rewriting member pointer type +courgette::LabelManagerTest_SimpleIndexAssigner_Test::TestBody()::TestCase::input +courgette::LabelManagerTest_SimpleIndexAssigner_Test::TestBody()::TestCase::expect_forward +courgette::LabelManagerTest_SimpleIndexAssigner_Test::TestBody()::TestCase::expect_backward +courgette::LabelManagerTest_SimpleIndexAssigner_Test::TestBody()::TestCase::expect_in + +# Populated manually - pointer to address in another process could be confused +# as a pointer to PA memory +sandbox::TargetProcess::base_address_ # https://crbug.com/1173374 + +# Populated manually - uninitialized memory reinterpret_cast to CheckedPtr +# causing AddRef/Deref mismatch. +sandbox::(anonymous namespace)::_HEAP_32::Heap +sandbox::(anonymous namespace)::_HEAP_64::Heap +sql::SandboxedVfsFileSqliteBridge::sandboxed_vfs_file +sandbox::SharedMemIPCServer::client_control_ +sandbox::SharedMemIPCServer::thread_provider_ +sandbox::SharedMemIPCServer::call_dispatcher_ + +# Populated manually - pointer to stale non-PA allocation could be confused as a +# pointer to PA memory when that address space is reused https://crbug.com/1173851 +base::PersistentMemoryAllocator::mem_base_ # https://crbug.com/1169582 +base::SharedMemoryMapping::memory_ # https://crbug.com/1169581 +mojo::core::PlatformSharedMemoryMapping::base_ # https://crbug.com/1173380 # Populated manually - other compile-time reasons iovec::iov_base # requires adding .get() in another repository, used e.g. in @@ -119,37 +141,38 @@ iovec::iov_base # requires adding .get() in another repository, used e.g. in net::SockaddrStorage::addr # .get() not added in reinterpret_cast, reason unknown views::internal::ClassPropertyValueSetter::property_ # passed to templated param T* (anonymous namespace)::ScopedFunctionHelper::function_ # function pointer template - -####### -# CheckedPtr2/MTECheckedPtr-specific sections -####### - -# Populated manually - these pointers are assigned invalid address (with top -# bits sets), which CheckedPtr is unable to handle, leading to run-time crashes. -(anonymous namespace)::TlsVectorEntry::data -blink::(anonymous namespace)::ThreadMarker::creating_thread_ -blink::ControlKey::name_ -performance_manager::frame_priority::BoostingVoteAggregator::Edge::src_ -performance_manager::frame_priority::BoostingVoteAggregator::Edge::dst_ - -# Populated manually - these pointers crash due to a mismatching tag. -# Under investigation. -blink::NGPhysicalContainerFragment::buffer_ -blink::DisplayItem::client_ -cc::FrameSequenceMetrics::throughput_ukm_reporter_ -cc::(anonymous namespace)::RasterTaskImpl::tile_tracing_id_ -content::RenderViewImpl::webview_ -mojo::core::WatcherDispatcher::last_watch_to_block_arming_ -net::IOBuffer::data_ -safe_browsing::RemoteSafeBrowsingDatabaseManager::ClientRequest::client_ - -# Populated manually - this pointer crashes, because assigned address appears -# unallocated. -x11::ReadBuffer::fds - -####### -# End of CheckedPtr2/MTECheckedPtr-specific sections -####### +KeyedServiceBaseFactory::service_name_ # used in decltype +OutOfMemoryTest::value_ # used to pass non const pointer to pointer + +# ELEMENT() treats the CheckedPtr as a void*, and so when a pointer is written +# AddRef() won't be called, causing AddRef/Deref mismatch. +device::AttestedCredentialData::ConsumeFromCtapResponse(base::span<const uint8_t>)::COSEKey::alg +device::AttestedCredentialData::ConsumeFromCtapResponse(base::span<const uint8_t>)::COSEKey::kty +device::cablev2::authenticator::(anonymous namespace)::MakeCredRequest::client_data_hash +device::cablev2::authenticator::(anonymous namespace)::MakeCredRequest::rp_id +device::cablev2::authenticator::(anonymous namespace)::MakeCredRequest::user_id +device::cablev2::authenticator::(anonymous namespace)::MakeCredRequest::cred_params +device::cablev2::authenticator::(anonymous namespace)::MakeCredRequest::excluded_credentials +device::cablev2::authenticator::(anonymous namespace)::MakeCredRequest::origin +device::cablev2::authenticator::(anonymous namespace)::MakeCredRequest::challenge +device::cablev2::authenticator::(anonymous namespace)::AttestationObject::fmt +device::cablev2::authenticator::(anonymous namespace)::AttestationObject::auth_data +device::cablev2::authenticator::(anonymous namespace)::AttestationObject::statement +device::cablev2::authenticator::(anonymous namespace)::GetAssertionRequest::rp_id +device::cablev2::authenticator::(anonymous namespace)::GetAssertionRequest::client_data_hash +device::cablev2::authenticator::(anonymous namespace)::GetAssertionRequest::allowed_credentials +device::cablev2::authenticator::(anonymous namespace)::GetAssertionRequest::origin +device::cablev2::authenticator::(anonymous namespace)::GetAssertionRequest::challenge +device::Ed25519PublicKey::ExtractFromCOSEKey(int32_t, base::span<const uint8_t>, const cbor::Value::MapValue &)::COSEKey::kty +device::Ed25519PublicKey::ExtractFromCOSEKey(int32_t, base::span<const uint8_t>, const cbor::Value::MapValue &)::COSEKey::crv +device::Ed25519PublicKey::ExtractFromCOSEKey(int32_t, base::span<const uint8_t>, const cbor::Value::MapValue &)::COSEKey::key +device::P256PublicKey::ExtractFromCOSEKey(int32_t, base::span<const uint8_t>, const cbor::Value::MapValue &)::COSEKey::kty +device::P256PublicKey::ExtractFromCOSEKey(int32_t, base::span<const uint8_t>, const cbor::Value::MapValue &)::COSEKey::crv +device::P256PublicKey::ExtractFromCOSEKey(int32_t, base::span<const uint8_t>, const cbor::Value::MapValue &)::COSEKey::x +device::P256PublicKey::ExtractFromCOSEKey(int32_t, base::span<const uint8_t>, const cbor::Value::MapValue &)::COSEKey::y +device::RSAPublicKey::ExtractFromCOSEKey(int32_t, base::span<const uint8_t>, const cbor::Value::MapValue &)::COSEKey::kty +device::RSAPublicKey::ExtractFromCOSEKey(int32_t, base::span<const uint8_t>, const cbor::Value::MapValue &)::COSEKey::n +device::RSAPublicKey::ExtractFromCOSEKey(int32_t, base::span<const uint8_t>, const cbor::Value::MapValue &)::COSEKey::e ####### # BackupRefPtr-specific sections @@ -180,3 +203,34 @@ gpu::gles2::PassthroughProgramCache::ProgramCacheValue::program_cache_ ####### # End of BackupRefPtr-specific sections ####### + +####### +# Performance-related exclusions +####### + +# Populated manually - on-stack pointer + a large number of non-PA pointees +base::AutoReset::scoped_variable_ + +# Populated manually - on-stack pointee +base::StackAllocator::source_ + +# Populated manually - on-stack pointer + a large number of non-PA pointees +mojo::core::ports::PortLocker::port_refs_ + +# Populated manually - static pointee +mojo::core::RequestContext::tls_context_ + +# Populated manually - on-stack pointee +mojo::internal::ValidationContext::ScopedDepthTracker::ctx_ + +# Populated manually - using mmap, MapViewOfFile or base::AllocPages directly +blink::GCInfoTable::table_ +cc::(anonymous namespace)::BitmapRasterBufferImpl::pixels_ +# TODO(bartekn): This one has a malloc() path, consider rewriting after all. +disk_cache::MappedFile::buffer_ +network::MojoToNetPendingBuffer::buffer_ +network::NetToMojoPendingBuffer::buffer_ + +####### +# End of performance-related exclusions +####### diff --git a/chromium/tools/clang/rewrite_raw_ptr_fields/manual-paths-to-ignore.txt b/chromium/tools/clang/rewrite_raw_ptr_fields/manual-paths-to-ignore.txt index de839fbde0e..3e0cc942410 100644 --- a/chromium/tools/clang/rewrite_raw_ptr_fields/manual-paths-to-ignore.txt +++ b/chromium/tools/clang/rewrite_raw_ptr_fields/manual-paths-to-ignore.txt @@ -4,6 +4,9 @@ # If a source file path contains any of the lines in the filter file below, # then such source file will not be rewritten. # +# Lines prefixed with "!" can be used to force include files that matched a file +# path to be ignored. +# # Note that the rewriter has a hardcoded logic for a handful of path-based # exclusions that cannot be expressed as substring matches: # - Excluding paths containing "third_party/", but still covering @@ -27,10 +30,24 @@ net/tools/ chrome/chrome_elf/ chrome/installer/mini_installer/ +# DEPS prohibits includes from base/ +chrome/install_static + +# Exclude pocdll.dll as it doesn't depend on //base and only used for testing. +sandbox/win/sandbox_poc/pocdll + +# Exclude directories that don't depend on //base, because nothing there uses +# anything from /base. +sandbox/linux/system_headers/ + # The folder holds headers that are duplicated in the Android source and need to # provide a stable C ABI. Can't depend on //base. android_webview/public/ +# Exclude dependences of checked_ptr.h +base/logging.h +base/synchronization/lock_impl.h + # Exclude code that only runs inside a renderer process - renderer # processes are excluded for now from the MiraclePtr project scope, # because they are sensitive to performance regressions (to a much higher @@ -40,12 +57,17 @@ android_webview/public/ # elsewhere - for example "v8/" is excluded in another part of this # file. # +# The common/ directories must be included in the rewrite as they contain code +# that is also used from the browser process. +# # Also, note that isInThirdPartyLocation AST matcher in # RewriteRawPtrFields.cpp explicitly includes third_party/blink # (because it is in the same git repository as the rest of Chromium), # but we go ahead and exclude it below. /renderer/ # (e.g. //content/renderer/ or //components/visitedlink/renderer/). third_party/blink/ +!third_party/blink/public/common/ +!third_party/blink/common/ # Exclude paths in separate repositories - i.e. in directories that # 1. Contain a ".git" subdirectory diff --git a/chromium/tools/clang/rewrite_raw_ptr_fields/rewrite.sh b/chromium/tools/clang/rewrite_raw_ptr_fields/rewrite.sh index 30eebc984ba..868968dad78 100755 --- a/chromium/tools/clang/rewrite_raw_ptr_fields/rewrite.sh +++ b/chromium/tools/clang/rewrite_raw_ptr_fields/rewrite.sh @@ -47,10 +47,16 @@ gn gen $OUT_DIR GEN_H_TARGETS=`ninja -C $OUT_DIR -t targets all | grep '^gen/.*\(\.h\|inc\|css_tokenizer_codepoints.cc\)' | cut -d : -f 1` time ninja -C $OUT_DIR $GEN_H_TARGETS +if grep -qE '^\s*target_os\s*=\s*("win"|win)' $OUT_DIR/args.gn +then + TARGET_OS_OPTION="--target_os=win" +fi + # A preliminary rewriter run in a special mode that generates a list of fields # to ignore. These fields would likely lead to compiler errors if rewritten. echo "*** Generating the ignore list ***" time tools/clang/scripts/run_tool.py \ + $TARGET_OS_OPTION \ --tool rewrite_raw_ptr_fields \ --tool-arg=--exclude-paths=$REWRITER_SRC_DIR/manual-paths-to-ignore.txt \ --generate-compdb \ @@ -61,11 +67,13 @@ cat ~/scratch/rewriter.out \ | sort | uniq > ~/scratch/automated-fields-to-ignore.txt cat ~/scratch/automated-fields-to-ignore.txt \ tools/clang/rewrite_raw_ptr_fields/manual-fields-to-ignore.txt \ - >> ~/scratch/combined-fields-to-ignore.txt + | grep -v "base::FileDescriptorWatcher::Controller::watcher_" \ + > ~/scratch/combined-fields-to-ignore.txt # Main rewrite. echo "*** Running the main rewrite phase ***" time tools/clang/scripts/run_tool.py \ + $TARGET_OS_OPTION \ --tool rewrite_raw_ptr_fields \ --tool-arg=--exclude-fields=$HOME/scratch/combined-fields-to-ignore.txt \ --tool-arg=--exclude-paths=$REWRITER_SRC_DIR/manual-paths-to-ignore.txt \ diff --git a/chromium/tools/clang/scripts/apply_edits.py b/chromium/tools/clang/scripts/apply_edits.py index a49a739792b..abf95df42ac 100755 --- a/chromium/tools/clang/scripts/apply_edits.py +++ b/chromium/tools/clang/scripts/apply_edits.py @@ -164,6 +164,59 @@ _INCLUDE_INSERTION_POINT_REGEX_TEMPLATE = r''' ''' +_NEWLINE_CHARACTERS = [ord('\n'), ord('\r')] + + +def _FindStartOfPreviousLine(contents, index): + """ Requires that `index` points to the start of a line. + Returns an index to the start of the previous line. + """ + assert (index > 0) + assert (contents[index - 1] in _NEWLINE_CHARACTERS) + + # Go back over the newline characters associated with the *single* end of a + # line just before `index`, despite of whether end of a line is designated by + # "\r", "\n" or "\r\n". Examples: + # 1. "... \r\n <new index> \r\n <old index> ... + # 2. "... \n <new index> \n <old index> ... + index = index - 1 + if index > 0 and contents[index - 1] in _NEWLINE_CHARACTERS and \ + contents[index - 1] != contents[index]: + index = index - 1 + + # Go back until `index` points right after an end of a line (or at the + # beginning of the `contents`). + while index > 0 and contents[index - 1] not in _NEWLINE_CHARACTERS: + index = index - 1 + + return index + + +def _SkipOverPreviousComment(contents, index): + """ Returns `index`, possibly moving it earlier so that it skips over comment + lines appearing in `contents` just before the old `index. + + Example: + <returned `index` points here>// Comment + // Comment + <original `index` points here>bar + """ + # If `index` points at the start of the file, or `index` doesn't point at the + # beginning of a line, then don't skip anything and just return `index`. + if index == 0 or contents[index - 1] not in _NEWLINE_CHARACTERS: + return index + + # Is the previous line a non-comment? If so, just return `index`. + new_index = _FindStartOfPreviousLine(contents, index) + prev_text = contents[new_index:index] + _COMMENT_START_REGEX = "^ \s* ( // | \* )" + if not re.search(_COMMENT_START_REGEX, prev_text, re.VERBOSE): + return index + + # Otherwise skip over the previous line + continue skipping via recursion. + return _SkipOverPreviousComment(contents, new_index) + + def _InsertNonSystemIncludeHeader(filepath, header_line_to_add, contents): """ Mutates |contents| (contents of |filepath|) to #include the |header_to_add @@ -185,7 +238,7 @@ def _InsertNonSystemIncludeHeader(filepath, header_line_to_add, contents): regex_text = _INCLUDE_INSERTION_POINT_REGEX_TEMPLATE % primary_header_basename match = re.search(regex_text, contents, re.MULTILINE | re.VERBOSE) assert (match is not None) - insertion_point = match.start() + insertion_point = _SkipOverPreviousComment(contents, match.start()) # Extra empty line is required if the addition is not adjacent to other # includes. diff --git a/chromium/tools/clang/scripts/apply_edits_test.py b/chromium/tools/clang/scripts/apply_edits_test.py index d8edd45f869..a2bc63b4fe2 100755 --- a/chromium/tools/clang/scripts/apply_edits_test.py +++ b/chromium/tools/clang/scripts/apply_edits_test.py @@ -56,6 +56,15 @@ def _InsertHeader(old_contents, class InsertIncludeHeaderTest(unittest.TestCase): + def _assertEqualContents(self, expected, actual): + if expected != actual: + print("####################### EXPECTED:") + print(expected) + print("####################### ACTUAL:") + print(actual) + print("####################### END.") + self.assertEqual(expected, actual) + def testSkippingCppComments(self): old_contents = ''' // Copyright info here. @@ -69,7 +78,144 @@ class InsertIncludeHeaderTest(unittest.TestCase): #include "old/header.h" ''' new_header_line = '#include "new/header.h' - self.assertEqual(expected_new_contents, _InsertHeader(old_contents)) + self._assertEqualContents(expected_new_contents, + _InsertHeader(old_contents)) + + def testSkippingCppComments_DocCommentForStruct(self): + """ This is a regression test for https://crbug.com/1175684 """ + old_contents = ''' +// Copyright blah blah... + +#ifndef SANDBOX_LINUX_SYSTEM_HEADERS_LINUX_FILTER_H_ +#define SANDBOX_LINUX_SYSTEM_HEADERS_LINUX_FILTER_H_ + +#include <stdint.h> + +// Doc comment for a struct. +// Multiline. +struct sock_filter { + uint16_t code; +}; + ''' + expected_new_contents = ''' +// Copyright blah blah... + +#ifndef SANDBOX_LINUX_SYSTEM_HEADERS_LINUX_FILTER_H_ +#define SANDBOX_LINUX_SYSTEM_HEADERS_LINUX_FILTER_H_ + +#include <stdint.h> + +#include "new/header.h" + +// Doc comment for a struct. +// Multiline. +struct sock_filter { + uint16_t code; +}; + ''' + new_header_line = '#include "new/header.h' + self._assertEqualContents(expected_new_contents, + _InsertHeader(old_contents)) + + def testSkippingCppComments_DocCommentForStruct2(self): + """ This is a regression test for https://crbug.com/1175684 """ + old_contents = ''' +// Copyright blah blah... + +// Doc comment for a struct. +struct sock_filter { + uint16_t code; +}; + ''' + expected_new_contents = ''' +// Copyright blah blah... + +#include "new/header.h" + +// Doc comment for a struct. +struct sock_filter { + uint16_t code; +}; + ''' + new_header_line = '#include "new/header.h' + self._assertEqualContents(expected_new_contents, + _InsertHeader(old_contents)) + + def testSkippingCppComments_DocCommentForStruct3(self): + """ This is a regression test for https://crbug.com/1175684 """ + old_contents = ''' +// Doc comment for a struct. +struct sock_filter { + uint16_t code; +}; + ''' + expected_new_contents = ''' +#include "new/header.h" + +// Doc comment for a struct. +struct sock_filter { + uint16_t code; +}; + ''' + new_header_line = '#include "new/header.h' + self._assertEqualContents(expected_new_contents, + _InsertHeader(old_contents)) + + def testSkippingCppComments_DocCommentForInclude(self): + """ This is a regression test for https://crbug.com/1175684 """ + old_contents = ''' +// Copyright blah blah... + +// System includes. +#include <stdint.h> + +// Doc comment for a struct. +struct sock_filter { + uint16_t code; +}; + ''' + expected_new_contents = ''' +// Copyright blah blah... + +// System includes. +#include <stdint.h> + +#include "new/header.h" + +// Doc comment for a struct. +struct sock_filter { + uint16_t code; +}; + ''' + new_header_line = '#include "new/header.h' + self._assertEqualContents(expected_new_contents, + _InsertHeader(old_contents)) + + def testSkippingCppComments_DocCommentForWholeFile(self): + """ This is a regression test for https://crbug.com/1175684 """ + old_contents = ''' +// Copyright blah blah... + +// Doc comment for the whole file. + +struct sock_filter { + uint16_t code; +}; + ''' + expected_new_contents = ''' +// Copyright blah blah... + +// Doc comment for the whole file. + +#include "new/header.h" + +struct sock_filter { + uint16_t code; +}; + ''' + new_header_line = '#include "new/header.h' + self._assertEqualContents(expected_new_contents, + _InsertHeader(old_contents)) def testSkippingOldStyleComments(self): old_contents = ''' @@ -87,7 +233,8 @@ class InsertIncludeHeaderTest(unittest.TestCase): #include "new/header.h" #include "old/header.h" ''' - self.assertEqual(expected_new_contents, _InsertHeader(old_contents)) + self._assertEqualContents(expected_new_contents, + _InsertHeader(old_contents)) def testSkippingOldStyleComments_NoWhitespaceAtLineStart(self): old_contents = ''' @@ -105,7 +252,8 @@ class InsertIncludeHeaderTest(unittest.TestCase): #include "new/header.h" #include "old/header.h" ''' - self.assertEqual(expected_new_contents, _InsertHeader(old_contents)) + self._assertEqualContents(expected_new_contents, + _InsertHeader(old_contents)) def testSkippingSystemHeaders(self): old_contents = ''' @@ -121,7 +269,8 @@ class InsertIncludeHeaderTest(unittest.TestCase): #include "new/header.h" #include "old/header.h" ''' - self.assertEqual(expected_new_contents, _InsertHeader(old_contents)) + self._assertEqualContents(expected_new_contents, + _InsertHeader(old_contents)) def testSkippingPrimaryHeader(self): old_contents = ''' @@ -139,7 +288,8 @@ class InsertIncludeHeaderTest(unittest.TestCase): #include "new/header.h" #include "old/header.h" ''' - self.assertEqual(expected_new_contents, _InsertHeader(old_contents)) + self._assertEqualContents(expected_new_contents, + _InsertHeader(old_contents)) def testSimilarNonPrimaryHeader_WithPrimaryHeader(self): old_contents = ''' @@ -159,7 +309,8 @@ class InsertIncludeHeaderTest(unittest.TestCase): #include "new/header.h" #include "zzz/foo.h" ''' - self.assertEqual(expected_new_contents, _InsertHeader(old_contents)) + self._assertEqualContents(expected_new_contents, + _InsertHeader(old_contents)) def testSimilarNonPrimaryHeader_NoPrimaryHeader(self): old_contents = ''' @@ -175,7 +326,8 @@ class InsertIncludeHeaderTest(unittest.TestCase): #include "new/header.h" #include "zzz/foo.h" ''' - self.assertEqual(expected_new_contents, _InsertHeader(old_contents)) + self._assertEqualContents(expected_new_contents, + _InsertHeader(old_contents)) def testSkippingIncludeGuards(self): old_contents = ''' @@ -195,8 +347,9 @@ class InsertIncludeHeaderTest(unittest.TestCase): #endif FOO_IMPL_H_ ''' - self.assertEqual(expected_new_contents, - _InsertHeader(old_contents, 'foo/impl.h', 'new/header.h')) + self._assertEqualContents( + expected_new_contents, + _InsertHeader(old_contents, 'foo/impl.h', 'new/header.h')) def testSkippingIncludeGuards2(self): # This test is based on base/third_party/valgrind/memcheck.h @@ -217,7 +370,8 @@ class InsertIncludeHeaderTest(unittest.TestCase): #endif ''' - self.assertEqual(expected_new_contents, _InsertHeader(old_contents)) + self._assertEqualContents(expected_new_contents, + _InsertHeader(old_contents)) def testSkippingIncludeGuards3(self): # This test is based on base/third_party/xdg_mime/xdgmime.h @@ -256,7 +410,8 @@ typedef void (*XdgMimeCallback) (void *user_data); #endif /* __cplusplus */ #endif /* __XDG_MIME_H__ */ ''' - self.assertEqual(expected_new_contents, _InsertHeader(old_contents)) + self._assertEqualContents(expected_new_contents, + _InsertHeader(old_contents)) def testSkippingIncludeGuards4(self): # This test is based on ash/first_run/desktop_cleaner.h and/or @@ -285,7 +440,8 @@ namespace ash { #endif // ASH_FIRST_RUN_DESKTOP_CLEANER_ ''' - self.assertEqual(expected_new_contents, _InsertHeader(old_contents)) + self._assertEqualContents(expected_new_contents, + _InsertHeader(old_contents)) def testSkippingIncludeGuards5(self): # This test is based on third_party/weston/include/GLES2/gl2.h (the |extern @@ -322,7 +478,8 @@ namespace ash { #endif ''' - self.assertEqual(expected_new_contents, _InsertHeader(old_contents)) + self._assertEqualContents(expected_new_contents, + _InsertHeader(old_contents)) def testSkippingIncludeGuards6(self): # This test is based on ios/third_party/blink/src/html_token.h @@ -353,7 +510,8 @@ namespace ash { #endif ''' - self.assertEqual(expected_new_contents, _InsertHeader(old_contents)) + self._assertEqualContents(expected_new_contents, + _InsertHeader(old_contents)) def testNoOpIfAlreadyPresent(self): # This tests that the new header won't be inserted (and duplicated) @@ -372,7 +530,8 @@ namespace ash { #include "new/header.h" #include "new/header2.h" ''' - self.assertEqual(expected_new_contents, _InsertHeader(old_contents)) + self._assertEqualContents(expected_new_contents, + _InsertHeader(old_contents)) def testNoOpIfAlreadyPresent_WithTrailingComment(self): # This tests that the new header won't be inserted (and duplicated) @@ -391,7 +550,8 @@ namespace ash { #include "new/header.h" // blah #include "new/header2.h" ''' - self.assertEqual(expected_new_contents, _InsertHeader(old_contents)) + self._assertEqualContents(expected_new_contents, + _InsertHeader(old_contents)) def testNoOldHeaders(self): # This tests that an extra new line is inserted after the new header @@ -408,7 +568,8 @@ struct S {}; struct S {}; ''' - self.assertEqual(expected_new_contents, _InsertHeader(old_contents)) + self._assertEqualContents(expected_new_contents, + _InsertHeader(old_contents)) def testPlatformIfDefs(self): # This test is based on @@ -454,7 +615,8 @@ inline void abort_noreturn() { abort(); } namespace double_conversion { ''' - self.assertEqual(expected_new_contents, _InsertHeader(old_contents)) + self._assertEqualContents(expected_new_contents, + _InsertHeader(old_contents)) def testNoOldIncludesAndIfDefs(self): # Artificial test: no old #includes + some #ifdefs. The main focus of the @@ -476,7 +638,8 @@ void foo(); void foo(); ''' - self.assertEqual(expected_new_contents, _InsertHeader(old_contents)) + self._assertEqualContents(expected_new_contents, + _InsertHeader(old_contents)) def testNoOldIncludesAndIfDefs2(self): # Artificial test: no old #includes + some #ifdefs. The main focus of the @@ -498,7 +661,8 @@ void foo(); void foo(); ''' - self.assertEqual(expected_new_contents, _InsertHeader(old_contents)) + self._assertEqualContents(expected_new_contents, + _InsertHeader(old_contents)) def testUtf8BomMarker(self): # Test based on @@ -523,12 +687,12 @@ void foo(); expected.extend(expected_new_contents.encode('utf-8')) # Test sanity check (i.e. not an assertion about code under test). utf8_bom = [0xef, 0xbb, 0xbf] - self.assertEqual(list(actual[0:3]), utf8_bom) - self.assertEqual(list(expected[0:3]), utf8_bom) + self._assertEqualContents(list(actual[0:3]), utf8_bom) + self._assertEqualContents(list(expected[0:3]), utf8_bom) # Actual test. edit = apply_edits.Edit('include-user-header', -1, -1, "new/header.h") apply_edits._ApplySingleEdit("foo/impl.cc", actual, edit, None) - self.assertEqual(expected, actual) + self._assertEqualContents(expected, actual) def _CreateReplacement(content_string, old_substring, new_substring): diff --git a/chromium/tools/clang/scripts/build.py b/chromium/tools/clang/scripts/build.py index d4793c26811..ba3cc0f6653 100755 --- a/chromium/tools/clang/scripts/build.py +++ b/chromium/tools/clang/scripts/build.py @@ -38,7 +38,6 @@ LLVM_BOOTSTRAP_INSTALL_DIR = os.path.join(THIRD_PARTY_DIR, LLVM_INSTRUMENTED_DIR = os.path.join(THIRD_PARTY_DIR, 'llvm-instrumented') LLVM_PROFDATA_FILE = os.path.join(LLVM_INSTRUMENTED_DIR, 'profdata.prof') CHROME_TOOLS_SHIM_DIR = os.path.join(LLVM_DIR, 'llvm', 'tools', 'chrometools') -COMPILER_RT_BUILD_DIR = os.path.join(LLVM_BUILD_DIR, 'compiler-rt') LLVM_BUILD_TOOLS_DIR = os.path.abspath( os.path.join(LLVM_DIR, '..', 'llvm-build-tools')) ANDROID_NDK_DIR = os.path.join( @@ -165,8 +164,9 @@ def UrlOpen(url): def GetLatestLLVMCommit(): """Get the latest commit hash in the LLVM monorepo.""" - ref = json.loads(UrlOpen(('https://api.github.com/repos/' - 'llvm/llvm-project/git/refs/heads/master'))) + ref = json.loads( + UrlOpen(('https://api.github.com/repos/' + 'llvm/llvm-project/git/refs/heads/main'))) assert ref['object']['type'] == 'commit' return ref['object']['sha'] @@ -175,8 +175,9 @@ def GetCommitDescription(commit): """Get the output of `git describe`. Needs to be called from inside the git repository dir.""" + git_exe = 'git.bat' if sys.platform.startswith('win') else 'git' return subprocess.check_output( - ['git', 'describe', '--long', '--abbrev=8', commit]).rstrip() + [git_exe, 'describe', '--long', '--abbrev=8', commit]).rstrip() def DeleteChromeToolsShim(): @@ -316,9 +317,9 @@ def MaybeDownloadHostGcc(args): """Download a modern GCC host compiler on Linux.""" if not sys.platform.startswith('linux') or args.gcc_toolchain: return - gcc_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'gcc530trusty') + gcc_dir = os.path.join(LLVM_BUILD_TOOLS_DIR, 'gcc-10.2.0-trusty') if not os.path.exists(gcc_dir): - DownloadAndUnpack(CDS_URL + '/tools/gcc530trusty.tgz', gcc_dir) + DownloadAndUnpack(CDS_URL + '/tools/gcc-10.2.0-trusty.tgz', gcc_dir) args.gcc_toolchain = gcc_dir @@ -378,6 +379,7 @@ def CopyLibstdcpp(args, build_dir): # The two fuzzer tests are weird in that they copy the fuzzer binary from bin/ # into the test tree under a different name. To make the relative rpath in # them work, copy libstdc++ to the copied location for now. + # There is also a compiler-rt test that copies llvm-symbolizer out of bin/. # TODO(thakis): Instead, make the upstream lit.local.cfg.py for these 2 tests # check if the binary contains an rpath and if so disable the tests. for d in ['lib', @@ -386,6 +388,17 @@ def CopyLibstdcpp(args, build_dir): EnsureDirExists(os.path.join(build_dir, d)) CopyFile(libstdcpp, os.path.join(build_dir, d)) + sanitizer_common_tests = os.path.join(build_dir, + 'projects/compiler-rt/test/sanitizer_common') + if os.path.exists(sanitizer_common_tests): + for d in ['asan-i386-Linux', 'asan-x86_64-Linux', 'lsan-i386-Linux', + 'lsan-x86_64-Linux', 'msan-x86_64-Linux', 'tsan-x86_64-Linux', + 'ubsan-i386-Linux', 'ubsan-x86_64-Linux']: + libpath = os.path.join(sanitizer_common_tests, d, 'Output', 'lib') + EnsureDirExists(libpath) + CopyFile(libstdcpp, libpath) + + def gn_arg(v): if v == 'True': return True @@ -416,6 +429,8 @@ def main(): help='do not build anything') parser.add_argument('--skip-checkout', action='store_true', help='do not create or update any checkouts') + parser.add_argument('--build-dir', + help='Override build directory') parser.add_argument('--extra-tools', nargs='*', default=[], help='select additional chrome tools to build') parser.add_argument('--use-system-cmake', action='store_true', @@ -454,7 +469,9 @@ def main(): print('Install the Fuchsia SDK by adding fuchsia to the ') print('target_os section in your .gclient and running hooks, ') print('or pass --without-fuchsia.') - print('https://chromium.googlesource.com/chromium/src/+/master/docs/fuchsia_build_instructions.md') + print( + 'https://chromium.googlesource.com/chromium/src/+/master/docs/fuchsia/build_instructions.md' + ) print('for general Fuchsia build instructions.') return 1 @@ -484,7 +501,11 @@ def main(): if sys.platform == 'darwin': isysroot = subprocess.check_output(['xcrun', '--show-sdk-path']).rstrip() - global CLANG_REVISION, PACKAGE_VERSION + global CLANG_REVISION, PACKAGE_VERSION, LLVM_BUILD_DIR + + if args.build_dir: + LLVM_BUILD_DIR = args.build_dir + if args.llvm_force_head_revision: checkout_revision = GetLatestLLVMCommit() else: @@ -570,8 +591,6 @@ def main(): '-DLIBCXX_INCLUDE_TESTS=OFF', '-DLIBCXX_ENABLE_EXPERIMENTAL_LIBRARY=OFF', ]) - # Prefer Python 2. TODO(crbug.com/1076834): Remove this. - base_cmake_args.append('-DPython3_EXECUTABLE=/nonexistent') if args.gcc_toolchain: # Force compiler-rt tests to use our gcc toolchain (including libstdc++.so) @@ -670,11 +689,7 @@ def main(): CopyLibstdcpp(args, LLVM_BOOTSTRAP_INSTALL_DIR) RunCommand(['ninja'], msvc_arch='x64') if args.run_tests: - test_targets = [ 'check-all' ] - if sys.platform == 'darwin': - # TODO(crbug.com/731375): Run check-all on Darwin too. - test_targets = [ 'check-llvm', 'check-clang', 'check-builtins' ] - RunCommand(['ninja'] + test_targets, msvc_arch='x64') + RunCommand(['ninja', 'check-all'], msvc_arch='x64') RunCommand(['ninja', 'install'], msvc_arch='x64') if sys.platform == 'win32': @@ -725,8 +740,6 @@ def main(): if cc is not None: instrument_args.append('-DCMAKE_C_COMPILER=' + cc) if cxx is not None: instrument_args.append('-DCMAKE_CXX_COMPILER=' + cxx) if lld is not None: instrument_args.append('-DCMAKE_LINKER=' + lld) - if args.thinlto: - instrument_args.append('-DLLVM_ENABLE_LTO=Thin') RunCommand(['cmake'] + instrument_args + [os.path.join(LLVM_DIR, 'llvm')], msvc_arch='x64') @@ -793,8 +806,8 @@ def main(): '-DDARWIN_iossim_ARCHS=i386;x86_64;arm64', ]) if args.bootstrap: - # mac/arm64 needs MacOSX11.0.sdk. System Xcode (+ SDK) on the chrome bots - # is something much older. + # mac/arm64 needs MacOSX11.0.sdk. System Xcode (+ SDK) may be something + # else, so use the hermetic Xcode. # Options: # - temporarily set system Xcode to Xcode 12 beta while running this # script, (cf build/swarming_xcode_install.py, but it looks unused) @@ -806,11 +819,11 @@ def main(): # LLVM build without it being system Xcode. # # The last option seems best, so let's go with that. We need to pass - # -isysroot to the 11.0 SDK and -B to the /usr/bin so that the new ld64 is + # -isysroot to the SDK and -B to the /usr/bin so that the new ld64 is # used. # The compiler-rt build overrides -isysroot flags set via cflags, and we # only need to use the 11 SDK for the compiler-rt build. So set only - # DARWIN_macosx_CACHED_SYSROOT to the 11.0 SDK and use the regular SDK + # DARWIN_macosx_CACHED_SYSROOT to the 11 SDK and use the regular SDK # for the rest of the build. (The new ld is used for all links.) sys.path.insert(1, os.path.join(CHROMIUM_DIR, 'build')) import mac_toolchain @@ -818,12 +831,12 @@ def main(): mac_toolchain.InstallXcodeBinaries(LLVM_XCODE) isysroot_11 = os.path.join(LLVM_XCODE, 'Contents', 'Developer', 'Platforms', 'MacOSX.platform', 'Developer', - 'SDKs', 'MacOSX11.0.sdk') + 'SDKs', 'MacOSX11.1.sdk') xcode_bin = os.path.join(LLVM_XCODE, 'Contents', 'Developer', 'Toolchains', 'XcodeDefault.xctoolchain', 'usr', 'bin') # Include an arm64 slice for libclang_rt.osx.a. This requires using - # MacOSX11.0.sdk (via -isysroot, via DARWIN_macosx_CACHED_SYSROOT) and + # MacOSX11.x.sdk (via -isysroot, via DARWIN_macosx_CACHED_SYSROOT) and # the new ld, via -B compiler_rt_args.extend([ # We don't need 32-bit intel support for macOS, we only ship 64-bit. @@ -926,10 +939,11 @@ def main(): # Do an out-of-tree build of compiler-rt for 32-bit Win clang_rt.profile.lib. if sys.platform == 'win32': - if os.path.isdir(COMPILER_RT_BUILD_DIR): - RmTree(COMPILER_RT_BUILD_DIR) - os.makedirs(COMPILER_RT_BUILD_DIR) - os.chdir(COMPILER_RT_BUILD_DIR) + compiler_rt_build_dir = os.path.join(LLVM_BUILD_DIR, 'compiler-rt') + if os.path.isdir(compiler_rt_build_dir): + RmTree(compiler_rt_build_dir) + os.makedirs(compiler_rt_build_dir) + os.chdir(compiler_rt_build_dir) if args.bootstrap: # The bootstrap compiler produces 64-bit binaries by default. cflags += ['-m32'] @@ -955,7 +969,7 @@ def main(): RunCommand(['ninja', 'compiler-rt'], msvc_arch='x86') # Copy select output to the main tree. - rt_lib_src_dir = os.path.join(COMPILER_RT_BUILD_DIR, 'lib', 'clang', + rt_lib_src_dir = os.path.join(compiler_rt_build_dir, 'lib', 'clang', RELEASE_VERSION, 'lib', platform) # Static and dynamic libraries: CopyDirectoryContents(rt_lib_src_dir, rt_lib_dst_dir) @@ -979,6 +993,10 @@ def main(): '--target=' + target_triple, '--sysroot=%s/sysroot' % toolchain_dir, '--gcc-toolchain=' + toolchain_dir, + # android_ndk/toolchains/llvm/prebuilt/linux-x86_64/aarch64-linux-android/bin/ld + # depends on a newer version of libxml2.so than what's available on + # the bots. To make things work, use our just-built lld as linker. + '-fuse-ld=lld', ] android_args = base_cmake_args + [ '-DCMAKE_C_COMPILER=' + os.path.join(LLVM_BUILD_DIR, 'bin/clang'), @@ -1071,6 +1089,27 @@ def main(): CopyFile(os.path.join(build_dir, 'lib', target_spec, builtins_a), fuchsia_lib_dst_dir) + # Build the Fuchsia profile runtime. + if target_arch == 'x86_64': + fuchsia_args.extend([ + '-DCOMPILER_RT_BUILD_BUILTINS=OFF', + '-DCOMPILER_RT_BUILD_PROFILE=ON', + '-DCMAKE_CXX_COMPILER_TARGET=%s-fuchsia' % target_arch, + '-DCMAKE_CXX_COMPILER_WORKS=ON', + ]) + profile_build_dir = os.path.join(LLVM_BUILD_DIR, + 'fuchsia-profile-' + target_arch) + if not os.path.exists(profile_build_dir): + os.mkdir(os.path.join(profile_build_dir)) + os.chdir(profile_build_dir) + RunCommand(['cmake'] + + fuchsia_args + + [COMPILER_RT_DIR]) + profile_a = 'libclang_rt.profile.a' + RunCommand(['ninja', profile_a]) + CopyFile(os.path.join(profile_build_dir, 'lib', target_spec, profile_a), + fuchsia_lib_dst_dir) + # Run tests. if args.run_tests or args.llvm_force_head_revision: RunCommand(['ninja', '-C', LLVM_BUILD_DIR, 'cr-check-all'], msvc_arch='x64') diff --git a/chromium/tools/clang/scripts/build_clang_tools_extra.py b/chromium/tools/clang/scripts/build_clang_tools_extra.py index 2026c845caa..7c49391af62 100755 --- a/chromium/tools/clang/scripts/build_clang_tools_extra.py +++ b/chromium/tools/clang/scripts/build_clang_tools_extra.py @@ -50,7 +50,7 @@ def FetchLLVM(checkout_dir, revision): except subprocess.CalledProcessError: # Otherwise, try to update it. print('-- Attempting to update existing repo') - args = ['git', 'pull', '--rebase', 'origin', 'master'] + args = ['git', 'pull', '--rebase', 'origin', 'main'] subprocess.check_call(args, cwd=checkout_dir, shell=sys.platform == 'win32') if revision: args = ['git', 'checkout', revision] diff --git a/chromium/tools/clang/scripts/generate_compdb.py b/chromium/tools/clang/scripts/generate_compdb.py index f981c36af9c..d5329532cda 100755 --- a/chromium/tools/clang/scripts/generate_compdb.py +++ b/chromium/tools/clang/scripts/generate_compdb.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # Copyright 2014 The Chromium Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. diff --git a/chromium/tools/clang/scripts/package.py b/chromium/tools/clang/scripts/package.py index cfed526d3b5..4ea0b6380b5 100755 --- a/chromium/tools/clang/scripts/package.py +++ b/chromium/tools/clang/scripts/package.py @@ -158,13 +158,6 @@ def main(): help='Upload the target archive to Google Cloud Storage.') args = parser.parse_args() - # Check that the script is not going to upload a toolchain built from HEAD. - use_head_revision = bool(int(os.environ.get('LLVM_FORCE_HEAD_REVISION', '0'))) - if args.upload and use_head_revision: - print ("--upload and LLVM_FORCE_HEAD_REVISION could not be used " - "at the same time.") - return 1 - expected_stamp = GetExpectedStamp() pdir = 'clang-' + expected_stamp print(pdir) @@ -189,7 +182,7 @@ def main(): os.path.join(THIS_DIR, 'build.py'), '--bootstrap', '--disable-asserts', '--run-tests', '--pgo' ] - if sys.platform.startswith('linux'): + if sys.platform != 'darwin': build_cmd.append('--thinlto') TeeCmd(build_cmd, log) @@ -226,6 +219,7 @@ def main(): # Include libclang_rt.builtins.a for Fuchsia targets. 'lib/clang/$V/lib/aarch64-fuchsia/libclang_rt.builtins.a', 'lib/clang/$V/lib/x86_64-fuchsia/libclang_rt.builtins.a', + 'lib/clang/$V/lib/x86_64-fuchsia/libclang_rt.profile.a', ]) if sys.platform == 'darwin': want.extend([ @@ -397,6 +391,8 @@ def main(): if sys.platform.startswith('linux'): os.symlink('lld', os.path.join(pdir, 'bin', 'ld.lld')) + os.symlink('lld', os.path.join(pdir, 'bin', 'ld64.lld')) + os.symlink('lld', os.path.join(pdir, 'bin', 'ld64.lld.darwinnew')) os.symlink('lld', os.path.join(pdir, 'bin', 'lld-link')) # Copy libc++ headers. @@ -473,8 +469,10 @@ def main(): os.path.join(llddir, 'bin')) shutil.copy(os.path.join(LLVM_RELEASE_DIR, 'bin', 'llvm-ar'), os.path.join(llddir, 'bin')) - os.symlink('lld', os.path.join(llddir, 'bin', 'lld-link')) os.symlink('lld', os.path.join(llddir, 'bin', 'ld.lld')) + os.symlink('lld', os.path.join(llddir, 'bin', 'ld64.lld')) + os.symlink('lld', os.path.join(llddir, 'bin', 'ld64.lld.darwinnew')) + os.symlink('lld', os.path.join(llddir, 'bin', 'lld-link')) with tarfile.open(llddir + '.tgz', 'w:gz') as tar: tar.add(os.path.join(llddir, 'bin'), arcname='bin', filter=PrintTarProgress) diff --git a/chromium/tools/clang/scripts/update.py b/chromium/tools/clang/scripts/update.py index 3fdf468b935..a9c0af0dc9f 100755 --- a/chromium/tools/clang/scripts/update.py +++ b/chromium/tools/clang/scripts/update.py @@ -38,12 +38,11 @@ import zipfile # https://chromium.googlesource.com/chromium/src/+/master/docs/updating_clang.md # Reverting problematic clang rolls is safe, though. # This is the output of `git describe` and is usable as a commit-ish. -CLANG_REVISION = 'llvmorg-12-init-11462-g418f18c6' -CLANG_SUB_REVISION = 1 +CLANG_REVISION = 'llvmorg-13-init-1559-g01b87444' +CLANG_SUB_REVISION = 3 PACKAGE_VERSION = '%s-%s' % (CLANG_REVISION, CLANG_SUB_REVISION) -RELEASE_VERSION = '12.0.0' - +RELEASE_VERSION = '13.0.0' CDS_URL = os.environ.get('CDS_CLANG_BUCKET_OVERRIDE', 'https://commondatastorage.googleapis.com/chromium-browser-clang') @@ -178,12 +177,28 @@ def DownloadAndUnpackPackage(package_file, output_dir, host_os): sys.exit(1) +def DownloadAndUnpackClangMacRuntime(output_dir): + cds_file = "clang-%s.tgz" % PACKAGE_VERSION + cds_full_url = GetPlatformUrlPrefix('mac') + cds_file + path_prefixes = [ + 'lib/clang/' + RELEASE_VERSION + '/lib/darwin', 'include/c++/v1' + ] + try: + DownloadAndUnpack(cds_full_url, output_dir, path_prefixes) + except URLError: + print('Failed to download prebuilt clang %s' % cds_file) + print('Use build.py if you want to build locally.') + print('Exiting.') + sys.exit(1) + + # TODO(hans): Create a clang-win-runtime package instead. def DownloadAndUnpackClangWinRuntime(output_dir): - cds_file = "clang-%s.tgz" % PACKAGE_VERSION + cds_file = "clang-%s.tgz" % PACKAGE_VERSION cds_full_url = GetPlatformUrlPrefix('win') + cds_file - path_prefixes = [ 'lib/clang/' + RELEASE_VERSION + '/lib/', - 'bin/llvm-symbolizer.exe' ] + path_prefixes = [ + 'lib/clang/' + RELEASE_VERSION + '/lib/windows', 'bin/llvm-symbolizer.exe' + ] try: DownloadAndUnpack(cds_full_url, output_dir, path_prefixes) except URLError: @@ -256,6 +271,8 @@ def UpdatePackage(package_name, host_os): DownloadAndUnpackPackage(package_file, LLVM_BUILD_DIR, host_os) + if package_name == 'clang' and 'mac' in target_os: + DownloadAndUnpackClangMacRuntime(LLVM_BUILD_DIR) if package_name == 'clang' and 'win' in target_os: # When doing win/cross builds on other hosts, get the Windows runtime # libraries, and llvm-symbolizer.exe (needed in asan builds). @@ -285,8 +302,6 @@ def main(): help='Which host OS to download for (default: %s)' % default_host_os, default=default_host_os, choices=('linux', 'mac', 'win')) - parser.add_argument('--force-local-build', action='store_true', - help='(no longer used)') parser.add_argument('--print-revision', action='store_true', help='Print current clang revision and exit.') parser.add_argument('--llvm-force-head-revision', action='store_true', @@ -298,11 +313,6 @@ def main(): help='Verify that clang has the passed-in version.') args = parser.parse_args() - if args.force_local_build: - print(('update.py --force-local-build is no longer used to build clang; ' - 'use build.py instead.')) - return 1 - if args.verify_version and args.verify_version != RELEASE_VERSION: print('RELEASE_VERSION is %s but --verify-version argument was %s.' % ( RELEASE_VERSION, args.verify_version)) diff --git a/chromium/tools/clang/scripts/upload_revision.py b/chromium/tools/clang/scripts/upload_revision.py index d05f5127246..2fb1aaa289d 100755 --- a/chromium/tools/clang/scripts/upload_revision.py +++ b/chromium/tools/clang/scripts/upload_revision.py @@ -36,7 +36,6 @@ Cq-Include-Trybots: chromium/try:dawn-win10-x86-deps-rel Cq-Include-Trybots: chromium/try:linux-chromeos-dbg Cq-Include-Trybots: chromium/try:linux_angle_deqp_rel_ng Cq-Include-Trybots: chromium/try:linux_chromium_cfi_rel_ng -Cq-Include-Trybots: chromium/try:linux_chromium_chromeos_asan_rel_ng Cq-Include-Trybots: chromium/try:linux_chromium_chromeos_msan_rel_ng Cq-Include-Trybots: chromium/try:linux_chromium_compile_dbg_32_ng Cq-Include-Trybots: chromium/try:linux_chromium_msan_rel_ng diff --git a/chromium/tools/clang/string_piece_rewriters/CMakeLists.txt b/chromium/tools/clang/string_piece_rewriters/CMakeLists.txt new file mode 100644 index 00000000000..14ed0c8b775 --- /dev/null +++ b/chromium/tools/clang/string_piece_rewriters/CMakeLists.txt @@ -0,0 +1,33 @@ +set(LLVM_LINK_COMPONENTS + BitReader + MCParser + Option + Support + X86AsmParser + X86CodeGen + X86Desc + X86Info + ) + +add_llvm_executable(string_piece_rewriters + StringPieceRewriters.cpp + ) + +target_link_libraries(string_piece_rewriters + clangAST + clangASTMatchers + clangAnalysis + clangBasic + clangDriver + clangEdit + clangFrontend + clangLex + clangParse + clangSema + clangSerialization + clangTooling + clangToolingCore + clangTransformer + ) + +cr_install(TARGETS string_piece_rewriters RUNTIME DESTINATION bin) diff --git a/chromium/tools/clang/string_piece_rewriters/StringPieceRewriters.cpp b/chromium/tools/clang/string_piece_rewriters/StringPieceRewriters.cpp new file mode 100644 index 00000000000..fa2a73b6d3b --- /dev/null +++ b/chromium/tools/clang/string_piece_rewriters/StringPieceRewriters.cpp @@ -0,0 +1,275 @@ +// Copyright 2021 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +#include <assert.h> +#include <stdlib.h> +#include <algorithm> +#include <memory> +#include <string> +#include <vector> + +#include "clang/AST/ASTContext.h" +#include "clang/AST/ParentMap.h" +#include "clang/ASTMatchers/ASTMatchFinder.h" +#include "clang/ASTMatchers/ASTMatchers.h" +#include "clang/ASTMatchers/ASTMatchersMacros.h" +#include "clang/Analysis/CFG.h" +#include "clang/Basic/SourceManager.h" +#include "clang/Frontend/FrontendActions.h" +#include "clang/Lex/Lexer.h" +#include "clang/Tooling/CommonOptionsParser.h" +#include "clang/Tooling/Refactoring.h" +#include "clang/Tooling/Refactoring/AtomicChange.h" +#include "clang/Tooling/Tooling.h" +#include "clang/Tooling/Transformer/RewriteRule.h" +#include "clang/Tooling/Transformer/SourceCodeBuilders.h" +#include "clang/Tooling/Transformer/Stencil.h" +#include "clang/Tooling/Transformer/Transformer.h" +#include "llvm/Support/CommandLine.h" +#include "llvm/Support/TargetSelect.h" + +using clang::tooling::AtomicChange; +using clang::tooling::AtomicChanges; +using clang::tooling::Transformer; +using namespace ::clang::ast_matchers; +using namespace ::clang::transformer; + +namespace { + +// Custom matcher to differentiate variable initializations based on the syntax. +AST_MATCHER_P(clang::VarDecl, + hasInitStyle, + clang::VarDecl::InitializationStyle, + InitStyle) { + return Node.getInitStyle() == InitStyle; +} + +// Like `maybeDeref`, but with support for smart pointers. Assumes that any type +// that overloads `->` also overloads `*`. +Stencil maybeDerefSmart(std::string ID) { + return run([ID = std::move(ID)](const MatchFinder::MatchResult& result) + -> llvm::Expected<std::string> { + if (const auto* op_call = + result.Nodes.getNodeAs<clang::CXXOperatorCallExpr>(ID)) { + if (op_call->getOperator() == clang::OO_Arrow && + op_call->getNumArgs() == 1) { + llvm::Optional<std::string> text = clang::tooling::buildDereference( + *op_call->getArg(0), *result.Context); + if (!text) { + return llvm::make_error<llvm::StringError>( + llvm::errc::invalid_argument, + "ID has no corresponding source: " + ID); + } + return *text; + } + } + return maybeDeref(std::move(ID))->eval(result); + }); +} + +// A matcher that matches the `as_string()` member function call on a +// StringPiece. Binds both the call to `as_string()`, as well as the +// StringPiece. +auto GetAsStringMatcher() { + return materializeTemporaryExpr( + has(ignoringParenImpCasts( + cxxBindTemporaryExpr(has(cxxMemberCallExpr( + on(expr().bind("piece")), + callee(cxxMethodDecl( + ofClass(hasName("::base::BasicStringPiece")), + hasName("as_string"))))))))) + .bind("as_string"); +} + +// Replaces calls of `piece.as_string()` and `piece_ptr->as_string()` with +// `std::string(piece)` and `std::string(*piece_ptr)` respectively. +RewriteRule ReplaceAsStringWithExplicitStringConversionRule() { + return makeRule(GetAsStringMatcher(), + changeTo(cat("std::string(", maybeDerefSmart("piece"), ")"))); +} + +// A rule that rewrites expressions like `std::string str = piece.as_string();` +// to `std::string str(foo);`, making use of the explicit conversion from +// base::StringPiece to std::string. +RewriteRule RewriteImplicitToExplicitStringConstructionRule() { + auto matcher = materializeTemporaryExpr( + GetAsStringMatcher(), hasParent(cxxConstructExpr( + hasDeclaration(cxxConstructorDecl( + ofClass(hasName("::std::basic_string")))), + hasParent(exprWithCleanups(hasParent( + varDecl(hasInitStyle(clang::VarDecl::CInit)) + .bind("varDecl"))))))); + return makeRule( + matcher, + // Remove the existing initialization via assignment and insert a new + // making use of explicit construction. + editList({ + remove(between(name("varDecl"), after(node("as_string")))), + insertAfter(name("varDecl"), cat("(", maybeDerefSmart("piece"), ")")), + })); +} + +// A rule that removes redundant calls to `as_string`. This can happen if: +// +// (1) the resulting string is converted to another string piece, +// (2) the resulting string is involved in a call to a member function (2a) or +// operator (2b) StringPiece also supports, or +// (3) the as_string call is part of the explicit construction of a std::string. +// This can either be a local variable that is explicitly constructed (3a), +// or a class member initialized by the constructor list (3b). +// +// The resulting rewrite rule will replace expressions like `piece.as_string()` +// simply with `piece`, and expressions like `piece_ptr->as_string()` with +// either `*piece_ptr` or `piece_ptr->`, depending on whether or not it is +// followed by a member expression. +RewriteRule RemoveAsStringRule() { + // List of std::string members that are also supported by base::StringPiece. + // Note: `data()` is absent from this list, because std::string::data is + // guaranteed to return a null-terminated string, while + // base::StringPiece::data is not. Furthermore, `substr()` is missing as well, + // due to the possibly breaking change in return type (std::string vs + // base::StringPiece). + static constexpr llvm::StringRef kMatchingStringMembers[] = { + "begin", + "cbegin", + "end", + "cend", + "rbegin", + "crbegin", + "rend", + "crend", + "at", + "front", + "back", + "size", + "length", + "max_size", + "empty", + "copy", + "compare", + "find", + "rfind", + "find_first_of", + "find_last_of", + "find_first_not_of", + "find_last_not_of", + "npos", + }; + + // List of std::string operators that are also supported by base::StringPiece. + // Note: `operator[]` is absent from this list, because string::operator[idx] + // is valid for idx == size(), while base::StringPiece::operator[] is not. + static constexpr llvm::StringRef kMatchingStringOperators[] = { + "==", "!=", "<", ">", "<=", ">=", "<<", + }; + + auto string_piece_construct_expr = cxxConstructExpr(hasDeclaration( + cxxConstructorDecl(ofClass(hasName("::base::BasicStringPiece"))))); + + auto matching_string_member_expr = + memberExpr(member(hasAnyName(kMatchingStringMembers))).bind("member"); + + auto matching_string_operator_call_expr = cxxOperatorCallExpr( + hasAnyOverloadedOperatorName(kMatchingStringOperators)); + + auto string_construct_expr = cxxConstructExpr(hasDeclaration( + cxxConstructorDecl(ofClass(hasName("::std::basic_string"))))); + + // Matches the explicit construction of a string variable, i.e. not making use + // of C-style assignment syntax. + auto explicit_string_var_construct_expr = cxxConstructExpr( + string_construct_expr, + hasParent(exprWithCleanups( + hasParent(varDecl(unless(hasInitStyle(clang::VarDecl::CInit))))))); + + auto string_class_member_construct_expr = cxxConstructExpr( + string_construct_expr, + hasParent(exprWithCleanups(hasParent(cxxConstructorDecl())))); + + auto matcher = materializeTemporaryExpr( + GetAsStringMatcher(), + anyOf( + // Case (1) + hasParent(string_piece_construct_expr), + // Case (2a) + hasParent(matching_string_member_expr), + // Const APIs like `size()` or `find()` add an extra implicit cast + // to const std::string here, that we need to ignore. + hasParent(implicitCastExpr(hasParent(matching_string_member_expr))), + // Case (2b) + hasParent(matching_string_operator_call_expr), + // Case (3a) + hasParent(explicit_string_var_construct_expr), + // Case (3b) + hasParent(string_class_member_construct_expr))); + return makeRule( + matcher, + // In case there is a bound member expression, construct an access + // expression into the string piece. This is required to handle + // expressions like `piece_ptr->as_string().some_member()` correctly. + ifBound("member", + changeTo(node("member"), access("piece", cat(member("member")))), + changeTo(maybeDerefSmart("piece")))); +} + +// Returns a consumer that adds `change` to `changes` if present. +Transformer::ChangeConsumer GetConsumer(AtomicChanges& changes) { + return [&changes](llvm::Expected<AtomicChange> change) { + if (change) + changes.push_back(*change); + }; +} + +} // namespace + +int main(int argc, const char* argv[]) { + llvm::InitializeNativeTarget(); + llvm::InitializeNativeTargetAsmParser(); + + static llvm::cl::OptionCategory tool_options("Tool options"); + clang::tooling::CommonOptionsParser options(argc, argv, tool_options); + clang::tooling::ClangTool tool(options.getCompilations(), + options.getSourcePathList()); + + // Combine the above rules into a single one and add an include for the right + // header. + RewriteRule as_string_rule = applyFirst({ + RemoveAsStringRule(), + RewriteImplicitToExplicitStringConstructionRule(), + ReplaceAsStringWithExplicitStringConversionRule(), + }); + addInclude(as_string_rule, "base/strings/string_piece.h"); + + AtomicChanges changes; + Transformer transformer(as_string_rule, GetConsumer(changes)); + + MatchFinder match_finder; + transformer.registerMatchers(&match_finder); + auto factory = clang::tooling::newFrontendActionFactory(&match_finder); + int result = tool.run(factory.get()); + if (result != 0) + return result; + + if (changes.empty()) + return 0; + + // Serialization format is documented in tools/clang/scripts/run_tool.py + llvm::outs() << "==== BEGIN EDITS ====\n"; + for (const auto& change : changes) { + for (const auto& r : change.getReplacements()) { + std::string replacement(r.getReplacementText()); + std::replace(replacement.begin(), replacement.end(), '\n', '\0'); + llvm::outs() << "r:::" << r.getFilePath() << ":::" << r.getOffset() + << ":::" << r.getLength() << ":::" << replacement << "\n"; + } + + for (const auto& header : change.getInsertedHeaders()) { + llvm::outs() << "include-user-header:::" << change.getFilePath() + << ":::-1:::-1:::" << header << "\n"; + } + } + llvm::outs() << "==== END EDITS ====\n"; + + return 0; +} diff --git a/chromium/tools/clang/translation_unit/TranslationUnitGenerator.cpp b/chromium/tools/clang/translation_unit/TranslationUnitGenerator.cpp index 278b40b7824..382b57cb597 100644 --- a/chromium/tools/clang/translation_unit/TranslationUnitGenerator.cpp +++ b/chromium/tools/clang/translation_unit/TranslationUnitGenerator.cpp @@ -281,7 +281,13 @@ static llvm::cl::extrahelp common_help(CommonOptionsParser::HelpMessage); int main(int argc, const char* argv[]) { llvm::cl::OptionCategory category("TranslationUnitGenerator Tool"); - CommonOptionsParser options(argc, argv, category); + auto ExpectedParser = CommonOptionsParser::create( + argc, argv, category, llvm::cl::OneOrMore, nullptr); + if (!ExpectedParser) { + llvm::errs() << ExpectedParser.takeError(); + return 1; + } + CommonOptionsParser& options = ExpectedParser.get(); std::unique_ptr<clang::tooling::FrontendActionFactory> frontend_factory = clang::tooling::newFrontendActionFactory<CompilationIndexerAction>(); clang::tooling::ClangTool tool(options.getCompilations(), |