[clang-scan-deps] add skip excluded conditional preprocessor block preprocessing optimization

This commit adds an optimization to clang-scan-deps and clang's preprocessor that skips excluded preprocessor
blocks by bumping the lexer pointer, and not lexing the tokens until reaching appropriate #else/#endif directive.
The skip positions and lexer offsets are computed when the file is minimized, directly from the minimized tokens.

On an 18-core iMacPro with macOS Catalina Beta I got 10-15% speed-up from this optimization when running clang-scan-deps on
the compilation database for a recent LLVM and Clang (3511 files).

Differential Revision: https://reviews.llvm.org/D67127

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@371656 91177308-0d34-0410-b5e6-96231b3b80d8

3 files changed, 26 insertions, 3 deletions

diff --git a/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h b/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
index cc78a587e4..94b3705200 100644
--- a/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
+++ b/include/clang/Tooling/DependencyScanning/DependencyScanningFilesystem.h
@@ -10,6 +10,7 @@
 #define LLVM_CLANG_TOOLING_DEPENDENCY_SCANNING_FILESYSTEM_H
 
 #include "clang/Basic/LLVM.h"
+#include "clang/Lex/PreprocessorExcludedConditionalDirectiveSkipMapping.h"
 #include "llvm/ADT/StringMap.h"
 #include "llvm/ADT/StringSet.h"
 #include "llvm/Support/Allocator.h"
@@ -76,6 +77,12 @@ public:
     return MaybeStat->getName();
   }
 
+  /// Return the mapping between location -> distance that is used to speed up
+  /// the block skipping in the preprocessor.
+  const PreprocessorSkippedRangeMapping &getPPSkippedRangeMapping() const {
+    return PPSkippedRangeMapping;
+  }
+
   CachedFileSystemEntry(CachedFileSystemEntry &&) = default;
   CachedFileSystemEntry &operator=(CachedFileSystemEntry &&) = default;
 
@@ -89,6 +96,7 @@ private:
   // Note: small size of 1 allows us to store an empty string with an implicit
   // null terminator without any allocations.
   llvm::SmallString<1> Contents;
+  PreprocessorSkippedRangeMapping PPSkippedRangeMapping;
 };
 
 /// This class is a shared cache, that caches the 'stat' and 'open' calls to the
@@ -133,8 +141,10 @@ class DependencyScanningWorkerFilesystem : public llvm::vfs::ProxyFileSystem {
 public:
   DependencyScanningWorkerFilesystem(
       DependencyScanningFilesystemSharedCache &SharedCache,
-      IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
-      : ProxyFileSystem(std::move(FS)), SharedCache(SharedCache) {}
+      IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS,
+      ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings)
+      : ProxyFileSystem(std::move(FS)), SharedCache(SharedCache),
+        PPSkipMappings(PPSkipMappings) {}
 
   llvm::ErrorOr<llvm::vfs::Status> status(const Twine &Path) override;
   llvm::ErrorOr<std::unique_ptr<llvm::vfs::File>>
@@ -159,6 +169,10 @@ private:
   /// The local cache is used by the worker thread to cache file system queries
   /// locally instead of querying the global cache every time.
   llvm::StringMap<const CachedFileSystemEntry *, llvm::BumpPtrAllocator> Cache;
+  /// The optional mapping structure which records information about the
+  /// excluded conditional directive skip mappings that are used by the
+  /// currently active preprocessor.
+  ExcludedPreprocessorDirectiveSkipMapping *PPSkipMappings;
 };
 
 } // end namespace dependencies
diff --git a/include/clang/Tooling/DependencyScanning/DependencyScanningService.h b/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
index c49f92d082..fd8ed80b14 100644
--- a/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
+++ b/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
@@ -34,12 +34,15 @@ enum class ScanningMode {
 /// the invidual dependency scanning workers.
 class DependencyScanningService {
 public:
-  DependencyScanningService(ScanningMode Mode, bool ReuseFileManager = true);
+  DependencyScanningService(ScanningMode Mode, bool ReuseFileManager = true,
+                            bool SkipExcludedPPRanges = true);
 
   ScanningMode getMode() const { return Mode; }
 
   bool canReuseFileManager() const { return ReuseFileManager; }
 
+  bool canSkipExcludedPPRanges() const { return SkipExcludedPPRanges; }
+
   DependencyScanningFilesystemSharedCache &getSharedCache() {
     return SharedCache;
   }
@@ -47,6 +50,10 @@ public:
 private:
   const ScanningMode Mode;
   const bool ReuseFileManager;
+  /// Set to true to use the preprocessor optimization that skips excluded PP
+  /// ranges by bumping the buffer pointer in the lexer instead of lexing the
+  /// tokens in the range until reaching the corresponding directive.
+  const bool SkipExcludedPPRanges;
   /// The global file system cache.
   DependencyScanningFilesystemSharedCache SharedCache;
 };
diff --git a/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h b/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
index 5b42741ecb..45c9fb4f02 100644
--- a/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
+++ b/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
@@ -13,6 +13,7 @@
 #include "clang/Basic/FileManager.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Frontend/PCHContainerOperations.h"
+#include "clang/Lex/PreprocessorExcludedConditionalDirectiveSkipMapping.h"
 #include "clang/Tooling/CompilationDatabase.h"
 #include "llvm/Support/Error.h"
 #include "llvm/Support/FileSystem.h"
@@ -62,6 +63,7 @@ public:
 private:
   IntrusiveRefCntPtr<DiagnosticOptions> DiagOpts;
   std::shared_ptr<PCHContainerOperations> PCHContainerOps;
+  std::unique_ptr<ExcludedPreprocessorDirectiveSkipMapping> PPSkipMappings;
 
   llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> RealFS;
   /// The file system that is used by each worker when scanning for