[clang-scan-deps] initial outline of the tool that runs preprocessor to find

dependencies over a JSON compilation database

This commit introduces an outline for the clang-scan-deps tool that will be
used to implement fast dependency discovery phase using implicit modules for
explicit module builds.

The initial version of the tool works by computing non-modular header dependencies
for files in the compilation database without any optimizations
(i.e. without source minimization from r362459).
The tool spawns a number of worker threads to run the clang compiler workers in parallel.

The immediate goal for clang-scan-deps is to create a ClangScanDeps library
which will be used to build up this tool to use the source minimization and
caching multi-threaded filesystem to implement the optimized non-incremental
dependency scanning phase for a non-modular build. This will allow us to do
benchmarks and comparisons for performance that the minimization and caching give us

Differential Revision: https://reviews.llvm.org/D60233

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@363204 91177308-0d34-0410-b5e6-96231b3b80d8

3 files changed, 245 insertions, 0 deletions

diff --git a/tools/CMakeLists.txt b/tools/CMakeLists.txt
index f5c90ba783..223f1f74f3 100644
--- a/tools/CMakeLists.txt
+++ b/tools/CMakeLists.txt
@@ -8,6 +8,7 @@ add_clang_subdirectory(clang-format-vs)
 add_clang_subdirectory(clang-fuzzer)
 add_clang_subdirectory(clang-import-test)
 add_clang_subdirectory(clang-offload-bundler)
+add_clang_subdirectory(clang-scan-deps)
 
 add_clang_subdirectory(c-index-test)
 
diff --git a/tools/clang-scan-deps/CMakeLists.txt b/tools/clang-scan-deps/CMakeLists.txt
new file mode 100644
index 0000000000..63488aaa8d
--- /dev/null
+++ b/tools/clang-scan-deps/CMakeLists.txt
@@ -0,0 +1,26 @@
+set(LLVM_LINK_COMPONENTS
+    Core
+    Support
+)
+
+add_clang_tool(clang-scan-deps
+  ClangScanDeps.cpp
+  )
+
+set(CLANG_SCAN_DEPS_LIB_DEPS
+  clangAST
+  clangBasic
+  clangCodeGen
+  clangDriver
+  clangFrontend
+  clangFrontendTool
+  clangLex
+  clangParse
+  clangTooling
+  )
+
+target_link_libraries(clang-scan-deps
+  PRIVATE
+  ${CLANG_SCAN_DEPS_LIB_DEPS}
+  )
+
diff --git a/tools/clang-scan-deps/ClangScanDeps.cpp b/tools/clang-scan-deps/ClangScanDeps.cpp
new file mode 100644
index 0000000000..ca9e05f8c9
--- /dev/null
+++ b/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -0,0 +1,218 @@
+//===-- ClangScanDeps.cpp - Implementation of clang-scan-deps -------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Frontend/CompilerInstance.h"
+#include "clang/Frontend/CompilerInvocation.h"
+#include "clang/Frontend/FrontendActions.h"
+#include "clang/Frontend/PCHContainerOperations.h"
+#include "clang/FrontendTool/Utils.h"
+#include "clang/Tooling/CommonOptionsParser.h"
+#include "clang/Tooling/JSONCompilationDatabase.h"
+#include "clang/Tooling/Tooling.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/InitLLVM.h"
+#include "llvm/Support/JSON.h"
+#include "llvm/Support/Options.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/Program.h"
+#include "llvm/Support/Signals.h"
+#include "llvm/Support/Threading.h"
+#include <thread>
+
+using namespace clang;
+
+namespace {
+
+/// A clang tool that runs the preprocessor only for the given compiler
+/// invocation.
+class PreprocessorOnlyTool : public tooling::ToolAction {
+public:
+  PreprocessorOnlyTool(StringRef WorkingDirectory)
+      : WorkingDirectory(WorkingDirectory) {}
+
+  bool runInvocation(std::shared_ptr<CompilerInvocation> Invocation,
+                     FileManager *FileMgr,
+                     std::shared_ptr<PCHContainerOperations> PCHContainerOps,
+                     DiagnosticConsumer *DiagConsumer) override {
+    // Create a compiler instance to handle the actual work.
+    CompilerInstance Compiler(std::move(PCHContainerOps));
+    Compiler.setInvocation(std::move(Invocation));
+    FileMgr->getFileSystemOpts().WorkingDir = WorkingDirectory;
+    Compiler.setFileManager(FileMgr);
+
+    // Create the compiler's actual diagnostics engine.
+    Compiler.createDiagnostics(DiagConsumer, /*ShouldOwnClient=*/false);
+    if (!Compiler.hasDiagnostics())
+      return false;
+
+    Compiler.createSourceManager(*FileMgr);
+
+    auto Action = llvm::make_unique<PreprocessOnlyAction>();
+    const bool Result = Compiler.ExecuteAction(*Action);
+    FileMgr->clearStatCache();
+    return Result;
+  }
+
+private:
+  StringRef WorkingDirectory;
+};
+
+/// A proxy file system that doesn't call `chdir` when changing the working
+/// directory of a clang tool.
+class ProxyFileSystemWithoutChdir : public llvm::vfs::ProxyFileSystem {
+public:
+  ProxyFileSystemWithoutChdir(
+      llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> FS)
+      : ProxyFileSystem(std::move(FS)) {}
+
+  llvm::ErrorOr<std::string> getCurrentWorkingDirectory() const override {
+    assert(!CWD.empty() && "empty CWD");
+    return CWD;
+  }
+
+  std::error_code setCurrentWorkingDirectory(const Twine &Path) override {
+    CWD = Path.str();
+    return {};
+  }
+
+private:
+  std::string CWD;
+};
+
+/// The high-level implementation of the dependency discovery tool that runs on
+/// an individual worker thread.
+class DependencyScanningTool {
+public:
+  /// Construct a dependency scanning tool.
+  ///
+  /// \param Compilations     The reference to the compilation database that's
+  /// used by the clang tool.
+  DependencyScanningTool(const tooling::CompilationDatabase &Compilations)
+      : Compilations(Compilations) {
+    PCHContainerOps = std::make_shared<PCHContainerOperations>();
+    BaseFS = new ProxyFileSystemWithoutChdir(llvm::vfs::getRealFileSystem());
+  }
+
+  /// Computes the dependencies for the given file.
+  ///
+  /// \returns True on error.
+  bool runOnFile(const std::string &Input, StringRef CWD) {
+    BaseFS->setCurrentWorkingDirectory(CWD);
+    tooling::ClangTool Tool(Compilations, Input, PCHContainerOps, BaseFS);
+    Tool.clearArgumentsAdjusters();
+    Tool.setRestoreWorkingDir(false);
+    PreprocessorOnlyTool Action(CWD);
+    return Tool.run(&Action);
+  }
+
+private:
+  const tooling::CompilationDatabase &Compilations;
+  std::shared_ptr<PCHContainerOperations> PCHContainerOps;
+  /// The real filesystem used as a base for all the operations performed by the
+  /// tool.
+  llvm::IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS;
+};
+
+llvm::cl::opt<bool> Help("h", llvm::cl::desc("Alias for -help"),
+                         llvm::cl::Hidden);
+
+llvm::cl::OptionCategory DependencyScannerCategory("Tool options");
+
+llvm::cl::opt<unsigned>
+    NumThreads("j", llvm::cl::Optional,
+               llvm::cl::desc("Number of worker threads to use (default: use "
+                              "all concurrent threads)"),
+               llvm::cl::init(0));
+
+llvm::cl::opt<std::string>
+    CompilationDB("compilation-database",
+                  llvm::cl::desc("Compilation database"), llvm::cl::Required,
+                  llvm::cl::cat(DependencyScannerCategory));
+
+} // end anonymous namespace
+
+int main(int argc, const char **argv) {
+  llvm::InitLLVM X(argc, argv);
+  llvm::cl::HideUnrelatedOptions(DependencyScannerCategory);
+  if (!llvm::cl::ParseCommandLineOptions(argc, argv))
+    return 1;
+
+  std::string ErrorMessage;
+  std::unique_ptr<tooling::JSONCompilationDatabase> Compilations =
+      tooling::JSONCompilationDatabase::loadFromFile(
+          CompilationDB, ErrorMessage,
+          tooling::JSONCommandLineSyntax::AutoDetect);
+  if (!Compilations) {
+    llvm::errs() << "error: " << ErrorMessage << "\n";
+    return 1;
+  }
+
+  llvm::cl::PrintOptionValues();
+
+  // By default the tool runs on all inputs in the CDB.
+  std::vector<std::pair<std::string, std::string>> Inputs;
+  for (const auto &Command : Compilations->getAllCompileCommands())
+    Inputs.emplace_back(Command.Filename, Command.Directory);
+
+  // The command options are rewritten to run Clang in preprocessor only mode.
+  auto AdjustingCompilations =
+      llvm::make_unique<tooling::ArgumentsAdjustingCompilations>(
+          std::move(Compilations));
+  AdjustingCompilations->appendArgumentsAdjuster(
+      [](const tooling::CommandLineArguments &Args, StringRef /*unused*/) {
+        tooling::CommandLineArguments AdjustedArgs = Args;
+        AdjustedArgs.push_back("-o");
+        AdjustedArgs.push_back("/dev/null");
+        AdjustedArgs.push_back("-Xclang");
+        AdjustedArgs.push_back("-Eonly");
+        AdjustedArgs.push_back("-Xclang");
+        AdjustedArgs.push_back("-sys-header-deps");
+        return AdjustedArgs;
+      });
+
+  unsigned NumWorkers =
+      NumThreads == 0 ? llvm::hardware_concurrency() : NumThreads;
+  std::vector<std::unique_ptr<DependencyScanningTool>> WorkerTools;
+  for (unsigned I = 0; I < NumWorkers; ++I)
+    WorkerTools.push_back(
+        llvm::make_unique<DependencyScanningTool>(*AdjustingCompilations));
+
+  std::vector<std::thread> WorkerThreads;
+  std::atomic<bool> HadErrors(false);
+  std::mutex Lock;
+  size_t Index = 0;
+
+  llvm::outs() << "Running clang-scan-deps on " << Inputs.size()
+               << " files using " << NumWorkers << " workers\n";
+  for (unsigned I = 0; I < NumWorkers; ++I) {
+    WorkerThreads.emplace_back(
+        [I, &Lock, &Index, &Inputs, &HadErrors, &WorkerTools]() {
+          while (true) {
+            std::string Input;
+            StringRef CWD;
+            // Take the next input.
+            {
+              std::unique_lock<std::mutex> LockGuard(Lock);
+              if (Index >= Inputs.size())
+                return;
+              const auto &Compilation = Inputs[Index++];
+              Input = Compilation.first;
+              CWD = Compilation.second;
+            }
+            // Run the tool on it.
+            if (WorkerTools[I]->runOnFile(Input, CWD))
+              HadErrors = true;
+          }
+        });
+  }
+  for (auto &W : WorkerThreads)
+    W.join();
+
+  return HadErrors;
+}