[Index] Add indexing support for MACROs.

Reviewers: akyrtzi, arphaman, sammccall

Reviewed By: sammccall

Subscribers: malaperle, sammccall, cfe-commits

Differential Revision: https://reviews.llvm.org/D48961

git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@336524 91177308-0d34-0410-b5e6-96231b3b80d8

8 files changed, 216 insertions, 53 deletions

diff --git a/include/clang/Index/IndexSymbol.h b/include/clang/Index/IndexSymbol.h
index 3355dd2f28..1e8439ed5a 100644
--- a/include/clang/Index/IndexSymbol.h
+++ b/include/clang/Index/IndexSymbol.h
@@ -11,6 +11,7 @@
 #define LLVM_CLANG_INDEX_INDEXSYMBOL_H
 
 #include "clang/Basic/LLVM.h"
+#include "clang/Lex/MacroInfo.h"
 #include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/DataTypes.h"
 
@@ -93,28 +94,31 @@ static const unsigned SymbolPropertyBitNum = 8;
 /// Low 9 bits of clang-c/include/Index.h CXSymbolRole mirrors this enum.
 enum class SymbolRole : uint32_t {
   Declaration = 1 << 0,
-  Definition  = 1 << 1,
-  Reference   = 1 << 2,
-  Read        = 1 << 3,
-  Write       = 1 << 4,
-  Call        = 1 << 5,
-  Dynamic     = 1 << 6,
-  AddressOf   = 1 << 7,
-  Implicit    = 1 << 8,
+  Definition = 1 << 1,
+  Reference = 1 << 2,
+  Read = 1 << 3,
+  Write = 1 << 4,
+  Call = 1 << 5,
+  Dynamic = 1 << 6,
+  AddressOf = 1 << 7,
+  Implicit = 1 << 8,
+  // FIXME: this is not mirrored in CXSymbolRole.
+  // Note that macro occurrences aren't currently supported in libclang.
+  Undefinition = 1 << 9, // macro #undef
 
   // Relation roles.
-  RelationChildOf     = 1 << 9,
-  RelationBaseOf      = 1 << 10,
-  RelationOverrideOf  = 1 << 11,
-  RelationReceivedBy  = 1 << 12,
-  RelationCalledBy    = 1 << 13,
-  RelationExtendedBy  = 1 << 14,
-  RelationAccessorOf  = 1 << 15,
-  RelationContainedBy = 1 << 16,
-  RelationIBTypeOf    = 1 << 17,
-  RelationSpecializationOf = 1 << 18,
+  RelationChildOf = 1 << 10,
+  RelationBaseOf = 1 << 11,
+  RelationOverrideOf = 1 << 12,
+  RelationReceivedBy = 1 << 13,
+  RelationCalledBy = 1 << 14,
+  RelationExtendedBy = 1 << 15,
+  RelationAccessorOf = 1 << 16,
+  RelationContainedBy = 1 << 17,
+  RelationIBTypeOf = 1 << 18,
+  RelationSpecializationOf = 1 << 19,
 };
-static const unsigned SymbolRoleBitNum = 19;
+static const unsigned SymbolRoleBitNum = 20;
 typedef unsigned SymbolRoleSet;
 
 /// Represents a relation to another symbol for a symbol occurrence.
@@ -135,6 +139,8 @@ struct SymbolInfo {
 
 SymbolInfo getSymbolInfo(const Decl *D);
 
+SymbolInfo getSymbolInfoForMacro(const MacroInfo &MI);
+
 bool isFunctionLocalSymbol(const Decl *D);
 
 void applyForEachSymbolRole(SymbolRoleSet Roles,
diff --git a/include/clang/Index/IndexingAction.h b/include/clang/Index/IndexingAction.h
index 98e2455417..48385b396f 100644
--- a/include/clang/Index/IndexingAction.h
+++ b/include/clang/Index/IndexingAction.h
@@ -11,6 +11,7 @@
 #define LLVM_CLANG_INDEX_INDEXINGACTION_H
 
 #include "clang/Basic/LLVM.h"
+#include "clang/Lex/PPCallbacks.h"
 #include "llvm/ADT/ArrayRef.h"
 #include <memory>
 
@@ -40,18 +41,30 @@ struct IndexingOptions {
   bool IndexFunctionLocals = false;
 };
 
+/// Creates a frontend action that indexes all symbols (macros and AST decls).
 /// \param WrappedAction another frontend action to wrap over or null.
 std::unique_ptr<FrontendAction>
 createIndexingAction(std::shared_ptr<IndexDataConsumer> DataConsumer,
                      IndexingOptions Opts,
                      std::unique_ptr<FrontendAction> WrappedAction);
 
+/// Recursively indexes all decls in the AST.
+/// Note that this does not index macros.
 void indexASTUnit(ASTUnit &Unit, IndexDataConsumer &DataConsumer,
                   IndexingOptions Opts);
 
+/// Recursively indexes \p Decls.
+/// Note that this does not index macros.
 void indexTopLevelDecls(ASTContext &Ctx, ArrayRef<const Decl *> Decls,
                         IndexDataConsumer &DataConsumer, IndexingOptions Opts);
 
+/// Creates a PPCallbacks that indexes macros and feeds macros to \p Consumer.
+/// The caller is responsible for calling `Consumer.setPreprocessor()`.
+std::unique_ptr<PPCallbacks> indexMacrosCallback(IndexDataConsumer &Consumer,
+                                                 IndexingOptions Opts);
+
+/// Recursively indexes all top-level decls in the module.
+/// FIXME: make this index macros as well.
 void indexModuleFile(serialization::ModuleFile &Mod, ASTReader &Reader,
                      IndexDataConsumer &DataConsumer, IndexingOptions Opts);
 
diff --git a/lib/Index/IndexSymbol.cpp b/lib/Index/IndexSymbol.cpp
index 733d4dbc2f..03b55ffe8a 100644
--- a/lib/Index/IndexSymbol.cpp
+++ b/lib/Index/IndexSymbol.cpp
@@ -12,6 +12,7 @@
 #include "clang/AST/DeclObjC.h"
 #include "clang/AST/DeclTemplate.h"
 #include "clang/AST/PrettyPrinter.h"
+#include "clang/Lex/MacroInfo.h"
 
 using namespace clang;
 using namespace clang::index;
@@ -348,6 +349,15 @@ SymbolInfo index::getSymbolInfo(const Decl *D) {
   return Info;
 }
 
+SymbolInfo index::getSymbolInfoForMacro(const MacroInfo &) {
+  SymbolInfo Info;
+  Info.Kind = SymbolKind::Macro;
+  Info.SubKind = SymbolSubKind::None;
+  Info.Properties = SymbolPropertySet();
+  Info.Lang = SymbolLanguage::C;
+  return Info;
+}
+
 bool index::applyForEachSymbolRoleInterruptible(SymbolRoleSet Roles,
                                    llvm::function_ref<bool(SymbolRole)> Fn) {
 #define APPLY_FOR_ROLE(Role) \
@@ -364,6 +374,7 @@ bool index::applyForEachSymbolRoleInterruptible(SymbolRoleSet Roles,
   APPLY_FOR_ROLE(Dynamic);
   APPLY_FOR_ROLE(AddressOf);
   APPLY_FOR_ROLE(Implicit);
+  APPLY_FOR_ROLE(Undefinition);
   APPLY_FOR_ROLE(RelationChildOf);
   APPLY_FOR_ROLE(RelationBaseOf);
   APPLY_FOR_ROLE(RelationOverrideOf);
@@ -405,6 +416,7 @@ void index::printSymbolRoles(SymbolRoleSet Roles, raw_ostream &OS) {
     case SymbolRole::Dynamic: OS << "Dyn"; break;
     case SymbolRole::AddressOf: OS << "Addr"; break;
     case SymbolRole::Implicit: OS << "Impl"; break;
+    case SymbolRole::Undefinition: OS << "Undef"; break;
     case SymbolRole::RelationChildOf: OS << "RelChild"; break;
     case SymbolRole::RelationBaseOf: OS << "RelBase"; break;
     case SymbolRole::RelationOverrideOf: OS << "RelOver"; break;
diff --git a/lib/Index/IndexingAction.cpp b/lib/Index/IndexingAction.cpp
index e378a092be..b8faf7075e 100644
--- a/lib/Index/IndexingAction.cpp
+++ b/lib/Index/IndexingAction.cpp
@@ -13,8 +13,11 @@
 #include "clang/Frontend/FrontendAction.h"
 #include "clang/Frontend/MultiplexConsumer.h"
 #include "clang/Index/IndexDataConsumer.h"
+#include "clang/Lex/PPCallbacks.h"
 #include "clang/Lex/Preprocessor.h"
 #include "clang/Serialization/ASTReader.h"
+#include "llvm/ADT/STLExtras.h"
+#include <memory>
 
 using namespace clang;
 using namespace clang::index;
@@ -43,21 +46,22 @@ namespace {
 
 class IndexASTConsumer : public ASTConsumer {
   std::shared_ptr<Preprocessor> PP;
-  IndexingContext &IndexCtx;
+  std::shared_ptr<IndexingContext> IndexCtx;
 
 public:
-  IndexASTConsumer(std::shared_ptr<Preprocessor> PP, IndexingContext &IndexCtx)
-      : PP(std::move(PP)), IndexCtx(IndexCtx) {}
+  IndexASTConsumer(std::shared_ptr<Preprocessor> PP,
+                   std::shared_ptr<IndexingContext> IndexCtx)
+      : PP(std::move(PP)), IndexCtx(std::move(IndexCtx)) {}
 
 protected:
   void Initialize(ASTContext &Context) override {
-    IndexCtx.setASTContext(Context);
-    IndexCtx.getDataConsumer().initialize(Context);
-    IndexCtx.getDataConsumer().setPreprocessor(PP);
+    IndexCtx->setASTContext(Context);
+    IndexCtx->getDataConsumer().initialize(Context);
+    IndexCtx->getDataConsumer().setPreprocessor(PP);
   }
 
   bool HandleTopLevelDecl(DeclGroupRef DG) override {
-    return IndexCtx.indexDeclGroupRef(DG);
+    return IndexCtx->indexDeclGroupRef(DG);
   }
 
   void HandleInterestingDecl(DeclGroupRef DG) override {
@@ -65,22 +69,50 @@ protected:
   }
 
   void HandleTopLevelDeclInObjCContainer(DeclGroupRef DG) override {
-    IndexCtx.indexDeclGroupRef(DG);
+    IndexCtx->indexDeclGroupRef(DG);
   }
 
   void HandleTranslationUnit(ASTContext &Ctx) override {
   }
 };
 
+class IndexPPCallbacks : public PPCallbacks {
+  std::shared_ptr<IndexingContext> IndexCtx;
+
+public:
+  IndexPPCallbacks(std::shared_ptr<IndexingContext> IndexCtx)
+      : IndexCtx(std::move(IndexCtx)) {}
+
+  void MacroExpands(const Token &MacroNameTok, const MacroDefinition &MD,
+                    SourceRange Range, const MacroArgs *Args) override {
+    IndexCtx->handleMacroReference(*MacroNameTok.getIdentifierInfo(),
+                                   Range.getBegin(), *MD.getMacroInfo());
+  }
+
+  void MacroDefined(const Token &MacroNameTok,
+                    const MacroDirective *MD) override {
+    IndexCtx->handleMacroDefined(*MacroNameTok.getIdentifierInfo(),
+                                 MacroNameTok.getLocation(),
+                                 *MD->getMacroInfo());
+  }
+
+  void MacroUndefined(const Token &MacroNameTok, const MacroDefinition &MD,
+                      const MacroDirective *Undef) override {
+    IndexCtx->handleMacroUndefined(*MacroNameTok.getIdentifierInfo(),
+                                   MacroNameTok.getLocation(),
+                                   *MD.getMacroInfo());
+  }
+};
+
 class IndexActionBase {
 protected:
   std::shared_ptr<IndexDataConsumer> DataConsumer;
-  IndexingContext IndexCtx;
+  std::shared_ptr<IndexingContext> IndexCtx;
 
   IndexActionBase(std::shared_ptr<IndexDataConsumer> dataConsumer,
                   IndexingOptions Opts)
-    : DataConsumer(std::move(dataConsumer)),
-      IndexCtx(Opts, *DataConsumer) {}
+      : DataConsumer(std::move(dataConsumer)),
+        IndexCtx(new IndexingContext(Opts, *DataConsumer)) {}
 
   std::unique_ptr<IndexASTConsumer>
   createIndexASTConsumer(CompilerInstance &CI) {
@@ -88,6 +120,10 @@ protected:
                                                IndexCtx);
   }
 
+  std::unique_ptr<PPCallbacks> createIndexPPCallbacks() {
+    return llvm::make_unique<IndexPPCallbacks>(IndexCtx);
+  }
+
   void finish() {
     DataConsumer->finish();
   }
@@ -105,6 +141,11 @@ protected:
     return createIndexASTConsumer(CI);
   }
 
+  bool BeginSourceFileAction(clang::CompilerInstance &CI) override {
+    CI.getPreprocessor().addPPCallbacks(createIndexPPCallbacks());
+    return true;
+  }
+
   void EndSourceFileAction() override {
     FrontendAction::EndSourceFileAction();
     finish();
@@ -123,32 +164,34 @@ public:
 
 protected:
   std::unique_ptr<ASTConsumer> CreateASTConsumer(CompilerInstance &CI,
-                                                 StringRef InFile) override;
-  void EndSourceFileAction() override;
-};
-
-} // anonymous namespace
+                                                 StringRef InFile) override {
+    auto OtherConsumer = WrapperFrontendAction::CreateASTConsumer(CI, InFile);
+    if (!OtherConsumer) {
+      IndexActionFailed = true;
+      return nullptr;
+    }
+
+    std::vector<std::unique_ptr<ASTConsumer>> Consumers;
+    Consumers.push_back(std::move(OtherConsumer));
+    Consumers.push_back(createIndexASTConsumer(CI));
+    return llvm::make_unique<MultiplexConsumer>(std::move(Consumers));
+  }
 
-void WrappingIndexAction::EndSourceFileAction() {
-  // Invoke wrapped action's method.
-  WrapperFrontendAction::EndSourceFileAction();
-  if (!IndexActionFailed)
-    finish();
-}
+  bool BeginSourceFileAction(clang::CompilerInstance &CI) override {
+    WrapperFrontendAction::BeginSourceFileAction(CI);
+    CI.getPreprocessor().addPPCallbacks(createIndexPPCallbacks());
+    return true;
+  }
 
-std::unique_ptr<ASTConsumer>
-WrappingIndexAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
-  auto OtherConsumer = WrapperFrontendAction::CreateASTConsumer(CI, InFile);
-  if (!OtherConsumer) {
-    IndexActionFailed = true;
-    return nullptr;
+  void EndSourceFileAction() override {
+    // Invoke wrapped action's method.
+    WrapperFrontendAction::EndSourceFileAction();
+    if (!IndexActionFailed)
+      finish();
   }
+};
 
-  std::vector<std::unique_ptr<ASTConsumer>> Consumers;
-  Consumers.push_back(std::move(OtherConsumer));
-  Consumers.push_back(createIndexASTConsumer(CI));
-  return llvm::make_unique<MultiplexConsumer>(std::move(Consumers));
-}
+} // anonymous namespace
 
 std::unique_ptr<FrontendAction>
 index::createIndexingAction(std::shared_ptr<IndexDataConsumer> DataConsumer,
@@ -161,7 +204,6 @@ index::createIndexingAction(std::shared_ptr<IndexDataConsumer> DataConsumer,
   return llvm::make_unique<IndexAction>(std::move(DataConsumer), Opts);
 }
 
-
 static bool topLevelDeclVisitor(void *context, const Decl *D) {
   IndexingContext &IndexCtx = *static_cast<IndexingContext*>(context);
   return IndexCtx.indexTopLevelDecl(D);
@@ -193,6 +235,12 @@ void index::indexTopLevelDecls(ASTContext &Ctx, ArrayRef<const Decl *> Decls,
   DataConsumer.finish();
 }
 
+std::unique_ptr<PPCallbacks>
+index::indexMacrosCallback(IndexDataConsumer &Consumer, IndexingOptions Opts) {
+  return llvm::make_unique<IndexPPCallbacks>(
+      std::make_shared<IndexingContext>(Opts, Consumer));
+}
+
 void index::indexModuleFile(serialization::ModuleFile &Mod, ASTReader &Reader,
                             IndexDataConsumer &DataConsumer,
                             IndexingOptions Opts) {
diff --git a/lib/Index/IndexingContext.cpp b/lib/Index/IndexingContext.cpp
index 97b3e10450..71b92bc6bf 100644
--- a/lib/Index/IndexingContext.cpp
+++ b/lib/Index/IndexingContext.cpp
@@ -8,6 +8,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "IndexingContext.h"
+#include "clang/Basic/SourceLocation.h"
 #include "clang/Index/IndexDataConsumer.h"
 #include "clang/AST/ASTContext.h"
 #include "clang/AST/DeclTemplate.h"
@@ -290,6 +291,7 @@ static bool shouldReportOccurrenceForSystemDeclOnlyMode(
       case SymbolRole::Dynamic:
       case SymbolRole::AddressOf:
       case SymbolRole::Implicit:
+      case SymbolRole::Undefinition:
       case SymbolRole::RelationReceivedBy:
       case SymbolRole::RelationCalledBy:
       case SymbolRole::RelationContainedBy:
@@ -406,3 +408,24 @@ bool IndexingContext::handleDeclOccurrence(const Decl *D, SourceLocation Loc,
   IndexDataConsumer::ASTNodeInfo Node{OrigE, OrigD, Parent, ContainerDC};
   return DataConsumer.handleDeclOccurence(D, Roles, FinalRelations, Loc, Node);
 }
+
+void IndexingContext::handleMacroDefined(const IdentifierInfo &Name,
+                                         SourceLocation Loc,
+                                         const MacroInfo &MI) {
+  SymbolRoleSet Roles = (unsigned)SymbolRole::Definition;
+  DataConsumer.handleMacroOccurence(&Name, &MI, Roles, Loc);
+}
+
+void IndexingContext::handleMacroUndefined(const IdentifierInfo &Name,
+                                           SourceLocation Loc,
+                                           const MacroInfo &MI) {
+  SymbolRoleSet Roles = (unsigned)SymbolRole::Undefinition;
+  DataConsumer.handleMacroOccurence(&Name, &MI, Roles, Loc);
+}
+
+void IndexingContext::handleMacroReference(const IdentifierInfo &Name,
+                                           SourceLocation Loc,
+                                           const MacroInfo &MI) {
+  SymbolRoleSet Roles = (unsigned)SymbolRole::Reference;
+  DataConsumer.handleMacroOccurence(&Name, &MI, Roles, Loc);
+}
diff --git a/lib/Index/IndexingContext.h b/lib/Index/IndexingContext.h
index 566651c83a..f05eaae696 100644
--- a/lib/Index/IndexingContext.h
+++ b/lib/Index/IndexingContext.h
@@ -10,9 +10,11 @@
 #ifndef LLVM_CLANG_LIB_INDEX_INDEXINGCONTEXT_H
 #define LLVM_CLANG_LIB_INDEX_INDEXINGCONTEXT_H
 
+#include "clang/Basic/IdentifierTable.h"
 #include "clang/Basic/LLVM.h"
 #include "clang/Index/IndexSymbol.h"
 #include "clang/Index/IndexingAction.h"
+#include "clang/Lex/MacroInfo.h"
 #include "llvm/ADT/ArrayRef.h"
 
 namespace clang {
@@ -80,6 +82,15 @@ public:
                        const Expr *RefE = nullptr,
                        const Decl *RefD = nullptr);
 
+  void handleMacroDefined(const IdentifierInfo &Name, SourceLocation Loc,
+                          const MacroInfo &MI);
+
+  void handleMacroUndefined(const IdentifierInfo &Name, SourceLocation Loc,
+                            const MacroInfo &MI);
+
+  void handleMacroReference(const IdentifierInfo &Name, SourceLocation Loc,
+                            const MacroInfo &MD);
+
   bool importedModule(const ImportDecl *ImportD);
 
   bool indexDecl(const Decl *D);
diff --git a/test/Index/Core/index-macros.c b/test/Index/Core/index-macros.c
new file mode 100644
index 0000000000..9e38c44e99
--- /dev/null
+++ b/test/Index/Core/index-macros.c
@@ -0,0 +1,12 @@
+// RUN: c-index-test core -print-source-symbols -- %s | FileCheck %s
+
+// CHECK: [[@LINE+1]]:9 | macro/C | X1 | c:index-macros.c@157@macro@X1 | Def |
+#define X1 1
+// CHECK: [[@LINE+1]]:9 | macro/C | DEF | c:index-macros.c@251@macro@DEF | Def |
+#define DEF(x) int x
+// CHECK: [[@LINE+1]]:8 | macro/C | X1 | c:index-macros.c@157@macro@X1 | Undef |
+#undef X1
+
+// CHECK: [[@LINE+2]]:1 | macro/C | DEF | c:index-macros.c@251@macro@DEF | Ref |
+// CHECK: [[@LINE+1]]:5 | variable/C | i | c:@i | i | Def | rel: 0
+DEF(i);
diff --git a/tools/c-index-test/core_main.cpp b/tools/c-index-test/core_main.cpp
index 6093df5285..a7732c09d5 100644
--- a/tools/c-index-test/core_main.cpp
+++ b/tools/c-index-test/core_main.cpp
@@ -7,6 +7,7 @@
 //
 //===----------------------------------------------------------------------===//
 
+#include "clang/Basic/LangOptions.h"
 #include "clang/CodeGen/ObjectFilePCHContainerOperations.h"
 #include "clang/Frontend/ASTUnit.h"
 #include "clang/Frontend/CompilerInstance.h"
@@ -16,6 +17,7 @@
 #include "clang/Index/IndexDataConsumer.h"
 #include "clang/Index/USRGeneration.h"
 #include "clang/Index/CodegenNameGenerator.h"
+#include "clang/Lex/Preprocessor.h"
 #include "clang/Serialization/ASTReader.h"
 #include "llvm/Support/CommandLine.h"
 #include "llvm/Support/Signals.h"
@@ -77,6 +79,7 @@ namespace {
 class PrintIndexDataConsumer : public IndexDataConsumer {
   raw_ostream &OS;
   std::unique_ptr<CodegenNameGenerator> CGNameGen;
+  std::shared_ptr<Preprocessor> PP;
 
 public:
   PrintIndexDataConsumer(raw_ostream &OS) : OS(OS) {
@@ -86,6 +89,10 @@ public:
     CGNameGen.reset(new CodegenNameGenerator(Ctx));
   }
 
+  void setPreprocessor(std::shared_ptr<Preprocessor> PP) override {
+    this->PP = std::move(PP);
+  }
+
   bool handleDeclOccurence(const Decl *D, SymbolRoleSet Roles,
                            ArrayRef<SymbolRelation> Relations,
                            SourceLocation Loc, ASTNodeInfo ASTNode) override {
@@ -145,6 +152,37 @@ public:
 
     return true;
   }
+
+  bool handleMacroOccurence(const IdentifierInfo *Name, const MacroInfo *MI,
+                            SymbolRoleSet Roles, SourceLocation Loc) override {
+    assert(PP);
+    SourceManager &SM = PP->getSourceManager();
+
+    Loc = SM.getFileLoc(Loc);
+    FileID FID = SM.getFileID(Loc);
+    unsigned Line = SM.getLineNumber(FID, SM.getFileOffset(Loc));
+    unsigned Col = SM.getColumnNumber(FID, SM.getFileOffset(Loc));
+    OS << Line << ':' << Col << " | ";
+
+    printSymbolInfo(getSymbolInfoForMacro(*MI), OS);
+    OS << " | ";
+
+    OS << Name->getName();
+    OS << " | ";
+
+    SmallString<256> USRBuf;
+    if (generateUSRForMacro(Name->getName(), MI->getDefinitionLoc(), SM,
+                            USRBuf)) {
+      OS << "<no-usr>";
+    } else {
+      OS << USRBuf;
+    }
+    OS << " | ";
+
+    printSymbolRoles(Roles, OS);
+    OS << " |\n";
+    return true;
+  }
 };
 
 } // anonymous namespace