PR37189 Fix incorrect end source location and spelling for a split '>>' token.

When a '>>' token is split into two '>' tokens (in C++11 onwards), or (as an
extension) when we do the same for other tokens starting with a '>', we can't
just use a location pointing to the first '>' as the location of the split
token, because that would result in our miscomputing the length and spelling
for the token. As a consequence, for example, a refactoring replacing 'A<X>'
with something else would sometimes replace one character too many, and
similarly diagnostics highlighting a template-id source range would highlight
one character too many.

Fix this by creating an expansion range covering the first character of the
'>>' token, whose spelling is '>'. For this to work, we generalize the
expansion range of a macro FileID to be either a token range (the common case)
or a character range (used in this new case).


git-svn-id: https://llvm.org/svn/llvm-project/cfe/trunk@331155 91177308-0d34-0410-b5e6-96231b3b80d8

1 files changed, 10 insertions, 9 deletions

diff --git a/lib/Rewrite/HTMLRewrite.cpp b/lib/Rewrite/HTMLRewrite.cpp
index 3a086537b4..d93961f358 100644
--- a/lib/Rewrite/HTMLRewrite.cpp
+++ b/lib/Rewrite/HTMLRewrite.cpp
@@ -30,7 +30,8 @@ using namespace clang;
 /// start/end tags are placed at the start/end of each line if the range is
 /// multiline.
 void html::HighlightRange(Rewriter &R, SourceLocation B, SourceLocation E,
-                          const char *StartTag, const char *EndTag) {
+                          const char *StartTag, const char *EndTag,
+                          bool IsTokenRange) {
   SourceManager &SM = R.getSourceMgr();
   B = SM.getExpansionLoc(B);
   E = SM.getExpansionLoc(E);
@@ -41,7 +42,8 @@ void html::HighlightRange(Rewriter &R, SourceLocation B, SourceLocation E,
   unsigned EOffset = SM.getFileOffset(E);
 
   // Include the whole end token in the range.
-  EOffset += Lexer::MeasureTokenLength(E, R.getSourceMgr(), R.getLangOpts());
+  if (IsTokenRange)
+    EOffset += Lexer::MeasureTokenLength(E, R.getSourceMgr(), R.getLangOpts());
 
   bool Invalid = false;
   const char *BufferStart = SM.getBufferData(FID, &Invalid).data();
@@ -588,16 +590,15 @@ void html::HighlightMacros(Rewriter &R, FileID FID, const Preprocessor& PP) {
     // Okay, we have the first token of a macro expansion: highlight the
     // expansion by inserting a start tag before the macro expansion and
     // end tag after it.
-    std::pair<SourceLocation, SourceLocation> LLoc =
-      SM.getExpansionRange(Tok.getLocation());
+    CharSourceRange LLoc = SM.getExpansionRange(Tok.getLocation());
 
     // Ignore tokens whose instantiation location was not the main file.
-    if (SM.getFileID(LLoc.first) != FID) {
+    if (SM.getFileID(LLoc.getBegin()) != FID) {
       TmpPP.Lex(Tok);
       continue;
     }
 
-    assert(SM.getFileID(LLoc.second) == FID &&
+    assert(SM.getFileID(LLoc.getEnd()) == FID &&
            "Start and end of expansion must be in the same ultimate file!");
 
     std::string Expansion = EscapeText(TmpPP.getSpelling(Tok));
@@ -612,7 +613,7 @@ void html::HighlightMacros(Rewriter &R, FileID FID, const Preprocessor& PP) {
     // instantiation.  It would be really nice to pop up a window with all the
     // spelling of the tokens or something.
     while (!Tok.is(tok::eof) &&
-           SM.getExpansionLoc(Tok.getLocation()) == LLoc.first) {
+           SM.getExpansionLoc(Tok.getLocation()) == LLoc.getBegin()) {
       // Insert a newline if the macro expansion is getting large.
       if (LineLen > 60) {
         Expansion += "<br>";
@@ -641,8 +642,8 @@ void html::HighlightMacros(Rewriter &R, FileID FID, const Preprocessor& PP) {
     // highlighted.
     Expansion = "<span class='expansion'>" + Expansion + "</span></span>";
 
-    HighlightRange(R, LLoc.first, LLoc.second,
-                   "<span class='macro'>", Expansion.c_str());
+    HighlightRange(R, LLoc.getBegin(), LLoc.getEnd(), "<span class='macro'>",
+                   Expansion.c_str(), LLoc.isTokenRange());
   }
 
   // Restore the preprocessor's old state.