C++: TranslationUnit::getPosition takes utf16char offsets

...and not byte offsets anymore. This is necessary in order to calculate
the line and column numbers correctly with respect to unicode code
points.

Change-Id: I5d79857b3eaefeb8d563b4f1e3938a64debc5e08
Reviewed-by: Erik Verbruggen <erik.verbruggen@digia.com>

11 files changed, 359 insertions, 77 deletions

diff --git a/src/libs/3rdparty/cplusplus/ASTVisitor.cpp b/src/libs/3rdparty/cplusplus/ASTVisitor.cpp
index 8fe4399673..f4119e9ce1 100644
--- a/src/libs/3rdparty/cplusplus/ASTVisitor.cpp
+++ b/src/libs/3rdparty/cplusplus/ASTVisitor.cpp
@@ -86,7 +86,7 @@ void ASTVisitor::getTokenPosition(unsigned index,
 { translationUnit()->getTokenPosition(index, line, column, fileName); }
 
 void ASTVisitor::getTokenStartPosition(unsigned index, unsigned *line, unsigned *column) const
-{ getPosition(tokenAt(index).bytesBegin(), line, column); }
+{ getPosition(tokenAt(index).utf16charsBegin(), line, column); }
 
 void ASTVisitor::getTokenEndPosition(unsigned index, unsigned *line, unsigned *column) const
-{ getPosition(tokenAt(index).bytesEnd(), line, column); }
+{ getPosition(tokenAt(index).utf16charsEnd(), line, column); }
diff --git a/src/libs/3rdparty/cplusplus/Bind.cpp b/src/libs/3rdparty/cplusplus/Bind.cpp
index 8abb3e4ce7..93c1932c54 100644
--- a/src/libs/3rdparty/cplusplus/Bind.cpp
+++ b/src/libs/3rdparty/cplusplus/Bind.cpp
@@ -1094,8 +1094,8 @@ void Bind::lambdaDeclarator(LambdaDeclaratorAST *ast)
 
 
     Function *fun = control()->newFunction(0, 0);
-    fun->setStartOffset(tokenAt(ast->firstToken()).bytesBegin());
-    fun->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
+    fun->setStartOffset(tokenAt(ast->firstToken()).utf16charsBegin());
+    fun->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
     if (ast->trailing_return_type)
         _type = this->trailingReturnType(ast->trailing_return_type, _type);
     fun->setReturnType(_type);
@@ -1192,8 +1192,8 @@ bool Bind::visit(CompoundStatementAST *ast)
 {
     Block *block = control()->newBlock(ast->firstToken());
     unsigned startScopeToken = ast->lbrace_token ? ast->lbrace_token : ast->firstToken();
-    block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
-    block->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
+    block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
+    block->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
     ast->symbol = block;
     _scope->addMember(block);
     Scope *previousScope = switchScope(block);
@@ -1235,8 +1235,8 @@ bool Bind::visit(ForeachStatementAST *ast)
 {
     Block *block = control()->newBlock(ast->firstToken());
     const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
-    block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
-    block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
+    block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
+    block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
     _scope->addMember(block);
     ast->symbol = block;
 
@@ -1285,8 +1285,8 @@ bool Bind::visit(RangeBasedForStatementAST *ast)
 {
     Block *block = control()->newBlock(ast->firstToken());
     const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
-    block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
-    block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
+    block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
+    block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
     _scope->addMember(block);
     ast->symbol = block;
 
@@ -1334,8 +1334,8 @@ bool Bind::visit(ForStatementAST *ast)
 {
     Block *block = control()->newBlock(ast->firstToken());
     const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
-    block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
-    block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
+    block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
+    block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
     _scope->addMember(block);
     ast->symbol = block;
 
@@ -1354,8 +1354,8 @@ bool Bind::visit(IfStatementAST *ast)
 {
     Block *block = control()->newBlock(ast->firstToken());
     const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
-    block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
-    block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
+    block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
+    block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
     _scope->addMember(block);
     ast->symbol = block;
 
@@ -1410,8 +1410,8 @@ bool Bind::visit(SwitchStatementAST *ast)
 {
     Block *block = control()->newBlock(ast->firstToken());
     const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
-    block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
-    block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
+    block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
+    block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
     _scope->addMember(block);
     ast->symbol = block;
 
@@ -1436,8 +1436,8 @@ bool Bind::visit(CatchClauseAST *ast)
 {
     Block *block = control()->newBlock(ast->firstToken());
     const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
-    block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
-    block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
+    block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
+    block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
     _scope->addMember(block);
     ast->symbol = block;
 
@@ -1453,8 +1453,8 @@ bool Bind::visit(WhileStatementAST *ast)
 {
     Block *block = control()->newBlock(ast->firstToken());
     const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
-    block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
-    block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
+    block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
+    block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
     _scope->addMember(block);
     ast->symbol = block;
 
@@ -1469,8 +1469,8 @@ bool Bind::visit(ObjCFastEnumerationAST *ast)
 {
     Block *block = control()->newBlock(ast->firstToken());
     const unsigned startScopeToken = ast->lparen_token ? ast->lparen_token : ast->firstToken();
-    block->setStartOffset(tokenAt(startScopeToken).bytesEnd());
-    block->setEndOffset(tokenAt(ast->lastToken()).bytesBegin());
+    block->setStartOffset(tokenAt(startScopeToken).utf16charsEnd());
+    block->setEndOffset(tokenAt(ast->lastToken()).utf16charsBegin());
     _scope->addMember(block);
     ast->symbol = block;
 
@@ -2139,7 +2139,7 @@ bool Bind::visit(FunctionDefinitionAST *ast)
 
     if (fun) {
         setDeclSpecifiers(fun, declSpecifiers);
-        fun->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
+        fun->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
 
         if (_scope->isClass()) {
             fun->setVisibility(_visibility);
@@ -2201,8 +2201,8 @@ bool Bind::visit(NamespaceAST *ast)
     }
 
     Namespace *ns = control()->newNamespace(sourceLocation, namespaceName);
-    ns->setStartOffset(tokenAt(sourceLocation).bytesEnd()); // the scope starts after the namespace or the identifier token.
-    ns->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
+    ns->setStartOffset(tokenAt(sourceLocation).utf16charsEnd()); // the scope starts after the namespace or the identifier token.
+    ns->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
     ns->setInline(ast->inline_token != 0);
     ast->symbol = ns;
     _scope->addMember(ns);
@@ -2261,8 +2261,8 @@ bool Bind::visit(ParameterDeclarationAST *ast)
 bool Bind::visit(TemplateDeclarationAST *ast)
 {
     Template *templ = control()->newTemplate(ast->firstToken(), 0);
-    templ->setStartOffset(tokenAt(ast->firstToken()).bytesBegin());
-    templ->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
+    templ->setStartOffset(tokenAt(ast->firstToken()).utf16charsBegin());
+    templ->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
     ast->symbol = templ;
     Scope *previousScope = switchScope(templ);
 
@@ -2375,34 +2375,34 @@ unsigned Bind::calculateScopeStart(ObjCClassDeclarationAST *ast) const
 {
     if (ast->inst_vars_decl)
         if (unsigned pos = ast->inst_vars_decl->lbrace_token)
-            return tokenAt(pos).bytesEnd();
+            return tokenAt(pos).utf16charsEnd();
 
     if (ast->protocol_refs)
         if (unsigned pos = ast->protocol_refs->lastToken())
-            return tokenAt(pos - 1).bytesEnd();
+            return tokenAt(pos - 1).utf16charsEnd();
 
     if (ast->superclass)
         if (unsigned pos = ast->superclass->lastToken())
-            return tokenAt(pos - 1).bytesEnd();
+            return tokenAt(pos - 1).utf16charsEnd();
 
     if (ast->colon_token)
-        return tokenAt(ast->colon_token).bytesEnd();
+        return tokenAt(ast->colon_token).utf16charsEnd();
 
     if (ast->rparen_token)
-        return tokenAt(ast->rparen_token).bytesEnd();
+        return tokenAt(ast->rparen_token).utf16charsEnd();
 
     if (ast->category_name)
         if (unsigned pos = ast->category_name->lastToken())
-            return tokenAt(pos - 1).bytesEnd();
+            return tokenAt(pos - 1).utf16charsEnd();
 
     if (ast->lparen_token)
-        return tokenAt(ast->lparen_token).bytesEnd();
+        return tokenAt(ast->lparen_token).utf16charsEnd();
 
     if (ast->class_name)
         if (unsigned pos = ast->class_name->lastToken())
-            return tokenAt(pos - 1).bytesEnd();
+            return tokenAt(pos - 1).utf16charsEnd();
 
-    return tokenAt(ast->firstToken()).bytesBegin();
+    return tokenAt(ast->firstToken()).utf16charsBegin();
 }
 
 bool Bind::visit(ObjCClassDeclarationAST *ast)
@@ -2420,7 +2420,7 @@ bool Bind::visit(ObjCClassDeclarationAST *ast)
     _scope->addMember(klass);
 
     klass->setStartOffset(calculateScopeStart(ast));
-    klass->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
+    klass->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
 
     if (ast->interface_token)
         klass->setInterface(true);
@@ -2479,12 +2479,12 @@ unsigned Bind::calculateScopeStart(ObjCProtocolDeclarationAST *ast) const
 {
     if (ast->protocol_refs)
         if (unsigned pos = ast->protocol_refs->lastToken())
-            return tokenAt(pos - 1).bytesEnd();
+            return tokenAt(pos - 1).utf16charsEnd();
     if (ast->name)
         if (unsigned pos = ast->name->lastToken())
-            return tokenAt(pos - 1).bytesEnd();
+            return tokenAt(pos - 1).utf16charsEnd();
 
-    return tokenAt(ast->firstToken()).bytesBegin();
+    return tokenAt(ast->firstToken()).utf16charsBegin();
 }
 
 bool Bind::visit(ObjCProtocolDeclarationAST *ast)
@@ -2499,7 +2499,7 @@ bool Bind::visit(ObjCProtocolDeclarationAST *ast)
     const unsigned sourceLocation = location(ast->name, ast->firstToken());
     ObjCProtocol *protocol = control()->newObjCProtocol(sourceLocation, name);
     protocol->setStartOffset(calculateScopeStart(ast));
-    protocol->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
+    protocol->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
     ast->symbol = protocol;
     _scope->addMember(protocol);
 
@@ -2920,7 +2920,7 @@ bool Bind::visit(ClassSpecifierAST *ast)
 {
     // unsigned classkey_token = ast->classkey_token;
     unsigned sourceLocation = ast->firstToken();
-    unsigned startScopeOffset = tokenAt(sourceLocation).bytesEnd(); // at the end of the class key
+    unsigned startScopeOffset = tokenAt(sourceLocation).utf16charsEnd(); // at the end of the class key
 
     for (SpecifierListAST *it = ast->attribute_list; it; it = it->next) {
         _type = this->specifier(it->value, _type);
@@ -2930,12 +2930,12 @@ bool Bind::visit(ClassSpecifierAST *ast)
 
     if (ast->name && ! ast->name->asAnonymousName()) {
         sourceLocation = location(ast->name, sourceLocation);
-        startScopeOffset = tokenAt(sourceLocation).bytesEnd(); // at the end of the class name
+        startScopeOffset = tokenAt(sourceLocation).utf16charsEnd(); // at the end of the class name
 
         if (QualifiedNameAST *q = ast->name->asQualifiedName()) {
             if (q->unqualified_name) {
                 sourceLocation = q->unqualified_name->firstToken();
-                startScopeOffset = tokenAt(q->unqualified_name->lastToken() - 1).bytesEnd(); // at the end of the unqualified name
+                startScopeOffset = tokenAt(q->unqualified_name->lastToken() - 1).utf16charsEnd(); // at the end of the unqualified name
             }
         }
 
@@ -2944,7 +2944,7 @@ bool Bind::visit(ClassSpecifierAST *ast)
 
     Class *klass = control()->newClass(sourceLocation, className);
     klass->setStartOffset(startScopeOffset);
-    klass->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
+    klass->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
     _scope->addMember(klass);
 
     if (_scope->isClass())
@@ -3003,8 +3003,8 @@ bool Bind::visit(EnumSpecifierAST *ast)
     const Name *enumName = this->name(ast->name);
 
     Enum *e = control()->newEnum(sourceLocation, enumName);
-    e->setStartOffset(tokenAt(sourceLocation).bytesEnd()); // at the end of the enum or identifier token.
-    e->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
+    e->setStartOffset(tokenAt(sourceLocation).utf16charsEnd()); // at the end of the enum or identifier token.
+    e->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
     if (ast->key_token)
         e->setScoped(true);
     ast->symbol = e;
@@ -3126,8 +3126,8 @@ bool Bind::visit(NestedDeclaratorAST *ast)
 bool Bind::visit(FunctionDeclaratorAST *ast)
 {
     Function *fun = control()->newFunction(0, 0);
-    fun->setStartOffset(tokenAt(ast->firstToken()).bytesBegin());
-    fun->setEndOffset(tokenAt(ast->lastToken() - 1).bytesEnd());
+    fun->setStartOffset(tokenAt(ast->firstToken()).utf16charsBegin());
+    fun->setEndOffset(tokenAt(ast->lastToken() - 1).utf16charsEnd());
     if (ast->trailing_return_type)
         _type = this->trailingReturnType(ast->trailing_return_type, _type);
     fun->setReturnType(_type);
diff --git a/src/libs/3rdparty/cplusplus/Lexer.cpp b/src/libs/3rdparty/cplusplus/Lexer.cpp
index 914b3c2275..8357b9e554 100644
--- a/src/libs/3rdparty/cplusplus/Lexer.cpp
+++ b/src/libs/3rdparty/cplusplus/Lexer.cpp
@@ -109,7 +109,7 @@ void Lexer::pushLineStartOffset()
     ++_currentLine;
 
     if (_translationUnit)
-        _translationUnit->pushLineOffset(_currentChar - _firstChar);
+        _translationUnit->pushLineOffset(_currentCharUtf16);
 }
 
 void Lexer::scan(Token *tok)
diff --git a/src/libs/3rdparty/cplusplus/Symbol.cpp b/src/libs/3rdparty/cplusplus/Symbol.cpp
index 5c262b79f5..ae20b14bb7 100644
--- a/src/libs/3rdparty/cplusplus/Symbol.cpp
+++ b/src/libs/3rdparty/cplusplus/Symbol.cpp
@@ -166,7 +166,7 @@ void Symbol::setSourceLocation(unsigned sourceLocation, TranslationUnit *transla
     if (translationUnit) {
         const Token &tk = translationUnit->tokenAt(sourceLocation);
         _isGenerated = tk.generated();
-        translationUnit->getPosition(tk.bytesBegin(), &_line, &_column, &_fileId);
+        translationUnit->getPosition(tk.utf16charsBegin(), &_line, &_column, &_fileId);
     } else {
         _isGenerated = false;
         _line = 0;
diff --git a/src/libs/3rdparty/cplusplus/TranslationUnit.cpp b/src/libs/3rdparty/cplusplus/TranslationUnit.cpp
index 91d5101955..fe309ed82f 100644
--- a/src/libs/3rdparty/cplusplus/TranslationUnit.cpp
+++ b/src/libs/3rdparty/cplusplus/TranslationUnit.cpp
@@ -348,10 +348,10 @@ void TranslationUnit::pushPreprocessorLine(unsigned offset,
                                            const StringLiteral *fileName)
 { _ppLines.push_back(PPLine(offset, line, fileName)); }
 
-unsigned TranslationUnit::findLineNumber(unsigned offset) const
+unsigned TranslationUnit::findLineNumber(unsigned utf16charOffset) const
 {
     std::vector<unsigned>::const_iterator it =
-        std::lower_bound(_lineOffsets.begin(), _lineOffsets.end(), offset);
+        std::lower_bound(_lineOffsets.begin(), _lineOffsets.end(), utf16charOffset);
 
     if (it != _lineOffsets.begin())
         --it;
@@ -370,31 +370,31 @@ TranslationUnit::PPLine TranslationUnit::findPreprocessorLine(unsigned offset) c
     return *it;
 }
 
-unsigned TranslationUnit::findColumnNumber(unsigned offset, unsigned lineNumber) const
+unsigned TranslationUnit::findColumnNumber(unsigned utf16CharOffset, unsigned lineNumber) const
 {
-    if (! offset)
+    if (! utf16CharOffset)
         return 0;
 
-    return offset - _lineOffsets[lineNumber];
+    return utf16CharOffset - _lineOffsets[lineNumber];
 }
 
 void TranslationUnit::getTokenPosition(unsigned index,
                                        unsigned *line,
                                        unsigned *column,
                                        const StringLiteral **fileName) const
-{ return getPosition(tokenAt(index).bytesBegin(), line, column, fileName); }
+{ return getPosition(tokenAt(index).utf16charsBegin(), line, column, fileName); }
 
 void TranslationUnit::getTokenStartPosition(unsigned index, unsigned *line,
                                             unsigned *column,
                                             const StringLiteral **fileName) const
-{ return getPosition(tokenAt(index).bytesBegin(), line, column, fileName); }
+{ return getPosition(tokenAt(index).utf16charsBegin(), line, column, fileName); }
 
 void TranslationUnit::getTokenEndPosition(unsigned index, unsigned *line,
                                           unsigned *column,
                                           const StringLiteral **fileName) const
-{ return getPosition(tokenAt(index).bytesEnd(), line, column, fileName); }
+{ return getPosition(tokenAt(index).utf16charsEnd(), line, column, fileName); }
 
-void TranslationUnit::getPosition(unsigned tokenOffset,
+void TranslationUnit::getPosition(unsigned utf16charOffset,
                                   unsigned *line,
                                   unsigned *column,
                                   const StringLiteral **fileName) const
@@ -405,20 +405,20 @@ void TranslationUnit::getPosition(unsigned tokenOffset,
 
     // If this token is expanded we already have the information directly from the expansion
     // section header. Otherwise, we need to calculate it.
-    TokenLineColumn::const_iterator it = _expandedLineColumn.find(tokenOffset);
+    TokenLineColumn::const_iterator it = _expandedLineColumn.find(utf16charOffset);
     if (it != _expandedLineColumn.end()) {
         lineNumber = it->second.first;
         columnNumber = it->second.second + 1;
         file = _fileId;
     } else {
         // Identify line within the entire translation unit.
-        lineNumber = findLineNumber(tokenOffset);
+        lineNumber = findLineNumber(utf16charOffset);
 
         // Identify column.
-        columnNumber = findColumnNumber(tokenOffset, lineNumber);
+        columnNumber = findColumnNumber(utf16charOffset, lineNumber);
 
         // Adjust the line in regards to the preprocessing markers.
-        const PPLine ppLine = findPreprocessorLine(tokenOffset);
+        const PPLine ppLine = findPreprocessorLine(utf16charOffset);
         lineNumber -= findLineNumber(ppLine.offset) + 1;
         lineNumber += ppLine.line;
 
@@ -508,7 +508,7 @@ void TranslationUnit::fatal(unsigned index, const char *format, ...)
 
 unsigned TranslationUnit::findPreviousLineOffset(unsigned tokenIndex) const
 {
-    unsigned lineOffset = _lineOffsets[findLineNumber(tokenAt(tokenIndex).bytesBegin())];
+    unsigned lineOffset = _lineOffsets[findLineNumber(tokenAt(tokenIndex).utf16charsBegin())];
     return lineOffset;
 }
 
@@ -522,13 +522,16 @@ bool TranslationUnit::maybeSplitGreaterGreaterToken(unsigned tokenIndex)
 
     tok.f.kind = T_GREATER;
     tok.f.bytes = 1;
+    tok.f.utf16chars = 1;
 
     Token newGreater;
     newGreater.f.kind = T_GREATER;
     newGreater.f.expanded = tok.expanded();
     newGreater.f.generated = tok.generated();
     newGreater.f.bytes = 1;
+    newGreater.f.utf16chars = 1;
     newGreater.byteOffset = tok.byteOffset + 1;
+    newGreater.utf16charOffset = tok.utf16charOffset + 1;
 
     _tokens->insert(_tokens->begin() + tokenIndex + 1, newGreater);
 
@@ -551,7 +554,7 @@ void TranslationUnit::releaseTokensAndComments()
 
 void TranslationUnit::showErrorLine(unsigned index, unsigned column, FILE *out)
 {
-    unsigned lineOffset = _lineOffsets[findLineNumber(tokenAt(index).bytesBegin())];
+    unsigned lineOffset = _lineOffsets[findLineNumber(tokenAt(index).utf16charsBegin())];
     for (const char *cp = _firstSourceChar + lineOffset + 1; *cp && *cp != '\n'; ++cp) {
         fputc(*cp, out);
     }
diff --git a/src/libs/3rdparty/cplusplus/TranslationUnit.h b/src/libs/3rdparty/cplusplus/TranslationUnit.h
index 668445c961..83213703c5 100644
--- a/src/libs/3rdparty/cplusplus/TranslationUnit.h
+++ b/src/libs/3rdparty/cplusplus/TranslationUnit.h
@@ -127,7 +127,7 @@ public:
                              unsigned *column = 0,
                              const StringLiteral **fileName = 0) const;
 
-    void getPosition(unsigned offset,
+    void getPosition(unsigned utf16charOffset,
                      unsigned *line,
                      unsigned *column = 0,
                      const StringLiteral **fileName = 0) const;
@@ -172,8 +172,8 @@ private:
     };
 
     void releaseTokensAndComments();
-    unsigned findLineNumber(unsigned offset) const;
-    unsigned findColumnNumber(unsigned offset, unsigned lineNumber) const;
+    unsigned findLineNumber(unsigned utf16charOffset) const;
+    unsigned findColumnNumber(unsigned utf16CharOffset, unsigned lineNumber) const;
     PPLine findPreprocessorLine(unsigned offset) const;
     void showErrorLine(unsigned index, unsigned column, FILE *out);
 
diff --git a/src/plugins/cppeditor/cppcodemodelinspectordialog.cpp b/src/plugins/cppeditor/cppcodemodelinspectordialog.cpp
index 706decd32d..dab0b46821 100644
--- a/src/plugins/cppeditor/cppcodemodelinspectordialog.cpp
+++ b/src/plugins/cppeditor/cppcodemodelinspectordialog.cpp
@@ -852,7 +852,7 @@ void TokensModel::configure(CPlusPlus::TranslationUnit *translationUnit)
     for (int i = 0, total = translationUnit->tokenCount(); i < total; ++i) {
         TokenInfo info;
         info.token = translationUnit->tokenAt(i);
-        translationUnit->getPosition(info.token.bytesBegin(), &info.line, &info.column);
+        translationUnit->getPosition(info.token.utf16charsBegin(), &info.line, &info.column);
         m_tokenInfos.append(info);
     }
     emit layoutChanged();
diff --git a/src/plugins/cppeditor/fileandtokenactions_test.cpp b/src/plugins/cppeditor/fileandtokenactions_test.cpp
index 41050e3f73..95eb390860 100644
--- a/src/plugins/cppeditor/fileandtokenactions_test.cpp
+++ b/src/plugins/cppeditor/fileandtokenactions_test.cpp
@@ -238,7 +238,7 @@ TestActionsTestCase::TestActionsTestCase(const Actions &tokenActions, const Acti
             } else {
                 // Position the cursor on the token
                 unsigned line, column;
-                translationUnit->getPosition(token.bytesBegin(), &line, &column);
+                translationUnit->getPosition(token.utf16charsBegin(), &line, &column);
                 editor->gotoLine(line, column - 1);
                 QApplication::processEvents();
 
@@ -293,7 +293,7 @@ void TestActionsTestCase::moveWordCamelCaseToToken(TranslationUnit *translationU
     QVERIFY(editorWidget);
 
     unsigned line, column;
-    translationUnit->getPosition(token.bytesBegin(), &line, &column);
+    translationUnit->getPosition(token.utf16charsBegin(), &line, &column);
 
     while (editor->currentLine() < (int) line
            || (editor->currentLine() == (int) line
diff --git a/src/plugins/cpptools/cpprefactoringchanges.cpp b/src/plugins/cpptools/cpprefactoringchanges.cpp
index 553b905ac3..4ea1ed5477 100644
--- a/src/plugins/cpptools/cpprefactoringchanges.cpp
+++ b/src/plugins/cpptools/cpprefactoringchanges.cpp
@@ -199,7 +199,7 @@ ChangeSet::Range CppRefactoringFile::range(unsigned tokenIndex) const
 {
     const Token &token = tokenAt(tokenIndex);
     unsigned line, column;
-    cppDocument()->translationUnit()->getPosition(token.bytesBegin(), &line, &column);
+    cppDocument()->translationUnit()->getPosition(token.utf16charsBegin(), &line, &column);
     const int start = document()->findBlockByNumber(line - 1).position() + column - 1;
     return ChangeSet::Range(start, start + token.bytes());
 }
@@ -212,7 +212,7 @@ ChangeSet::Range CppRefactoringFile::range(AST *ast) const
 int CppRefactoringFile::startOf(unsigned index) const
 {
     unsigned line, column;
-    cppDocument()->translationUnit()->getPosition(tokenAt(index).bytesBegin(), &line, &column);
+    cppDocument()->translationUnit()->getPosition(tokenAt(index).utf16charsBegin(), &line, &column);
     return document()->findBlockByNumber(line - 1).position() + column - 1;
 }
 
@@ -224,7 +224,7 @@ int CppRefactoringFile::startOf(const AST *ast) const
 int CppRefactoringFile::endOf(unsigned index) const
 {
     unsigned line, column;
-    cppDocument()->translationUnit()->getPosition(tokenAt(index).bytesEnd(), &line, &column);
+    cppDocument()->translationUnit()->getPosition(tokenAt(index).utf16charsEnd(), &line, &column);
     return document()->findBlockByNumber(line - 1).position() + column - 1;
 }
 
@@ -239,7 +239,7 @@ void CppRefactoringFile::startAndEndOf(unsigned index, int *start, int *end) con
 {
     unsigned line, column;
     Token token(tokenAt(index));
-    cppDocument()->translationUnit()->getPosition(token.bytesBegin(), &line, &column);
+    cppDocument()->translationUnit()->getPosition(token.utf16charsBegin(), &line, &column);
     *start = document()->findBlockByNumber(line - 1).position() + column - 1;
     *end = *start + token.bytes();
 }
diff --git a/src/plugins/todo/cpptodoitemsscanner.cpp b/src/plugins/todo/cpptodoitemsscanner.cpp
index a3df596a04..f4f9e7b4bd 100644
--- a/src/plugins/todo/cpptodoitemsscanner.cpp
+++ b/src/plugins/todo/cpptodoitemsscanner.cpp
@@ -98,7 +98,7 @@ void CppTodoItemsScanner::processDocument(CPlusPlus::Document::Ptr doc)
         const QStringList commentLines =
             QString::fromUtf8(source).split(QLatin1Char('\n'), QString::SkipEmptyParts);
         unsigned lineNumber = 0;
-        translationUnit->getPosition(token.bytesBegin(), &lineNumber);
+        translationUnit->getPosition(token.utf16charsBegin(), &lineNumber);
         for (int j = 0; j < commentLines.count(); ++j) {
             const QString &commentLine = commentLines.at(j);
             processCommentLine(doc->fileName(), commentLine, lineNumber + j, itemList);
diff --git a/tests/auto/cplusplus/translationunit/tst_translationunit.cpp b/tests/auto/cplusplus/translationunit/tst_translationunit.cpp
index d89d979bab..e3a0f7e2c8 100644
--- a/tests/auto/cplusplus/translationunit/tst_translationunit.cpp
+++ b/tests/auto/cplusplus/translationunit/tst_translationunit.cpp
@@ -36,6 +36,15 @@
 #include <QtTest>
 #include <QDebug>
 
+struct LineColumn
+{
+    LineColumn(unsigned line = 0, unsigned column = 0) : line(line), column(column) {}
+    unsigned line;
+    unsigned column;
+};
+typedef QList<LineColumn> LineColumnList;
+Q_DECLARE_METATYPE(LineColumnList)
+
 //TESTED_COMPONENT=src/libs/cplusplus
 using namespace CPlusPlus;
 
@@ -58,6 +67,9 @@ private slots:
     void unicodeStringLiteral();
     void unicodeStringLiteral_data();
 
+    void locationOfUtf16CharOffset();
+    void locationOfUtf16CharOffset_data();
+
 private:
     class Document
     {
@@ -132,8 +144,49 @@ private:
             }
         } m_diagnosticClient;
     };
+
+    class TokenGetter
+    {
+    public:
+        typedef QSharedPointer<TokenGetter> Ptr;
+    public:
+        TokenGetter(TranslationUnit *translationUnit) : m_translationUnit(translationUnit) {}
+        virtual ~TokenGetter() {}
+        virtual unsigned tokenCount() { return m_translationUnit->tokenCount(); }
+        virtual Token tokenAt(unsigned index) { return m_translationUnit->tokenAt(index); }
+    protected:
+        TranslationUnit *m_translationUnit;
+    };
+
+    class CommentTokenGetter : public TokenGetter
+    {
+    public:
+        CommentTokenGetter(TranslationUnit *translationUnit) : TokenGetter(translationUnit) {}
+        unsigned tokenCount() { return m_translationUnit->commentCount(); }
+        Token tokenAt(unsigned index) { return m_translationUnit->commentAt(index); }
+    };
+
+    static void compareTokenLocations(TranslationUnit *translationUnit,
+                                      tst_TranslationUnit::TokenGetter *tokenGetter,
+                                      const LineColumnList &expectedLinesColumns);
 };
 
+void tst_TranslationUnit::compareTokenLocations(TranslationUnit *translationUnit,
+                                                tst_TranslationUnit::TokenGetter *tokenGetter,
+                                                const LineColumnList &expectedLinesColumns)
+{
+    QCOMPARE(tokenGetter->tokenCount(), (unsigned) expectedLinesColumns.count());
+    for (unsigned i = 0, tokenCount = tokenGetter->tokenCount(); i < tokenCount; ++i) {
+        const LineColumn expected = expectedLinesColumns.at(i);
+        const unsigned utf16CharOffset = tokenGetter->tokenAt(i).utf16charsBegin();
+        unsigned line, column;
+        translationUnit->getPosition(utf16CharOffset, &line, &column);
+//        qDebug("%d: LineColumn(%u, %u)", i, line, column);
+        QCOMPARE(line, expected.line);
+        QCOMPARE(column, expected.column);
+    }
+}
+
 void tst_TranslationUnit::unicodeIdentifier()
 {
     QFETCH(QByteArray, identifierText);
@@ -221,5 +274,231 @@ void tst_TranslationUnit::unicodeStringLiteral_data()
     QTest::newRow("non-latin1 literal 10") << _("U\"\U00010302\u00FC\u4E8C\"");
 }
 
+void tst_TranslationUnit::locationOfUtf16CharOffset()
+{
+    QFETCH(QByteArray, source);
+    QFETCH(LineColumnList, expectedNonCommentLinesColumns);
+    QFETCH(LineColumnList, expectedCommentLinesColumns);
+
+    Document::Ptr document = Document::create(source);
+    TranslationUnit *translationUnit = document->translationUnit();
+
+    const TokenGetter::Ptr nonCommentTokenGetter(new TokenGetter(translationUnit));
+    compareTokenLocations(translationUnit, nonCommentTokenGetter.data(),
+                          expectedNonCommentLinesColumns);
+
+    const TokenGetter::Ptr commentTokenGetter(new CommentTokenGetter(translationUnit));
+    compareTokenLocations(translationUnit, commentTokenGetter.data(), expectedCommentLinesColumns);
+}
+
+void tst_TranslationUnit::locationOfUtf16CharOffset_data()
+{
+    QTest::addColumn<QByteArray>("source");
+    QTest::addColumn<LineColumnList>("expectedNonCommentLinesColumns");
+    QTest::addColumn<LineColumnList>("expectedCommentLinesColumns");
+
+    typedef QByteArray _;
+
+    QTest::newRow("empty")
+        << _("")
+        << (LineColumnList()
+            << LineColumn(0, 0)
+            << LineColumn(0, 0)
+           )
+        << LineColumnList();
+
+    // --- Identifiers ---------------------------------------------------------------------------
+
+    QTest::newRow("latin1 identifiers")
+        << _("int i;")
+        << (LineColumnList()
+            << LineColumn(0, 0)
+            << LineColumn(1, 1) // int
+            << LineColumn(1, 5) // i
+            << LineColumn(1, 6) // ;
+            << LineColumn(1, 7)
+           )
+        << LineColumnList();
+
+    QTest::newRow("latin1 identifiers")
+        << _("int i;\n"
+             "int jj;\n")
+        << (LineColumnList()
+            << LineColumn(0, 0)
+            << LineColumn(1, 1) // int 1
+            << LineColumn(1, 5) // i
+            << LineColumn(1, 6) // ;
+            << LineColumn(2, 1) // int 2
+            << LineColumn(2, 5) // jj
+            << LineColumn(2, 7) // ;
+            << LineColumn(3, 1)
+           )
+        << LineColumnList();
+
+    QTest::newRow("non-latin1 identifier")
+        << _("int \u00FC;")
+        << (LineColumnList()
+            << LineColumn(0, 0)
+            << LineColumn(1, 1) // int
+            << LineColumn(1, 5) // non-latin1 identifier
+            << LineColumn(1, 6) // ;
+            << LineColumn(1, 7)
+           )
+        << LineColumnList();
+
+    QTest::newRow("non-latin1 identifiers 1")
+        << _("int \u00FC;\n"
+             "int \u00FC;")
+        << (LineColumnList()
+            << LineColumn(0, 0)
+            << LineColumn(1, 1) // int 1
+            << LineColumn(1, 5) // non-latin1 identifier 1
+            << LineColumn(1, 6) // ;
+            << LineColumn(2, 1) // int 2
+            << LineColumn(2, 5) // non-latin1 identifier 2
+            << LineColumn(2, 6) // ;
+            << LineColumn(2, 7) // ;
+           )
+        << LineColumnList();
+
+    QTest::newRow("non-latin1 identifiers 2")
+        << _("int \u00FC\u4E8C\U00010302;\n"
+             "int v;\n"
+             "int \U00010302\u4E8C;")
+        << (LineColumnList()
+            << LineColumn(0, 0)
+            << LineColumn(1, 1) // int 1
+            << LineColumn(1, 5) // non-latin1 identifier 1
+            << LineColumn(1, 9) // ;
+            << LineColumn(2, 1) // int 2
+            << LineColumn(2, 5) // non-latin1 identifier 2
+            << LineColumn(2, 6) // ;
+            << LineColumn(3, 1) // int 3
+            << LineColumn(3, 5) // non-latin1 identifier 3
+            << LineColumn(3, 8) // ;
+            << LineColumn(3, 9)
+           )
+        << LineColumnList();
+
+    // --- String literals -----------------------------------------------------------------------
+
+    QTest::newRow("latin1 string literal")
+        << _("char t[] = \"foo\";")
+        << (LineColumnList()
+            << LineColumn(0, 0)
+            << LineColumn(1, 1)  // char
+            << LineColumn(1, 6)  // t
+            << LineColumn(1, 7)  // [
+            << LineColumn(1, 8)  // ]
+            << LineColumn(1, 10) // =
+            << LineColumn(1, 12) // latin1 string literal
+            << LineColumn(1, 17) // ;
+            << LineColumn(1, 18)
+           )
+        << LineColumnList();
+
+    QTest::newRow("non-latin1 string literal")
+        << _("char t[] = \"i\u00FC\u4E8C\U00010302\";")
+        << (LineColumnList()
+            << LineColumn(0, 0)
+            << LineColumn(1, 1)  // char
+            << LineColumn(1, 6)  // t
+            << LineColumn(1, 7)  // [
+            << LineColumn(1, 8)  // ]
+            << LineColumn(1, 10) // =
+            << LineColumn(1, 12) // non-latin1 string literal
+            << LineColumn(1, 19) // ;
+            << LineColumn(1, 20)
+           )
+        << LineColumnList();
+
+    QTest::newRow("non-latin1 string literal multiple lines")
+        << _("char t[] = \"i\u00FC\u4E8C\U00010302 \\\n"
+             "\";")
+        << (LineColumnList()
+            << LineColumn(0, 0)
+            << LineColumn(1, 1)  // char
+            << LineColumn(1, 6)  // t
+            << LineColumn(1, 7)  // [
+            << LineColumn(1, 8)  // ]
+            << LineColumn(1, 10) // =
+            << LineColumn(1, 12) // non-latin1 string literal
+            << LineColumn(2, 2)  // ;
+            << LineColumn(2, 3)
+           )
+        << LineColumnList();
+
+    // --- Comments ------------------------------------------------------------------------------
+
+    QTest::newRow("latin1 c++ comment line")
+        << _("// comment line\n"
+             "int i;")
+        << (LineColumnList()
+            << LineColumn(0, 0)
+            << LineColumn(2, 1) // int
+            << LineColumn(2, 5) // i
+            << LineColumn(2, 6) // ;
+            << LineColumn(2, 7)
+           )
+        << (LineColumnList()
+            << LineColumn(1, 1) // comment
+           );
+
+    QTest::newRow("latin1 c comment line")
+        << _("/* comment line */ int i;")
+        << (LineColumnList()
+            << LineColumn(0, 0)
+            << LineColumn(1, 20) // int
+            << LineColumn(1, 24) // i
+            << LineColumn(1, 25) // ;
+            << LineColumn(1, 26)
+           )
+        << (LineColumnList()
+            << LineColumn(1, 1)  // comment
+           );
+
+    QTest::newRow("latin1 c comment lines")
+        << _("/* comment line 1\n"
+             "   comment line 2 */ int i;")
+        << (LineColumnList()
+            << LineColumn(0, 0)
+            << LineColumn(2, 22) // int
+            << LineColumn(2, 26) // i
+            << LineColumn(2, 27) // ;
+            << LineColumn(2, 28)
+           )
+        << (LineColumnList()
+            << LineColumn(1, 1)  // comment
+           );
+
+    QTest::newRow("non-latin1 c++ comment line")
+        << _("// comment line \u00FC\u4E8C\U00010302\n"
+             "int i;")
+        << (LineColumnList()
+            << LineColumn(0, 0)
+            << LineColumn(2, 1) // int
+            << LineColumn(2, 5) // i
+            << LineColumn(2, 6) // ;
+            << LineColumn(2, 7)
+           )
+        << (LineColumnList()
+            << LineColumn(1, 1) // comment
+           );
+
+    QTest::newRow("non-latin1 c comment lines")
+        << _("/* comment line 1\n"
+             "   comment line 2 */ int i;")
+        << (LineColumnList()
+            << LineColumn(0, 0)
+            << LineColumn(2, 22) // int
+            << LineColumn(2, 26) // i
+            << LineColumn(2, 27) // ;
+            << LineColumn(2, 28)
+           )
+        << (LineColumnList()
+            << LineColumn(1, 1)  // comment
+           );
+}
+
 QTEST_APPLESS_MAIN(tst_TranslationUnit)
 #include "tst_translationunit.moc"