C++/CppTools: Pass UTF-8 encoded source to Lexer

The Lexer can handle it now.

Task-number: QTCREATORBUG-7356
Change-Id: I8c4b03a247656e013d44c3cedca4835e133d4036
Reviewed-by: Erik Verbruggen <erik.verbruggen@digia.com>

6 files changed, 6 insertions, 34 deletions

diff --git a/src/libs/cplusplus/FastPreprocessor.cpp b/src/libs/cplusplus/FastPreprocessor.cpp
index c663375a8b..5c52764cf5 100644
--- a/src/libs/cplusplus/FastPreprocessor.cpp
+++ b/src/libs/cplusplus/FastPreprocessor.cpp
@@ -41,18 +41,6 @@ FastPreprocessor::FastPreprocessor(const Snapshot &snapshot)
     , _preproc(this, &_env)
 { }
 
-// This is a temporary fix to handle non-ascii characters. This can be removed when the lexer can
-// handle multi-byte characters.
-static QByteArray convertToLatin1(const QByteArray &contents)
-{
-    const char *p = contents.constData();
-    while (char ch = *p++)
-        if (ch & 0x80)
-            return QString::fromUtf8(contents).toLatin1();
-
-    return contents;
-}
-
 QByteArray FastPreprocessor::run(Document::Ptr newDoc, const QByteArray &source)
 {
     std::swap(newDoc, _currentDoc);
@@ -72,9 +60,7 @@ QByteArray FastPreprocessor::run(Document::Ptr newDoc, const QByteArray &source)
             mergeEnvironment(i.resolvedFileName());
     }
 
-    QByteArray src = convertToLatin1(source);
-
-    const QByteArray preprocessed = _preproc.run(fileName, src);
+    const QByteArray preprocessed = _preproc.run(fileName, source);
 //    qDebug("FastPreprocessor::run for %s produced [[%s]]", fileName.toUtf8().constData(), preprocessed.constData());
     std::swap(newDoc, _currentDoc);
     return preprocessed;
diff --git a/src/libs/cplusplus/SimpleLexer.cpp b/src/libs/cplusplus/SimpleLexer.cpp
index 95c6c051a5..7ce50a06c3 100644
--- a/src/libs/cplusplus/SimpleLexer.cpp
+++ b/src/libs/cplusplus/SimpleLexer.cpp
@@ -61,11 +61,11 @@ bool SimpleLexer::endedJoined() const
     return _endedJoined;
 }
 
-QList<Token> SimpleLexer::operator()(const QString &text, int state, bool convertToUtf8)
+QList<Token> SimpleLexer::operator()(const QString &text, int state)
 {
     QList<Token> tokens;
 
-    const QByteArray bytes = convertToUtf8 ? text.toUtf8() : text.toLatin1();
+    const QByteArray bytes = text.toUtf8();
     const char *firstChar = bytes.constData();
     const char *lastChar = firstChar + bytes.size();
 
diff --git a/src/libs/cplusplus/SimpleLexer.h b/src/libs/cplusplus/SimpleLexer.h
index a5b7d3e4ac..1eb4ab6c3b 100644
--- a/src/libs/cplusplus/SimpleLexer.h
+++ b/src/libs/cplusplus/SimpleLexer.h
@@ -54,7 +54,7 @@ public:
 
     bool endedJoined() const;
 
-    QList<Token> operator()(const QString &text, int state = 0, bool convertToUtf8 = false);
+    QList<Token> operator()(const QString &text, int state = 0);
 
     int state() const
     { return _lastState; }
diff --git a/src/libs/cplusplus/pp-engine.cpp b/src/libs/cplusplus/pp-engine.cpp
index b32bb9dda4..c6aabbe608 100644
--- a/src/libs/cplusplus/pp-engine.cpp
+++ b/src/libs/cplusplus/pp-engine.cpp
@@ -749,7 +749,7 @@ Preprocessor::Preprocessor(Client *client, Environment *env)
 
 QByteArray Preprocessor::run(const QString &fileName, const QString &source)
 {
-    return run(fileName, source.toLatin1());
+    return run(fileName, source.toUtf8());
 }
 
 QByteArray Preprocessor::run(const QString &fileName,
diff --git a/src/plugins/cpptools/cpppreprocessor.cpp b/src/plugins/cpptools/cpppreprocessor.cpp
index e5c9c9dbd3..c6dfc5012b 100644
--- a/src/plugins/cpptools/cpppreprocessor.cpp
+++ b/src/plugins/cpptools/cpppreprocessor.cpp
@@ -378,18 +378,6 @@ void CppPreprocessor::stopSkippingBlocks(unsigned offset)
         m_currentDoc->stopSkippingBlocks(offset);
 }
 
-// This is a temporary fix to handle non-ascii characters. This can be removed when the lexer can
-// handle multi-byte characters.
-static QByteArray convertToLatin1(const QByteArray &contents)
-{
-    const char *p = contents.constData();
-    while (char ch = *p++)
-        if (ch & 0x80)
-            return QString::fromUtf8(contents).toLatin1();
-
-    return contents;
-}
-
 void CppPreprocessor::sourceNeeded(unsigned line, const QString &fileName, IncludeType type)
 {
     typedef Document::DiagnosticMessage Message;
@@ -424,7 +412,6 @@ void CppPreprocessor::sourceNeeded(unsigned line, const QString &fileName, Inclu
     unsigned editorRevision = 0;
     QByteArray contents;
     const bool gotFileContents = getFileContents(absoluteFileName, &contents, &editorRevision);
-    contents = convertToLatin1(contents);
     if (m_currentDoc && !gotFileContents) {
         const QString text = QCoreApplication::translate(
             "CppPreprocessor", "%1: Could not get file contents").arg(fileName);
diff --git a/tests/auto/cplusplus/lexer/tst_lexer.cpp b/tests/auto/cplusplus/lexer/tst_lexer.cpp
index 7218f07fe8..0b80f9b2b7 100644
--- a/tests/auto/cplusplus/lexer/tst_lexer.cpp
+++ b/tests/auto/cplusplus/lexer/tst_lexer.cpp
@@ -114,8 +114,7 @@ void tst_SimpleLexer::run(const QByteArray &source,
     QVERIFY(compareFlags);
 
     SimpleLexer lexer;
-    const QList<Token> tokenList = lexer(source, preserveState ? _state : 0,
-                                         /*convertToUtf8=*/ true);
+    const QList<Token> tokenList = lexer(source, preserveState ? _state : 0);
     if (preserveState)
         _state = lexer.state();