Keep the original encoded text around while preprocessing.

12 files changed, 67 insertions, 46 deletions

diff --git a/src/libs/cplusplus/CppDocument.cpp b/src/libs/cplusplus/CppDocument.cpp
index 540f386d53..619b70799f 100644
--- a/src/libs/cplusplus/CppDocument.cpp
+++ b/src/libs/cplusplus/CppDocument.cpp
@@ -356,7 +356,7 @@ void Snapshot::insert(Document::Ptr doc)
         insert(doc->fileName(), doc);
 }
 
-QByteArray Snapshot::preprocessedCode(const QByteArray &source, const QString &fileName) const
+QByteArray Snapshot::preprocessedCode(const QString &source, const QString &fileName) const
 {
     FastPreprocessor pp(*this);
     return pp.run(fileName, source);
diff --git a/src/libs/cplusplus/CppDocument.h b/src/libs/cplusplus/CppDocument.h
index 71d203f88a..33aeec6f26 100644
--- a/src/libs/cplusplus/CppDocument.h
+++ b/src/libs/cplusplus/CppDocument.h
@@ -272,7 +272,7 @@ public:
     Snapshot();
     ~Snapshot();
 
-    QByteArray preprocessedCode(const QByteArray &source,
+    QByteArray preprocessedCode(const QString &source,
                                 const QString &fileName) const;
 
     Document::Ptr documentFromSource(const QByteArray &preprocessedCode,
diff --git a/src/libs/cplusplus/FastPreprocessor.cpp b/src/libs/cplusplus/FastPreprocessor.cpp
index 09afbaf340..4472949bc5 100644
--- a/src/libs/cplusplus/FastPreprocessor.cpp
+++ b/src/libs/cplusplus/FastPreprocessor.cpp
@@ -36,7 +36,7 @@ FastPreprocessor::FastPreprocessor(const Snapshot &snapshot)
       _preproc(this, &_env)
 { }
 
-QByteArray FastPreprocessor::run(QString fileName, const QByteArray &source)
+QByteArray FastPreprocessor::run(QString fileName, const QString &source)
 {
     const QByteArray preprocessed = _preproc(fileName, source);
     return preprocessed;
diff --git a/src/libs/cplusplus/FastPreprocessor.h b/src/libs/cplusplus/FastPreprocessor.h
index fc4a2e6304..0c0e0831dd 100644
--- a/src/libs/cplusplus/FastPreprocessor.h
+++ b/src/libs/cplusplus/FastPreprocessor.h
@@ -51,7 +51,7 @@ class CPLUSPLUS_EXPORT FastPreprocessor: public Client
 public:
     FastPreprocessor(const Snapshot &snapshot);
 
-    QByteArray run(QString fileName, const QByteArray &source);
+    QByteArray run(QString fileName, const QString &source);
 
     // CPlusPlus::Client
     virtual void sourceNeeded(QString &fileName, IncludeType, unsigned)
diff --git a/src/libs/cplusplus/pp-engine.cpp b/src/libs/cplusplus/pp-engine.cpp
index 239db98545..2714e28405 100644
--- a/src/libs/cplusplus/pp-engine.cpp
+++ b/src/libs/cplusplus/pp-engine.cpp
@@ -564,11 +564,21 @@ void Preprocessor::popState()
     _savedStates.removeLast();
 }
 
-QByteArray Preprocessor::operator()(const QString &filename,
+QByteArray Preprocessor::operator()(const QString &fileName, const QString &source)
+{
+    const QString previousOriginalSource = _originalSource;
+    _originalSource = source;
+    const QByteArray bytes = source.toLatin1();
+    const QByteArray preprocessedCode = operator()(fileName, bytes);
+    _originalSource = previousOriginalSource;
+    return preprocessedCode;
+}
+
+QByteArray Preprocessor::operator()(const QString &fileName,
                                     const QByteArray &source)
 {
     QByteArray preprocessed;
-    preprocess(filename, source, &preprocessed);
+    preprocess(fileName, source, &preprocessed);
     return preprocessed;
 }
 
@@ -1098,7 +1108,7 @@ void Preprocessor::processInclude(bool, TokenIterator firstToken,
         const char *beginOfPath = endOfToken(*start);
         const char *endOfPath = startOfToken(*tk);
 
-        QString fn = QString::fromUtf8(beginOfPath, endOfPath - beginOfPath);
+        QString fn = string(beginOfPath, endOfPath - beginOfPath);
         client->sourceNeeded(fn, Client::IncludeGlobal, firstToken->lineno);
 
     } else if (tk->is(T_ANGLE_STRING_LITERAL) || tk->is(T_STRING_LITERAL)) {
@@ -1111,7 +1121,7 @@ void Preprocessor::processInclude(bool, TokenIterator firstToken,
         if (beginOfPath + 1 != endOfPath && ((quote == '"' && endOfPath[-1] == '"') ||
                                               (quote == '<' && endOfPath[-1] == '>'))) {
 
-            QString fn = QString::fromUtf8(beginOfPath + 1, spell.length() - 2);
+            QString fn = string(beginOfPath + 1, spell.length() - 2);
             client->sourceNeeded(fn, Client::IncludeLocal, firstToken->lineno);
         }
     }
@@ -1422,3 +1432,12 @@ bool Preprocessor::isQtReservedWord(const QByteArray &macroId) const
         return true;
     return false;
 }
+
+QString Preprocessor::string(const char *first, int length) const
+{
+    if (_originalSource.isEmpty())
+        return QString::fromUtf8(first, length);
+
+    const int position = first - _source.constData();
+    return _originalSource.mid(position, length);
+}
diff --git a/src/libs/cplusplus/pp-engine.h b/src/libs/cplusplus/pp-engine.h
index 46ea0bc061..9930a59ceb 100644
--- a/src/libs/cplusplus/pp-engine.h
+++ b/src/libs/cplusplus/pp-engine.h
@@ -50,25 +50,23 @@
 #define CPLUSPLUS_PP_ENGINE_H
 
 #include "PreprocessorClient.h"
+#include "pp-macro-expander.h"
 
 #include <Token.h>
 #include <QVector>
 
 namespace CPlusPlus {
-    class Token;
-}
-
-namespace CPlusPlus {
 
 struct Value;
+class Environment;
 
 class CPLUSPLUS_EXPORT Preprocessor
 {
 public:
     Preprocessor(Client *client, Environment *env);
 
-    QByteArray operator()(const QString &filename,
-                          const QByteArray &source);
+    QByteArray operator()(const QString &filename, const QString &source);
+    QByteArray operator()(const QString &filename, const QByteArray &source);
 
     void preprocess(const QString &filename,
                     const QByteArray &source,
@@ -169,6 +167,8 @@ private:
     void out(char ch);
     void out(const char *s);
 
+    QString string(const char *first, int len) const;
+
 private:
     Client *client;
     Environment *env;
@@ -186,6 +186,8 @@ private:
 
     QByteArray *_result;
     bool _markGeneratedTokens;
+
+    QString _originalSource;
 };
 
 } // namespace CPlusPlus
diff --git a/src/plugins/cppeditor/cppeditor.cpp b/src/plugins/cppeditor/cppeditor.cpp
index 59aa1b863c..28c9e1a40e 100644
--- a/src/plugins/cppeditor/cppeditor.cpp
+++ b/src/plugins/cppeditor/cppeditor.cpp
@@ -495,7 +495,7 @@ void CPPEditor::reformatDocument()
     QByteArray source = toPlainText().toUtf8();
 
     Control control;
-    StringLiteral *fileId = control.findOrInsertFileName("<file>");
+    StringLiteral *fileId = control.findOrInsertStringLiteral("<file>");
     TranslationUnit unit(&control, fileId);
     unit.setQtMocRunEnabled(true);
     unit.setSource(source.constData(), source.length());
diff --git a/src/plugins/cpptools/cppmodelmanager.cpp b/src/plugins/cpptools/cppmodelmanager.cpp
index 0e10f9dc3c..89a3e70618 100644
--- a/src/plugins/cpptools/cppmodelmanager.cpp
+++ b/src/plugins/cpptools/cppmodelmanager.cpp
@@ -171,7 +171,7 @@ public:
     CppPreprocessor(QPointer<CppModelManager> modelManager);
     virtual ~CppPreprocessor();
 
-    void setWorkingCopy(const QMap<QString, QByteArray> &workingCopy);
+    void setWorkingCopy(const QMap<QString, QString> &workingCopy);
     void setIncludePaths(const QStringList &includePaths);
     void setFrameworkPaths(const QStringList &frameworkPaths);
     void setProjectFiles(const QStringList &files);
@@ -190,8 +190,8 @@ public: // attributes
 protected:
     CPlusPlus::Document::Ptr switchDocument(CPlusPlus::Document::Ptr doc);
 
-    bool includeFile(const QString &absoluteFilePath, QByteArray *result);
-    QByteArray tryIncludeFile(QString &fileName, IncludeType type);
+    bool includeFile(const QString &absoluteFilePath, QString *result);
+    QString tryIncludeFile(QString &fileName, IncludeType type);
 
     void mergeEnvironment(CPlusPlus::Document::Ptr doc);
 
@@ -212,7 +212,7 @@ private:
     Preprocessor preprocess;
     QStringList m_includePaths;
     QStringList m_systemIncludePaths;
-    QMap<QString, QByteArray> m_workingCopy;
+    QMap<QString, QString> m_workingCopy;
     QStringList m_projectFiles;
     QStringList m_frameworkPaths;
     QSet<QString> m_included;
@@ -233,7 +233,7 @@ CppPreprocessor::CppPreprocessor(QPointer<CppModelManager> modelManager)
 CppPreprocessor::~CppPreprocessor()
 { }
 
-void CppPreprocessor::setWorkingCopy(const QMap<QString, QByteArray> &workingCopy)
+void CppPreprocessor::setWorkingCopy(const QMap<QString, QString> &workingCopy)
 { m_workingCopy = workingCopy; }
 
 void CppPreprocessor::setIncludePaths(const QStringList &includePaths)
@@ -543,13 +543,13 @@ class Process: public std::unary_function<Document::Ptr, void>
 {
     QPointer<CppModelManager> _modelManager;
     Snapshot _snapshot;
-    QMap<QString, QByteArray> _workingCopy;
+    QMap<QString, QString> _workingCopy;
     Document::Ptr _doc;
 
 public:
     Process(QPointer<CppModelManager> modelManager,
             Snapshot snapshot,
-            const QMap<QString, QByteArray> &workingCopy)
+            const QMap<QString, QString> &workingCopy)
         : _modelManager(modelManager),
           _snapshot(snapshot),
           _workingCopy(workingCopy)
@@ -605,7 +605,7 @@ void CppPreprocessor::resetEnvironment()
     m_processed.clear();
 }
 
-bool CppPreprocessor::includeFile(const QString &absoluteFilePath, QByteArray *result)
+bool CppPreprocessor::includeFile(const QString &absoluteFilePath, QString *result)
 {
     if (absoluteFilePath.isEmpty() || m_included.contains(absoluteFilePath)) {
         return true;
@@ -634,11 +634,11 @@ bool CppPreprocessor::includeFile(const QString &absoluteFilePath, QByteArray *r
     return false;
 }
 
-QByteArray CppPreprocessor::tryIncludeFile(QString &fileName, IncludeType type)
+QString CppPreprocessor::tryIncludeFile(QString &fileName, IncludeType type)
 {
     QFileInfo fileInfo(fileName);
     if (fileName == QLatin1String(pp_configuration_file) || fileInfo.isAbsolute()) {
-        QByteArray contents;
+        QString contents;
         includeFile(fileName, &contents);
         return contents;
     }
@@ -649,7 +649,7 @@ QByteArray CppPreprocessor::tryIncludeFile(QString &fileName, IncludeType type)
         path += QLatin1Char('/');
         path += fileName;
         path = QDir::cleanPath(path);
-        QByteArray contents;
+        QString contents;
         if (includeFile(path, &contents)) {
             fileName = path;
             return contents;
@@ -661,7 +661,7 @@ QByteArray CppPreprocessor::tryIncludeFile(QString &fileName, IncludeType type)
         path += QLatin1Char('/');
         path += fileName;
         path = QDir::cleanPath(path);
-        QByteArray contents;
+        QString contents;
         if (includeFile(path, &contents)) {
             fileName = path;
             return contents;
@@ -674,7 +674,7 @@ QByteArray CppPreprocessor::tryIncludeFile(QString &fileName, IncludeType type)
         path += QLatin1Char('/');
         path += fileName;
         path = QDir::cleanPath(path);
-        QByteArray contents;
+        QString contents;
         if (includeFile(path, &contents)) {
             fileName = path;
             return contents;
@@ -693,7 +693,7 @@ QByteArray CppPreprocessor::tryIncludeFile(QString &fileName, IncludeType type)
             path += QLatin1String(".framework/Headers/");
             path += name;
             path = QDir::cleanPath(path);
-            QByteArray contents;
+            QString contents;
             if (includeFile(path, &contents)) {
                 fileName = path;
                 return contents;
@@ -708,14 +708,14 @@ QByteArray CppPreprocessor::tryIncludeFile(QString &fileName, IncludeType type)
     foreach (const QString &projectFile, m_projectFiles) {
         if (projectFile.endsWith(path)) {
             fileName = projectFile;
-            QByteArray contents;
+            QString contents;
             includeFile(fileName, &contents);
             return contents;
         }
     }
 
     //qDebug() << "**** file" << fileName << "not found!";
-    return QByteArray();
+    return QString();
 }
 
 void CppPreprocessor::macroAdded(const Macro &macro)
@@ -790,7 +790,7 @@ void CppPreprocessor::sourceNeeded(QString &fileName, IncludeType type,
     if (fileName.isEmpty())
         return;
 
-    QByteArray contents = tryIncludeFile(fileName, type);
+    QString contents = tryIncludeFile(fileName, type);
 
     if (m_currentDoc) {
         m_currentDoc->addIncludeFile(fileName, line);
@@ -822,7 +822,7 @@ void CppPreprocessor::sourceNeeded(QString &fileName, IncludeType type,
 
     Document::Ptr previousDoc = switchDocument(doc);
 
-    const QByteArray preprocessedCode = preprocess(fileName.toUtf8(), contents);
+    const QByteArray preprocessedCode = preprocess(fileName, contents);
 
     doc->setSource(preprocessedCode);
     doc->tokenize();
@@ -984,9 +984,9 @@ void CppModelManager::removeEditorSupport(AbstractEditorSupport *editorSupport)
     m_addtionalEditorSupport.remove(editorSupport);
 }
 
-QMap<QString, QByteArray> CppModelManager::buildWorkingCopyList()
+QMap<QString, QString> CppModelManager::buildWorkingCopyList()
 {
-    QMap<QString, QByteArray> workingCopy;
+    QMap<QString, QString> workingCopy;
     QMapIterator<TextEditor::ITextEditor *, CppEditorSupport *> it(m_editorSupport);
     while (it.hasNext()) {
         it.next();
@@ -1041,7 +1041,7 @@ void CppModelManager::updateProjectInfo(const ProjectInfo &pinfo)
 QFuture<void> CppModelManager::refreshSourceFiles(const QStringList &sourceFiles)
 {
     if (! sourceFiles.isEmpty() && qgetenv("QTCREATOR_NO_CODE_INDEXER").isNull()) {
-        const QMap<QString, QByteArray> workingCopy = buildWorkingCopyList();
+        const QMap<QString, QString> workingCopy = buildWorkingCopyList();
 
         CppPreprocessor *preproc = new CppPreprocessor(this);
         preproc->setProjectFiles(projectFiles());
diff --git a/src/plugins/cpptools/cppmodelmanager.h b/src/plugins/cpptools/cppmodelmanager.h
index 2ef684878a..cd00f3937d 100644
--- a/src/plugins/cpptools/cppmodelmanager.h
+++ b/src/plugins/cpptools/cppmodelmanager.h
@@ -89,7 +89,7 @@ public:
     CppEditorSupport *editorSupport(TextEditor::ITextEditor *editor) const
     { return m_editorSupport.value(editor); }
 
-    QMap<QString, QByteArray> buildWorkingCopyList();
+    QMap<QString, QString> buildWorkingCopyList();
 
     void emitDocumentUpdated(CPlusPlus::Document::Ptr doc);
 
diff --git a/src/plugins/cpptools/cppsemanticsearch.cpp b/src/plugins/cpptools/cppsemanticsearch.cpp
index 2782c40fac..dc8e565ab0 100644
--- a/src/plugins/cpptools/cppsemanticsearch.cpp
+++ b/src/plugins/cpptools/cppsemanticsearch.cpp
@@ -262,7 +262,7 @@ SemanticSearch *SearchFunctionCallFactory::create(QFutureInterface<Core::Utils::
 
 static void semanticSearch_helper(QFutureInterface<Core::Utils::FileSearchResult> &future,
                                   QPointer<CppModelManager> modelManager,
-                                  QMap<QString, QByteArray> wl,
+                                  QMap<QString, QString> wl,
                                   SemanticSearchFactory::Ptr factory)
 {
     const Snapshot snapshot = modelManager->snapshot();
@@ -277,14 +277,14 @@ static void semanticSearch_helper(QFutureInterface<Core::Utils::FileSearchResult
         QByteArray source;
 
         if (wl.contains(fileName))
-            source = wl.value(fileName);
+            source = snapshot.preprocessedCode(wl.value(fileName), fileName);
         else {
             QFile file(fileName);
             if (! file.open(QFile::ReadOnly))
                 continue;
 
             const QString contents = QTextStream(&file).readAll(); // ### FIXME
-            source = snapshot.preprocessedCode(contents.toUtf8(), fileName);
+            source = snapshot.preprocessedCode(contents, fileName);
         }
 
         Document::Ptr newDoc = snapshot.documentFromSource(source, fileName);
diff --git a/src/plugins/cpptools/cpptoolseditorsupport.cpp b/src/plugins/cpptools/cpptoolseditorsupport.cpp
index 8352b40806..17dbde9521 100644
--- a/src/plugins/cpptools/cpptoolseditorsupport.cpp
+++ b/src/plugins/cpptools/cpptoolseditorsupport.cpp
@@ -348,12 +348,12 @@ void CppEditorSupport::setTextEditor(TextEditor::ITextEditor *textEditor)
     updateDocument();
 }
 
-QByteArray CppEditorSupport::contents()
+QString CppEditorSupport::contents()
 {
     if (! _textEditor)
-        return QByteArray();
+        return QString();
     else if (! _cachedContents.isEmpty())
-        _cachedContents = _textEditor->contents().toUtf8();
+        _cachedContents = _textEditor->contents();
 
     return _cachedContents;
 }
@@ -403,7 +403,7 @@ void CppEditorSupport::checkDocumentNow()
             qobject_cast<TextEditor::BaseTextEditor *>(_textEditor->widget());
 
     Snapshot snapshot = _modelManager->snapshot();
-    const QByteArray plainText = contents();
+    const QString plainText = contents();
     const QString fileName = _textEditor->file()->fileName();
     const QByteArray preprocessedCode = snapshot.preprocessedCode(plainText, fileName);
 
diff --git a/src/plugins/cpptools/cpptoolseditorsupport.h b/src/plugins/cpptools/cpptoolseditorsupport.h
index 6c9a0da6aa..0726ee6bbe 100644
--- a/src/plugins/cpptools/cpptoolseditorsupport.h
+++ b/src/plugins/cpptools/cpptoolseditorsupport.h
@@ -112,7 +112,7 @@ public:
     int updateDocumentInterval() const;
     void setUpdateDocumentInterval(int updateDocumentInterval);
 
-    QByteArray contents(); // UTF-8 encoded
+    QString contents();
 
 Q_SIGNALS:
     void contentsChanged();
@@ -132,7 +132,7 @@ private:
     QTimer *_updateDocumentTimer;
     int _updateDocumentInterval;
     QFuture<void> _documentParser;
-    QByteArray _cachedContents;
+    QString _cachedContents;
 
     QTimer *_quickFixTimer;
     TextEditor::ITextMark *_quickFixMark;