C++: change working-copy to work on UTF-8 encoded QByteArrays.

These not only take less space than UTF-16 encoded QStrings, but due to
the caching in the CppEditorSupport also take less time to build.

This patch also fixes a number of possible encoding issues, where files
and constant strings were (falsely) assumed to be UTF-8.

Change-Id: Ib6f91c9a94ebed5b5dfbd4eb2998825c62c72784
Reviewed-by: Nikolai Kosjar <nikolai.kosjar@digia.com>
Reviewed-by: hjk <hjk121@nokiamail.com>

6 files changed, 40 insertions, 4 deletions

diff --git a/src/libs/cplusplus/CppDocument.cpp b/src/libs/cplusplus/CppDocument.cpp
index fd299f7ca0..6ba998a6e4 100644
--- a/src/libs/cplusplus/CppDocument.cpp
+++ b/src/libs/cplusplus/CppDocument.cpp
@@ -736,7 +736,8 @@ void Snapshot::insert(Document::Ptr doc)
         _documents.insert(doc->fileName(), doc);
 }
 
-Document::Ptr Snapshot::preprocessedDocument(const QString &source, const QString &fileName) const
+Document::Ptr Snapshot::preprocessedDocument(const QByteArray &source,
+                                             const QString &fileName) const
 {
     Document::Ptr newDoc = Document::create(fileName);
     if (Document::Ptr thisDocument = document(fileName)) {
diff --git a/src/libs/cplusplus/CppDocument.h b/src/libs/cplusplus/CppDocument.h
index 980535bb4d..071f9485e8 100644
--- a/src/libs/cplusplus/CppDocument.h
+++ b/src/libs/cplusplus/CppDocument.h
@@ -398,7 +398,7 @@ public:
 
     Snapshot simplified(Document::Ptr doc) const;
 
-    Document::Ptr preprocessedDocument(const QString &source,
+    Document::Ptr preprocessedDocument(const QByteArray &source,
                                        const QString &fileName) const;
 
     Document::Ptr documentFromSource(const QByteArray &preprocessedDocument,
diff --git a/src/libs/cplusplus/FastPreprocessor.cpp b/src/libs/cplusplus/FastPreprocessor.cpp
index 70f271e13b..95d7f1cc9f 100644
--- a/src/libs/cplusplus/FastPreprocessor.cpp
+++ b/src/libs/cplusplus/FastPreprocessor.cpp
@@ -41,7 +41,7 @@ FastPreprocessor::FastPreprocessor(const Snapshot &snapshot)
     , _preproc(this, &_env)
 { }
 
-QByteArray FastPreprocessor::run(Document::Ptr newDoc, const QString &source)
+QByteArray FastPreprocessor::run(Document::Ptr newDoc, const QByteArray &source)
 {
     std::swap(newDoc, _currentDoc);
     const QString fileName = _currentDoc->fileName();
diff --git a/src/libs/cplusplus/FastPreprocessor.h b/src/libs/cplusplus/FastPreprocessor.h
index 8e1a818597..ec585e28ca 100644
--- a/src/libs/cplusplus/FastPreprocessor.h
+++ b/src/libs/cplusplus/FastPreprocessor.h
@@ -54,7 +54,7 @@ class CPLUSPLUS_EXPORT FastPreprocessor: public Client
 public:
     FastPreprocessor(const Snapshot &snapshot);
 
-    QByteArray run(Document::Ptr newDoc, const QString &source);
+    QByteArray run(Document::Ptr newDoc, const QByteArray &source);
 
     // CPlusPlus::Client
     virtual void sourceNeeded(unsigned line, const QString &fileName, IncludeType mode);
diff --git a/src/libs/utils/textfileformat.cpp b/src/libs/utils/textfileformat.cpp
index df657b7803..26dac75df3 100644
--- a/src/libs/utils/textfileformat.cpp
+++ b/src/libs/utils/textfileformat.cpp
@@ -271,6 +271,39 @@ TextFileFormat::ReadResult
     return result;
 }
 
+TextFileFormat::ReadResult TextFileFormat::readFileUTF8(const QString &fileName,
+                                                        QByteArray *plainText, QString *errorString)
+{
+    QByteArray data;
+    try {
+        Utils::FileReader reader;
+        if (!reader.fetch(fileName, errorString))
+            return TextFileFormat::ReadIOError;
+        data = reader.data();
+    } catch (const std::bad_alloc &) {
+        *errorString = QCoreApplication::translate("Utils::TextFileFormat", "Out of memory.");
+        return TextFileFormat::ReadMemoryAllocationError;
+    }
+
+    TextFileFormat format = TextFileFormat::detect(data);
+    if (!format.codec)
+        format.codec = QTextCodec::codecForLocale();
+    if (format.codec->name() == "UTF-8") {
+        if (format.hasUtf8Bom)
+            data.remove(0, 3);
+        *plainText = data;
+        return TextFileFormat::ReadSuccess;
+    }
+
+    QString target;
+    if (!format.decode(data, &target)) {
+        *errorString = QCoreApplication::translate("Utils::TextFileFormat", "An encoding error was encountered.");
+        return TextFileFormat::ReadEncodingError;
+    }
+    *plainText = target.toUtf8();
+    return TextFileFormat::ReadSuccess;
+}
+
 /*!
     \brief Write out a text file.
 */
diff --git a/src/libs/utils/textfileformat.h b/src/libs/utils/textfileformat.h
index 9f3825b51f..91121c9a9d 100644
--- a/src/libs/utils/textfileformat.h
+++ b/src/libs/utils/textfileformat.h
@@ -76,6 +76,8 @@ public:
     static ReadResult readFile(const QString &fileName, const QTextCodec *defaultCodec,
                                QString *plainText, TextFileFormat *format, QString *errorString,
                                QByteArray *decodingErrorSample = 0);
+    static ReadResult readFileUTF8(const QString &fileName, QByteArray *plainText,
+                                   QString *errorString);
 
     bool writeFile(const QString &fileName, QString plainText, QString *errorString) const;