C++: change working-copy to work on UTF-8 encoded QByteArrays.

These not only take less space than UTF-16 encoded QStrings, but due to
the caching in the CppEditorSupport also take less time to build.

This patch also fixes a number of possible encoding issues, where files
and constant strings were (falsely) assumed to be UTF-8.

Change-Id: Ib6f91c9a94ebed5b5dfbd4eb2998825c62c72784
Reviewed-by: Nikolai Kosjar <nikolai.kosjar@digia.com>
Reviewed-by: hjk <hjk121@nokiamail.com>

1 files changed, 33 insertions, 0 deletions

diff --git a/src/libs/utils/textfileformat.cpp b/src/libs/utils/textfileformat.cpp
index df657b7803..26dac75df3 100644
--- a/src/libs/utils/textfileformat.cpp
+++ b/src/libs/utils/textfileformat.cpp
@@ -271,6 +271,39 @@ TextFileFormat::ReadResult
     return result;
 }
 
+TextFileFormat::ReadResult TextFileFormat::readFileUTF8(const QString &fileName,
+                                                        QByteArray *plainText, QString *errorString)
+{
+    QByteArray data;
+    try {
+        Utils::FileReader reader;
+        if (!reader.fetch(fileName, errorString))
+            return TextFileFormat::ReadIOError;
+        data = reader.data();
+    } catch (const std::bad_alloc &) {
+        *errorString = QCoreApplication::translate("Utils::TextFileFormat", "Out of memory.");
+        return TextFileFormat::ReadMemoryAllocationError;
+    }
+
+    TextFileFormat format = TextFileFormat::detect(data);
+    if (!format.codec)
+        format.codec = QTextCodec::codecForLocale();
+    if (format.codec->name() == "UTF-8") {
+        if (format.hasUtf8Bom)
+            data.remove(0, 3);
+        *plainText = data;
+        return TextFileFormat::ReadSuccess;
+    }
+
+    QString target;
+    if (!format.decode(data, &target)) {
+        *errorString = QCoreApplication::translate("Utils::TextFileFormat", "An encoding error was encountered.");
+        return TextFileFormat::ReadEncodingError;
+    }
+    *plainText = target.toUtf8();
+    return TextFileFormat::ReadSuccess;
+}
+
 /*!
     \brief Write out a text file.
 */