lupdate: Add support for parsing C++11 raw string literals

[ChangeLog][lupdate] Added support for parsing C++11
raw string literals

Task-number: QTBUG-42736
Change-Id: Ia246c7e7208580182a1e0413bb38bc2c8ff3f61e
Reviewed-by: Andy Shaw <andy.shaw@qt.io>
Reviewed-by: Kai Koehne <kai.koehne@qt.io>
Reviewed-by: Friedemann Kleint <Friedemann.Kleint@qt.io>

3 files changed, 119 insertions, 5 deletions

diff --git a/src/linguist/lupdate/cpp.cpp b/src/linguist/lupdate/cpp.cpp
index 3836bfcfd..eb36b3c80 100644
--- a/src/linguist/lupdate/cpp.cpp
+++ b/src/linguist/lupdate/cpp.cpp
@@ -228,7 +228,7 @@ private:
     enum TokenType {
         Tok_Eof, Tok_class, Tok_friend, Tok_namespace, Tok_using, Tok_return,
         Tok_Q_OBJECT, Tok_Access, Tok_Cancel,
-        Tok_Ident, Tok_String, Tok_Arrow, Tok_Colon, Tok_ColonColon,
+        Tok_Ident, Tok_String, Tok_RawString, Tok_Arrow, Tok_Colon, Tok_ColonColon,
         Tok_Equals, Tok_LeftBracket, Tok_RightBracket, Tok_QuestionMark,
         Tok_LeftBrace, Tok_RightBrace, Tok_LeftParen, Tok_RightParen, Tok_Comma, Tok_Semicolon,
         Tok_Null, Tok_Integer,
@@ -739,6 +739,60 @@ CppParser::TokenType CppParser::getToken()
                 break;
             }
 
+            // a C++11 raw string literal?
+            if (yyCh == '"' && (
+                        yyWord == QLatin1String("R") || yyWord == QLatin1String("LR") || yyWord == QLatin1String("u8R") ||
+                        yyWord == QLatin1String("uR") || yyWord == QLatin1String("UR")
+                        )) {
+                ptr = reinterpret_cast<ushort *>(const_cast<QChar *>(yyWord.unicode()));
+                //get delimiter
+                QString delimiter;
+                for (yyCh = getChar(); yyCh != EOF && yyCh != '('; yyCh = getChar())
+                    delimiter += QLatin1Char(yyCh);
+                if (yyCh != EOF)
+                    yyCh = getChar(); // throw away the opening parentheses
+                bool is_end = false;
+                ushort *ptr_past_end = nullptr;
+                while (yyCh != EOF && !is_end) {
+                    *ptr++ = yyCh;
+                    if (ptr_past_end != nullptr) {
+                        if (delimiter.size() == ptr - ptr_past_end
+                                && memcmp(delimiter.unicode(), ptr_past_end, (ptr - ptr_past_end) * sizeof (ushort)) == 0
+                           ) {
+                            // we've got the delimiter, check if " follows
+                            yyCh = getChar();
+                            if (yyCh == '"')
+                                is_end = true;
+                            else
+                                ptr_past_end = nullptr;
+                            continue;
+                        }
+                    }
+                    if (yyCh == ')') {
+                        ptr_past_end = ptr;
+                        if (delimiter.isEmpty()) {
+                            // no delimiter, check if " follows
+                            yyCh = getChar();
+                            if (yyCh == '"')
+                                is_end = true;
+                            else
+                                ptr_past_end = nullptr;
+                            continue;
+                        }
+                    }
+                    yyCh = getChar();
+                }
+                if (is_end)
+                    yyWord.resize(ptr_past_end - 1 - reinterpret_cast<const ushort *>(yyWord.unicode()));
+                else
+                    yyWord.resize(ptr - reinterpret_cast<const ushort *>(yyWord.unicode()));
+                if (yyCh != '"')
+                    yyMsg() << qPrintable(LU::tr("Unterminated/mismatched C++ Raw string\n"));
+                else
+                    yyCh = getChar();
+                return Tok_RawString;
+            }
+
             return Tok_Ident;
         } else {
             switch (yyCh) {
@@ -1393,10 +1447,13 @@ bool CppParser::matchString(QString *s)
     bool matches = false;
     s->clear();
     forever {
-        if (yyTok != Tok_String)
+        if (yyTok != Tok_String && yyTok != Tok_RawString)
             return matches;
         matches = true;
-        *s += yyWord;
+        if (yyTok == Tok_String)
+            *s += transcode(yyWord);
+        else
+            *s += yyWord;
         s->detach();
         yyTok = getToken();
     }
@@ -1530,7 +1587,7 @@ void CppParser::recordMessage(int line, const QString &context, const QString &t
     const QString &extracomment, const QString &msgid, const TranslatorMessage::ExtraData &extra, bool plural)
 {
     TranslatorMessage msg(
-        transcode(context), transcode(text), transcode(comment), QString(),
+        transcode(context), text, transcode(comment), QString(),
         yyFileName, line, QStringList(),
         TranslatorMessage::Unfinished, plural);
     msg.setExtraComment(transcode(extracomment.simplified()));
@@ -1703,7 +1760,7 @@ void CppParser::handleTrId(bool plural)
     yyTok = getToken();
     if (matchString(&msgid) && !msgid.isEmpty()) {
         plural |= match(Tok_Comma);
-        recordMessage(line, QString(), sourcetext, QString(), extracomment,
+        recordMessage(line, QString(), transcode(sourcetext), QString(), extracomment,
                       msgid, extra, plural);
     }
     sourcetext.clear();
diff --git a/tests/auto/linguist/lupdate/testdata/good/parsecpp2/main.cpp b/tests/auto/linguist/lupdate/testdata/good/parsecpp2/main.cpp
index fec916ce6..dec5232fc 100644
--- a/tests/auto/linguist/lupdate/testdata/good/parsecpp2/main.cpp
+++ b/tests/auto/linguist/lupdate/testdata/good/parsecpp2/main.cpp
@@ -117,3 +117,25 @@ QObject::tr("Hello World");
 
 // QTBUG-11843: complain about missing source in id-based messages
 qtTrId("no_source");
+
+QObject::tr(R"(simple one)" R"delim(enter
+)delim" R"delim(with delimiter )delim inside)delim" u8R"(with quote " inside)");
+
+QLatin1String not_translated(R"(
+                            This is a test string
+)");
+const char valid[] = QT_TRANSLATE_NOOP("global", R"(
+"The time has come," the Walrus said,
+"To talk of many things:
+Of shoes - and ships - and sealing-wax -
+Of cabbages - and kings -
+And why the sea is boiling hot -
+And whether pigs have wings."
+)");
+
+const QString nodelimiter(QObject::tr(R"(
+                            This is a test string
+)"));
+const Qstring withdelimiter = QObject::tr(R"delim(
+This is a test string
+)delim");
diff --git a/tests/auto/linguist/lupdate/testdata/good/parsecpp2/project.ts.result b/tests/auto/linguist/lupdate/testdata/good/parsecpp2/project.ts.result
index d2f5ff29f..0710915de 100644
--- a/tests/auto/linguist/lupdate/testdata/good/parsecpp2/project.ts.result
+++ b/tests/auto/linguist/lupdate/testdata/good/parsecpp2/project.ts.result
@@ -28,6 +28,26 @@
         <source>Hello World</source>
         <translation type="unfinished"></translation>
     </message>
+    <message>
+        <location filename="main.cpp" line="121"/>
+        <source>simple oneenter
+with delimiter )delim insidewith quote &quot; inside</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <location filename="main.cpp" line="136"/>
+        <source>
+                            This is a test string
+</source>
+        <translation type="unfinished"></translation>
+    </message>
+    <message>
+        <location filename="main.cpp" line="139"/>
+        <source>
+This is a test string
+</source>
+        <translation type="unfinished"></translation>
+    </message>
 </context>
 <context>
     <name>TopLevel</name>
@@ -45,4 +65,19 @@
         <translation type="unfinished"></translation>
     </message>
 </context>
+<context>
+    <name>global</name>
+    <message>
+        <location filename="main.cpp" line="127"/>
+        <source>
+&quot;The time has come,&quot; the Walrus said,
+&quot;To talk of many things:
+Of shoes - and ships - and sealing-wax -
+Of cabbages - and kings -
+And why the sea is boiling hot -
+And whether pigs have wings.&quot;
+</source>
+        <translation type="unfinished"></translation>
+    </message>
+</context>
 </TS>