From fea1fbc0576bb9d6d1dc84ac8cc3825e0a4e5232 Mon Sep 17 00:00:00 2001
From: Henrik Lievonen <henrik.lievonen@hotmail.com>
Date: Sun, 20 Jun 2021 12:38:00 +0300
Subject: Fix CFamilyLexer preprocessor tokenization errors (#1830)

CFamilyLexer failed to tokenize preprocessor macros when they were
preceded by line break surrounded by spaces. This was the case because
prerpocessor regex rule expected to start at the beginning of the line,
but the space regex rule matched also the whitespace after the line
break. Now the space rule has been refined not to match the line break.
Because of this, the preprocessor regex rule correctly matches
prerpocessor tokens even when they are preceded by white spaces, at the
cost of adding some more tokens in the token stream in some cases. This
change preserves the behavior of invalid preprocessor usage failing to
tokenize.
---
 tests/examplefiles/cpp/example.cpp.output   | 25 ++++++++++++++++++++-----
 tests/examplefiles/cpp/namespace.cpp.output |  3 ++-
 2 files changed, 22 insertions(+), 6 deletions(-)

(limited to 'tests/examplefiles/cpp')

diff --git a/tests/examplefiles/cpp/example.cpp.output b/tests/examplefiles/cpp/example.cpp.output
index aea47bec..0e88b682 100644
--- a/tests/examplefiles/cpp/example.cpp.output
+++ b/tests/examplefiles/cpp/example.cpp.output
@@ -11295,7 +11295,10 @@
 '              ' Text
 '// BEGIN Content of ASBeautifier.cpp.BITFIELD.patch:\n' Comment.Single
 
-'              \n                ' Text
+'              ' Text
+'\n'          Text
+
+'                ' Text
 'unsigned'    Keyword.Type
 ' '           Text
 'int'         Keyword.Type
@@ -11477,7 +11480,10 @@
 
 '\t\t\t\t\t'  Text
 '}'           Punctuation
-'\t\t\t\t\t\t\t\t\t\n\t\t\t\t' Text
+'\t\t\t\t\t\t\t\t\t' Text
+'\n'          Text
+
+'\t\t\t\t'    Text
 '}'           Punctuation
 '\n'          Text
 
@@ -11528,7 +11534,10 @@
 '              ' Text
 '// END Content of ASBeautifier.cpp.BITFIELD.patch:\n' Comment.Single
 
-'                \n                ' Text
+'                ' Text
+'\n'          Text
+
+'                ' Text
 '-'           Operator
 '-'           Operator
 'tabCount'    Name
@@ -11722,7 +11731,10 @@
 
 '                  ' Text
 '}'           Punctuation
-' \n              ' Text
+' '           Text
+'\n'          Text
+
+'              ' Text
 '// BEGIN content of ASBeautifier.cpp.BITFIELD.patch.bz2\n' Comment.Single
 
 '              ' Text
@@ -11925,7 +11937,10 @@
 
 '\t\t\t\t\t'  Text
 '}'           Punctuation
-'\t\t\t\t\t\t\t\t\t\n\t\t\t\t' Text
+'\t\t\t\t\t\t\t\t\t' Text
+'\n'          Text
+
+'\t\t\t\t'    Text
 '}'           Punctuation
 '\n'          Text
 
diff --git a/tests/examplefiles/cpp/namespace.cpp.output b/tests/examplefiles/cpp/namespace.cpp.output
index d929cb03..fbe050d5 100644
--- a/tests/examplefiles/cpp/namespace.cpp.output
+++ b/tests/examplefiles/cpp/namespace.cpp.output
@@ -164,7 +164,8 @@
 
 '}'           Punctuation
 ';'           Punctuation
-'  \n'        Text
+'  '          Text
+'\n'          Text
 
 '}'           Punctuation
 '\n'          Text
-- 
cgit v1.2.1