diff options
Diffstat (limited to 'pycparser/c_lexer.py')
| -rw-r--r-- | pycparser/c_lexer.py | 36 |
1 files changed, 36 insertions, 0 deletions
diff --git a/pycparser/c_lexer.py b/pycparser/c_lexer.py index 8fdd3d7..f861b5c 100644 --- a/pycparser/c_lexer.py +++ b/pycparser/c_lexer.py @@ -139,10 +139,16 @@ class CLexer(object): 'FLOAT_CONST', 'HEX_FLOAT_CONST', 'CHAR_CONST', 'WCHAR_CONST', + 'U8CHAR_CONST', + 'U16CHAR_CONST', + 'U32CHAR_CONST', # String literals 'STRING_LITERAL', 'WSTRING_LITERAL', + 'U8STRING_LITERAL', + 'U16STRING_LITERAL', + 'U32STRING_LITERAL', # Operators 'PLUS', 'MINUS', 'TIMES', 'DIVIDE', 'MOD', @@ -244,6 +250,9 @@ class CLexer(object): cconst_char = r"""([^'\\\n]|"""+escape_sequence+')' char_const = "'"+cconst_char+"'" wchar_const = 'L'+char_const + u8char_const = 'u8'+char_const + u16char_const = 'u'+char_const + u32char_const = 'U'+char_const multicharacter_constant = "'"+cconst_char+"{2,4}'" unmatched_quote = "('"+cconst_char+"*\\n)|('"+cconst_char+"*$)" bad_char_const = r"""('"""+cconst_char+"""[^'\n]+')|('')|('"""+bad_escape+r"""[^'\n]*')""" @@ -252,6 +261,9 @@ class CLexer(object): string_char = r"""([^"\\\n]|"""+escape_sequence_start_in_string+')' string_literal = '"'+string_char+'*"' wstring_literal = 'L'+string_literal + u8string_literal = 'u8'+string_literal + u16string_literal = 'u'+string_literal + u32string_literal = 'U'+string_literal bad_string_literal = '"'+string_char+'*'+bad_escape+string_char+'*"' # floating constants (K&R2: A.2.5.3) @@ -486,6 +498,18 @@ class CLexer(object): def t_WCHAR_CONST(self, t): return t + @TOKEN(u8char_const) + def t_U8CHAR_CONST(self, t): + return t + + @TOKEN(u16char_const) + def t_U16CHAR_CONST(self, t): + return t + + @TOKEN(u32char_const) + def t_U32CHAR_CONST(self, t): + return t + @TOKEN(unmatched_quote) def t_UNMATCHED_QUOTE(self, t): msg = "Unmatched '" @@ -500,6 +524,18 @@ class CLexer(object): def t_WSTRING_LITERAL(self, t): return t + @TOKEN(u8string_literal) + def t_U8STRING_LITERAL(self, t): + return t + + @TOKEN(u16string_literal) + def t_U16STRING_LITERAL(self, t): + return t + + @TOKEN(u32string_literal) + def t_U32STRING_LITERAL(self, t): + return t + # unmatched string literals are caught by the preprocessor @TOKEN(bad_string_literal) |