diff options
Diffstat (limited to 'Include/token.h')
| -rw-r--r-- | Include/token.h | 11 |
1 files changed, 3 insertions, 8 deletions
diff --git a/Include/token.h b/Include/token.h index cd1cd00f09..2d491e6927 100644 --- a/Include/token.h +++ b/Include/token.h @@ -1,3 +1,4 @@ +/* Auto-generated by Tools/scripts/generate_token.py */ /* Token types */ #ifndef Py_LIMITED_API @@ -62,25 +63,19 @@ extern "C" { #define ATEQUAL 50 #define RARROW 51 #define ELLIPSIS 52 -/* Don't forget to update the table _PyParser_TokenNames in tokenizer.c! */ #define OP 53 #define ERRORTOKEN 54 -/* These aren't used by the C tokenizer but are needed for tokenize.py */ -#define COMMENT 55 -#define NL 56 -#define ENCODING 57 #define N_TOKENS 58 +#define NT_OFFSET 256 /* Special definitions for cooperation with parser */ -#define NT_OFFSET 256 - #define ISTERMINAL(x) ((x) < NT_OFFSET) #define ISNONTERMINAL(x) ((x) >= NT_OFFSET) #define ISEOF(x) ((x) == ENDMARKER) -PyAPI_DATA(const char *) _PyParser_TokenNames[]; /* Token names */ +PyAPI_DATA(const char * const) _PyParser_TokenNames[]; /* Token names */ PyAPI_FUNC(int) PyToken_OneChar(int); PyAPI_FUNC(int) PyToken_TwoChars(int, int); PyAPI_FUNC(int) PyToken_ThreeChars(int, int, int); |