bpo-30455: Generate all token related code and docs from Grammar/Tokens. (GH-10370)

"Include/token.h", "Lib/token.py" (containing now some data moved from
"Lib/tokenize.py") and new files "Parser/token.c" (containing the code
moved from "Parser/tokenizer.c") and "Doc/library/token-list.inc" (included
in "Doc/library/token.rst") are now generated from "Grammar/Tokens" by
"Tools/scripts/generate_token.py". The script overwrites files only if
needed and can be used on the read-only sources tree.

"Lib/symbol.py" is now generated by "Tools/scripts/generate_symbol_py.py"
instead of been executable itself.

Added new make targets "regen-token" and "regen-symbol" which are now
dependencies of "regen-all".

The documentation contains now strings for operators and punctuation tokens.

2 files changed, 233 insertions, 237 deletions

diff --git a/Parser/token.c b/Parser/token.c
new file mode 100644
index 0000000000..35519aa4b6
--- /dev/null
+++ b/Parser/token.c
@@ -0,0 +1,233 @@
+/* Auto-generated by Tools/scripts/generate_token.py */
+
+#include "Python.h"
+#include "token.h"
+
+/* Token names */
+
+const char * const _PyParser_TokenNames[] = {
+    "ENDMARKER",
+    "NAME",
+    "NUMBER",
+    "STRING",
+    "NEWLINE",
+    "INDENT",
+    "DEDENT",
+    "LPAR",
+    "RPAR",
+    "LSQB",
+    "RSQB",
+    "COLON",
+    "COMMA",
+    "SEMI",
+    "PLUS",
+    "MINUS",
+    "STAR",
+    "SLASH",
+    "VBAR",
+    "AMPER",
+    "LESS",
+    "GREATER",
+    "EQUAL",
+    "DOT",
+    "PERCENT",
+    "LBRACE",
+    "RBRACE",
+    "EQEQUAL",
+    "NOTEQUAL",
+    "LESSEQUAL",
+    "GREATEREQUAL",
+    "TILDE",
+    "CIRCUMFLEX",
+    "LEFTSHIFT",
+    "RIGHTSHIFT",
+    "DOUBLESTAR",
+    "PLUSEQUAL",
+    "MINEQUAL",
+    "STAREQUAL",
+    "SLASHEQUAL",
+    "PERCENTEQUAL",
+    "AMPEREQUAL",
+    "VBAREQUAL",
+    "CIRCUMFLEXEQUAL",
+    "LEFTSHIFTEQUAL",
+    "RIGHTSHIFTEQUAL",
+    "DOUBLESTAREQUAL",
+    "DOUBLESLASH",
+    "DOUBLESLASHEQUAL",
+    "AT",
+    "ATEQUAL",
+    "RARROW",
+    "ELLIPSIS",
+    "OP",
+    "<ERRORTOKEN>",
+    "<COMMENT>",
+    "<NL>",
+    "<ENCODING>",
+    "<N_TOKENS>",
+};
+
+/* Return the token corresponding to a single character */
+
+int
+PyToken_OneChar(int c1)
+{
+    switch (c1) {
+    case '%': return PERCENT;
+    case '&': return AMPER;
+    case '(': return LPAR;
+    case ')': return RPAR;
+    case '*': return STAR;
+    case '+': return PLUS;
+    case ',': return COMMA;
+    case '-': return MINUS;
+    case '.': return DOT;
+    case '/': return SLASH;
+    case ':': return COLON;
+    case ';': return SEMI;
+    case '<': return LESS;
+    case '=': return EQUAL;
+    case '>': return GREATER;
+    case '@': return AT;
+    case '[': return LSQB;
+    case ']': return RSQB;
+    case '^': return CIRCUMFLEX;
+    case '{': return LBRACE;
+    case '|': return VBAR;
+    case '}': return RBRACE;
+    case '~': return TILDE;
+    }
+    return OP;
+}
+
+int
+PyToken_TwoChars(int c1, int c2)
+{
+    switch (c1) {
+    case '!':
+        switch (c2) {
+        case '=': return NOTEQUAL;
+        }
+        break;
+    case '%':
+        switch (c2) {
+        case '=': return PERCENTEQUAL;
+        }
+        break;
+    case '&':
+        switch (c2) {
+        case '=': return AMPEREQUAL;
+        }
+        break;
+    case '*':
+        switch (c2) {
+        case '*': return DOUBLESTAR;
+        case '=': return STAREQUAL;
+        }
+        break;
+    case '+':
+        switch (c2) {
+        case '=': return PLUSEQUAL;
+        }
+        break;
+    case '-':
+        switch (c2) {
+        case '=': return MINEQUAL;
+        case '>': return RARROW;
+        }
+        break;
+    case '/':
+        switch (c2) {
+        case '/': return DOUBLESLASH;
+        case '=': return SLASHEQUAL;
+        }
+        break;
+    case '<':
+        switch (c2) {
+        case '<': return LEFTSHIFT;
+        case '=': return LESSEQUAL;
+        case '>': return NOTEQUAL;
+        }
+        break;
+    case '=':
+        switch (c2) {
+        case '=': return EQEQUAL;
+        }
+        break;
+    case '>':
+        switch (c2) {
+        case '=': return GREATEREQUAL;
+        case '>': return RIGHTSHIFT;
+        }
+        break;
+    case '@':
+        switch (c2) {
+        case '=': return ATEQUAL;
+        }
+        break;
+    case '^':
+        switch (c2) {
+        case '=': return CIRCUMFLEXEQUAL;
+        }
+        break;
+    case '|':
+        switch (c2) {
+        case '=': return VBAREQUAL;
+        }
+        break;
+    }
+    return OP;
+}
+
+int
+PyToken_ThreeChars(int c1, int c2, int c3)
+{
+    switch (c1) {
+    case '*':
+        switch (c2) {
+        case '*':
+            switch (c3) {
+            case '=': return DOUBLESTAREQUAL;
+            }
+            break;
+        }
+        break;
+    case '.':
+        switch (c2) {
+        case '.':
+            switch (c3) {
+            case '.': return ELLIPSIS;
+            }
+            break;
+        }
+        break;
+    case '/':
+        switch (c2) {
+        case '/':
+            switch (c3) {
+            case '=': return DOUBLESLASHEQUAL;
+            }
+            break;
+        }
+        break;
+    case '<':
+        switch (c2) {
+        case '<':
+            switch (c3) {
+            case '=': return LEFTSHIFTEQUAL;
+            }
+            break;
+        }
+        break;
+    case '>':
+        switch (c2) {
+        case '>':
+            switch (c3) {
+            case '=': return RIGHTSHIFTEQUAL;
+            }
+            break;
+        }
+        break;
+    }
+    return OP;
+}
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
index c246ee204c..0e6c1a85e0 100644
--- a/Parser/tokenizer.c
+++ b/Parser/tokenizer.c
@@ -48,72 +48,6 @@ static int tok_nextc(struct tok_state *tok);
 static void tok_backup(struct tok_state *tok, int c);
 
 
-/* Token names */
-
-const char *_PyParser_TokenNames[] = {
-    "ENDMARKER",
-    "NAME",
-    "NUMBER",
-    "STRING",
-    "NEWLINE",
-    "INDENT",
-    "DEDENT",
-    "LPAR",
-    "RPAR",
-    "LSQB",
-    "RSQB",
-    "COLON",
-    "COMMA",
-    "SEMI",
-    "PLUS",
-    "MINUS",
-    "STAR",
-    "SLASH",
-    "VBAR",
-    "AMPER",
-    "LESS",
-    "GREATER",
-    "EQUAL",
-    "DOT",
-    "PERCENT",
-    "LBRACE",
-    "RBRACE",
-    "EQEQUAL",
-    "NOTEQUAL",
-    "LESSEQUAL",
-    "GREATEREQUAL",
-    "TILDE",
-    "CIRCUMFLEX",
-    "LEFTSHIFT",
-    "RIGHTSHIFT",
-    "DOUBLESTAR",
-    "PLUSEQUAL",
-    "MINEQUAL",
-    "STAREQUAL",
-    "SLASHEQUAL",
-    "PERCENTEQUAL",
-    "AMPEREQUAL",
-    "VBAREQUAL",
-    "CIRCUMFLEXEQUAL",
-    "LEFTSHIFTEQUAL",
-    "RIGHTSHIFTEQUAL",
-    "DOUBLESTAREQUAL",
-    "DOUBLESLASH",
-    "DOUBLESLASHEQUAL",
-    "AT",
-    "ATEQUAL",
-    "RARROW",
-    "ELLIPSIS",
-    /* This table must match the #defines in token.h! */
-    "OP",
-    "<ERRORTOKEN>",
-    "COMMENT",
-    "NL",
-    "ENCODING",
-    "<N_TOKENS>"
-};
-
-
 /* Create and initialize a new tok_state structure */
 
 static struct tok_state *
@@ -1114,177 +1048,6 @@ tok_backup(struct tok_state *tok, int c)
 }
 
 
-/* Return the token corresponding to a single character */
-
-int
-PyToken_OneChar(int c)
-{
-    switch (c) {
-    case '(':           return LPAR;
-    case ')':           return RPAR;
-    case '[':           return LSQB;
-    case ']':           return RSQB;
-    case ':':           return COLON;
-    case ',':           return COMMA;
-    case ';':           return SEMI;
-    case '+':           return PLUS;
-    case '-':           return MINUS;
-    case '*':           return STAR;
-    case '/':           return SLASH;
-    case '|':           return VBAR;
-    case '&':           return AMPER;
-    case '<':           return LESS;
-    case '>':           return GREATER;
-    case '=':           return EQUAL;
-    case '.':           return DOT;
-    case '%':           return PERCENT;
-    case '{':           return LBRACE;
-    case '}':           return RBRACE;
-    case '^':           return CIRCUMFLEX;
-    case '~':           return TILDE;
-    case '@':           return AT;
-    default:            return OP;
-    }
-}
-
-
-int
-PyToken_TwoChars(int c1, int c2)
-{
-    switch (c1) {
-    case '=':
-        switch (c2) {
-        case '=':               return EQEQUAL;
-        }
-        break;
-    case '!':
-        switch (c2) {
-        case '=':               return NOTEQUAL;
-        }
-        break;
-    case '<':
-        switch (c2) {
-        case '>':               return NOTEQUAL;
-        case '=':               return LESSEQUAL;
-        case '<':               return LEFTSHIFT;
-        }
-        break;
-    case '>':
-        switch (c2) {
-        case '=':               return GREATEREQUAL;
-        case '>':               return RIGHTSHIFT;
-        }
-        break;
-    case '+':
-        switch (c2) {
-        case '=':               return PLUSEQUAL;
-        }
-        break;
-    case '-':
-        switch (c2) {
-        case '=':               return MINEQUAL;
-        case '>':               return RARROW;
-        }
-        break;
-    case '*':
-        switch (c2) {
-        case '*':               return DOUBLESTAR;
-        case '=':               return STAREQUAL;
-        }
-        break;
-    case '/':
-        switch (c2) {
-        case '/':               return DOUBLESLASH;
-        case '=':               return SLASHEQUAL;
-        }
-        break;
-    case '|':
-        switch (c2) {
-        case '=':               return VBAREQUAL;
-        }
-        break;
-    case '%':
-        switch (c2) {
-        case '=':               return PERCENTEQUAL;
-        }
-        break;
-    case '&':
-        switch (c2) {
-        case '=':               return AMPEREQUAL;
-        }
-        break;
-    case '^':
-        switch (c2) {
-        case '=':               return CIRCUMFLEXEQUAL;
-        }
-        break;
-    case '@':
-        switch (c2) {
-        case '=':               return ATEQUAL;
-        }
-        break;
-    }
-    return OP;
-}
-
-int
-PyToken_ThreeChars(int c1, int c2, int c3)
-{
-    switch (c1) {
-    case '<':
-        switch (c2) {
-        case '<':
-            switch (c3) {
-            case '=':
-                return LEFTSHIFTEQUAL;
-            }
-            break;
-        }
-        break;
-    case '>':
-        switch (c2) {
-        case '>':
-            switch (c3) {
-            case '=':
-                return RIGHTSHIFTEQUAL;
-            }
-            break;
-        }
-        break;
-    case '*':
-        switch (c2) {
-        case '*':
-            switch (c3) {
-            case '=':
-                return DOUBLESTAREQUAL;
-            }
-            break;
-        }
-        break;
-    case '/':
-        switch (c2) {
-        case '/':
-            switch (c3) {
-            case '=':
-                return DOUBLESLASHEQUAL;
-            }
-            break;
-        }
-        break;
-    case '.':
-        switch (c2) {
-        case '.':
-            switch (c3) {
-            case '.':
-                return ELLIPSIS;
-            }
-            break;
-        }
-        break;
-    }
-    return OP;
-}
-
 static int
 syntaxerror(struct tok_state *tok, const char *format, ...)
 {