From 495da292255b92dd73758fdd0e4c7d27d82b1e57 Mon Sep 17 00:00:00 2001 From: Guido van Rossum Date: Thu, 7 Mar 2019 12:38:08 -0800 Subject: bpo-35975: Support parsing earlier minor versions of Python 3 (GH-12086) This adds a `feature_version` flag to `ast.parse()` (documented) and `compile()` (hidden) that allow tweaking the parser to support older versions of the grammar. In particular if `feature_version` is 5 or 6, the hacks for the `async` and `await` keyword from PEP 492 are reinstated. (For 7 or higher, these are unconditionally treated as keywords, but they are still special tokens rather than `NAME` tokens that the parser driver recognizes.) https://bugs.python.org/issue35975 --- Lib/token.py | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) (limited to 'Lib/token.py') diff --git a/Lib/token.py b/Lib/token.py index 9bf80a5950..493bf04265 100644 --- a/Lib/token.py +++ b/Lib/token.py @@ -58,14 +58,16 @@ RARROW = 51 ELLIPSIS = 52 COLONEQUAL = 53 OP = 54 -TYPE_IGNORE = 55 -TYPE_COMMENT = 56 +AWAIT = 55 +ASYNC = 56 +TYPE_IGNORE = 57 +TYPE_COMMENT = 58 # These aren't used by the C tokenizer but are needed for tokenize.py -ERRORTOKEN = 57 -COMMENT = 58 -NL = 59 -ENCODING = 60 -N_TOKENS = 61 +ERRORTOKEN = 59 +COMMENT = 60 +NL = 61 +ENCODING = 62 +N_TOKENS = 63 # Special definitions for cooperation with parser NT_OFFSET = 256 -- cgit v1.2.1