diff options
author | Pablo Galindo Salgado <Pablogsal@gmail.com> | 2021-08-24 17:50:05 +0100 |
---|---|---|
committer | GitHub <noreply@github.com> | 2021-08-24 17:50:05 +0100 |
commit | a24676bedcd332dd7e6fa5521d0449206391d190 (patch) | |
tree | 45ccbb5c30c4debd12c08df5edbdb87e38353348 /Lib/tokenize.py | |
parent | 9ed523159c7ba840dbf403e02498eeae1b5d3ed9 (diff) | |
download | cpython-git-a24676bedcd332dd7e6fa5521d0449206391d190.tar.gz |
Add tests for the C tokenizer and expose it as a private module (GH-27924)
Diffstat (limited to 'Lib/tokenize.py')
-rw-r--r-- | Lib/tokenize.py | 8 |
1 files changed, 8 insertions, 0 deletions
diff --git a/Lib/tokenize.py b/Lib/tokenize.py index 7d7736fe98..0b9e238310 100644 --- a/Lib/tokenize.py +++ b/Lib/tokenize.py @@ -680,5 +680,13 @@ def main(): perror("unexpected error: %s" % err) raise +def _generate_tokens_from_c_tokenizer(source): + """Tokenize a source reading Python code as unicode strings using the internal C tokenizer""" + import _tokenize as c_tokenizer + for info in c_tokenizer.TokenizerIter(source): + tok, type, lineno, end_lineno, col_off, end_col_off, line = info + yield TokenInfo(type, tok, (lineno, col_off), (end_lineno, end_col_off), line) + + if __name__ == "__main__": main() |