summaryrefslogtreecommitdiff
path: root/Lib/tokenize.py
diff options
context:
space:
mode:
authorPablo Galindo Salgado <Pablogsal@gmail.com>2021-08-24 17:50:05 +0100
committerGitHub <noreply@github.com>2021-08-24 17:50:05 +0100
commita24676bedcd332dd7e6fa5521d0449206391d190 (patch)
tree45ccbb5c30c4debd12c08df5edbdb87e38353348 /Lib/tokenize.py
parent9ed523159c7ba840dbf403e02498eeae1b5d3ed9 (diff)
downloadcpython-git-a24676bedcd332dd7e6fa5521d0449206391d190.tar.gz
Add tests for the C tokenizer and expose it as a private module (GH-27924)
Diffstat (limited to 'Lib/tokenize.py')
-rw-r--r--Lib/tokenize.py8
1 files changed, 8 insertions, 0 deletions
diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 7d7736fe98..0b9e238310 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -680,5 +680,13 @@ def main():
perror("unexpected error: %s" % err)
raise
+def _generate_tokens_from_c_tokenizer(source):
+ """Tokenize a source reading Python code as unicode strings using the internal C tokenizer"""
+ import _tokenize as c_tokenizer
+ for info in c_tokenizer.TokenizerIter(source):
+ tok, type, lineno, end_lineno, col_off, end_col_off, line = info
+ yield TokenInfo(type, tok, (lineno, col_off), (end_lineno, end_col_off), line)
+
+
if __name__ == "__main__":
main()