Add tests for the C tokenizer and expose it as a private module (GH-27924)

1 files changed, 8 insertions, 0 deletions

diff --git a/Lib/tokenize.py b/Lib/tokenize.py
index 7d7736fe98..0b9e238310 100644
--- a/Lib/tokenize.py
+++ b/Lib/tokenize.py
@@ -680,5 +680,13 @@ def main():
         perror("unexpected error: %s" % err)
         raise
 
+def _generate_tokens_from_c_tokenizer(source):
+    """Tokenize a source reading Python code as unicode strings using the internal C tokenizer"""
+    import _tokenize as c_tokenizer
+    for info in c_tokenizer.TokenizerIter(source):
+        tok, type, lineno, end_lineno, col_off, end_col_off, line = info
+        yield TokenInfo(type, tok, (lineno, col_off), (end_lineno, end_col_off), line)
+
+
 if __name__ == "__main__":
     main()