a tool for exploring abstract syntax trees generated by ``tokenize.generate_tokens()``

git-svn-id: http://svn.code.sf.net/p/docutils/code/trunk@999 929543f6-e4f2-0310-98a6-ba3bd3dd1d04

1 files changed, 34 insertions, 0 deletions

diff --git a/docutils/test/test_readers/test_python/showtok b/docutils/test/test_readers/test_python/showtok
new file mode 100755
index 000000000..b844b6bc0
--- /dev/null
+++ b/docutils/test/test_readers/test_python/showtok
@@ -0,0 +1,34 @@
+#! /usr/bin/env python
+
+
+"""
+This is a tool for exploring token lists generated by
+``tokenize.generate_tokens()`` from test data in
+docutils/test/test_readers/test_python/test_parser.
+
+Usage::
+
+    showtok <key> <index>
+
+Where ``<key>`` is the key to the ``totest`` dictionary, and ``<index>`` is
+the index of the list ``totest[key]``.
+"""
+
+import sys
+import tokenize
+import pprint
+from token import tok_name
+import test_parser
+
+def name_tokens(tokens):
+    for i in range(len(tokens)):
+        tup = tokens[i]
+        tokens[i] = (tok_name[tup[0]],) + tup
+
+key, caseno = sys.argv[1:]
+print 'totest["%s"][%s][0]:\n' % (key, caseno)
+input_text = test_parser.totest[key][int(caseno)][0]
+print input_text
+tokens = list(tokenize.generate_tokens(iter(input_text.splitlines(1)).next))
+name_tokens(tokens)
+pprint.pprint(tokens)