making good progress

git-svn-id: http://svn.code.sf.net/p/docutils/code/trunk@1020 929543f6-e4f2-0310-98a6-ba3bd3dd1d04

4 files changed, 534 insertions, 239 deletions

diff --git a/docutils/docutils/readers/python/moduleparser.py b/docutils/docutils/readers/python/moduleparser.py
index 9ab3eea79..cbca876a7 100644
--- a/docutils/docutils/readers/python/moduleparser.py
+++ b/docutils/docutils/readers/python/moduleparser.py
@@ -12,7 +12,7 @@ Ideas:
 * Tokenize the module in parallel to extract initial values, comments, etc.
 
 * Merge the compiler & tokenize output such that the raw text hangs off of
-  nodes?  Especially assignment expressions (RHS).
+  nodes.  Useful for assignment expressions (RHS).
 
 What I'd like to do is to take a module, read in the text, run it through the
 module parser (using compiler.py and tokenize.py) and produce a high-level AST
@@ -79,7 +79,12 @@ The module parser should produce a high-level AST, something like this::
                     1
                 <Docstring>
                     class_attribute's docstring
-            <Method name="__init__" argnames=['self', ('text', 'None')]>
+            <Method name="__init__">
+                <Parameters>
+                    <Parameter name="self">
+                    <Parameter name="text">
+                        <Expression>
+                            None
                 <Docstring>
                     __init__'s docstring
                 <Attribute name="instance_attribute" instance=True>
@@ -109,10 +114,10 @@ The module parser should produce a high-level AST, something like this::
                 <Docstring>
                     f.function_attribute's docstring
 
-compiler.parse() provides most of what's needed for this AST.  I think that
-"tokenize" can be used to get the rest, and all that's left is to hunker down
-and figure out how.  We can determine the line number from the
-compiler.parse() AST, and a get_rhs(lineno) method would provide the rest.
+compiler.parse() provides most of what's needed for this AST, and "tokenize"
+can be used to get the rest.  We can determine the line number from the
+compiler.parse() AST, and the TokenParser.rhs(lineno) method provides the
+rest.
 
 The Docutils Python reader component will transform this AST into a
 Python-specific doctree, and then a `stylist transform`_ would further
@@ -174,17 +179,17 @@ from types import StringType, UnicodeType
 
 def parse_module(module_text, filename):
     ast = compiler.parse(module_text)
-    visitor = ModuleVisitor(filename)
+    token_parser = TokenParser(module_text)
+    visitor = ModuleVisitor(filename, token_parser)
     compiler.walk(ast, visitor, walker=visitor)
     return visitor.module
 
 
-class ModuleVisitor(ASTVisitor):
+class BaseVisitor(ASTVisitor):
 
-    def __init__(self, filename):
+    def __init__(self, token_parser):
         ASTVisitor.__init__(self)
-        self.filename = filename
-        self.module = None
+        self.token_parser = token_parser
         self.context = []
         self.documentable = None
 
@@ -193,22 +198,12 @@ class ModuleVisitor(ASTVisitor):
         #print 'in default (%s)' % node.__class__.__name__
         #ASTVisitor.default(self, node, *args)
 
-    def default_ignore(self, node, *args):
-        #print 'in default_ignore (%s)' % node.__class__.__name__
+    def default_visit(self, node, *args):
+        #print 'in default_visit (%s)' % node.__class__.__name__
         ASTVisitor.default(self, node, *args)
 
-    def visitModule(self, node):
-        #print dir(node)
-        self.module = module = Module(node, self.filename)
-        if node.doc is not None:
-            module.append(Docstring(node, node.doc))
-        self.context.append(module)
-        self.documentable = module
-        self.visit(node.node)
-        self.context.pop()
 
-    def visitStmt(self, node):
-        self.default_ignore(node)
+class DocstringVisitor(BaseVisitor):
 
     def visitDiscard(self, node):
         if self.documentable:
@@ -221,17 +216,14 @@ class ModuleVisitor(ASTVisitor):
             else:
                 self.documentable = None
 
-    def visitImport(self, node):
-        self.context[-1].append(Import(node, node.names))
-        self.documentable = None
+    def visitStmt(self, node):
+        self.default_visit(node)
 
-    def visitFrom(self, node):
-        self.context[-1].append(
-            Import(node, node.names, from_name=node.modname))
-        self.documentable = None
+
+class AssignmentVisitor(DocstringVisitor):
 
     def visitAssign(self, node):
-        visitor = AssignmentVisitor()
+        visitor = AttributeVisitor(self.token_parser)
         compiler.walk(node, visitor, walker=visitor)
         if visitor.attributes:
             self.context[-1].extend(visitor.attributes)
@@ -241,66 +233,111 @@ class ModuleVisitor(ASTVisitor):
             self.documentable = None
 
 
-class AssignmentVisitor(ASTVisitor):
+class ModuleVisitor(AssignmentVisitor):
 
-    """
-    Tried reconstructing expressions (the RHS of assignments) by
-    visiting the compiler.parse() tree, but a lot of information is
-    missing, like parenthesis-grouping of expressions.
+    def __init__(self, filename, token_parser):
+        AssignmentVisitor.__init__(self, token_parser)
+        self.filename = filename
+        self.module = None
 
-    Gotta do it by parsing tokens.
-    """
+    def visitModule(self, node):
+        self.module = module = Module(node, self.filename)
+        if node.doc is not None:
+            module.append(Docstring(node, node.doc))
+        self.context.append(module)
+        self.documentable = module
+        self.visit(node.node)
+        self.context.pop()
 
-    def __init__(self):
-        ASTVisitor.__init__(self)
-        self.attributes = []
-        self.parts = []
+    def visitImport(self, node):
+        self.context[-1].append(Import(node, node.names))
+        self.documentable = None
 
-    def default(self, node, *args):
-        print >>sys.stderr, '%s not visited!' % node.__class__.__name__
-        ASTVisitor.default(self, node)
+    def visitFrom(self, node):
+        self.context[-1].append(
+            Import(node, node.names, from_name=node.modname))
+        self.documentable = None
+
+    def visitFunction(self, node):
+        visitor = FunctionVisitor(self.token_parser)
+        compiler.walk(node, visitor, walker=visitor)
+        self.context[-1].append(visitor.function)
+
+
+class AttributeVisitor(BaseVisitor):
+
+    def __init__(self, token_parser):
+        BaseVisitor.__init__(self, token_parser)
+        self.attributes = []
 
     def visitAssign(self, node):
-        ASTVisitor.default(self, node)
-        self.attributes[-1].append(Expression(node, ''.join(self.parts)))
+        # Don't visit the expression itself, just the attribute nodes:
+        for child in node.nodes:
+            self.dispatch(child)
+        expression_text = self.token_parser.rhs(node.lineno)
+        expression = Expression(node, expression_text)
+        for attribute in self.attributes:
+            attribute.append(expression)
 
     def visitAssName(self, node):
         self.attributes.append(Attribute(node, node.name))
 
-    def visitAdd(self, node):
-        ASTVisitor.default(self, node)
-        self.parts[-2:] = ' + '.join(self.parts[-2:])
-
-    def visitAnd(self, node):
-        ASTVisitor.default(self, node)
-        self.parts.insert(len(self.parts) - 1, ' and ')
+    def visitAssTuple(self, node):
+        attributes = self.attributes
+        self.attributes = []
+        self.default_visit(node)
+        names = [attribute.name for attribute in self.attributes]
+        att_tuple = AttributeTuple(node, names)
+        att_tuple.lineno = self.attributes[0].lineno
+        self.attributes = attributes
+        self.attributes.append(att_tuple)
 
-    def visitBackquote(self, node):
-        self.parts.append('`')
-        ASTVisitor.default(self, node)
-        self.parts.append('`')
+    def visitAssAttr(self, node):
+        self.default_visit(node, node.attrname)
 
-    def visitBitand(self, node):
-        ASTVisitor.default(self, node)
-        self.parts.insert(len(self.parts) - 1, ' & ')
+    def visitGetattr(self, node, suffix):
+        self.default_visit(node, node.attrname + '.' + suffix)
 
-    def visitBitor(self, node):
-        ASTVisitor.default(self, node)
-        self.parts.insert(len(self.parts) - 1, ' | ')
+    def visitName(self, node, suffix):
+        self.attributes.append(Attribute(node, node.name + '.' + suffix))
 
-    def visitBitxor(self, node):
-        ASTVisitor.default(self, node)
-        self.parts.insert(len(self.parts) - 1, ' ^ ')
 
-    def visitConst(self, node):
-        self.parts.append(repr(node.value))
+class FunctionVisitor(DocstringVisitor):
 
-    def visitConst(self, node):
-        self.parts.append(repr(node.value))
+    def visitFunction(self, node):
+        self.function = function = Function(node, node.name)
+        if node.doc is not None:
+            function.append(Docstring(node, node.doc))
+        self.context.append(function)
+        self.documentable = function
+        self.parse_parameter_list(node)
+        self.visit(node.code)
+        self.context.pop()
 
-    def visitInvert(self, node):
-        self.parts.append('~ ')
-        ASTVisitor.default(self, node)
+    def parse_parameter_list(self, node):
+        parameters = []
+        special = []
+        argnames = list(node.argnames)
+        if node.kwargs:
+            special.append(ExcessKeywordArguments(node, argnames[-1]))
+            argnames.pop()
+        if node.varargs:
+            special.append(ExcessPositionalArguments(node, argnames[-1]))
+            argnames.pop()
+        defaults = list(node.defaults)
+        defaults = [None] * (len(argnames) - len(defaults)) + defaults
+        for argname, default in zip(argnames, defaults):
+            parameter = Parameter(node, argname)
+            if default:
+                default_text = self.token_parser.default(node.lineno)
+                parameter.append(Default(node, default_text))
+            parameters.append(parameter)
+        if parameters or special:
+            special.reverse()
+            parameters.extend(special)
+            parameter_list = ParameterList(node)
+            parameter_list.extend(parameters)
+            self.function.append(parameter_list)
 
 
 class Node:
@@ -395,6 +432,16 @@ class Attribute(Node):
         return Node.attlist(self, name=self.name)
 
 
+class AttributeTuple(Node):
+
+    def __init__(self, node, names):
+        Node.__init__(self, node)
+        self.names = names
+
+    def attlist(self):
+        return Node.attlist(self, names=' '.join(self.names))
+
+
 class Expression(Node):
 
     def __init__(self, node, text):
@@ -404,40 +451,98 @@ class Expression(Node):
     def __str__(self, indent='    ', level=0):
         prefix = indent * (level + 1)
         return '%s%s%s\n' % (Node.__str__(self, indent, level),
-                             prefix, self.text)
+                             prefix, self.text.encode('unicode-escape'))
 
 
-class TokenReader:
+class Function(Attribute): pass
+
+
+class ParameterList(Node): pass
+
+
+class Parameter(Attribute): pass
+
+
+class ExcessPositionalArguments(Parameter): pass
+
+
+class ExcessKeywordArguments(Parameter): pass
+
+
+class Default(Expression): pass
+
+
+class TokenParser:
 
     def __init__(self, text):
-        self.text = text
-        self.lines = text.splitlines(1)
+        self.text = text + '\n\n'
+        self.lines = self.text.splitlines(1)
         self.generator = tokenize.generate_tokens(iter(self.lines).next)
+        self.next()
 
     def __iter__(self):
         return self
 
     def next(self):
-        token = self.generator.next()
-        self.type, self.string, self.start, self.end, self.line = token
-        return token
+        self.token = self.generator.next()
+        self.type, self.string, self.start, self.end, self.line = self.token
+        return self.token
 
     def goto_line(self, lineno):
-        for token in self:
-            if self.start[0] >= lineno:
-                return token
-        else:
-            raise IndexError
+        while self.start[0] < lineno:
+            self.next()
+        return token
 
-    def rhs(self, name, lineno):
+    def rhs(self, lineno):
+        """
+        Return a whitespace-normalized expression string from the right-hand
+        side of an assignment at line `lineno`.
+        """
         self.goto_line(lineno)
-        while self.start[0] == lineno:
-            if self.type == token.OP and self.string == '=':
-                break
+        while self.string != '=':
             self.next()
-        else:
-            raise IndexError
-        
+        while self.type != token.NEWLINE and self.string != ';':
+            append = 1
+            append_ws = 1
+            del_ws = 0
+            if self.string == '=':
+                start_row, start_col = self.end
+                tokens = []
+                last_type = None
+                last_string = None
+                backquote = 0
+                append = 0
+            elif self.string == '.':
+                del_ws = 1
+                append_ws = 0
+            elif self.string in ('(', '[', '{'):
+                append_ws = 0
+                if self.string in '([' and (last_type == token.NAME or
+                                            last_string in (')', ']', '}')):
+                    del_ws = 1
+            elif self.string in (')', ']', '}', ':', ','):
+                    del_ws = 1
+            elif self.string == '`':
+                if backquote:
+                    del_ws = 1
+                else:
+                    append_ws = 0
+                backquote = not backquote
+            elif self.type == tokenize.NL:
+                append = 0
+            if append:
+                if del_ws and tokens and tokens[-1] == ' ':
+                    del tokens[-1]
+                tokens.append(self.string)
+                last_type = self.type
+                last_string = self.string
+                if append_ws:
+                    tokens.append(' ')
+            self.next()
+        self.next()
+        text = ''.join(tokens)
+        return text.strip()
+
 
 def trim_docstring(text):
     """
diff --git a/docutils/test/test_readers/test_python/test_functions.py b/docutils/test/test_readers/test_python/test_functions.py
index c219517d0..d521b2203 100644
--- a/docutils/test/test_readers/test_python/test_functions.py
+++ b/docutils/test/test_readers/test_python/test_functions.py
@@ -12,7 +12,7 @@ Tests for PySource Reader functions.
 
 import unittest
 from __init__ import DocutilsTestSupport
-from docutils.readers.python.moduleparser import trim_docstring, TokenReader
+from docutils.readers.python.moduleparser import trim_docstring
 
 
 class MiscTests(unittest.TestCase):
@@ -51,10 +51,6 @@ Last line unindented."""),
             self.assertEquals(trim_docstring('\n    ' + docstring),
                               expected)
 
-#     def test_token_reader(self):
-#         tr = TokenReader('a = 1')
-#         self.assertEquals(tr.rhs('a', 1), '1')
-
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/docutils/test/test_readers/test_python/test_parser.py b/docutils/test/test_readers/test_python/test_parser.py
index 0621337de..f45bb6ad1 100644
--- a/docutils/test/test_readers/test_python/test_parser.py
+++ b/docutils/test/test_readers/test_python/test_parser.py
@@ -181,160 +181,308 @@ from __future__ import division
 '''],
 ]
 
-# totest['assign'] = [
-# ['''\
-# a = 1
-# ''',
-# '''\
-# <Module filename="test data">
-#     <Attribute lineno="1" name="a">
-#         <Expression lineno="1">
-#             1
-# '''],
-# ['''\
-# a = 1
-# """a's docstring"""
-# ''', #"
-# '''\
-# <Module filename="test data">
-#     <Attribute lineno="1" name="a">
-#         <Expression lineno="1">
-#             1
-#         <Docstring lineno="2">
-#             a's docstring
-# '''], #'
-# ['''\
-# a = 1
-# """a's docstring"""
-# """additional docstring"""
-# ''', #"
-# '''\
-# <Module filename="test data">
-#     <Attribute lineno="1" name="a">
-#         <Expression lineno="1">
-#             1
-#         <Docstring lineno="2">
-#             a's docstring
-#         <Docstring lineno="3">
-#             additional docstring
-# '''], #'
-# ['''\
-# a = 1 + 2 * 3 / 4 ** 5
-# ''',
-# '''\
-# <Module filename="test data">
-#     <Attribute lineno="1" name="a">
-#         <Expression lineno="1">
-#             1 + 2 * 3 / 4 ** 5
-# '''],
-# ['''\
-# a = 1 \\
-#     + 2
-# ''',
-# '''\
-# <Module filename="test data">
-#     <Attribute lineno="1" name="a">
-#         <Expression lineno="1">
-#             1 + 2
-# '''],
-# ['''\
-# a = not 1 and 2 or 3
-# ''',
-# '''\
-# <Module filename="test data">
-#     <Attribute lineno="1" name="a">
-#         <Expression lineno="1">
-#             not 1 and 2 or 3
-# '''],
-# ['''\
-# a = ~ 1 & 2 | 3 ^ 4
-# ''',
-# '''\
-# <Module filename="test data">
-#     <Attribute lineno="1" name="a">
-#         <Expression lineno="1">
-#             ~ 1 & 2 | 3 ^ 4
-# '''],
-# ['''\
-# a = `1 & 2`
-# ''',
-# '''\
-# <Module filename="test data">
-#     <Attribute lineno="1" name="a">
-#         <Expression lineno="1">
-#             `1 & 2`
-# '''],
-# ['''\
-# very_long_name = \\
-#     x
-# ''',
-# '''\
-# <Module filename="test data">
-#     <Attribute lineno="1" name="very_long_name">
-#         <Expression lineno="1">
-#             x
-# '''],
-# ['''\
-# very_long_name \\
-#     = x
-# ''',
-# '''\
-# <Module filename="test data">
-#     <Attribute lineno="1" name="very_long_name">
-#         <Expression lineno="1">
-#             x
-# '''],
+totest['assign'] = [
+['''\
+a = 1
+''',
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="a">
+        <Expression lineno="1">
+            1
+'''],
+['''a = 1''',
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="a">
+        <Expression lineno="1">
+            1
+'''],
+['''\
+a = 1
+"""a's docstring"""
+''', #"
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="a">
+        <Expression lineno="1">
+            1
+        <Docstring lineno="2">
+            a's docstring
+'''], #'
+['''\
+a = 1
+"""a's docstring"""
+"""additional docstring"""
+''', #"
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="a">
+        <Expression lineno="1">
+            1
+        <Docstring lineno="2">
+            a's docstring
+        <Docstring lineno="3">
+            additional docstring
+'''], #'
+['''\
+a = 1 + 2 * 3 / 4 ** 5
+''',
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="a">
+        <Expression lineno="1">
+            1 + 2 * 3 / 4 ** 5
+'''],
+['''\
+a = 1 \\
+    + 2
+''',
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="a">
+        <Expression lineno="1">
+            1 + 2
+'''],
+['''\
+a = not 1 and 2 or 3
+''',
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="a">
+        <Expression lineno="1">
+            not 1 and 2 or 3
+'''],
+['''\
+a = ~ 1 & 2 | 3 ^ 4
+''',
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="a">
+        <Expression lineno="1">
+            ~ 1 & 2 | 3 ^ 4
+'''],
+['''\
+a = `1 & 2`
+''',
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="a">
+        <Expression lineno="1">
+            `1 & 2`
+'''],
+['''\
+very_long_name = \\
+    x
+''',
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="very_long_name">
+        <Expression lineno="1">
+            x
+'''],
+['''\
+very_long_name \\
+    = x
+''',
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="very_long_name">
+        <Expression lineno="2">
+            x
+'''],
+['''\
+very_long_name = \\
+    another_long_name = \\
+    x
+''',
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="very_long_name">
+        <Expression lineno="1">
+            x
+    <Attribute lineno="2" name="another_long_name">
+        <Expression lineno="1">
+            x
+'''],
+['''\
+a = (1
+     + 2)
+b = a.b[1 +
+        fn(x, y,
+           z, {'key': (1 + 2
+                       + 3)})][4]
+c = """first line
+second line
+    third"""
+''',
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="a">
+        <Expression lineno="1">
+            (1 + 2)
+    <Attribute lineno="3" name="b">
+        <Expression lineno="3">
+            a.b[1 + fn(x, y, z, {'key': (1 + 2 + 3)})][4]
+    <Attribute lineno="7" name="c">
+        <Expression lineno="7">
+            """first line\\nsecond line\\n    third"""
+'''],
+['''\
+a, b, c = range(3)
+(d, e,
+ f) = a, b, c
+g, h, i = j = a, b, c
+k.a, k.b.c, k.d.e.f = a, b, c
+''',
+'''\
+<Module filename="test data">
+    <AttributeTuple lineno="1" names="a b c">
+        <Expression lineno="1">
+            range(3)
+    <AttributeTuple lineno="2" names="d e f">
+        <Expression lineno="3">
+            a, b, c
+    <AttributeTuple lineno="4" names="g h i">
+        <Expression lineno="4">
+            a, b, c
+    <Attribute lineno="4" name="j">
+        <Expression lineno="4">
+            a, b, c
+    <AttributeTuple lineno="5" names="k.a k.b.c k.d.e.f">
+        <Expression lineno="5">
+            a, b, c
+'''],
+['''\
+a = 1 ; b = 2
+print ; c = 3
+''',
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="a">
+        <Expression lineno="1">
+            1
+    <Attribute lineno="1" name="b">
+        <Expression lineno="1">
+            2
+    <Attribute lineno="2" name="c">
+        <Expression lineno="2">
+            3
+'''],
+['''\
+a.b = 1
+"""This assignment is noted but ignored unless ``a`` is a function."""
+''',
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="a.b">
+        <Expression lineno="1">
+            1
+        <Docstring lineno="2">
+            This assignment is noted but ignored unless ``a`` is a function.
+'''],
+['''\
+a[b] = 1
+"""This assignment is noted but ignored unless ``a`` is a function."""
+''',
+'''\
+<Module filename="test data">
+'''],
+]
+
+totest['def'] = [
+['''\
+def f():
+    """Function f's docstring"""
+    """Additional docstring"""
+    local = 1
+    """Not a docstring, since ``local`` is local."""
+''', # "
+'''\
+<Module filename="test data">
+    <Function lineno="1" name="f">
+        <Docstring lineno="1">
+            Function f's docstring
+        <Docstring lineno="3">
+            Additional docstring
+'''], # '
+['''\
+def f(a, b):
+    local = 1
+''',
+'''\
+<Module filename="test data">
+    <Function lineno="1" name="f">
+        <ParameterList lineno="1">
+            <Parameter lineno="1" name="a">
+            <Parameter lineno="1" name="b">
+'''],
 # ['''\
-# very_long_name = \\
-#     another_long_name = \\
-#     x
+# def f(a=None, b=1):
+#     local = 1
 # ''',
 # '''\
 # <Module filename="test data">
-#     <Attribute lineno="1" name="very_long_name">
-#         <Expression lineno="1">
-#             x
+#     <Function lineno="1" name="f">
+#         <ParameterList lineno="1">
+#             <Parameter lineno="1" name="a">
+#                 <Default lineno="1">
+#                     None
+#             <Parameter lineno="1" name="b">
+#                 <Default lineno="1">
+#                     1
 # '''],
+['''\
+def f(*args):
+    local = 1
+''',
+'''\
+<Module filename="test data">
+    <Function lineno="1" name="f">
+        <ParameterList lineno="1">
+            <ExcessPositionalArguments lineno="1" name="args">
+'''],
+['''\
+def f(**kwargs):
+    local = 1
+''',
+'''\
+<Module filename="test data">
+    <Function lineno="1" name="f">
+        <ParameterList lineno="1">
+            <ExcessKeywordArguments lineno="1" name="kwargs">
+'''],
 # ['''\
-# a = (1
-#      + 2)
-# b = a.b[1 +
-#         fn(x, y,
-#            z, {'key': (1 + 2
-#                        + 3)})]
-# c = """first line
-# second line
-#    third"""
+# def f(a, b=None, *args, **kwargs):
+#     local = 1
 # ''',
 # '''\
 # <Module filename="test data">
-#     <Attribute lineno="1" name="a">
-#         <Expression lineno="1">
-#             (1 + 2)
-#     <Attribute lineno="3" name="b">
-#         <Expression lineno="3">
-#             a.b[1 + fn(x, y, z, {'key': (1 + 2 + 3)})]
-#     <Attribute lineno="7" name="c">
-#         <Expression lineno="7">
-#             """first line\\nsecond line\\n    third"""
+#     <Function lineno="1" name="f">
+#         <ParameterList lineno="1">
+#             <Parameter lineno="1" name="a">
+#             <Parameter lineno="1" name="b">
+#                 <Default lineno="1">
+#                     None
+#             <ExcessPositionalArguments lineno="1" name="args">
+#             <ExcessKeywordArguments lineno="1" name="kwargs">
 # '''],
 # ['''\
-# a, b, c = range(3)
-# (d, e,
-#  f) = a, b, c
-# ''',
-# '''\
-# '''],
-# ]
-
-# totest['def'] = [
-# ['''\
 # def f():
 #     pass
-# ''',
+# f.attrib = 1
+# """f.attrib's docstring"""
+# ''', # "
 # '''\
-# '''],
-# ]
+# <Module filename="test data">
+#     <Function lineno="1" name="f">
+#         <Attribute lineno="3" name="attrib">
+#             <Expression lineno="3">
+#                 1
+#             <Docstring lineno="4">
+#                 f.attrib's docstring
+# '''], # '
+]
 
 totest['ignore'] = [
 ['''\
diff --git a/docutils/test/test_readers/test_python/test_token_parser.py b/docutils/test/test_readers/test_python/test_token_parser.py
new file mode 100644
index 000000000..23015a7dc
--- /dev/null
+++ b/docutils/test/test_readers/test_python/test_token_parser.py
@@ -0,0 +1,46 @@
+#! /usr/bin/env python
+
+# Author: David Goodger
+# Contact: goodger@users.sourceforge.net
+# Revision: $Revision$
+# Date: $Date$
+# Copyright: This module has been placed in the public domain.
+
+"""
+Tests for docutils/readers/python/moduleparser.py.
+"""
+
+from __init__ import DocutilsTestSupport
+
+
+def suite():
+    s = DocutilsTestSupport.PythonModuleParserTestSuite()
+    s.generateTests(totest, testmethod='test_token_parser_rhs')
+    return s
+
+totest = {}
+
+totest['expressions'] = [
+['''a = 1''', '''1'''],
+['''a = b = 1''', '''1'''],
+['''\
+a = (
+     1 + 2
+     + 3
+     )
+''',
+'''(1 + 2 + 3)'''],
+['''\
+a = """\\
+line one
+line two"""
+''',
+'''"""\\\nline one\nline two"""'''],
+['''a = `1`''', '''`1`'''],
+['''a = `1`+`2`''', '''`1` + `2`'''],
+]
+
+
+if __name__ == '__main__':
+    import unittest
+    unittest.main(defaultTest='suite')