making good progress

git-svn-id: http://svn.code.sf.net/p/docutils/code/trunk@1020 929543f6-e4f2-0310-98a6-ba3bd3dd1d04

3 files changed, 335 insertions, 145 deletions

diff --git a/docutils/test/test_readers/test_python/test_functions.py b/docutils/test/test_readers/test_python/test_functions.py
index c219517d0..d521b2203 100644
--- a/docutils/test/test_readers/test_python/test_functions.py
+++ b/docutils/test/test_readers/test_python/test_functions.py
@@ -12,7 +12,7 @@ Tests for PySource Reader functions.
 
 import unittest
 from __init__ import DocutilsTestSupport
-from docutils.readers.python.moduleparser import trim_docstring, TokenReader
+from docutils.readers.python.moduleparser import trim_docstring
 
 
 class MiscTests(unittest.TestCase):
@@ -51,10 +51,6 @@ Last line unindented."""),
             self.assertEquals(trim_docstring('\n    ' + docstring),
                               expected)
 
-#     def test_token_reader(self):
-#         tr = TokenReader('a = 1')
-#         self.assertEquals(tr.rhs('a', 1), '1')
-
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/docutils/test/test_readers/test_python/test_parser.py b/docutils/test/test_readers/test_python/test_parser.py
index 0621337de..f45bb6ad1 100644
--- a/docutils/test/test_readers/test_python/test_parser.py
+++ b/docutils/test/test_readers/test_python/test_parser.py
@@ -181,160 +181,308 @@ from __future__ import division
 '''],
 ]
 
-# totest['assign'] = [
-# ['''\
-# a = 1
-# ''',
-# '''\
-# <Module filename="test data">
-#     <Attribute lineno="1" name="a">
-#         <Expression lineno="1">
-#             1
-# '''],
-# ['''\
-# a = 1
-# """a's docstring"""
-# ''', #"
-# '''\
-# <Module filename="test data">
-#     <Attribute lineno="1" name="a">
-#         <Expression lineno="1">
-#             1
-#         <Docstring lineno="2">
-#             a's docstring
-# '''], #'
-# ['''\
-# a = 1
-# """a's docstring"""
-# """additional docstring"""
-# ''', #"
-# '''\
-# <Module filename="test data">
-#     <Attribute lineno="1" name="a">
-#         <Expression lineno="1">
-#             1
-#         <Docstring lineno="2">
-#             a's docstring
-#         <Docstring lineno="3">
-#             additional docstring
-# '''], #'
-# ['''\
-# a = 1 + 2 * 3 / 4 ** 5
-# ''',
-# '''\
-# <Module filename="test data">
-#     <Attribute lineno="1" name="a">
-#         <Expression lineno="1">
-#             1 + 2 * 3 / 4 ** 5
-# '''],
-# ['''\
-# a = 1 \\
-#     + 2
-# ''',
-# '''\
-# <Module filename="test data">
-#     <Attribute lineno="1" name="a">
-#         <Expression lineno="1">
-#             1 + 2
-# '''],
-# ['''\
-# a = not 1 and 2 or 3
-# ''',
-# '''\
-# <Module filename="test data">
-#     <Attribute lineno="1" name="a">
-#         <Expression lineno="1">
-#             not 1 and 2 or 3
-# '''],
-# ['''\
-# a = ~ 1 & 2 | 3 ^ 4
-# ''',
-# '''\
-# <Module filename="test data">
-#     <Attribute lineno="1" name="a">
-#         <Expression lineno="1">
-#             ~ 1 & 2 | 3 ^ 4
-# '''],
-# ['''\
-# a = `1 & 2`
-# ''',
-# '''\
-# <Module filename="test data">
-#     <Attribute lineno="1" name="a">
-#         <Expression lineno="1">
-#             `1 & 2`
-# '''],
-# ['''\
-# very_long_name = \\
-#     x
-# ''',
-# '''\
-# <Module filename="test data">
-#     <Attribute lineno="1" name="very_long_name">
-#         <Expression lineno="1">
-#             x
-# '''],
-# ['''\
-# very_long_name \\
-#     = x
-# ''',
-# '''\
-# <Module filename="test data">
-#     <Attribute lineno="1" name="very_long_name">
-#         <Expression lineno="1">
-#             x
-# '''],
+totest['assign'] = [
+['''\
+a = 1
+''',
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="a">
+        <Expression lineno="1">
+            1
+'''],
+['''a = 1''',
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="a">
+        <Expression lineno="1">
+            1
+'''],
+['''\
+a = 1
+"""a's docstring"""
+''', #"
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="a">
+        <Expression lineno="1">
+            1
+        <Docstring lineno="2">
+            a's docstring
+'''], #'
+['''\
+a = 1
+"""a's docstring"""
+"""additional docstring"""
+''', #"
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="a">
+        <Expression lineno="1">
+            1
+        <Docstring lineno="2">
+            a's docstring
+        <Docstring lineno="3">
+            additional docstring
+'''], #'
+['''\
+a = 1 + 2 * 3 / 4 ** 5
+''',
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="a">
+        <Expression lineno="1">
+            1 + 2 * 3 / 4 ** 5
+'''],
+['''\
+a = 1 \\
+    + 2
+''',
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="a">
+        <Expression lineno="1">
+            1 + 2
+'''],
+['''\
+a = not 1 and 2 or 3
+''',
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="a">
+        <Expression lineno="1">
+            not 1 and 2 or 3
+'''],
+['''\
+a = ~ 1 & 2 | 3 ^ 4
+''',
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="a">
+        <Expression lineno="1">
+            ~ 1 & 2 | 3 ^ 4
+'''],
+['''\
+a = `1 & 2`
+''',
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="a">
+        <Expression lineno="1">
+            `1 & 2`
+'''],
+['''\
+very_long_name = \\
+    x
+''',
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="very_long_name">
+        <Expression lineno="1">
+            x
+'''],
+['''\
+very_long_name \\
+    = x
+''',
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="very_long_name">
+        <Expression lineno="2">
+            x
+'''],
+['''\
+very_long_name = \\
+    another_long_name = \\
+    x
+''',
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="very_long_name">
+        <Expression lineno="1">
+            x
+    <Attribute lineno="2" name="another_long_name">
+        <Expression lineno="1">
+            x
+'''],
+['''\
+a = (1
+     + 2)
+b = a.b[1 +
+        fn(x, y,
+           z, {'key': (1 + 2
+                       + 3)})][4]
+c = """first line
+second line
+    third"""
+''',
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="a">
+        <Expression lineno="1">
+            (1 + 2)
+    <Attribute lineno="3" name="b">
+        <Expression lineno="3">
+            a.b[1 + fn(x, y, z, {'key': (1 + 2 + 3)})][4]
+    <Attribute lineno="7" name="c">
+        <Expression lineno="7">
+            """first line\\nsecond line\\n    third"""
+'''],
+['''\
+a, b, c = range(3)
+(d, e,
+ f) = a, b, c
+g, h, i = j = a, b, c
+k.a, k.b.c, k.d.e.f = a, b, c
+''',
+'''\
+<Module filename="test data">
+    <AttributeTuple lineno="1" names="a b c">
+        <Expression lineno="1">
+            range(3)
+    <AttributeTuple lineno="2" names="d e f">
+        <Expression lineno="3">
+            a, b, c
+    <AttributeTuple lineno="4" names="g h i">
+        <Expression lineno="4">
+            a, b, c
+    <Attribute lineno="4" name="j">
+        <Expression lineno="4">
+            a, b, c
+    <AttributeTuple lineno="5" names="k.a k.b.c k.d.e.f">
+        <Expression lineno="5">
+            a, b, c
+'''],
+['''\
+a = 1 ; b = 2
+print ; c = 3
+''',
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="a">
+        <Expression lineno="1">
+            1
+    <Attribute lineno="1" name="b">
+        <Expression lineno="1">
+            2
+    <Attribute lineno="2" name="c">
+        <Expression lineno="2">
+            3
+'''],
+['''\
+a.b = 1
+"""This assignment is noted but ignored unless ``a`` is a function."""
+''',
+'''\
+<Module filename="test data">
+    <Attribute lineno="1" name="a.b">
+        <Expression lineno="1">
+            1
+        <Docstring lineno="2">
+            This assignment is noted but ignored unless ``a`` is a function.
+'''],
+['''\
+a[b] = 1
+"""This assignment is noted but ignored unless ``a`` is a function."""
+''',
+'''\
+<Module filename="test data">
+'''],
+]
+
+totest['def'] = [
+['''\
+def f():
+    """Function f's docstring"""
+    """Additional docstring"""
+    local = 1
+    """Not a docstring, since ``local`` is local."""
+''', # "
+'''\
+<Module filename="test data">
+    <Function lineno="1" name="f">
+        <Docstring lineno="1">
+            Function f's docstring
+        <Docstring lineno="3">
+            Additional docstring
+'''], # '
+['''\
+def f(a, b):
+    local = 1
+''',
+'''\
+<Module filename="test data">
+    <Function lineno="1" name="f">
+        <ParameterList lineno="1">
+            <Parameter lineno="1" name="a">
+            <Parameter lineno="1" name="b">
+'''],
 # ['''\
-# very_long_name = \\
-#     another_long_name = \\
-#     x
+# def f(a=None, b=1):
+#     local = 1
 # ''',
 # '''\
 # <Module filename="test data">
-#     <Attribute lineno="1" name="very_long_name">
-#         <Expression lineno="1">
-#             x
+#     <Function lineno="1" name="f">
+#         <ParameterList lineno="1">
+#             <Parameter lineno="1" name="a">
+#                 <Default lineno="1">
+#                     None
+#             <Parameter lineno="1" name="b">
+#                 <Default lineno="1">
+#                     1
 # '''],
+['''\
+def f(*args):
+    local = 1
+''',
+'''\
+<Module filename="test data">
+    <Function lineno="1" name="f">
+        <ParameterList lineno="1">
+            <ExcessPositionalArguments lineno="1" name="args">
+'''],
+['''\
+def f(**kwargs):
+    local = 1
+''',
+'''\
+<Module filename="test data">
+    <Function lineno="1" name="f">
+        <ParameterList lineno="1">
+            <ExcessKeywordArguments lineno="1" name="kwargs">
+'''],
 # ['''\
-# a = (1
-#      + 2)
-# b = a.b[1 +
-#         fn(x, y,
-#            z, {'key': (1 + 2
-#                        + 3)})]
-# c = """first line
-# second line
-#    third"""
+# def f(a, b=None, *args, **kwargs):
+#     local = 1
 # ''',
 # '''\
 # <Module filename="test data">
-#     <Attribute lineno="1" name="a">
-#         <Expression lineno="1">
-#             (1 + 2)
-#     <Attribute lineno="3" name="b">
-#         <Expression lineno="3">
-#             a.b[1 + fn(x, y, z, {'key': (1 + 2 + 3)})]
-#     <Attribute lineno="7" name="c">
-#         <Expression lineno="7">
-#             """first line\\nsecond line\\n    third"""
+#     <Function lineno="1" name="f">
+#         <ParameterList lineno="1">
+#             <Parameter lineno="1" name="a">
+#             <Parameter lineno="1" name="b">
+#                 <Default lineno="1">
+#                     None
+#             <ExcessPositionalArguments lineno="1" name="args">
+#             <ExcessKeywordArguments lineno="1" name="kwargs">
 # '''],
 # ['''\
-# a, b, c = range(3)
-# (d, e,
-#  f) = a, b, c
-# ''',
-# '''\
-# '''],
-# ]
-
-# totest['def'] = [
-# ['''\
 # def f():
 #     pass
-# ''',
+# f.attrib = 1
+# """f.attrib's docstring"""
+# ''', # "
 # '''\
-# '''],
-# ]
+# <Module filename="test data">
+#     <Function lineno="1" name="f">
+#         <Attribute lineno="3" name="attrib">
+#             <Expression lineno="3">
+#                 1
+#             <Docstring lineno="4">
+#                 f.attrib's docstring
+# '''], # '
+]
 
 totest['ignore'] = [
 ['''\
diff --git a/docutils/test/test_readers/test_python/test_token_parser.py b/docutils/test/test_readers/test_python/test_token_parser.py
new file mode 100644
index 000000000..23015a7dc
--- /dev/null
+++ b/docutils/test/test_readers/test_python/test_token_parser.py
@@ -0,0 +1,46 @@
+#! /usr/bin/env python
+
+# Author: David Goodger
+# Contact: goodger@users.sourceforge.net
+# Revision: $Revision$
+# Date: $Date$
+# Copyright: This module has been placed in the public domain.
+
+"""
+Tests for docutils/readers/python/moduleparser.py.
+"""
+
+from __init__ import DocutilsTestSupport
+
+
+def suite():
+    s = DocutilsTestSupport.PythonModuleParserTestSuite()
+    s.generateTests(totest, testmethod='test_token_parser_rhs')
+    return s
+
+totest = {}
+
+totest['expressions'] = [
+['''a = 1''', '''1'''],
+['''a = b = 1''', '''1'''],
+['''\
+a = (
+     1 + 2
+     + 3
+     )
+''',
+'''(1 + 2 + 3)'''],
+['''\
+a = """\\
+line one
+line two"""
+''',
+'''"""\\\nline one\nline two"""'''],
+['''a = `1`''', '''`1`'''],
+['''a = `1`+`2`''', '''`1` + `2`'''],
+]
+
+
+if __name__ == '__main__':
+    import unittest
+    unittest.main(defaultTest='suite')