More progress; functions done.

git-svn-id: http://svn.code.sf.net/p/docutils/code/trunk@1027 929543f6-e4f2-0310-98a6-ba3bd3dd1d04

2 files changed, 230 insertions, 85 deletions

diff --git a/docutils/docutils/readers/python/moduleparser.py b/docutils/docutils/readers/python/moduleparser.py
index cbca876a7..8dcf432b2 100644
--- a/docutils/docutils/readers/python/moduleparser.py
+++ b/docutils/docutils/readers/python/moduleparser.py
@@ -163,6 +163,14 @@ It's for exactly this reason:
 ... but it goes past docstring processing.  It's also important to keep style
 decisions and tool-specific data transforms out of this module parser.
 
+
+Issues
+======
+
+* At what point should namespaces be computed?  Should they be part of the
+  basic AST produced by the ASTVisitor walk, or generated by another tree
+  traversal?
+
 """
 
 __docformat__ = 'reStructuredText'
@@ -174,7 +182,7 @@ import tokenize
 import token
 from compiler.consts import OP_ASSIGN
 from compiler.visitor import ASTVisitor
-from types import StringType, UnicodeType
+from types import StringType, UnicodeType, TupleType
 
 
 def parse_module(module_text, filename):
@@ -304,7 +312,13 @@ class AttributeVisitor(BaseVisitor):
 
 class FunctionVisitor(DocstringVisitor):
 
+    in_function = 0
+
     def visitFunction(self, node):
+        if self.in_function:
+            # Don't bother with nested function definitions.
+            return
+        self.in_function = 1
         self.function = function = Function(node, node.name)
         if node.doc is not None:
             function.append(Docstring(node, node.doc))
@@ -326,11 +340,17 @@ class FunctionVisitor(DocstringVisitor):
             argnames.pop()
         defaults = list(node.defaults)
         defaults = [None] * (len(argnames) - len(defaults)) + defaults
+        function_parameters = self.token_parser.function_parameters(
+            node.lineno)
+        #print >>sys.stderr, function_parameters
         for argname, default in zip(argnames, defaults):
-            parameter = Parameter(node, argname)
+            if type(argname) is TupleType:
+                parameter = ParameterTuple(node, argname)
+                argname = normalize_parameter_name(argname)
+            else:
+                parameter = Parameter(node, argname)
             if default:
-                default_text = self.token_parser.default(node.lineno)
-                parameter.append(Default(node, default_text))
+                parameter.append(Default(node, function_parameters[argname]))
             parameters.append(parameter)
         if parameters or special:
             special.reverse()
@@ -463,6 +483,12 @@ class ParameterList(Node): pass
 class Parameter(Attribute): pass
 
 
+class ParameterTuple(AttributeTuple):
+
+    def attlist(self):
+        return Node.attlist(self, names=normalize_parameter_name(self.names))
+
+
 class ExcessPositionalArguments(Parameter): pass
 
 
@@ -502,47 +528,129 @@ class TokenParser:
         while self.string != '=':
             self.next()
         while self.type != token.NEWLINE and self.string != ';':
-            append = 1
-            append_ws = 1
-            del_ws = 0
             if self.string == '=':
-                start_row, start_col = self.end
-                tokens = []
-                last_type = None
-                last_string = None
-                backquote = 0
-                append = 0
-            elif self.string == '.':
-                del_ws = 1
-                append_ws = 0
-            elif self.string in ('(', '[', '{'):
-                append_ws = 0
-                if self.string in '([' and (last_type == token.NAME or
-                                            last_string in (')', ']', '}')):
-                    del_ws = 1
-            elif self.string in (')', ']', '}', ':', ','):
-                    del_ws = 1
-            elif self.string == '`':
-                if backquote:
-                    del_ws = 1
-                else:
-                    append_ws = 0
-                backquote = not backquote
-            elif self.type == tokenize.NL:
-                append = 0
-            if append:
-                if del_ws and tokens and tokens[-1] == ' ':
-                    del tokens[-1]
-                tokens.append(self.string)
-                last_type = self.type
-                last_string = self.string
-                if append_ws:
-                    tokens.append(' ')
+                self.tokens = []
+                self.stack = []
+                self._type = None
+                self._string = None
+                self._backquote = 0
+            else:
+                self.note_token()
             self.next()
         self.next()
-        text = ''.join(tokens)
+        text = ''.join(self.tokens)
         return text.strip()
 
+    openers = {')': '(', ']': '[', '}': '{'}
+
+    def note_token(self):
+        append = 1
+        append_ws = 1
+        del_ws = 0
+        if self.string == '.':
+            del_ws = 1
+            append_ws = 0
+        elif self.string in ('(', '[', '{'):
+            append_ws = 0
+            if self.string in '([' and (self._type == token.NAME or
+                                        self._string in (')', ']', '}')):
+                del_ws = 1
+            self.stack.append(self.string)
+        elif self.string in (')', ']', '}'):
+            del_ws = 1
+            assert self.stack[-1] == self.openers[self.string]
+            self.stack.pop()
+        elif self.string in (':', ','):
+            del_ws = 1
+        elif self.string == '`':
+            if self._backquote:
+                del_ws = 1
+                assert self.stack[-1] == self.string
+                self.stack.pop()
+            else:
+                append_ws = 0
+                self.stack.append(self.string)
+            self._backquote = not self._backquote
+        elif self.type == tokenize.NL:
+            append = 0
+        if append:
+            if del_ws and self.tokens and self.tokens[-1] == ' ':
+                del self.tokens[-1]
+            self.tokens.append(self.string)
+            self._type = self.type
+            self._string = self.string
+            if append_ws:
+                self.tokens.append(' ')
+
+    def function_parameters(self, lineno):
+        """
+        Return a dictionary mapping parameters to defaults
+        (whitespace-normalized strings).
+        """
+        self.goto_line(lineno)
+        while self.string != 'def':
+            self.next()
+        while self.string != '(':
+            self.next()
+        name = None
+        default = None
+        parameter_tuple = None
+        self.tokens = []
+        parameters = {}
+        self.stack = [self.string]
+        self.next()
+        while 1:
+            if len(self.stack) == 1:
+                if parameter_tuple:
+                    # Just encountered ")".
+                    #print >>sys.stderr, 'parameter_tuple: %r' % self.tokens
+                    name = ''.join(self.tokens).strip()
+                    self.tokens = []
+                    parameter_tuple = None
+                if self.string in (')', ','):
+                    if name:
+                        if self.tokens:
+                            default_text = ''.join(self.tokens).strip()
+                        else:
+                            default_text = None
+                        parameters[name] = default_text
+                        self.tokens = []
+                        name = None
+                        default = None
+                    if self.string == ')':
+                        break
+                elif self.type == token.NAME:
+                    if name and default:
+                        self.note_token()
+                    else:
+                        assert name is None, (
+                            'token=%r name=%r parameters=%r stack=%r'
+                            % (self.token, name, parameters, self.stack))
+                        name = self.string
+                        #print >>sys.stderr, 'name=%r' % name
+                elif self.string == '=':
+                    assert name is not None, 'token=%r' % (self.token,)
+                    assert default is None, 'token=%r' % (self.token,)
+                    assert self.tokens == [], 'token=%r' % (self.token,)
+                    default = 1
+                    self._type = None
+                    self._string = None
+                    self._backquote = 0
+                elif name:
+                    self.note_token()
+                elif self.string == '(':
+                    parameter_tuple = 1
+                    self._type = None
+                    self._string = None
+                    self._backquote = 0
+                    self.note_token()
+                else:                   # ignore these tokens:
+                    assert self.string in ('*', '**', '\n'), (
+                        'token=%r' % (self.token,))
+            else:
+                self.note_token()
+            self.next()
+        return parameters
 
 def trim_docstring(text):
     """
@@ -573,3 +681,12 @@ def trim_docstring(text):
         trimmed.pop(0)
     # Return a single string:
     return '\n'.join(trimmed)
+
+def normalize_parameter_name(name):
+    """
+    Converts a tuple like ``('a', ('b', 'c'), 'd')`` into ``'(a, (b, c), d)'``
+    """
+    if type(name) is TupleType:
+        return '(%s)' % ', '.join([normalize_parameter_name(n) for n in name])
+    else:
+        return name
diff --git a/docutils/test/test_readers/test_python/test_parser.py b/docutils/test/test_readers/test_python/test_parser.py
index f45bb6ad1..021ad3438 100644
--- a/docutils/test/test_readers/test_python/test_parser.py
+++ b/docutils/test/test_readers/test_python/test_parser.py
@@ -417,21 +417,38 @@ def f(a, b):
             <Parameter lineno="1" name="a">
             <Parameter lineno="1" name="b">
 '''],
-# ['''\
-# def f(a=None, b=1):
-#     local = 1
-# ''',
-# '''\
-# <Module filename="test data">
-#     <Function lineno="1" name="f">
-#         <ParameterList lineno="1">
-#             <Parameter lineno="1" name="a">
-#                 <Default lineno="1">
-#                     None
-#             <Parameter lineno="1" name="b">
-#                 <Default lineno="1">
-#                     1
-# '''],
+['''\
+def f(a=None, b=1):
+    local = 1
+''',
+'''\
+<Module filename="test data">
+    <Function lineno="1" name="f">
+        <ParameterList lineno="1">
+            <Parameter lineno="1" name="a">
+                <Default lineno="1">
+                    None
+            <Parameter lineno="1" name="b">
+                <Default lineno="1">
+                    1
+'''],
+['''\
+def f(a, (b, c, d)=range(3),
+      e=None):
+    local = 1
+''',
+'''\
+<Module filename="test data">
+    <Function lineno="1" name="f">
+        <ParameterList lineno="1">
+            <Parameter lineno="1" name="a">
+            <ParameterTuple lineno="1" names="(b, c, d)">
+                <Default lineno="1">
+                    range(3)
+            <Parameter lineno="1" name="e">
+                <Default lineno="1">
+                    None
+'''],
 ['''\
 def f(*args):
     local = 1
@@ -452,36 +469,47 @@ def f(**kwargs):
         <ParameterList lineno="1">
             <ExcessKeywordArguments lineno="1" name="kwargs">
 '''],
-# ['''\
-# def f(a, b=None, *args, **kwargs):
-#     local = 1
-# ''',
-# '''\
-# <Module filename="test data">
-#     <Function lineno="1" name="f">
-#         <ParameterList lineno="1">
-#             <Parameter lineno="1" name="a">
-#             <Parameter lineno="1" name="b">
-#                 <Default lineno="1">
-#                     None
-#             <ExcessPositionalArguments lineno="1" name="args">
-#             <ExcessKeywordArguments lineno="1" name="kwargs">
-# '''],
-# ['''\
-# def f():
-#     pass
-# f.attrib = 1
-# """f.attrib's docstring"""
-# ''', # "
-# '''\
-# <Module filename="test data">
-#     <Function lineno="1" name="f">
-#         <Attribute lineno="3" name="attrib">
-#             <Expression lineno="3">
-#                 1
-#             <Docstring lineno="4">
-#                 f.attrib's docstring
-# '''], # '
+['''\
+def f(a, b=None, *args, **kwargs):
+    local = 1
+''',
+'''\
+<Module filename="test data">
+    <Function lineno="1" name="f">
+        <ParameterList lineno="1">
+            <Parameter lineno="1" name="a">
+            <Parameter lineno="1" name="b">
+                <Default lineno="1">
+                    None
+            <ExcessPositionalArguments lineno="1" name="args">
+            <ExcessKeywordArguments lineno="1" name="kwargs">
+'''],
+['''\
+def f():
+    pass
+f.attrib = 1
+"""f.attrib's docstring"""
+''', # "
+# @@@ When should the Attribute move inside the Function?
+'''\
+<Module filename="test data">
+    <Function lineno="1" name="f">
+    <Attribute lineno="3" name="f.attrib">
+        <Expression lineno="3">
+            1
+        <Docstring lineno="4">
+            f.attrib's docstring
+'''], # '
+['''\
+def f():
+    def g():
+        pass
+    local = 1
+''',
+'''\
+<Module filename="test data">
+    <Function lineno="1" name="f">
+'''],
 ]
 
 totest['ignore'] = [