Reader parses docstrings (according to __docformat__) and produces full

output.  The reader should thus be "done".  Run readers/python/__init__.py
with a filename argument to get output in the DOM format.

A transformer will be necessary to translate this into the standard
docutils DOM.


git-svn-id: http://svn.code.sf.net/p/docutils/code/trunk/docutils@1881 929543f6-e4f2-0310-98a6-ba3bd3dd1d04

3 files changed, 178 insertions, 54 deletions

diff --git a/docutils/readers/python/__init__.py b/docutils/readers/python/__init__.py
index 3087f9147..0da4ba938 100644
--- a/docutils/readers/python/__init__.py
+++ b/docutils/readers/python/__init__.py
@@ -14,14 +14,113 @@ __docformat__ = 'reStructuredText'
 import sys
 import docutils.readers
 from docutils.readers.python import moduleparser
-
+from docutils import parsers
+from docutils import nodes
+from docutils.readers.python import pynodes
+from docutils import readers
 
 class Reader(docutils.readers.Reader):
 
     config_section = 'python reader'
     config_section_dependencies = ('readers',)
 
+    default_parser = 'restructuredtext'
+
     def parse(self):
         """Parse `self.input` into a document tree."""
-        self.document = document = moduleparser.parse_module(self.input)
-        document.current_source = document.current_line = None
+        self.document = document = self.new_document()
+        module_section = moduleparser.parse_module(self.input,
+                                                   self.source.source_path)
+        module_section.walk(DocformatVisitor(self.document))
+        visitor = DocstringFormattingVisitor(
+            document=document,
+            default_parser=self.default_parser)
+        module_section.walk(visitor)
+        self.document.append(module_section)
+
+class DocformatVisitor(nodes.SparseNodeVisitor):
+
+    """
+    This sets docformat attributes in a module.  Wherever an assignment
+    to __docformat__ is found, we look for the enclosing scope -- a class,
+    a module, or a function -- and set the docformat attribute there.
+
+    We can't do this during the DocstringFormattingVisitor walking,
+    because __docformat__ may appear below a docstring in that format
+    (typically below the module docstring).
+    """
+
+    def visit_attribute(self, node):
+        assert isinstance(node[0], pynodes.object_name)
+        name = node[0][0].data
+        if name != '__docformat__':
+            return
+        value = None
+        for child in children:
+            if isinstance(child, pynodes.expression_value):
+                value = child[0].data
+                break
+        assert value.startswith("'") or value.startswith('"'), "__docformat__ must be assigned a string literal (not %s); line: %s" % (value, node['lineno'])
+        name = name[1:-1]
+        looking_in = node.parent
+        while not isinstance(looking_in, (pynodes.module_section,
+                                          pynodes.function_section,
+                                          pynodes.class_section)):
+            looking_in = looking_in.parent
+        looking_in['docformat'] = name
+
+class DocstringFormattingVisitor(nodes.SparseNodeVisitor):
+
+    def __init__(self, document, default_parser):
+        self.document = document
+        self.default_parser = default_parser
+        self.parsers = {}
+
+    def visit_docstring(self, node):
+        text = node[0].data
+        docformat = self.find_docformat(node)
+        del node[0]
+        node['docformat'] = docformat
+        parser = self.get_parser(docformat)
+        parser.parse(text, self.document)
+        for child in self.document.get_children():
+            node.append(child)
+        self.document.current_source = self.document.current_line = None
+        del self.document[:]
+
+    def get_parser(self, parser_name):
+        """
+        Get a parser based on its name.  We reuse parsers during this
+        visitation, so parser instances are cached.
+        """
+        parser_name = parsers._parser_aliases.get(parser_name, parser_name)
+        if not self.parsers.has_key(parser_name):
+            cls = parsers.get_parser_class(parser_name)
+            self.parsers[parser_name] = cls()
+        return self.parsers[parser_name]
+
+    def find_docformat(self, node):
+        """
+        Find the __docformat__ closest to this node (i.e., look in the
+        class or module)
+        """
+        while node:
+            if node.get('docformat'):
+                return node['docformat']
+            node = node.parent
+        return self.default_parser
+    
+if __name__ == '__main__':
+    import locale
+    try:
+        locale.setlocale(locale.LC_ALL, '')
+    except:
+        pass
+
+    from docutils.core import publish_cmdline, default_description
+
+    description = ('Generates pseudo-XML from standalone reStructuredText '
+                   'sources (for testing purposes).  ' + default_description)
+
+    publish_cmdline(description=description,
+                    reader=Reader())
diff --git a/docutils/readers/python/moduleparser.py b/docutils/readers/python/moduleparser.py
index c95d997c8..7f965e6e2 100644
--- a/docutils/readers/python/moduleparser.py
+++ b/docutils/readers/python/moduleparser.py
@@ -7,11 +7,10 @@
 """
 Parser for Python modules.
 
-The `parse_module()` function takes a module's text and file name, runs it
-through the module parser (using compiler.py and tokenize.py) and produces a
-"module documentation tree": a high-level AST full of nodes that are
-interesting from an auto-documentation standpoint.  For example, given this
-module (x.py)::
+The `parse_module()` function takes a module's text and file name,
+runs it through the module parser (using compiler.py and tokenize.py)
+and produces a parse tree of the source code, using the nodes as found
+in pynodes.py.  For example, given this module (x.py)::
 
     # comment
 
@@ -50,69 +49,95 @@ module (x.py)::
 
 The module parser will produce this module documentation tree::
 
-    <Module filename="test data">
-        <Comment lineno=1>
-            comment
-        <Docstring>
+    <module_section filename="test data">
+        <docstring>
             Docstring
-        <Docstring lineno="5">
+        <docstring lineno="5">
             Additional docstring
-        <Attribute lineno="7" name="__docformat__">
-            <Expression lineno="7">
+        <attribute lineno="7">
+	    <object_name>
+	        __docformat__
+            <expression_value lineno="7">
                 'reStructuredText'
-        <Attribute lineno="9" name="a">
-            <Expression lineno="9">
+        <attribute lineno="9">
+	    <object_name>
+	        a
+            <expression_value lineno="9">
                 1
-            <Docstring lineno="10">
+            <docstring lineno="10">
                 Attribute docstring
-        <Class bases="Super" lineno="12" name="C">
-            <Docstring lineno="12">
+        <class_section lineno="12">
+	    <object_name>
+	        C
+            <class_base>
+	        Super
+            <docstring lineno="12">
                 C's docstring
-            <Attribute lineno="16" name="class_attribute">
-                <Expression lineno="16">
+            <attribute lineno="16">
+	        <object_name>
+		    class_attribute
+                <expression_value lineno="16">
                     1
-                <Docstring lineno="17">
+                <docstring lineno="17">
                     class_attribute's docstring
-            <Method lineno="19" name="__init__">
-                <Docstring lineno="19">
+            <method_section lineno="19">
+	        <object_name>
+		    __init__
+                <docstring lineno="19">
                     __init__'s docstring
-                <ParameterList lineno="19">
-                    <Parameter lineno="19" name="self">
-                    <Parameter lineno="19" name="text">
-                        <Default lineno="19">
+                <parameter_list lineno="19">
+                    <parameter lineno="19">
+		        <object_name>
+			    self
+                    <parameter lineno="19">
+		        <object_name>
+			    text
+                        <parameter_default lineno="19">
                             None
-                <Attribute lineno="22" name="self.instance_attribute">
-                    <Expression lineno="22">
+                <attribute lineno="22">
+		    <object_name>
+		        self.instance_attribute
+                    <expression_value lineno="22">
                         (text * 7 + ' whaddyaknow')
-                    <Docstring lineno="24">
+                    <docstring lineno="24">
                         instance_attribute's docstring
-        <Function lineno="27" name="f">
-            <Docstring lineno="27">
+        <function_section lineno="27">
+	    <object_name>
+	        f
+            <docstring lineno="27">
                 f's docstring
-            <ParameterList lineno="27">
-                <Parameter lineno="27" name="x">
-                    <Comment>
+            <parameter_list lineno="27">
+                <parameter lineno="27">
+		    <object_name>
+		        x
+                    <comment>
                         # parameter x
-                <Parameter lineno="27" name="y">
-                    <Default lineno="27">
+                <parameter lineno="27">
+		    <object_name>
+		        y
+                    <parameter_default lineno="27">
                         a * 5
-                    <Comment>
+                    <comment>
                         # parameter y
-                <ExcessPositionalArguments lineno="27" name="args">
-                    <Comment>
+                <parameter excess_positional="1" lineno="27">
+		    <object_name>
+		        args
+                    <comment>
                         # parameter args
-        <Attribute lineno="33" name="f.function_attribute">
-            <Expression lineno="33">
+        <attribute lineno="33">
+	    <object_name>
+	        f.function_attribute
+            <expression_value lineno="33">
                 1
-            <Docstring lineno="34">
+            <docstring lineno="34">
                 f.function_attribute's docstring
 
 (Comments are not implemented yet.)
 
 compiler.parse() provides most of what's needed for this doctree, and
-"tokenize" can be used to get the rest.  We can determine the line number from
-the compiler.parse() AST, and the TokenParser.rhs(lineno) method provides the
-rest.
+"tokenize" can be used to get the rest.  We can determine the line
+number from the compiler.parse() AST, and the TokenParser.rhs(lineno)
+method provides the rest.
 
 The Docutils Python reader component will transform this module doctree into a
 Python-specific Docutils doctree, and then a `stylist transform`_ will
diff --git a/docutils/readers/python/pynodes.py b/docutils/readers/python/pynodes.py
index 40db6975e..61e21f10e 100644
--- a/docutils/readers/python/pynodes.py
+++ b/docutils/readers/python/pynodes.py
@@ -74,14 +74,14 @@ class attribute_tuple(PythonStructural, TextElement): pass
 
 
 # Collect all the classes we've written above
-node_class_names = []
-def build_node_class_names():
+def install_node_class_names():
+    node_class_names = []
     for name, var in globals().items():
         if (type(var) is types.ClassType
             and issubclass(var, PythonStructural) \
             and name.lower() == name):
             node_class_names.append(var.tagname or name)
-
-# Register the new node names with GenericNodeVisitor and
-# SpecificNodeVisitor:
-nodes._add_node_class_names(node_class_names)
+    # Register the new node names with GenericNodeVisitor and
+    # SpecificNodeVisitor:
+    nodes._add_node_class_names(node_class_names)
+install_node_class_names()