Added document extraction system; made recursive documentable

5 files changed, 435 insertions, 23 deletions

diff --git a/paste/docsupport/__init__.py b/paste/docsupport/__init__.py
new file mode 100644
index 0000000..792d600
--- /dev/null
+++ b/paste/docsupport/__init__.py
@@ -0,0 +1 @@
+#
diff --git a/paste/docsupport/extract.py b/paste/docsupport/extract.py
new file mode 100644
index 0000000..effd509
--- /dev/null
+++ b/paste/docsupport/extract.py
@@ -0,0 +1,275 @@
+import types
+import inspect
+from cStringIO import StringIO
+import textwrap
+import findmodules
+from paste.docsupport import metadata
+from paste.util.classinit import ClassInitMeta
+
+extractors = []
+
+class Extractor(object):
+
+    __metaclass__ = ClassInitMeta
+    match_type = None
+
+    def __classinit__(cls, new_attrs):
+        if cls.__bases__ != (object,):
+            extractors.append(cls)
+
+    def __init__(self, obj, context):
+        self.obj = obj
+        self.context = context
+
+    def extract(self):
+        raise NotImplementedError
+
+    def applies(cls, obj, context):
+        return isinstance(obj, cls.match_type)
+    applies = classmethod(applies)
+
+class ModuleExtractor(Extractor):
+    
+    match_type = types.ModuleType
+
+    def extract(self):
+        objs = getattr(self.obj, '__all__', [])
+        if not objs:
+            return
+        self.context.writeheader(self.obj.__name__, type='Module')
+        self.context.writedoc(self.obj.__doc__)
+        for name in objs:
+            self.context.push_name(name)
+            self.context.extract(getattr(self.obj, name))
+            self.context.pop_name(name)
+        self.context.endheader()
+
+class ClassExtractor(Extractor):
+
+    match_type = type
+        
+    def extract(self):
+        self.context.writeheader(self.context.last_name, type='Class')
+        self.context.writedoc(self.obj.__doc__)
+        attrs = self.getattrs(self.obj)
+        for name, value in attrs:
+            self.context.push_name(name)
+            self.context.extract(value)
+            self.context.pop_name(name)
+        methods = self.getmethods(self.obj)
+        for name, value in methods:
+            self.context.push_name(name)
+            self.context.extract(value)
+            self.context.pop_name(name)
+        self.context.endheader()
+
+    def getattrs(self, cls):
+        bases = inspect.getmro(cls)
+        attrs = {}
+        for i, base in enumerate(bases):
+            for name, value in base.__dict__.items():
+                if not isinstance(value, metadata.DocItem):
+                    continue
+                if name in attrs:
+                    continue
+                attrs[name] = (i, value.__creationorder__, value)
+        attrs = attrs.items()
+        attrs.sort(lambda a, b: cmp(a[1], b[1]))
+        return [
+            (m[0], m[1][2]) for m in attrs]
+
+    def getmethods(self, cls):
+        bases = inspect.getmro(cls)
+        methods = {}
+        for i, base in enumerate(bases):
+            if base.__dict__.has_key('__all__'):
+                all = base.__all__ or []
+            else:
+                all = None
+            for name, value in base.__dict__.items():
+                if all is not None and name not in all:
+                    continue
+                if not isinstance(value, types.FunctionType):
+                    continue
+                if name in methods:
+                    continue
+                methods[name] = (i, value.func_code.co_firstlineno, value)
+        methods = methods.items()
+        methods.sort(lambda a, b: cmp(a[1], b[1]))
+        return [
+            (m[0], m[1][2]) for m in methods]
+    
+class DetectedExtractor(Extractor):
+
+    def applies(cls, obj, context):
+        return isinstance(obj, metadata.DocItem)
+    applies = classmethod(applies)
+    
+    def extract(self):
+        self.obj.writeto(self.context)
+
+class MethodExtractor(Extractor):
+
+    match_type = types.FunctionType
+
+    def extract(self):
+        if not self.obj.__doc__:
+            return
+        sig = self.make_sig(self.obj)
+        self.context.writekey('def %s(%s)' % (self.obj.func_name, sig),
+                              monospace=False)
+        self.context.writedoc(self.obj.__doc__)
+        if getattr(self.obj, 'returns', None):
+            returns = self.obj.returns
+            if isinstance(returns, str):
+                returns = self.obj.func_globals[returns]
+            self.context.extract(self.obj.returns)
+        self.context.endkey()
+
+    def make_sig(self, func):
+        args, varargs, varkw, defaults = inspect.getargspec(func)
+        sig = []
+        args.reverse()
+        for arg in args:
+            if defaults:
+                sig.append('%s=%r' % (arg, defaults[-1]))
+                defaults = defaults[:-1]
+            else:
+                sig.append(arg)
+        sig.reverse()
+        if varargs:
+            sig.append('*%s' % varargs)
+        if varkw:
+            sig.append('**%s' % varkw)
+        return ', '.join(sig)
+
+
+############################################################
+## Context
+############################################################
+
+class DocContext(object):
+
+    headerchars = '+=-~.\'_`'
+
+    def __init__(self):
+        self.out = StringIO()
+        self.header_level = 0
+        self.indent_level = 0
+        self.names = []
+
+    def push_name(self, name):
+        self.names.append(name)
+
+    def pop_name(self, name=None):
+        if name is not None:
+            assert self.names[-1] == name, (
+                "Out of order pop (popping %r; expected %r)"
+                % (self.names[-1], name))
+        self.names.pop()
+
+    def last_name__get(self):
+        return self.names[-1]
+    last_name = property(last_name__get)
+
+    def writeheader(self, name, type=None):
+        if self.indent_level:
+            self.writekey(name, type=type, monospace=False)
+            return
+        if type:
+            name = '%s: %s' % (type, name)
+        self.write(name + '\n')
+        self.write(self.headerchars[self.header_level]
+                   * len(name))
+        self.write('\n\n')
+        self.header_level += 1
+
+    def endheader(self):
+        if self.indent_level:
+            self.endkey()
+            return
+        self.header_level -= 1
+        assert self.header_level >= 0, (
+            "Too many endheader() calls.")
+
+    def writedoc(self, doc):
+        if doc is None:
+            return
+        doc = self.clean(doc)
+        if not doc:
+            return
+        self.write(doc)
+        self.write('\n\n')
+
+    def writelist(self, seq, header=None, with_titles=False):
+        seq = list(seq)
+        if not seq:
+            return
+        if header:
+            self.writeheader(header)
+        for name, value in seq:
+            if with_titles:
+                self.writeheader(name)
+            value.write(self)
+            if with_titles:
+                self.endheader()
+        if header:
+            self.endheader()
+
+    def writekey(self, key, type=None, monospace=True):
+        if monospace:
+            key = '``%s``' % key
+        if type:
+            key = '%s: %s' % (type, key)
+        self.write('%s:\n' % key)
+        self.indent_level += 2
+
+    def endkey(self):
+        self.indent_level -= 2
+        assert self.indent_level >= 0, (
+            "Too many endkeys or dedents (indent %s)" % self.indent_level)
+
+    def write(self, s):
+        if self.indent_level:
+            self.out.write(self.indent(s, self.indent_level))
+        else:
+            self.out.write(s)
+
+    def clean(self, s):
+        return textwrap.dedent(s).rstrip().lstrip('\n')
+
+    def indent(self, s, indent=2):
+        new = '\n'.join([' '*indent + l for l in s.splitlines()])
+        if s.endswith('\n'):
+            new += '\n'
+        return new
+    
+    def capture(self, obj):
+        old_out = self.out
+        self.out = StringIO()
+        obj.write(self)
+        result = self.out.getvalue()
+        self.out = old_out
+        return result
+
+    def extract(self, obj):
+        for extractor in extractors:
+            if extractor.applies(obj, self):
+                ext = extractor(obj, self)
+                ext.extract()
+                break
+        else:
+            print >> sys.stderr, 'No extractor applies to %r\n' % obj
+
+def build_doc(package):
+    context = DocContext()
+    for module in findmodules.find_modules(package):
+        context.extract(module)
+    return context.out.getvalue()
+
+if __name__ == '__main__':
+    import sys
+    from paste.util.import_string import import_module
+    base = import_module(sys.argv[1])
+    print build_doc(base)
+    
diff --git a/paste/docsupport/findmodules.py b/paste/docsupport/findmodules.py
new file mode 100644
index 0000000..4797310
--- /dev/null
+++ b/paste/docsupport/findmodules.py
@@ -0,0 +1,25 @@
+"""
+Finds all modules in a packages, loads them, and returns them.
+"""
+
+import os
+from paste.util.import_string import import_module
+
+def find_modules(package):
+    pkg_name = package.__name__
+    modules = []
+    base = os.path.abspath(package.__file__)
+    if os.path.basename(os.path.splitext(base)[0]) == '__init__':
+        base = os.path.dirname(base)
+    if os.path.isdir(base):
+        for module_fn in os.listdir(base):
+            base, ext = os.path.splitext(module_fn)
+            full = os.path.join(base, module_fn)
+            if (os.path.isdir(full)
+                and os.path.exists(os.path.join(full, '__ini__.py'))):
+                modules.extend(import_module(pkg_name + '.' + base))
+            elif ext == '.py':
+                modules.append(import_module(pkg_name + '.' + base))
+    else:
+        modules.append(package)
+    return modules
diff --git a/paste/docsupport/metadata.py b/paste/docsupport/metadata.py
new file mode 100644
index 0000000..aa8879b
--- /dev/null
+++ b/paste/docsupport/metadata.py
@@ -0,0 +1,78 @@
+import sys
+import copy
+import types
+import inspect
+from itertools import count
+from paste.util.classinit import ClassInitMeta
+from paste.util.classinstance import classinstancemethod
+
+doc_count = count()
+
+class DocItem(object):
+
+    __metaclass__ = ClassInitMeta
+
+    def __classinit__(cls, new_attrs):
+        cls.__creationorder__ = doc_count.next()
+
+    def __init__(self):
+        self.__creationorder__ = doc_count.next()
+        stack = inspect.stack()
+        try:
+            while 1:
+                name = stack[0][0].f_globals['__name__']
+                if name != __name__:
+                    break
+                stack.pop(0)
+            self.call_module_name = name
+        finally:
+            # Break reference to frames
+            stack = None
+
+    def get_object(self, name):
+        return getattr(sys.modules[self.call_module_name], name)
+
+    def writeto(self, context):
+        raise NotImplementedError
+
+    def writeobj(self, name, context):
+        """
+        Write the named object to the context
+        """
+        if name is None:
+            return
+        obj = self.get_object(name)
+        context.push_name(name)
+        context.extract(obj)
+        context.pop_name(name)
+        
+
+class WSGIKey(DocItem):
+
+    def __init__(self, name, doc=None, interface=None):
+        self.name = name
+        self.doc = doc
+        self.interface = interface
+        super(WSGIKey, self).__init__()
+
+    def writeto(self, context):
+        context.writekey(self.name, type='WSGI Environment Key')
+        context.writedoc(self.doc)
+        self.writeobj(self.interface, context)
+        context.endkey()
+
+class Attribute(DocItem):
+
+    def __init__(self, doc, name=None, interface=None):
+        self.doc = doc
+        self.name = name
+        self.interface = interface
+        super(Attribute, self).__init__()
+
+    def writeto(self, context):
+        name = self.name or context.last_name
+        context.writekey(self.name, type='Attribute')
+        context.writedoc(self.doc)
+        context.writeobj(self.interface, context)
+        context.endkey()
+
diff --git a/paste/recursive.py b/paste/recursive.py
index 68bc380..f21c91a 100644
--- a/paste/recursive.py
+++ b/paste/recursive.py
@@ -1,30 +1,20 @@
-"""
-A WSGI middleware that allows for recursive and forwarded calls.
-All these calls go to the same 'application', but presumably that
-application acts differently with different URLs.  The forwarded
-URLs must be relative to this container.
-
-The forwarder is available through
-``environ['paste.recursive.forward'](path, extra_environ=None)``,
-the second argument is a dictionary of values to be added to the
-request, overwriting any keys.  The forward will call start_response;
-thus you must *not* call it after you have sent any output to the
-server.  Also, it will return an iterator that must be returned up the
-stack.  You may need to use exceptions to guarantee that this iterator
-will be passed back through the application.
-
-The includer is available through
-``environ['paste.recursive.include'](path, extra_environ=None)``.
-It is like forwarder, except it completes the request and returns a
-response object.  The response object has three public attributes:
-status, headers, and body.  The status is a string, headers is a list
-of (header_name, header_value) tuples, and the body is a string.
-"""
-
 from cStringIO import StringIO
+from paste.docsupport import metadata
+
+__all__ = ['RecursiveMiddleware']
 
 class RecursiveMiddleware(object):
 
+    """
+    A WSGI middleware that allows for recursive and forwarded calls.
+    All these calls go to the same 'application', but presumably that
+    application acts differently with different URLs.  The forwarded
+    URLs must be relative to this container.
+
+    Interface is entirely through the ``paste.recursive.forward`` and
+    ``paste.recursive.include`` environmental keys.
+    """
+
     def __init__(self, application):
         self.application = application
 
@@ -35,6 +25,11 @@ class RecursiveMiddleware(object):
             self.application, environ, start_response)
         return self.application(environ, start_response)
 
+    _wsgi_add1 = metadata.WSGIKey('paste.recursive.forward',
+                                  interface='Forwarder')
+    _wsgi_add2 = metadata.WSGIKey('paste.recursive.include',
+                                  interface='Includer')
+
 class Recursive(object):
 
     def __init__(self, application, environ, start_response):
@@ -44,6 +39,11 @@ class Recursive(object):
         self.start_response = start_response
 
     def __call__(self, path, new_environ=None):
+        """
+        `extra_environ` is an optional dictionary that is also added
+        to the forwarded request.  E.g., ``{'HTTP_HOST': 'new.host'}``
+        could be used to forward to a different virtual host.
+        """
         environ = self.original_environ.copy()
         if new_environ:
             environ.update(new_environ)
@@ -67,10 +67,32 @@ class Recursive(object):
 
 class Forwarder(Recursive):
 
+    """
+    The forwarder will try to restart the request, except with
+    the new `path` (replacing ``PATH_INFO`` in the request).
+
+    It must not be called after and headers have been returned.
+    It returns an iterator that must be returned back up the call
+    stack, so it must be used like::
+
+        return environ['paste.recursive.forward'](path)
+
+    Meaningful transformations cannot be done, since headers are
+    sent directly to the server and cannot be inspected or
+    rewritten.
+    """
+
     def activate(self, environ):
         return self.application(environ, self.start_response)
+    
 
 class Includer(Recursive):
+
+    """
+    Starts another request with the given path and adding or
+    overwriting any values in the `extra_environ` dictionary.
+    Returns an IncludeResponse object.
+    """
     
     def activate(self, environ):
         response = IncludedResponse
@@ -98,6 +120,16 @@ class IncludedResponse(object):
         self.output = StringIO()
         self.str = None
 
+    _attr_headers = metadata.Attribute("""
+    A list of ``[(header_name), (header_value)]``
+    """)
+    _attr_status = metadata.Attribute("""
+    The status code returned (a string), like ``'200 OK'``
+    """)
+    _attr_body = metadata.Attribute("""
+    The body of the response.
+    """)
+
     def close(self):
         self.str = self.output.getvalue()
         self.output.close()
@@ -118,3 +150,4 @@ class IncludedResponse(object):
         else:
             return self.str
     body = property(body__get)
+