path: root/paste/debug/doctest_webapp.py

# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org)
# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php
#!/usr/bin/env python2.4
# (c) 2005 Ian Bicking and contributors; written for Paste (http://pythonpaste.org)
# Licensed under the MIT license: http://www.opensource.org/licenses/mit-license.php

"""
These are functions for use when doctest-testing a document.
"""

try:
    import subprocess
except ImportError:
    from paste.util import subprocess24 as subprocess
import doctest
import os
import sys
import shutil
import re
import cgi
import rfc822
from cStringIO import StringIO
from paste.util import PySourceColor


here = os.path.abspath(__file__)
paste_parent = os.path.dirname(
    os.path.dirname(os.path.dirname(here)))

def run(command):
    data = run_raw(command)
    if data:
        print(data)

def run_raw(command):
    """
    Runs the string command, returns any output.
    """
    proc = subprocess.Popen(command, shell=True,
                            stderr=subprocess.STDOUT,
                            stdout=subprocess.PIPE, env=_make_env())
    data = proc.stdout.read()
    proc.wait()
    while data.endswith('\n') or data.endswith('\r'):
        data = data[:-1]
    if data:
        data = '\n'.join(
            [l for l in data.splitlines() if l])
        return data
    else:
        return ''

def run_command(command, name, and_print=False):
    output = run_raw(command)
    data = '$ %s\n%s' % (command, output)
    show_file('shell-command', name, description='shell transcript',
              data=data)
    if and_print and output:
        print(output)

def _make_env():
    env = os.environ.copy()
    env['PATH'] = (env.get('PATH', '')
                   + ':'
                   + os.path.join(paste_parent, 'scripts')
                   + ':'
                   + os.path.join(paste_parent, 'paste', '3rd-party',
                                  'sqlobject-files', 'scripts'))
    env['PYTHONPATH'] = (env.get('PYTHONPATH', '')
                         + ':'
                         + paste_parent)
    return env

def clear_dir(dir):
    """
    Clears (deletes) the given directory
    """
    shutil.rmtree(dir, True)

def ls(dir=None, recurse=False, indent=0):
    """
    Show a directory listing
    """
    dir = dir or os.getcwd()
    fns = os.listdir(dir)
    fns.sort()
    for fn in fns:
        full = os.path.join(dir, fn)
        if os.path.isdir(full):
            fn = fn + '/'
        print(' '*indent + fn)
        if os.path.isdir(full) and recurse:
            ls(dir=full, recurse=True, indent=indent+2)

default_app = None
default_url = None

def set_default_app(app, url):
    global default_app
    global default_url
    default_app = app
    default_url = url

def resource_filename(fn):
    """
    Returns the filename of the resource -- generally in the directory
    resources/DocumentName/fn
    """
    return os.path.join(
        os.path.dirname(sys.testing_document_filename),
        'resources',
        os.path.splitext(os.path.basename(sys.testing_document_filename))[0],
        fn)

def show(path_info, example_name):
    fn = resource_filename(example_name + '.html')
    out = StringIO()
    assert default_app is not None, (
        "No default_app set")
    url = default_url + path_info
    out.write('<span class="doctest-url"><a href="%s">%s</a></span><br>\n'
              % (url, url))
    out.write('<div class="doctest-example">\n')
    proc = subprocess.Popen(
        ['paster', 'serve' '--server=console', '--no-verbose',
         '--url=' + path_info],
        stderr=subprocess.PIPE,
        stdout=subprocess.PIPE,
        env=_make_env())
    stdout, errors = proc.communicate()
    stdout = StringIO(stdout)
    headers = rfc822.Message(stdout)
    content = stdout.read()
    for header, value in headers.items():
        if header.lower() == 'status' and int(value.split()[0]) == 200:
            continue
        if header.lower() in ('content-type', 'content-length'):
            continue
        if (header.lower() == 'set-cookie'
            and value.startswith('_SID_')):
            continue
        out.write('<span class="doctest-header">%s: %s</span><br>\n'
                  % (header, value))
    lines = [l for l in content.splitlines() if l.strip()]
    for line in lines:
        out.write(line + '\n')
    if errors:
        out.write('<pre class="doctest-errors">%s</pre>'
                  % errors)
    out.write('</div>\n')
    result = out.getvalue()
    if not os.path.exists(fn):
        f = open(fn, 'wb')
        f.write(result)
        f.close()
    else:
        f = open(fn, 'rb')
        expected = f.read()
        f.close()
        if not html_matches(expected, result):
            print('Pages did not match.  Expected from %s:' % fn)
            print('-'*60)
            print(expected)
            print('='*60)
            print('Actual output:')
            print('-'*60)
            print(result)

def html_matches(pattern, text):
    regex = re.escape(pattern)
    regex = regex.replace(r'\.\.\.', '.*')
    regex = re.sub(r'0x[0-9a-f]+', '.*', regex)
    regex = '^%s$' % regex
    return re.search(regex, text)

def convert_docstring_string(data):
    if data.startswith('\n'):
        data = data[1:]
    lines = data.splitlines()
    new_lines = []
    for line in lines:
        if line.rstrip() == '.':
            new_lines.append('')
        else:
            new_lines.append(line)
    data = '\n'.join(new_lines) + '\n'
    return data

def create_file(path, version, data):
    data = convert_docstring_string(data)
    write_data(path, data)
    show_file(path, version)

def append_to_file(path, version, data):
    data = convert_docstring_string(data)
    f = open(path, 'a')
    f.write(data)
    f.close()
    # I think these appends can happen so quickly (in less than a second)
    # that the .pyc file doesn't appear to be expired, even though it
    # is after we've made this change; so we have to get rid of the .pyc
    # file:
    if path.endswith('.py'):
        pyc_file = path + 'c'
        if os.path.exists(pyc_file):
            os.unlink(pyc_file)
    show_file(path, version, description='added to %s' % path,
              data=data)

def show_file(path, version, description=None, data=None):
    ext = os.path.splitext(path)[1]
    if data is None:
        f = open(path, 'rb')
        data = f.read()
        f.close()
    if ext == '.py':
        html = ('<div class="source-code">%s</div>' 
                % PySourceColor.str2html(data, PySourceColor.dark))
    else:
        html = '<pre class="source-code">%s</pre>' % cgi.escape(data, 1)
    html = '<span class="source-filename">%s</span><br>%s' % (
        description or path, html)
    write_data(resource_filename('%s.%s.gen.html' % (path, version)),
               html)

def call_source_highlight(input, format):
    proc = subprocess.Popen(['source-highlight', '--out-format=html',
                             '--no-doc', '--css=none',
                             '--src-lang=%s' % format], shell=False,
                            stdout=subprocess.PIPE)
    stdout, stderr = proc.communicate(input)
    result = stdout
    proc.wait()
    return result


def write_data(path, data):
    dir = os.path.dirname(os.path.abspath(path))
    if not os.path.exists(dir):
        os.makedirs(dir)
    f = open(path, 'wb')
    f.write(data)
    f.close()
    

def change_file(path, changes):
    f = open(os.path.abspath(path), 'rb')
    lines = f.readlines()
    f.close()
    for change_type, line, text in changes:
        if change_type == 'insert':
            lines[line:line] = [text]
        elif change_type == 'delete':
            lines[line:text] = []
        else:
            assert 0, (
                "Unknown change_type: %r" % change_type)
    f = open(path, 'wb')
    f.write(''.join(lines))
    f.close()

class LongFormDocTestParser(doctest.DocTestParser):

    """
    This parser recognizes some reST comments as commands, without
    prompts or expected output, like:

    .. run:

        do_this(...
        ...)
    """

    _EXAMPLE_RE = re.compile(r"""
        # Source consists of a PS1 line followed by zero or more PS2 lines.
        (?: (?P<source>
                (?:^(?P<indent> [ ]*) >>>    .*)    # PS1 line
                (?:\n           [ ]*  \.\.\. .*)*)  # PS2 lines
            \n?
            # Want consists of any non-blank lines that do not start with PS1.
            (?P<want> (?:(?![ ]*$)    # Not a blank line
                         (?![ ]*>>>)  # Not a line starting with PS1
                         .*$\n?       # But any other line
                      )*))
        | 
        (?: # This is for longer commands that are prefixed with a reST
            # comment like '.. run:' (two colons makes that a directive).
            # These commands cannot have any output.

            (?:^\.\.[ ]*(?P<run>run):[ ]*\n) # Leading command/command
            (?:[ ]*\n)?         # Blank line following
            (?P<runsource>
                (?:(?P<runindent> [ ]+)[^ ].*$)
                (?:\n [ ]+ .*)*)
            )
        |
        (?: # This is for shell commands

            (?P<shellsource>
                (?:^(P<shellindent> [ ]*) [$] .*)   # Shell line
                (?:\n               [ ]*  [>] .*)*) # Continuation
            \n?
            # Want consists of any non-blank lines that do not start with $
            (?P<shellwant> (?:(?![ ]*$)
                              (?![ ]*[$]$)
                              .*$\n?
                           )*))
        """, re.MULTILINE | re.VERBOSE)

    def _parse_example(self, m, name, lineno):
        r"""
        Given a regular expression match from `_EXAMPLE_RE` (`m`),
        return a pair `(source, want)`, where `source` is the matched
        example's source code (with prompts and indentation stripped);
        and `want` is the example's expected output (with indentation
        stripped).

        `name` is the string's name, and `lineno` is the line number
        where the example starts; both are used for error messages.

        >>> def parseit(s):
        ...     p = LongFormDocTestParser()
        ...     return p._parse_example(p._EXAMPLE_RE.search(s), '<string>', 1)
        >>> parseit('>>> 1\n1')
        ('1', {}, '1', None)
        >>> parseit('>>> (1\n... +1)\n2')
        ('(1\n+1)', {}, '2', None)
        >>> parseit('.. run:\n\n    test1\n    test2\n')
        ('test1\ntest2', {}, '', None)
        """
        # Get the example's indentation level.
        runner = m.group('run') or ''
        indent = len(m.group('%sindent' % runner))
        
        # Divide source into lines; check that they're properly
        # indented; and then strip their indentation & prompts.
        source_lines = m.group('%ssource' % runner).split('\n')
        if runner:
            self._check_prefix(source_lines[1:], ' '*indent, name, lineno)
        else:
            self._check_prompt_blank(source_lines, indent, name, lineno)
            self._check_prefix(source_lines[2:], ' '*indent + '.', name, lineno)
        if runner:
            source = '\n'.join([sl[indent:] for sl in source_lines])
        else:
            source = '\n'.join([sl[indent+4:] for sl in source_lines])

        if runner:
            want = ''
            exc_msg = None
        else:
            # Divide want into lines; check that it's properly indented; and
            # then strip the indentation.  Spaces before the last newline should
            # be preserved, so plain rstrip() isn't good enough.
            want = m.group('want')
            want_lines = want.split('\n')
            if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]):
                del want_lines[-1]  # forget final newline & spaces after it
            self._check_prefix(want_lines, ' '*indent, name,
                               lineno + len(source_lines))
            want = '\n'.join([wl[indent:] for wl in want_lines])

            # If `want` contains a traceback message, then extract it.
            m = self._EXCEPTION_RE.match(want)
            if m:
                exc_msg = m.group('msg')
            else:
                exc_msg = None

        # Extract options from the source.
        options = self._find_options(source, name, lineno)

        return source, options, want, exc_msg


    def parse(self, string, name='<string>'):
        """
        Divide the given string into examples and intervening text,
        and return them as a list of alternating Examples and strings.
        Line numbers for the Examples are 0-based.  The optional
        argument `name` is a name identifying this string, and is only
        used for error messages.
        """
        string = string.expandtabs()
        # If all lines begin with the same indentation, then strip it.
        min_indent = self._min_indent(string)
        if min_indent > 0:
            string = '\n'.join([l[min_indent:] for l in string.split('\n')])

        output = []
        charno, lineno = 0, 0
        # Find all doctest examples in the string:
        for m in self._EXAMPLE_RE.finditer(string):
            # Add the pre-example text to `output`.
            output.append(string[charno:m.start()])
            # Update lineno (lines before this example)
            lineno += string.count('\n', charno, m.start())
            # Extract info from the regexp match.
            (source, options, want, exc_msg) = \
                     self._parse_example(m, name, lineno)
            # Create an Example, and add it to the list.
            if not self._IS_BLANK_OR_COMMENT(source):
                # @@: Erg, this is the only line I need to change...
                output.append(doctest.Example(
                    source, want, exc_msg,
                    lineno=lineno,
                    indent=min_indent+len(m.group('indent') or m.group('runindent')),
                    options=options))
            # Update lineno (lines inside this example)
            lineno += string.count('\n', m.start(), m.end())
            # Update charno.
            charno = m.end()
        # Add any remaining post-example text to `output`.
        output.append(string[charno:])
        return output



if __name__ == '__main__':
    if sys.argv[1:] and sys.argv[1] == 'doctest':
        doctest.testmod()
        sys.exit()
    if not paste_parent in sys.path:
        sys.path.append(paste_parent)
    for fn in sys.argv[1:]:
        fn = os.path.abspath(fn)
        # @@: OK, ick; but this module gets loaded twice
        sys.testing_document_filename = fn
        doctest.testfile(
            fn, module_relative=False,
            optionflags=doctest.ELLIPSIS|doctest.REPORT_ONLY_FIRST_FAILURE,
            parser=LongFormDocTestParser())
        new = os.path.splitext(fn)[0] + '.html'
        assert new != fn
        os.system('rst2html.py %s > %s' % (fn, new))