#!/usr/bin/env python2.4
"""
These are functions for use when doctest-testing a document.
"""

import subprocess
import doctest
import os
import sys
import shutil
import re
import cgi
import rfc822
from cStringIO import StringIO
from paste import server
from paste import wsgilib
from paste.util.thirdparty import add_package
add_package('PySourceColor')
import PySourceColor


here = os.path.abspath(__file__)
paste_parent = os.path.dirname(
    os.path.dirname(os.path.dirname(here)))

def run(command):
    data = run_raw(command)
    if data:
        print data

def run_raw(command):
    """
    Runs the string command, returns any output.
    """
    proc = subprocess.Popen(command, shell=True,
                            stderr=subprocess.STDOUT,
                            stdout=subprocess.PIPE, env=_make_env())
    data = proc.stdout.read()
    proc.wait()
    while data.endswith('\n') or data.endswith('\r'):
        data = data[:-1]
    if data:
        data = '\n'.join(
            [l for l in data.splitlines() if l])
        return data
    else:
        return ''

def run_command(command, name, and_print=False):
    output = run_raw(command)
    data = '$ %s\n%s' % (command, output)
    show_file('shell-command', name, description='shell transcript',
              data=data)
    if and_print and output:
        print output

def _make_env():
    env = os.environ.copy()
    env['PATH'] = (env.get('PATH', '')
                   + ':'
                   + os.path.join(paste_parent, 'scripts')
                   + ':'
                   + os.path.join(paste_parent, 'paste', '3rd-party',
                                  'sqlobject-files', 'scripts'))
    env['PYTHONPATH'] = (env.get('PYTHONPATH', '')
                         + ':'
                         + paste_parent)
    return env

def clear_dir(dir):
    """
    Clears (deletes) the given directory
    """
    shutil.rmtree(dir, True)

def ls(dir=None, recurse=False, indent=0):
    """
    Show a directory listing
    """
    dir = dir or os.getcwd()
    fns = os.listdir(dir)
    fns.sort()
    for fn in fns:
        full = os.path.join(dir, fn)
        if os.path.isdir(full):
            fn = fn + '/'
        print ' '*indent + fn
        if os.path.isdir(full) and recurse:
            ls(dir=full, recurse=True, indent=indent+2)

def make_app(dir):
    os.chdir(dir)
    sys.path.append(os.path.dirname(dir))
    conf, app = server.load_commandline((), allow_reload=False)
    assert conf is not None, (
        "server.load_commandline requested exit with code %r"
        % app)
    return app

default_app = None
default_url = None

def set_default_app(app, url):
    global default_app
    global default_url
    default_app = app
    default_url = url

def resource_filename(fn):
    """
    Returns the filename of the resource -- generally in the directory
    resources/DocumentName/fn
    """
    return os.path.join(
        os.path.dirname(sys.testing_document_filename),
        'resources',
        os.path.splitext(os.path.basename(sys.testing_document_filename))[0],
        fn)

def show(path_info, example_name):
    fn = resource_filename(example_name + '.html')
    out = StringIO()
    assert default_app is not None, (
        "No default_app set")
    url = default_url + path_info
    out.write('<span class="doctest-url"><a href="%s">%s</a></span><br>\n'
              % (url, url))
    out.write('<div class="doctest-example">\n')
    proc = subprocess.Popen(
        ['paster', 'serve' '--server=console', '--no-verbose',
         '--url=' + path_info],
        stderr=subprocess.PIPE,
        stdout=subprocess.PIPE,
        env=_make_env())
    stdout, errors = proc.communicate()
    stdout = StringIO(stdout)
    headers = rfc822.Message(stdout)
    content = stdout.read()
    for header, value in headers.items():
        if header.lower() == 'status' and int(value.split()[0]) == 200:
            continue
        if header.lower() in ('content-type', 'content-length'):
            continue
        if (header.lower() == 'set-cookie'
            and value.startswith('_SID_')):
            continue
        out.write('<span class="doctest-header">%s: %s</span><br>\n'
                  % (header, value))
    lines = [l for l in content.splitlines() if l.strip()]
    for line in lines:
        out.write(line + '\n')
    if errors:
        out.write('<pre class="doctest-errors">%s</pre>'
                  % errors)
    out.write('</div>\n')
    result = out.getvalue()
    if not os.path.exists(fn):
        f = open(fn, 'wb')
        f.write(result)
        f.close()
    else:
        f = open(fn, 'rb')
        expected = f.read()
        f.close()
        if not html_matches(expected, result):
            print 'Pages did not match.  Expected from %s:' % fn
            print '-'*60
            print expected
            print '='*60
            print 'Actual output:'
            print '-'*60
            print result

def html_matches(pattern, text):
    return True
    regex = re.escape(pattern)
    regex = regex.replace(r'\.\.\.', '.*')
    regex = re.sub(r'0x[0-9a-f]+', '.*', regex)
    regex = '^%s$' % regex
    return re.search(regex, text)

def convert_docstring_string(data):
    if data.startswith('\n'):
        data = data[1:]
    lines = data.splitlines()
    new_lines = []
    for line in lines:
        if line.rstrip() == '.':
            new_lines.append('')
        else:
            new_lines.append(line)
    data = '\n'.join(new_lines) + '\n'
    return data

def create_file(path, version, data):
    data = convert_docstring_string(data)
    write_data(path, data)
    show_file(path, version)

def append_to_file(path, version, data):
    data = convert_docstring_string(data)
    f = open(path, 'a')
    f.write(data)
    f.close()
    # I think these appends can happen so quickly (in less than a second)
    # that the .pyc file doesn't appear to be expired, even though it
    # is after we've made this change; so we have to get rid of the .pyc
    # file:
    if path.endswith('.py'):
        pyc_file = path + 'c'
        if os.path.exists(pyc_file):
            os.unlink(pyc_file)
    show_file(path, version, description='added to %s' % path,
              data=data)

def show_file(path, version, description=None, data=None):
    ext = os.path.splitext(path)[1]
    if data is None:
        f = open(path, 'rb')
        data = f.read()
        f.close()
    if ext == '.py':
        html = ('<div class="source-code">%s</div>' 
                % PySourceColor.str2html(data, PySourceColor.dark))
    else:
        html = '<pre class="source-code">%s</pre>' % cgi.escape(data, 1)
    html = '<span class="source-filename">%s</span><br>%s' % (
        description or path, html)
    write_data(resource_filename('%s.%s.gen.html' % (path, version)),
               html)

def call_source_highlight(input, format):
    proc = subprocess.Popen(['source-highlight', '--out-format=html',
                             '--no-doc', '--css=none',
                             '--src-lang=%s' % format], shell=False,
                            stdout=subprocess.PIPE)
    stdout, stderr = proc.communicate(input)
    result = stdout
    proc.wait()
    return result


def write_data(path, data):
    dir = os.path.dirname(os.path.abspath(path))
    if not os.path.exists(dir):
        os.makedirs(dir)
    f = open(path, 'wb')
    f.write(data)
    f.close()
    

def change_file(path, changes):
    f = open(os.path.abspath(path), 'rb')
    lines = f.readlines()
    f.close()
    for change_type, line, text in changes:
        if change_type == 'insert':
            lines[line:line] = [text]
        elif change_type == 'delete':
            lines[line:text] = []
        else:
            assert 0, (
                "Unknown change_type: %r" % change_type)
    f = open(path, 'wb')
    f.write(''.join(lines))
    f.close()

class LongFormDocTestParser(doctest.DocTestParser):

    """
    This parser recognizes some reST comments as commands, without
    prompts or expected output, like:

    .. run:

        do_this(...
        ...)
    """

    _EXAMPLE_RE = re.compile(r"""
        # Source consists of a PS1 line followed by zero or more PS2 lines.
        (?: (?P<source>
                (?:^(?P<indent> [ ]*) >>>    .*)    # PS1 line
                (?:\n           [ ]*  \.\.\. .*)*)  # PS2 lines
            \n?
            # Want consists of any non-blank lines that do not start with PS1.
            (?P<want> (?:(?![ ]*$)    # Not a blank line
                         (?![ ]*>>>)  # Not a line starting with PS1
                         .*$\n?       # But any other line
                      )*))
        | 
        (?: # This is for longer commands that are prefixed with a reST
            # comment like '.. run:' (two colons makes that a directive).
            # These commands cannot have any output.

            (?:^\.\.[ ]*(?P<run>run):[ ]*\n) # Leading command/command
            (?:[ ]*\n)?         # Blank line following
            (?P<runsource>
                (?:(?P<runindent> [ ]+)[^ ].*$)
                (?:\n [ ]+ .*)*)
            )
        |
        (?: # This is for shell commands

            (?P<shellsource>
                (?:^(P<shellindent> [ ]*) [$] .*)   # Shell line
                (?:\n               [ ]*  [>] .*)*) # Continuation
            \n?
            # Want consists of any non-blank lines that do not start with $
            (?P<shellwant> (?:(?![ ]*$)
                              (?![ ]*[$]$)
                              .*$\n?
                           )*))
        """, re.MULTILINE | re.VERBOSE)

    def _parse_example(self, m, name, lineno):
        r"""
        Given a regular expression match from `_EXAMPLE_RE` (`m`),
        return a pair `(source, want)`, where `source` is the matched
        example's source code (with prompts and indentation stripped);
        and `want` is the example's expected output (with indentation
        stripped).

        `name` is the string's name, and `lineno` is the line number
        where the example starts; both are used for error messages.

        >>> def parseit(s):
        ...     p = LongFormDocTestParser()
        ...     return p._parse_example(p._EXAMPLE_RE.search(s), '<string>', 1)
        >>> parseit('>>> 1\n1')
        ('1', {}, '1', None)
        >>> parseit('>>> (1\n... +1)\n2')
        ('(1\n+1)', {}, '2', None)
        >>> parseit('.. run:\n\n    test1\n    test2\n')
        ('test1\ntest2', {}, '', None)
        """
        # Get the example's indentation level.
        runner = m.group('run') or ''
        indent = len(m.group('%sindent' % runner))
        
        # Divide source into lines; check that they're properly
        # indented; and then strip their indentation & prompts.
        source_lines = m.group('%ssource' % runner).split('\n')
        if runner:
            self._check_prefix(source_lines[1:], ' '*indent, name, lineno)
        else:
            self._check_prompt_blank(source_lines, indent, name, lineno)
            self._check_prefix(source_lines[2:], ' '*indent + '.', name, lineno)
        if runner:
            source = '\n'.join([sl[indent:] for sl in source_lines])
        else:
            source = '\n'.join([sl[indent+4:] for sl in source_lines])

        if runner:
            want = ''
            exc_msg = None
        else:
            # Divide want into lines; check that it's properly indented; and
            # then strip the indentation.  Spaces before the last newline should
            # be preserved, so plain rstrip() isn't good enough.
            want = m.group('want')
            want_lines = want.split('\n')
            if len(want_lines) > 1 and re.match(r' *$', want_lines[-1]):
                del want_lines[-1]  # forget final newline & spaces after it
            self._check_prefix(want_lines, ' '*indent, name,
                               lineno + len(source_lines))
            want = '\n'.join([wl[indent:] for wl in want_lines])

            # If `want` contains a traceback message, then extract it.
            m = self._EXCEPTION_RE.match(want)
            if m:
                exc_msg = m.group('msg')
            else:
                exc_msg = None

        # Extract options from the source.
        options = self._find_options(source, name, lineno)

        return source, options, want, exc_msg


    def parse(self, string, name='<string>'):
        """
        Divide the given string into examples and intervening text,
        and return them as a list of alternating Examples and strings.
        Line numbers for the Examples are 0-based.  The optional
        argument `name` is a name identifying this string, and is only
        used for error messages.
        """
        string = string.expandtabs()
        # If all lines begin with the same indentation, then strip it.
        min_indent = self._min_indent(string)
        if min_indent > 0:
            string = '\n'.join([l[min_indent:] for l in string.split('\n')])

        output = []
        charno, lineno = 0, 0
        # Find all doctest examples in the string:
        for m in self._EXAMPLE_RE.finditer(string):
            # Add the pre-example text to `output`.
            output.append(string[charno:m.start()])
            # Update lineno (lines before this example)
            lineno += string.count('\n', charno, m.start())
            # Extract info from the regexp match.
            (source, options, want, exc_msg) = \
                     self._parse_example(m, name, lineno)
            # Create an Example, and add it to the list.
            if not self._IS_BLANK_OR_COMMENT(source):
                # @@: Erg, this is the only line I need to change...
                output.append( doctest.Example(source, want, exc_msg,
                                    lineno=lineno,
                                    indent=min_indent+len(m.group('indent') or m.group('runindent')),
                                    options=options) )
            # Update lineno (lines inside this example)
            lineno += string.count('\n', m.start(), m.end())
            # Update charno.
            charno = m.end()
        # Add any remaining post-example text to `output`.
        output.append(string[charno:])
        return output



if __name__ == '__main__':
    import sys
    if sys.argv[1:] and sys.argv[1] == 'doctest':
        doctest.testmod()
        sys.exit()
    if not paste_parent in sys.path:
        sys.path.append(paste_parent)
    for fn in sys.argv[1:]:
        fn = os.path.abspath(fn)
        # @@: OK, ick; but this module gets loaded twice
        sys.testing_document_filename = fn
        doctest.testfile(
            fn, module_relative=False,
            optionflags=doctest.ELLIPSIS|doctest.REPORT_ONLY_FIRST_FAILURE,
            parser=LongFormDocTestParser())
        new = os.path.splitext(fn)[0] + '.html'
        assert new != fn
        os.system('rst2html.py %s > %s' % (fn, new))