Merge branch 'master' of github.com:goosemo/pycco

17 files changed, 862 insertions, 309 deletions

diff --git a/.editorconfig b/.editorconfig
new file mode 100644
index 0000000..fbee8fc
--- /dev/null
+++ b/.editorconfig
@@ -0,0 +1,15 @@
+; This file is for unifying the coding style for different editors and IDEs.
+; More information at http://EditorConfig.org
+
+root = true
+
+[*]
+end_of_line = LF
+
+[*.py]
+indent_style = space
+indent_size = 4
+
+[*.css]
+indent_style = space
+indent_size = 2
diff --git a/.gitignore b/.gitignore
index f749ab2..e1c9655 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,2 +1,13 @@
+.coverage
 *.pyc
 /Pycco.egg-info
+build/*
+dist/*
+docs/*
+/tags
+
+.cache
+.hypothesis
+.ropeproject
+
+.DS_Store
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..62e7c6b
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,12 @@
+language: python
+python:
+    - '2.7'
+    - '3.5'
+install:
+    - 'pip install -r requirements.txt'
+    - 'pip install -r requirements.test.txt'
+script:
+    - 'py.test --cov=pycco tests/'
+    - 'python -m pycco.main pycco/main.py'
+after_success:
+    - coveralls
diff --git a/AUTHORS b/AUTHORS
index 9c50a4a..188d852 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -1,6 +1,11 @@
 Alexis Metaireau
 Anders Bergh
 Antti Kaihola
+Christopher Gateley
 Jack Miller
+Morgan Goose
 Nick Fitzgerald
+Steffen Kampmann
+Zach Smith
+goosemo
 khamidou
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
new file mode 100644
index 0000000..eafb7ed
--- /dev/null
+++ b/CONTRIBUTING.md
@@ -0,0 +1,7 @@
+# Contributing to Pycco
+
+[Zach Smith](http://zdsmith.com) is the current maintainer of this project. 
+
+## Help Us Out
+
+Feel free to contribute by opening a pull request on this project's [GitHub repo](https://github.com/pycco-docs/pycco). All requests with documented and tested code will be gladly reviewed.
diff --git a/LICENSE b/LICENSE
index fbb0a92..de4c3a7 100644
--- a/LICENSE
+++ b/LICENSE
@@ -1,4 +1,5 @@
 Copyright (c) 2010 Nick Fitzgerald
+Copyright (c) 2016 Zachary Smith
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal
@@ -19,4 +20,4 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
 THE SOFTWARE.
 
 Parts of Pycco are taken from Docco, see http://github.com/jashkenas/docco for
-more information.
\ No newline at end of file
+more information.
diff --git a/README b/README
deleted file mode 100644
index 0ee0935..0000000
--- a/README
+++ /dev/null
@@ -1,25 +0,0 @@
-888888b.
-888   Y88b
-888    888
-888   d88P  888  888   .d8888b  .d8888b  .d88b.
-8888888P"   888  888  d88P"    d88P"    d88""88b
-888         888  888  888      888      888  888
-888         Y88b 888  Y88b.    Y88b.    Y88..88P
-888          "Y88888   "Y8888P  "Y8888P  "Y88P"
-                 888
-            Y8b d88P
-             "Y88P"
-
-Pycco is a Python port of Docco: the original quick-and-dirty,
-hundred-line-long, literate-programming-style documentation generator. For more
-information, see:
-
-http://fitzgen.github.com/pycco/
-
-Others:
-
-CoffeeScript (Original) - http://jashkenas.github.com/docco/
-
-Ruby - http://rtomayko.github.com/rocco/
-
-Sh - http://rtomayko.github.com/shocco/
\ No newline at end of file
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..73bca0a
--- /dev/null
+++ b/README.md
@@ -0,0 +1,30 @@
+```
+888888b.
+888   Y88b
+888    888
+888   d88P  888  888   .d8888b  .d8888b  .d88b.
+8888888P"   888  888  d88P"    d88P"    d88""88b
+888         888  888  888      888      888  888
+888         Y88b 888  Y88b.    Y88b.    Y88..88P
+888          "Y88888   "Y8888P  "Y8888P  "Y88P"
+                 888
+            Y8b d88P
+             "Y88P"
+```
+
+Pycco is a Python port of Docco: the original quick-and-dirty, hundred-line-
+long, literate-programming-style documentation generator. For more information,
+see:
+
+https://pycco-docs.github.io/pycco/
+
+Others:
+
+CoffeeScript (Original) - http://jashkenas.github.com/docco/
+
+Ruby - http://rtomayko.github.com/rocco/
+
+Sh - http://rtomayko.github.com/shocco/
+
+[![Build Status](https://travis-ci.org/pycco-docs/pycco.svg?branch=master)](https://travis-ci.org/pycco-docs/pycco)
+[![Coverage Status](https://coveralls.io/repos/pycco-docs/pycco/badge.svg?branch=master&service=github)](https://coveralls.io/github/pycco-docs/pycco?branch=master)
diff --git a/pycco/compat.py b/pycco/compat.py
new file mode 100644
index 0000000..68233a4
--- /dev/null
+++ b/pycco/compat.py
@@ -0,0 +1,10 @@
+try:
+    pycco_unichr = unichr
+except NameError:
+    pycco_unichr = chr
+
+def compat_items(d):
+    try:
+        return d.iteritems()
+    except AttributeError:
+        return d.items()
diff --git a/pycco/generate_index.py b/pycco/generate_index.py
new file mode 100644
index 0000000..ab0ead0
--- /dev/null
+++ b/pycco/generate_index.py
@@ -0,0 +1,76 @@
+"""
+This is the module responsible for automatically generating an HTML index of
+all documentation files generated by Pycco.
+"""
+import re
+from os import path
+
+from pycco.compat import compat_items
+from pycco_resources import pycco_template
+
+
+__all__ = ('generate_index',)
+
+
+def build_tree(file_paths, outdir):
+    tree = {}
+    for file_path in file_paths:
+        entry = {
+            'path': file_path,
+            'relpath': path.relpath(file_path, outdir)
+        }
+        path_steps = entry['relpath'].split(path.sep)
+        add_file(entry, path_steps, tree)
+
+    return tree
+
+
+def add_file(entry, path_steps, tree):
+    """
+    :param entry: A dictionary containing a path to a documentation file, and a
+    relative path to the same file.
+    :param path_steps: A list of steps in a file path to look within.
+    """
+    node, subpath = path_steps[0], path_steps[1:]
+    if node not in tree:
+        tree[node] = {}
+
+    if subpath:
+        add_file(entry, subpath, tree[node])
+
+    else:
+        tree[node]['entry'] = entry
+
+
+def generate_tree_html(tree):
+    """
+    Given a tree representing HTML file paths, return an HTML table plotting
+    those paths.
+    """
+    items = []
+    for node, subtree in sorted(compat_items(tree)):
+        if 'entry' in subtree:
+            html = u'<li><a href="{}">{}</a></li>'.format(subtree['entry']['relpath'], node)
+        else:
+            html = u'<dl><dt>{}</dt><dd><ul>{}</ul></dd></dl>'.format(node, generate_tree_html(subtree))
+
+        items.append(html)
+
+    return ''.join(items)
+
+
+def generate_index(files, outdir):
+    """
+    Given a list of generated documentation files, generate HTML to display
+    index of all files.
+    """
+    tree = build_tree(files, outdir)
+
+    rendered = pycco_template({
+        "title": 'Index',
+        "stylesheet": 'pycco.css',
+        "sections": {'docs_html': generate_tree_html(tree)},
+        "source": '',
+    })
+
+    return re.sub(r"__DOUBLE_OPEN_STACHE__", "{{", rendered).encode("utf-8")
diff --git a/pycco/main.py b/pycco/main.py
index c853959..e95ad73 100644
--- a/pycco/main.py
+++ b/pycco/main.py
@@ -1,53 +1,83 @@
 #!/usr/bin/env python
+from __future__ import print_function
+
+# This module contains all of our static resources.
+from pycco_resources import pycco_template, css as pycco_css
+
+"""
+"**Pycco**" is a Python port of [Docco](http://jashkenas.github.com/docco/):
+the original quick-and-dirty, hundred-line-long, literate-programming-style
+documentation generator. It produces HTML that displays your comments
+alongside your code. Comments are passed through
+[Markdown](http://daringfireball.net/projects/markdown/syntax) and
+[SmartyPants](http://daringfireball.net/projects/smartypants), while code is
+passed through [Pygments](http://pygments.org/) for syntax highlighting.
+This page is the result of running Pycco against its own source file.
+
+If you install Pycco, you can run it from the command-line:
+
+    pycco src/*.py
+
+This will generate linked HTML documentation for the named source files,
+saving it into a `docs` folder by default.
+
+The [source for Pycco](https://github.com/pycco-docs/pycco) is available on GitHub,
+and released under the MIT license.
+
+To install Pycco, simply
+
+    pip install pycco
+
+Or, to install the latest source
+
+    git clone git://github.com/pycco-docs/pycco.git
+    cd pycco
+    python setup.py install
+"""
+
+# === Main Documentation Generation Functions ===
+
+
+def generate_documentation(source, outdir=None, preserve_paths=True,
+                           language=None, encoding="utf8"):
+    """
+    Generate the documentation for a source file by reading it in, splitting it
+    up into comment/code sections, highlighting them for the appropriate
+    language, and merging them into an HTML template.
+    """
+
+    if not outdir:
+        raise TypeError("Missing the required 'outdir' keyword argument.")
+    code = open(source, "rb").read().decode(encoding)
+    return _generate_documentation(source, code, outdir, preserve_paths, language)
+
+
+def _generate_documentation(file_path, code, outdir, preserve_paths, language):
+    """
+    Helper function to allow documentation generation without file handling.
+    """
+    language = get_language(file_path, code, language=language)
+    sections = parse(code, language)
+    highlight(sections, language, preserve_paths=preserve_paths, outdir=outdir)
+    return generate_html(file_path, sections, preserve_paths=preserve_paths, outdir=outdir)
+
+
+def parse(code, language):
+    """
+    Given a string of source code, parse out each comment and the code that
+    follows it, and create an individual **section** for it.
+    Sections take the form:
+
+        { "docs_text": ...,
+          "docs_html": ...,
+          "code_text": ...,
+          "code_html": ...,
+          "num":       ...
+        }
+    """
 
-__all__ = ("process",)
-
-# **Pycco** is a Python port of [Docco](http://jashkenas.github.com/docco/ ):
-# the original quick-and-dirty, hundred-line-long, literate-programming-style
-# documentation generator. It produces HTML that displays your comments
-# alongside your code. Comments are passed through
-# [Markdown](http://daringfireball.net/projects/markdown/syntax), and code is
-# passed through [Pygments](http://pygments.org/) syntax highlighting.  This
-# page is the result of running Pycco against its own source file.
-#
-# If you install Pycco, you can run it from the command-line:
-#
-#     pycco src/*.py
-#
-# ...will generate linked HTML documentation for the named source files, saving
-# it into a `docs` folder by default.
-#
-# To install Pycco, simply
-#
-#     sudo setup.py install
-#
-
-#### Main Documentation Generation Functions
-
-# Generate the documentation for a source file by reading it in, splitting it
-# up into comment/code sections, highlighting them for the appropriate language,
-# and merging them into an HTML template.
-def generate_documentation(source, options):
-    fh = open(source, "r")
-    sections = parse(source, fh.read())
-    highlight(source, sections, options)
-    generate_html(source, sections, options=options)
-
-# Given a string of source code, parse out each comment and the code that
-# follows it, and create an individual **section** for it.
-# Sections take the form:
-#
-#     { "docs_text": ...,
-#       "docs_html": ...,
-#       "code_text": ...,
-#       "code_html": ...,
-#       "num":       ...
-#     }
-#
-def parse(source, code):
     lines = code.split("\n")
     sections = []
-    language = get_language(source)
     has_code = docs_text = code_text = ""
 
     if lines[0].startswith("#!"):
@@ -59,120 +89,155 @@ def parse(source, code):
                 lines.pop(linenum)
                 break
 
-
     def save(docs, code):
-        sections.append({
-            "docs_text": docs,
-            "code_text": code
-        })
+        if docs or code:
+            sections.append({
+                "docs_text": docs,
+                "code_text": code
+            })
 
     # Setup the variables to get ready to check for multiline comments
-    preformatted = multi_line = False
-    last_scope = 0
-    multi_line_delimeters = [language["multistart"], language["multiend"]]
+    multi_line = False
+    multi_string = False
+    multistart, multiend = language.get("multistart"), language.get("multiend")
+    comment_matcher = language['comment_matcher']
 
     for line in lines:
-
-        # Only go into multiline comments section when one of the delimeters is
+        process_as_code = False
+        # Only go into multiline comments section when one of the delimiters is
         # found to be at the start of a line
-        if any([line.lstrip().startswith(delim) for delim in multi_line_delimeters]):
-            if not multi_line:
-                multi_line = True
-
-            else:
+        if multistart and multiend \
+           and any(line.lstrip().startswith(delim) or line.rstrip().endswith(delim)
+                   for delim in (multistart, multiend)):
+            multi_line = not multi_line
+
+            if multi_line \
+               and line.strip().endswith(multiend) \
+               and len(line.strip()) > len(multiend):
                 multi_line = False
 
-            # Get rid of the delimeters so that they aren't in the final docs
-            line = re.sub(language["multistart"],'',line)
-            line = re.sub(language["multiend"],'',line)
-            docs_text += line.strip() + '\n'
-
-            if has_code and docs_text.strip():
-                save(docs_text, code_text[:-1])
-                code_text = code_text.split('\n')[-1]
-                last_scope = 0
-                has_code = docs_text = ''
+            if not line.strip().startswith(multistart) and not multi_line \
+               or multi_string:
 
-        elif multi_line:
-            line_striped = line.rstrip()
-            current_scope = line_striped.count("    ")
+                process_as_code = True
 
-            # This section will parse if the line is indented at least four
-            # places, and if so know to have the final text treat it as a
-            # preformatted text block.
-            if line_striped.startswith("    ") and last_scope:
-                if current_scope > last_scope and not preformatted:
-                    preformatted = True
-                    docs_text += "<pre>"
+                if multi_string:
+                    multi_line = False
+                    multi_string = False
+                else:
+                    multi_string = True
 
             else:
-                if preformatted:
-                    preformatted = False
-                    docs_text += "</pre>"
+                # Get rid of the delimiters so that they aren't in the final
+                # docs
+                line = line.replace(multistart, '')
+                line = line.replace(multiend, '')
+                docs_text += line.strip() + '\n'
+                indent_level = re.match("\s*", line).group(0)
+
+                if has_code and docs_text.strip():
+                    save(docs_text, code_text[:-1])
+                    code_text = code_text.split('\n')[-1]
+                    has_code = docs_text = ''
 
-            # Keep a tracker var to see if the scope increases, that way later
-            # the code can decided if a section is indented more than 4 spaces
-            # from the leading code.
-            last_scope = current_scope if current_scope > last_scope else last_scope
-            docs_text += line.strip() + '\n'
+        elif multi_line:
+            # Remove leading spaces
+            if re.match(r' {{{:d}}}'.format(len(indent_level)), line):
+                docs_text += line[len(indent_level):] + '\n'
+            else:
+                docs_text += line + '\n'
 
-        elif re.match(language["comment_matcher"], line):
+        elif re.match(comment_matcher, line):
             if has_code:
                 save(docs_text, code_text)
                 has_code = docs_text = code_text = ''
-            docs_text += re.sub(language["comment_matcher"], "", line) + "\n"
+            docs_text += re.sub(comment_matcher, "", line) + "\n"
 
         else:
-            if code_text and any([line.lstrip().startswith(x) for x in ['class ', 'def ']]):
-                save(docs_text, code_text)
-                code_text = has_code = docs_text = ''
+            process_as_code = True
+
+        if process_as_code:
+            if code_text and any(line.lstrip().startswith(x)
+                                 for x in ['class ', 'def ', '@']):
+                if not code_text.lstrip().startswith("@"):
+                    save(docs_text, code_text)
+                    code_text = has_code = docs_text = ''
 
             has_code = True
             code_text += line + '\n'
 
-
     save(docs_text, code_text)
+
     return sections
 
 # === Preprocessing the comments ===
-#
-# Add cross-references before having the text processed by markdown.
-# It's possible to reference another file, like this : [[pycco.py]] or a specific section of
-# another file, like this: [[pycco.py#Highlighting]]. Of course, sections have to be manually declared before,
-# A section name is written on a single line, and surrounded by equals signs, === like this ===
-def preprocess(comment, section_nr, options):
+
+
+def preprocess(comment, preserve_paths=True, outdir=None):
+    """
+    Add cross-references before having the text processed by markdown.  It's
+    possible to reference another file, like this : `[[main.py]]` which renders
+    [[main.py]]. You can also reference a specific section of another file, like
+    this: `[[main.py#highlighting-the-source-code]]` which renders as
+    [[main.py#highlighting-the-source-code]]. Sections have to be manually
+    declared; they are written on a single line, and surrounded by equals signs:
+    `=== like this ===`
+    """
+
+    if not outdir:
+        raise TypeError("Missing the required 'outdir' keyword argument.")
+
     def sanitize_section_name(name):
-        return name.strip().split(" ")[0]
+        return "-".join(name.lower().strip().split(" "))
 
     def replace_crossref(match):
         # Check if the match contains an anchor
         if '#' in match.group(1):
             name, anchor = match.group(1).split('#')
-            return "[%s](%s#%s)" % (name, path.basename(destination(name, options)), anchor)
+            return " [{}]({}#{})".format(name,
+                                         path.basename(destination(name,
+                                                                   preserve_paths=preserve_paths,
+                                                                   outdir=outdir)),
+                                         anchor)
 
         else:
-            return "[%s](%s)" % (match.group(1), path.basename(destination(match.group(1), options)))
+            return " [{}]({})".format(match.group(1),
+                                      path.basename(destination(match.group(1),
+                                                                preserve_paths=preserve_paths,
+                                                                outdir=outdir)))
 
     def replace_section_name(match):
-        return '<a name="%s">*%s*</a>' % (sanitize_section_name(match.group(1)), match.group(1))
-
-    comment = re.sub('===(.+)===\\n', replace_section_name, comment)
-    comment = re.sub('\[\[(.+)\]\]', replace_crossref, comment)
+        """
+        Replace equals-sign-formatted section names with anchor links.
+        """
+        return '{lvl} <span id="{id}" href="{id}">{name}</span>'.format(
+            lvl=re.sub('=', '#', match.group(1)),
+            id=sanitize_section_name(match.group(2)),
+            name=match.group(2)
+        )
+
+    comment = re.sub('^([=]+)([^=]+)[=]*\s*$', replace_section_name, comment)
+    comment = re.sub('(?<!`)\[\[(.+?)\]\]', replace_crossref, comment)
 
     return comment
 
 # === Highlighting the source code ===
-#
-# Highlights a single chunk of code using the **Pygments** module, and runs the
-# text of its corresponding comment through **Markdown**.
-#
-# We process the entire file in a single call to Pygments by inserting little
-# marker comments between each section and then splitting the result string
-# wherever our markers occur.
-def highlight(source, sections, options):
-    language = get_language(source)
-
-    output = pygments.highlight(language["divider_text"].join(section["code_text"] for section in sections),
+
+
+def highlight(sections, language, preserve_paths=True, outdir=None):
+    """
+    Highlights a single chunk of code using the **Pygments** module, and runs
+    the text of its corresponding comment through **Markdown**.
+
+    We process the entire file in a single call to Pygments by inserting little
+    marker comments between each section and then splitting the result string
+    wherever our markers occur.
+    """
+
+    if not outdir:
+        raise TypeError("Missing the required 'outdir' keyword argument.")
+
+    output = pygments.highlight(language["divider_text"].join(section["code_text"].rstrip() for section in sections),
                                 language["lexer"],
                                 formatters.get_formatter_by_name("html"))
 
@@ -184,46 +249,59 @@ def highlight(source, sections, options):
             docs_text = unicode(section["docs_text"])
         except UnicodeError:
             docs_text = unicode(section["docs_text"].decode('utf-8'))
-        section["docs_html"] = markdown(preprocess(docs_text, i, options))
+        except NameError:
+            docs_text = section['docs_text']
+        section["docs_html"] = markdown(preprocess(docs_text,
+                                                   preserve_paths=preserve_paths,
+                                                   outdir=outdir))
         section["num"] = i
 
+    return sections
+
 # === HTML Code generation ===
-# Once all of the code is finished highlighting, we can generate the HTML file
-# and write out the documentation. Pass the completed sections into the template
-# found in `resources/pycco.html`
-def generate_html(source, sections, options):
+
+
+def generate_html(source, sections, preserve_paths=True, outdir=None):
+    """
+    Once all of the code is finished highlighting, we can generate the HTML file
+    and write out the documentation. Pass the completed sections into the
+    template found in `resources/pycco.html`.
+
+    Pystache will attempt to recursively render context variables, so we must
+    replace any occurences of `{{`, which is valid in some languages, with a
+    "unique enough" identifier before rendering, and then post-process the
+    rendered template and change the identifier back to `{{`.
+    """
+
+    if not outdir:
+        raise TypeError("Missing the required 'outdir' keyword argument")
     title = path.basename(source)
-    dest = destination(source, options)
-    html = pycco_template({
-        "title":       title,
-        "stylesheet":  path.relpath(path.join(options['dir'], "pycco.css"),
-                                    path.split(dest)[0]),
-        "sections":    sections,
-        "source":     source,
-        "path":        path,
-        "destination": destination
+    dest = destination(source, preserve_paths=preserve_paths, outdir=outdir)
+    csspath = path.relpath(path.join(outdir, "pycco.css"), path.split(dest)[0])
+
+    for sect in sections:
+        sect["code_html"] = re.sub(r"\{\{", r"__DOUBLE_OPEN_STACHE__", sect["code_html"])
+
+    rendered = pycco_template({
+        "title": title,
+        "stylesheet": csspath,
+        "sections": sections,
+        "source": source,
     })
-    print "pycco = %s -> %s" % (source, dest)
-    try:
-        os.makedirs(path.split(dest)[0])
-    except OSError:
-        pass
-    fh = open(dest, "w")
-    fh.write(html.encode("utf-8"))
-    fh.close()
 
-#### Helpers & Setup
+    return re.sub(r"__DOUBLE_OPEN_STACHE__", "{{", rendered).encode("utf-8")
 
-# This module contains all of our static resources.
-import pycco_resources
+# === Helpers & Setup ===
 
 # Import our external dependencies.
 import optparse
 import os
 import pygments
-import pystache
 import re
 import sys
+import time
+import pycco.generate_index as generate_index
+
 from markdown import markdown
 from os import path
 from pygments import lexers, formatters
@@ -232,30 +310,43 @@ from pygments import lexers, formatters
 # the name of the Pygments lexer and the symbol that indicates a comment. To
 # add another language to Pycco's repertoire, add it here.
 languages = {
-    ".coffee": { "name": "coffee-script", "symbol": "#" },
+    ".coffee": {"name": "coffee-script", "symbol": "#",
+                "multistart": '###', "multiend": '###'},
+
+    ".pl":  {"name": "perl", "symbol": "#"},
 
-    ".pl":  { "name": "perl", "symbol": "#" },
+    ".sql": {"name": "sql", "symbol": "--"},
 
-    ".sql": { "name": "sql", "symbol": "--" },
+    ".c":   {"name": "c", "symbol": "//",
+             "multistart": "/*", "multiend": "*/"},
+    ".h":   {"name": "c", "symbol": "//",
+             "multistart": "/*", "multiend": "*/"},
 
-    ".c":   { "name": "c", "symbol": "//"},
+    ".cpp": {"name": "cpp", "symbol": "//"},
 
-    ".cpp": { "name": "cpp", "symbol": "//"},
+    ".cl":   {"name": "c", "symbol": "//",
+              "multistart": "/*", "multiend": "*/"},
 
-    ".js": { "name": "javascript", "symbol": "//", 
-        "multistart": "/*", "multiend": "*/"},
+    ".js": {"name": "javascript", "symbol": "//",
+            "multistart": "/*", "multiend": "*/"},
+    ".rb": {"name": "ruby", "symbol": "#",
+            "multistart": "=begin", "multiend": "=end"},
 
-    ".rb": { "name": "ruby", "symbol": "#",
-        "multistart": "=begin", "multiend": "=end"},
+    ".py": {"name": "python", "symbol": "#",
+            "multistart": '"""', "multiend": '"""'},
 
-    ".py": { "name": "python", "symbol": "#",  
-        "multistart": '"""', "multiend": '"""' },
+    ".scm": {"name": "scheme", "symbol": ";;",
+             "multistart": "#|", "multiend": "|#"},
 
-    ".scm": { "name": "scheme", "symbol": ";;",
-        "multistart": "#|", "multiend": "|#"},
+    ".lua": {"name": "lua", "symbol": "--",
+             "multistart": "--[[", "multiend": "--]]"},
 
-    ".lua": { "name": "lua", "symbol": "--",
-        "multistart": "--[[", "mutliend": "--]]"},
+    ".erl": {"name": "erlang", "symbol": "%%"},
+
+    ".tcl":  {"name": "tcl", "symbol": "#"},
+
+    ".hs": {"name": "haskell", "symbol": "--",
+            "multistart": "{-", "multiend": "-}"},
 }
 
 # Build out the appropriate matchers and delimiters for each language.
@@ -274,54 +365,90 @@ for ext, l in languages.items():
     # Get the Pygments Lexer for this language.
     l["lexer"] = lexers.get_lexer_by_name(l["name"])
 
-# Get the current language we're documenting, based on the extension.
-def get_language(source):
-    try:
-        return languages[ source[source.rindex("."):] ]
-    except ValueError:
-        source = open(source, "r")
-        code = source.read()
-        source.close()
-        lang = lexers.guess_lexer(code).name.lower()
+
+def get_language(source, code, language=None):
+    """Get the current language we're documenting, based on the extension."""
+
+    if language is not None:
         for l in languages.values():
-            if l["name"] == lang:
+            if l["name"] == language:
                 return l
         else:
+            raise ValueError("Unknown forced language: " + language)
+
+    m = re.match(r'.*(\..+)', os.path.basename(source)) if source else None
+    if m and m.group(1) in languages:
+        return languages[m.group(1)]
+    else:
+        try:
+            lang = lexers.guess_lexer(code).name.lower()
+            for l in languages.values():
+                if l["name"] == lang:
+                    return l
+            else:
+                raise ValueError()
+        except ValueError:
+            # If pygments can't find any lexers, it will raise its own
+            # subclass of ValueError. We will catch it and raise ours
+            # for consistency.
             raise ValueError("Can't figure out the language!")
 
-# Compute the destination HTML path for an input source file path. If the source
-# is `lib/example.py`, the HTML will be at `docs/example.html`
-def destination(filepath, options):
-    preserve_paths = options['paths']
+
+def destination(filepath, preserve_paths=True, outdir=None):
+    """
+    Compute the destination HTML path for an input source file path. If the
+    source is `lib/example.py`, the HTML will be at `docs/example.html`
+    """
+
+    dirname, filename = path.split(filepath)
+    if not outdir:
+        raise TypeError("Missing the required 'outdir' keyword argument.")
     try:
-        name = filepath.replace(filepath[ filepath.rindex("."): ], "")
+        name = re.sub(r"\.[^.]*$", "", filename)
     except ValueError:
-        name = filepath
-    if not preserve_paths:
-        name = path.basename(name)
-    return path.join(options['dir'], "%s.html" % name)
+        name = filename
+    if preserve_paths:
+        name = path.join(dirname, name)
+    dest = path.join(outdir, u"{}.html".format(name))
+    # If `join` is passed an absolute path, it will ignore any earlier path
+    # elements. We will force outdir to the beginning of the path to avoid
+    # writing outside our destination.
+    if not dest.startswith(outdir):
+        dest = outdir + os.sep + dest
+    return dest
+
 
-# Shift items off the front of the `list` until it is empty, then return
-# `default`.
 def shift(list, default):
+    """
+    Shift items off the front of the `list` until it is empty, then return
+    `default`.
+    """
+
     try:
         return list.pop(0)
     except IndexError:
         return default
 
-# Ensure that the destination directory exists.
+
+def remove_control_chars(s):
+    # Sanitization regexp copied from
+    # http://stackoverflow.com/questions/92438/stripping-non-printable-characters-from-a-string-in-python
+    from pycco.compat import pycco_unichr
+    control_chars = ''.join(map(pycco_unichr, list(range(0, 32)) + list(range(127, 160))))
+    control_char_re = re.compile(u'[{}]'.format(re.escape(control_chars)))
+    return control_char_re.sub('', s)
+
+
 def ensure_directory(directory):
+    """
+    Sanitize directory string and ensure that the destination directory exists.
+    """
+    directory = remove_control_chars(directory)
     if not os.path.isdir(directory):
-        os.mkdir(directory)
-
-def template(source):
-    return lambda context: pystache.render(source, context)
+        os.makedirs(directory)
 
-# Create the template that we will use to generate the Pycco HTML page.
-pycco_template = template(pycco_resources.html)
+    return directory
 
-# The CSS styles we"d like to apply to the documentation.
-pycco_styles = pycco_resources.css
 
 # The start of each Pygments highlight block.
 highlight_start = "<div class=\"highlight\"><pre>"
@@ -329,37 +456,142 @@ highlight_start = "<div class=\"highlight\"><pre>"
 # The end of each Pygments highlight block.
 highlight_end = "</pre></div>"
 
-# The bulk of the work is done here
-# For each source file passed in as an argument, generate the documentation.
-def process(sources, options):
-    sources.sort()
+
+def process(sources, preserve_paths=True, outdir=None, language=None, encoding="utf8", index=False):
+    """For each source file passed as argument, generate the documentation."""
+
+    if not outdir:
+        raise TypeError("Missing the required 'directory' keyword argument.")
+
+    # Make a copy of sources given on the command line. `main()` needs the
+    # original list when monitoring for changed files.
+    sources = sorted(sources)
+
+    # Proceed to generating the documentation.
     if sources:
-        ensure_directory(options['dir'])
-        css = open(path.join(options['dir'], "pycco.css"), "w")
-        css.write(pycco_styles)
+        outdir = ensure_directory(outdir)
+        css = open(path.join(outdir, "pycco.css"), "wb")
+        css.write(pycco_css.encode(encoding))
         css.close()
 
+        generated_files = []
+
         def next_file():
-            generate_documentation(sources.pop(0), options)
+            s = sources.pop(0)
+            dest = destination(s, preserve_paths=preserve_paths, outdir=outdir)
+
+            try:
+                os.makedirs(path.split(dest)[0])
+            except OSError:
+                pass
+
+            with open(dest, "wb") as f:
+                f.write(generate_documentation(s, preserve_paths=preserve_paths,
+                                               outdir=outdir,
+                                               language=language,
+                                               encoding=encoding))
+
+            print("pycco: {} -> {}".format(s, dest))
+            generated_files.append(dest)
+
             if sources:
                 next_file()
         next_file()
 
+        if index:
+            with open(path.join(outdir, "index.html"), "wb") as f:
+                f.write(generate_index.generate_index(generated_files, outdir))
+
+__all__ = ("process", "generate_documentation")
+
+
+def monitor(sources, opts):
+    """Monitor each source file and re-generate documentation on change."""
+
+    # The watchdog modules are imported in `main()` but we need to re-import
+    # here to bring them into the local namespace.
+    import watchdog.events
+    import watchdog.observers
+
+    # Watchdog operates on absolute paths, so map those to original paths
+    # as specified on the command line.
+    absolute_sources = dict((os.path.abspath(source), source)
+                            for source in sources)
+
+    class RegenerateHandler(watchdog.events.FileSystemEventHandler):
+
+        """A handler for recompiling files which triggered watchdog events"""
+
+        def on_modified(self, event):
+            """Regenerate documentation for a file which triggered an event"""
+            # Re-generate documentation from a source file if it was listed on
+            # the command line. Watchdog monitors whole directories, so other
+            # files may cause notifications as well.
+            if event.src_path in absolute_sources:
+                process([absolute_sources[event.src_path]],
+                        outdir=opts.outdir,
+                        preserve_paths=opts.paths)
+
+    # Set up an observer which monitors all directories for files given on
+    # the command line and notifies the handler defined above.
+    event_handler = RegenerateHandler()
+    observer = watchdog.observers.Observer()
+    directories = set(os.path.split(source)[0] for source in sources)
+    for directory in directories:
+        observer.schedule(event_handler, path=directory)
+
+    # Run the file change monitoring loop until the user hits Ctrl-C.
+    observer.start()
+    try:
+        while True:
+            time.sleep(1)
+    except KeyboardInterrupt:
+        observer.stop()
+        observer.join()
+
 
-# Hook spot for the console script
 def main():
+    """Hook spot for the console script."""
+
     parser = optparse.OptionParser()
     parser.add_option('-p', '--paths', action='store_true',
                       help='Preserve path structure of original files')
 
     parser.add_option('-d', '--directory', action='store', type='string',
-                      dest='dir', default='docs',
+                      dest='outdir', default='docs',
                       help='The output directory that the rendered files should go to.')
 
+    parser.add_option('-w', '--watch', action='store_true',
+                      help='Watch original files and re-generate documentation on changes')
+
+    parser.add_option('-l', '--force-language', action='store', type='string',
+                      dest='language', default=None,
+                      help='Force the language for the given files')
+
+    parser.add_option('-i', '--generate_index', action='store_true',
+                      help='Generate an index.html document with sitemap content')
+
     opts, sources = parser.parse_args()
-    process(sources, opts.__dict__)
+    if opts.outdir == '':
+        outdir = '.'
+    else:
+        outdir = opts.outdir
+
+    process(sources, outdir=outdir, preserve_paths=opts.paths,
+            language=opts.language, index=opts.generate_index)
+
+    # If the -w / --watch option was present, monitor the source directories
+    # for changes and re-generate documentation for source files whenever they
+    # are modified.
+    if opts.watch:
+        try:
+            import watchdog.events
+            import watchdog.observers
+        except ImportError:
+            sys.exit('The -w/--watch option requires the watchdog package.')
+
+        monitor(sources, opts)
 
 # Run the script.
 if __name__ == "__main__":
     main()
-
diff --git a/pycco_resources/__init__.py b/pycco_resources/__init__.py
index 4f6af45..a0eb757 100644
--- a/pycco_resources/__init__.py
+++ b/pycco_resources/__init__.py
@@ -6,6 +6,7 @@ body {
   line-height: 24px;
   color: #252519;
   margin: 0; padding: 0;
+  background: #f5f5ff;
 }
 a {
   color: #261a3b;
@@ -19,18 +20,21 @@ p {
 h1, h2, h3, h4, h5, h6 {
   margin: 40px 0 15px 0;
 }
-  h3, h4, h5, h6 {
-    margin-top: 20px;
+h2, h3, h4, h5, h6 {
+    margin-top: 0;
   }
 #container {
+  background: white;
+ }
+#container, div.section {
   position: relative;
 }
 #background {
-  position: fixed;
+  position: absolute;
   top: 0; left: 580px; right: 0; bottom: 0;
   background: #f5f5ff;
   border-left: 1px solid #e5e5ee;
-  z-index: -1;
+  z-index: 0;
 }
 #jump_to, #jump_page {
   background: white;
@@ -68,56 +72,57 @@ h1, h2, h3, h4, h5, h6 {
         }
         #jump_page .source:first-child {
         }
-table td {
-  border: 0;
-  outline: 0;
+div.docs {
+  float: left;
+  max-width: 500px;
+  min-width: 500px;
+  min-height: 5px;
+  padding: 10px 25px 1px 50px;
+  vertical-align: top;
+  text-align: left;
 }
-  td.docs, th.docs {
-    max-width: 500px;
-    min-width: 500px;
-    min-height: 5px;
-    padding: 10px 25px 1px 50px;
-    vertical-align: top;
-    text-align: left;
+  .docs pre {
+    margin: 15px 0 15px;
+    padding-left: 15px;
   }
-    .docs pre {
-      margin: 15px 0 15px;
-      padding-left: 15px;
-    }
-    .docs p tt, .docs p code {
-      background: #f8f8ff;
-      border: 1px solid #dedede;
-      font-size: 12px;
-      padding: 0 0.2em;
-    }
-    .octowrap {
-      position: relative;
+  .docs p tt, .docs p code {
+    background: #f8f8ff;
+    border: 1px solid #dedede;
+    font-size: 12px;
+    padding: 0 0.2em;
+  }
+  .octowrap {
+    position: relative;
+  }
+    .octothorpe {
+      font: 12px Arial;
+      text-decoration: none;
+      color: #454545;
+      position: absolute;
+      top: 3px; left: -20px;
+      padding: 1px 2px;
+      opacity: 0;
+      -webkit-transition: opacity 0.2s linear;
     }
-      .octothorpe {
-        font: 12px Arial;
-        text-decoration: none;
-        color: #454545;
-        position: absolute;
-        top: 3px; left: -20px;
-        padding: 1px 2px;
-        opacity: 0;
-        -webkit-transition: opacity 0.2s linear;
+      div.docs:hover .octothorpe {
+        opacity: 1;
       }
-        td.docs:hover .octothorpe {
-          opacity: 1;
-        }
-  td.code, th.code {
-    padding: 14px 15px 16px 50px;
-    width: 100%;
-    vertical-align: top;
-    background: #f5f5ff;
-    border-left: 1px solid #e5e5ee;
+div.code {
+  margin-left: 580px;
+  padding: 14px 15px 16px 50px;
+  vertical-align: top;
+}
+  .code pre, .docs p code {
+    font-size: 12px;
   }
     pre, tt, code {
-      font-size: 12px; line-height: 18px;
+      line-height: 18px;
       font-family: Monaco, Consolas, "Lucida Console", monospace;
       margin: 0; padding: 0;
     }
+div.clearall {
+    clear: both;
+}
 
 
 /*---------------------- Syntax Highlighting -----------------------------*/
@@ -209,28 +214,32 @@ html = """\
     </div>
   </div>
   {{/sources?}}
-  <table cellspacing=0 cellpadding=0>
-  <thead>
-    <tr>
-      <th class=docs><h1>{{ title }}</h1></th>
-      <th class=code></th>
-    </tr>
-  </thead>
-  <tbody>
-    {{#sections}}
-    <tr id='section-{{ num }}'>
-      <td class=docs>
-        <div class="octowrap">
-          <a class="octothorpe" href="#section-{{ num }}">#</a>
-        </div>
-        {{{ docs_html }}}
-      </td>
-      <td class=code>
-        <div class='highlight'><pre>{{{ code_html }}}</pre></div>
-      </td>
-    </tr>
-    {{/sections}}
-  </table>
+  <div class='section'>
+    <div class='docs'><h1>{{ title }}</h1></div>
+  </div>
+  <div class='clearall'>
+  {{#sections}}
+  <div class='section' id='section-{{ num }}'>
+    <div class='docs'>
+      <div class='octowrap'>
+        <a class='octothorpe' href='#section-{{ num }}'>#</a>
+      </div>
+      {{{ docs_html }}}
+    </div>
+    <div class='code'>
+      {{{ code_html }}}
+    </div>
+  </div>
+  <div class='clearall'></div>
+  {{/sections}}
 </div>
 </body>
 """
+
+import pystache
+
+
+def template(source):
+    return lambda context: pystache.render(source, context)
+# Create the template that we will use to generate the Pycco HTML page.
+pycco_template = template(html)
diff --git a/requirements.test.txt b/requirements.test.txt
new file mode 100644
index 0000000..5db5948
--- /dev/null
+++ b/requirements.test.txt
@@ -0,0 +1,3 @@
+hypothesis==1.18.1
+pytest-cov==2.2.0
+coveralls==1.1
diff --git a/requirements.txt b/requirements.txt
new file mode 100644
index 0000000..38964da
--- /dev/null
+++ b/requirements.txt
@@ -0,0 +1,3 @@
+pystache==0.5.4
+Pygments==2.0.2
+markdown==2.6.3
diff --git a/setup.py b/setup.py
index cffb9e8..6405ca9 100644
--- a/setup.py
+++ b/setup.py
@@ -1,19 +1,20 @@
 from setuptools import setup, find_packages
 
 setup(
-        name = "Pycco",
-        version = "0.1.2",
-        description = """A Python port of Docco: the original quick-and-dirty, 
+    name="Pycco",
+    version="0.4.1",
+    description="""A Python port of Docco: the original quick-and-dirty,
         hundred-line-long, literate-programming-style documentation generator.
         """,
-        author = "Nick Fitzgerald",
-        author_email = "fitzgen@gmail.com",
-        url = "http://fitzgen.github.com/pycco",
-        packages = find_packages(),
-        entry_points = {
-            'console_scripts': [
-                'pycco = pycco.main:main',
-                ]
-            },
-        install_requires = ['markdown', 'pygments', 'pystache'],
-        )
+    author="Zach Smith",
+    author_email="subsetpark@gmail.com",
+    url="https://pycco-docs.github.io/pycco/",
+    packages=find_packages(),
+    entry_points={
+        'console_scripts': [
+            'pycco = pycco.main:main',
+        ]
+    },
+    install_requires=['markdown', 'pygments', 'pystache', 'smartypants'],
+    extras_require={'monitoring': 'watchdog'},
+)
diff --git a/tests/__init__.py b/tests/__init__.py
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/tests/__init__.py
diff --git a/tests/test_pycco.py b/tests/test_pycco.py
new file mode 100644
index 0000000..02ef008
--- /dev/null
+++ b/tests/test_pycco.py
@@ -0,0 +1,163 @@
+import copy
+import os
+import tempfile
+import time
+import os.path
+import pytest
+from hypothesis import given, example, assume
+from hypothesis.strategies import lists, text, booleans, choices, none
+
+import pycco.generate_index as generate_index
+import pycco.main as p
+
+
+PYTHON = p.languages['.py']
+PYCCO_SOURCE = 'pycco/main.py'
+FOO_FUNCTION = """def foo():\n    return True"""
+
+
+def get_language(choice):
+    return choice(list(p.languages.values()))
+
+
+@given(lists(text()), text())
+def test_shift(fragments, default):
+    if fragments == []:
+        assert p.shift(fragments, default) == default
+    else:
+        fragments2 = copy.copy(fragments)
+        head = p.shift(fragments, default)
+        assert [head] + fragments == fragments2
+
+
+@given(text(), booleans(), text(min_size=1))
+@example("/foo", True, "0")
+def test_destination(filepath, preserve_paths, outdir):
+    dest = p.destination(filepath, preserve_paths=preserve_paths, outdir=outdir)
+    assert dest.startswith(outdir)
+    assert dest.endswith(".html")
+
+
+@given(choices(), text())
+def test_parse(choice, source):
+    l = get_language(choice)
+    parsed = p.parse(source, l)
+    for s in parsed:
+        assert {"code_text", "docs_text"} == set(s.keys())
+
+
+def test_skip_coding_directive():
+    source = "# -*- coding: utf-8 -*-\n" + FOO_FUNCTION
+    parsed = p.parse(source, PYTHON)
+    for section in parsed:
+        assert "coding" not in section['code_text']
+
+
+def test_multi_line_leading_spaces():
+    source = "# This is a\n# comment that\n# is indented\n"
+    source += FOO_FUNCTION
+    parsed = p.parse(source, PYTHON)
+    # The resulting comment has leading spaces stripped out.
+    assert parsed[0]["docs_text"] == "This is a\ncomment that\nis indented\n"
+
+
+def test_comment_with_only_cross_ref():
+    source = '''# ==Link Target==\n\ndef test_link():\n    """[[testing.py#link-target]]"""\n    pass'''
+    sections = p.parse(source, PYTHON)
+    p.highlight(sections, PYTHON, outdir=tempfile.gettempdir())
+    assert sections[1]['docs_html'] == '<p><a href="testing.html#link-target">testing.py</a></p>'
+
+
+@given(text(), text())
+def test_get_language_specify_language(source, code):
+    assert p.get_language(source, code, language="python") == p.languages['.py']
+
+    with pytest.raises(ValueError):
+        p.get_language(source, code, language="non-existent")
+
+
+@given(text() | none())
+def test_get_language_bad_source(source):
+    code = "#!/usr/bin/python\n"
+    code += FOO_FUNCTION
+    assert p.get_language(source, code) == PYTHON
+    with pytest.raises(ValueError) as e:
+        assert p.get_language(source, "badlang")
+
+    msg = "Can't figure out the language!"
+    try:
+        assert e.value.message == msg
+    except AttributeError:
+        assert e.value.args[0] == msg
+
+
+@given(text() | none())
+def test_get_language_bad_code(code):
+    source = "test.py"
+    assert p.get_language(source, code) == PYTHON
+
+
+@given(text(max_size=64))
+def test_ensure_directory(dir_name):
+    tempdir = os.path.join(tempfile.gettempdir(), str(int(time.time())), dir_name)
+
+    # Use sanitization from function, but only for housekeeping. We
+    # pass in the unsanitized string to the function.
+    safe_name = p.remove_control_chars(dir_name)
+
+    if not os.path.isdir(safe_name) and os.access(safe_name, os.W_OK):
+        p.ensure_directory(tempdir)
+        assert os.path.isdir(safe_name)
+
+
+def test_ensure_multiline_string_support():
+    code = '''x = """
+multi-line-string
+"""
+
+y = z  # comment
+
+# *comment with formatting*
+
+def x():
+    """multi-line-string
+    """'''
+
+    docs_code_tuple_list = p.parse(code, PYTHON)
+
+    assert docs_code_tuple_list[0]['docs_text'] == ''
+    assert "#" not in docs_code_tuple_list[1]['docs_text']
+
+
+def test_indented_block():
+
+    code = '''"""To install Pycco, simply
+
+    pip install pycco
+"""
+'''
+    parsed = p.parse(code, PYTHON)
+    highlighted = p.highlight(parsed, PYTHON, outdir=tempfile.gettempdir())
+    pre_block = highlighted[0]['docs_html']
+    assert '<pre>' in pre_block
+    assert '</pre>' in pre_block
+
+
+def test_generate_documentation():
+    p.generate_documentation(PYCCO_SOURCE, outdir=tempfile.gettempdir())
+
+
+@given(booleans(), booleans(), choices())
+def test_process(preserve_paths, index, choice):
+    lang_name = choice([l["name"] for l in p.languages.values()])
+    p.process([PYCCO_SOURCE], preserve_paths=preserve_paths,
+              index=index,
+              outdir=tempfile.gettempdir(),
+              language=lang_name)
+
+
+@given(lists(lists(text(min_size=1), min_size=1, max_size=30), min_size=1), lists(text(min_size=1), min_size=1))
+def test_generate_index(path_lists, outdir_list):
+    file_paths = [os.path.join(*path_list) for path_list in path_lists]
+    outdir = os.path.join(*outdir_list)
+    generate_index.generate_index(file_paths, outdir=outdir)