diff options
Diffstat (limited to 'vendor/tornado')
89 files changed, 16531 insertions, 0 deletions
diff --git a/vendor/tornado/MANIFEST.in b/vendor/tornado/MANIFEST.in new file mode 100644 index 0000000000..c7a51e4094 --- /dev/null +++ b/vendor/tornado/MANIFEST.in @@ -0,0 +1,2 @@ +recursive-include demos *.py *.yaml *.html *.css *.png *.js *.xml *.sql README +include tornado/epoll.c diff --git a/vendor/tornado/README b/vendor/tornado/README new file mode 100644 index 0000000000..d504022243 --- /dev/null +++ b/vendor/tornado/README @@ -0,0 +1,27 @@ +Tornado +======= +Tornado is an open source version of the scalable, non-blocking web server +and and tools that power FriendFeed. Documentation and downloads are +available at http://www.tornadoweb.org/ + +Tornado is licensed under the Apache Licence, Version 2.0 +(http://www.apache.org/licenses/LICENSE-2.0.html). + +Installation +============ +To install: + + python setup.py build + sudo python setup.py install + +Tornado has been tested on Python 2.5 and 2.6. To use all of the features +of Tornado, you need to have PycURL and a JSON library like simplejson +installed. + +On Mac OS X, you can install the packages with: + + sudo easy_install setuptools pycurl==7.16.2.1 simplejson + +On Ubuntu Linux, you can install the packages with: + + sudo apt-get install python-pycurl python-simplejson diff --git a/vendor/tornado/demos/appengine/README b/vendor/tornado/demos/appengine/README new file mode 100644 index 0000000000..e4aead6701 --- /dev/null +++ b/vendor/tornado/demos/appengine/README @@ -0,0 +1,48 @@ +Running the Tornado AppEngine example +===================================== +This example is designed to run in Google AppEngine, so there are a couple +of steps to get it running. You can download the Google AppEngine Python +development environment at http://code.google.com/appengine/downloads.html. + +1. Link or copy the tornado code directory into this directory: + + ln -s ../../tornado tornado + + AppEngine doesn't use the Python modules installed on this machine. + You need to have the 'tornado' module copied or linked for AppEngine + to find it. + +3. Install and run dev_appserver + + If you don't already have the App Engine SDK, download it from + http://code.google.com/appengine/downloads.html + + To start the tornado demo, run the dev server on this directory: + + dev_appserver.py . + +4. Visit http://localhost:8080/ in your browser + + If you sign in as an administrator, you will be able to create and + edit blog posts. If you sign in as anybody else, you will only see + the existing blog posts. + + +If you want to deploy the blog in production: + +1. Register a new appengine application and put its id in app.yaml + + First register a new application at http://appengine.google.com/. + Then edit app.yaml in this directory and change the "application" + setting from "tornado-appenginge" to your new application id. + +2. Deploy to App Engine + + If you registered an application id, you can now upload your new + Tornado blog by running this command: + + appcfg update . + + After that, visit application_id.appspot.com, where application_id + is the application you registered. + diff --git a/vendor/tornado/demos/appengine/app.yaml b/vendor/tornado/demos/appengine/app.yaml new file mode 100644 index 0000000000..2d00c586dd --- /dev/null +++ b/vendor/tornado/demos/appengine/app.yaml @@ -0,0 +1,11 @@ +application: tornado-appengine +version: 1 +runtime: python +api_version: 1 + +handlers: +- url: /static/ + static_dir: static + +- url: /.* + script: blog.py diff --git a/vendor/tornado/demos/appengine/blog.py b/vendor/tornado/demos/appengine/blog.py new file mode 100644 index 0000000000..ccaabd5392 --- /dev/null +++ b/vendor/tornado/demos/appengine/blog.py @@ -0,0 +1,169 @@ +#!/usr/bin/env python +# +# Copyright 2009 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import functools +import markdown +import os.path +import re +import tornado.web +import tornado.wsgi +import unicodedata +import wsgiref.handlers + +from google.appengine.api import users +from google.appengine.ext import db + + +class Entry(db.Model): + """A single blog entry.""" + author = db.UserProperty() + title = db.StringProperty(required=True) + slug = db.StringProperty(required=True) + markdown = db.TextProperty(required=True) + html = db.TextProperty(required=True) + published = db.DateTimeProperty(auto_now_add=True) + updated = db.DateTimeProperty(auto_now=True) + + +def administrator(method): + """Decorate with this method to restrict to site admins.""" + @functools.wraps(method) + def wrapper(self, *args, **kwargs): + if not self.current_user: + if self.request.method == "GET": + self.redirect(self.get_login_url()) + return + raise tornado.web.HTTPError(403) + elif not self.current_user.administrator: + if self.request.method == "GET": + self.redirect("/") + return + raise tornado.web.HTTPError(403) + else: + return method(self, *args, **kwargs) + return wrapper + + +class BaseHandler(tornado.web.RequestHandler): + """Implements Google Accounts authentication methods.""" + def get_current_user(self): + user = users.get_current_user() + if user: user.administrator = users.is_current_user_admin() + return user + + def get_login_url(self): + return users.create_login_url(self.request.uri) + + def render_string(self, template_name, **kwargs): + # Let the templates access the users module to generate login URLs + return tornado.web.RequestHandler.render_string( + self, template_name, users=users, **kwargs) + + +class HomeHandler(BaseHandler): + def get(self): + entries = db.Query(Entry).order('-published').fetch(limit=5) + if not entries: + if not self.current_user or self.current_user.administrator: + self.redirect("/compose") + return + self.render("home.html", entries=entries) + + +class EntryHandler(BaseHandler): + def get(self, slug): + entry = db.Query(Entry).filter("slug =", slug).get() + if not entry: raise tornado.web.HTTPError(404) + self.render("entry.html", entry=entry) + + +class ArchiveHandler(BaseHandler): + def get(self): + entries = db.Query(Entry).order('-published') + self.render("archive.html", entries=entries) + + +class FeedHandler(BaseHandler): + def get(self): + entries = db.Query(Entry).order('-published').fetch(limit=10) + self.set_header("Content-Type", "application/atom+xml") + self.render("feed.xml", entries=entries) + + +class ComposeHandler(BaseHandler): + @administrator + def get(self): + key = self.get_argument("key", None) + entry = Entry.get(key) if key else None + self.render("compose.html", entry=entry) + + @administrator + def post(self): + key = self.get_argument("key", None) + if key: + entry = Entry.get(key) + entry.title = self.get_argument("title") + entry.markdown = self.get_argument("markdown") + entry.html = markdown.markdown(self.get_argument("markdown")) + else: + title = self.get_argument("title") + slug = unicodedata.normalize("NFKD", title).encode( + "ascii", "ignore") + slug = re.sub(r"[^\w]+", " ", slug) + slug = "-".join(slug.lower().strip().split()) + if not slug: slug = "entry" + while True: + existing = db.Query(Entry).filter("slug =", slug).get() + if not existing or str(existing.key()) == key: + break + slug += "-2" + entry = Entry( + author=self.current_user, + title=title, + slug=slug, + markdown=self.get_argument("markdown"), + html=markdown.markdown(self.get_argument("markdown")), + ) + entry.put() + self.redirect("/entry/" + entry.slug) + + +class EntryModule(tornado.web.UIModule): + def render(self, entry): + return self.render_string("modules/entry.html", entry=entry) + + +settings = { + "blog_title": u"Tornado Blog", + "template_path": os.path.join(os.path.dirname(__file__), "templates"), + "ui_modules": {"Entry": EntryModule}, + "xsrf_cookies": True, +} +application = tornado.wsgi.WSGIApplication([ + (r"/", HomeHandler), + (r"/archive", ArchiveHandler), + (r"/feed", FeedHandler), + (r"/entry/([^/]+)", EntryHandler), + (r"/compose", ComposeHandler), +], **settings) + + +def main(): + wsgiref.handlers.CGIHandler().run(application) + + +if __name__ == "__main__": + main() diff --git a/vendor/tornado/demos/appengine/markdown.py b/vendor/tornado/demos/appengine/markdown.py new file mode 100644 index 0000000000..59ba731bf0 --- /dev/null +++ b/vendor/tornado/demos/appengine/markdown.py @@ -0,0 +1,1877 @@ +#!/usr/bin/env python +# Copyright (c) 2007-2008 ActiveState Corp. +# License: MIT (http://www.opensource.org/licenses/mit-license.php) + +r"""A fast and complete Python implementation of Markdown. + +[from http://daringfireball.net/projects/markdown/] +> Markdown is a text-to-HTML filter; it translates an easy-to-read / +> easy-to-write structured text format into HTML. Markdown's text +> format is most similar to that of plain text email, and supports +> features such as headers, *emphasis*, code blocks, blockquotes, and +> links. +> +> Markdown's syntax is designed not as a generic markup language, but +> specifically to serve as a front-end to (X)HTML. You can use span-level +> HTML tags anywhere in a Markdown document, and you can use block level +> HTML tags (like <div> and <table> as well). + +Module usage: + + >>> import markdown2 + >>> markdown2.markdown("*boo!*") # or use `html = markdown_path(PATH)` + u'<p><em>boo!</em></p>\n' + + >>> markdowner = Markdown() + >>> markdowner.convert("*boo!*") + u'<p><em>boo!</em></p>\n' + >>> markdowner.convert("**boom!**") + u'<p><strong>boom!</strong></p>\n' + +This implementation of Markdown implements the full "core" syntax plus a +number of extras (e.g., code syntax coloring, footnotes) as described on +<http://code.google.com/p/python-markdown2/wiki/Extras>. +""" + +cmdln_desc = """A fast and complete Python implementation of Markdown, a +text-to-HTML conversion tool for web writers. +""" + +# Dev Notes: +# - There is already a Python markdown processor +# (http://www.freewisdom.org/projects/python-markdown/). +# - Python's regex syntax doesn't have '\z', so I'm using '\Z'. I'm +# not yet sure if there implications with this. Compare 'pydoc sre' +# and 'perldoc perlre'. + +__version_info__ = (1, 0, 1, 14) # first three nums match Markdown.pl +__version__ = '1.0.1.14' +__author__ = "Trent Mick" + +import os +import sys +from pprint import pprint +import re +import logging +try: + from hashlib import md5 +except ImportError: + from md5 import md5 +import optparse +from random import random +import codecs + + + +#---- Python version compat + +if sys.version_info[:2] < (2,4): + from sets import Set as set + def reversed(sequence): + for i in sequence[::-1]: + yield i + def _unicode_decode(s, encoding, errors='xmlcharrefreplace'): + return unicode(s, encoding, errors) +else: + def _unicode_decode(s, encoding, errors='strict'): + return s.decode(encoding, errors) + + +#---- globals + +DEBUG = False +log = logging.getLogger("markdown") + +DEFAULT_TAB_WIDTH = 4 + +# Table of hash values for escaped characters: +def _escape_hash(s): + # Lame attempt to avoid possible collision with someone actually + # using the MD5 hexdigest of one of these chars in there text. + # Other ideas: random.random(), uuid.uuid() + #return md5(s).hexdigest() # Markdown.pl effectively does this. + return 'md5-'+md5(s).hexdigest() +g_escape_table = dict([(ch, _escape_hash(ch)) + for ch in '\\`*_{}[]()>#+-.!']) + + + +#---- exceptions + +class MarkdownError(Exception): + pass + + + +#---- public api + +def markdown_path(path, encoding="utf-8", + html4tags=False, tab_width=DEFAULT_TAB_WIDTH, + safe_mode=None, extras=None, link_patterns=None, + use_file_vars=False): + text = codecs.open(path, 'r', encoding).read() + return Markdown(html4tags=html4tags, tab_width=tab_width, + safe_mode=safe_mode, extras=extras, + link_patterns=link_patterns, + use_file_vars=use_file_vars).convert(text) + +def markdown(text, html4tags=False, tab_width=DEFAULT_TAB_WIDTH, + safe_mode=None, extras=None, link_patterns=None, + use_file_vars=False): + return Markdown(html4tags=html4tags, tab_width=tab_width, + safe_mode=safe_mode, extras=extras, + link_patterns=link_patterns, + use_file_vars=use_file_vars).convert(text) + +class Markdown(object): + # The dict of "extras" to enable in processing -- a mapping of + # extra name to argument for the extra. Most extras do not have an + # argument, in which case the value is None. + # + # This can be set via (a) subclassing and (b) the constructor + # "extras" argument. + extras = None + + urls = None + titles = None + html_blocks = None + html_spans = None + html_removed_text = "[HTML_REMOVED]" # for compat with markdown.py + + # Used to track when we're inside an ordered or unordered list + # (see _ProcessListItems() for details): + list_level = 0 + + _ws_only_line_re = re.compile(r"^[ \t]+$", re.M) + + def __init__(self, html4tags=False, tab_width=4, safe_mode=None, + extras=None, link_patterns=None, use_file_vars=False): + if html4tags: + self.empty_element_suffix = ">" + else: + self.empty_element_suffix = " />" + self.tab_width = tab_width + + # For compatibility with earlier markdown2.py and with + # markdown.py's safe_mode being a boolean, + # safe_mode == True -> "replace" + if safe_mode is True: + self.safe_mode = "replace" + else: + self.safe_mode = safe_mode + + if self.extras is None: + self.extras = {} + elif not isinstance(self.extras, dict): + self.extras = dict([(e, None) for e in self.extras]) + if extras: + if not isinstance(extras, dict): + extras = dict([(e, None) for e in extras]) + self.extras.update(extras) + assert isinstance(self.extras, dict) + self._instance_extras = self.extras.copy() + self.link_patterns = link_patterns + self.use_file_vars = use_file_vars + self._outdent_re = re.compile(r'^(\t|[ ]{1,%d})' % tab_width, re.M) + + def reset(self): + self.urls = {} + self.titles = {} + self.html_blocks = {} + self.html_spans = {} + self.list_level = 0 + self.extras = self._instance_extras.copy() + if "footnotes" in self.extras: + self.footnotes = {} + self.footnote_ids = [] + + def convert(self, text): + """Convert the given text.""" + # Main function. The order in which other subs are called here is + # essential. Link and image substitutions need to happen before + # _EscapeSpecialChars(), so that any *'s or _'s in the <a> + # and <img> tags get encoded. + + # Clear the global hashes. If we don't clear these, you get conflicts + # from other articles when generating a page which contains more than + # one article (e.g. an index page that shows the N most recent + # articles): + self.reset() + + if not isinstance(text, unicode): + #TODO: perhaps shouldn't presume UTF-8 for string input? + text = unicode(text, 'utf-8') + + if self.use_file_vars: + # Look for emacs-style file variable hints. + emacs_vars = self._get_emacs_vars(text) + if "markdown-extras" in emacs_vars: + splitter = re.compile("[ ,]+") + for e in splitter.split(emacs_vars["markdown-extras"]): + if '=' in e: + ename, earg = e.split('=', 1) + try: + earg = int(earg) + except ValueError: + pass + else: + ename, earg = e, None + self.extras[ename] = earg + + # Standardize line endings: + text = re.sub("\r\n|\r", "\n", text) + + # Make sure $text ends with a couple of newlines: + text += "\n\n" + + # Convert all tabs to spaces. + text = self._detab(text) + + # Strip any lines consisting only of spaces and tabs. + # This makes subsequent regexen easier to write, because we can + # match consecutive blank lines with /\n+/ instead of something + # contorted like /[ \t]*\n+/ . + text = self._ws_only_line_re.sub("", text) + + if self.safe_mode: + text = self._hash_html_spans(text) + + # Turn block-level HTML blocks into hash entries + text = self._hash_html_blocks(text, raw=True) + + # Strip link definitions, store in hashes. + if "footnotes" in self.extras: + # Must do footnotes first because an unlucky footnote defn + # looks like a link defn: + # [^4]: this "looks like a link defn" + text = self._strip_footnote_definitions(text) + text = self._strip_link_definitions(text) + + text = self._run_block_gamut(text) + + if "footnotes" in self.extras: + text = self._add_footnotes(text) + + text = self._unescape_special_chars(text) + + if self.safe_mode: + text = self._unhash_html_spans(text) + + text += "\n" + return text + + _emacs_oneliner_vars_pat = re.compile(r"-\*-\s*([^\r\n]*?)\s*-\*-", re.UNICODE) + # This regular expression is intended to match blocks like this: + # PREFIX Local Variables: SUFFIX + # PREFIX mode: Tcl SUFFIX + # PREFIX End: SUFFIX + # Some notes: + # - "[ \t]" is used instead of "\s" to specifically exclude newlines + # - "(\r\n|\n|\r)" is used instead of "$" because the sre engine does + # not like anything other than Unix-style line terminators. + _emacs_local_vars_pat = re.compile(r"""^ + (?P<prefix>(?:[^\r\n|\n|\r])*?) + [\ \t]*Local\ Variables:[\ \t]* + (?P<suffix>.*?)(?:\r\n|\n|\r) + (?P<content>.*?\1End:) + """, re.IGNORECASE | re.MULTILINE | re.DOTALL | re.VERBOSE) + + def _get_emacs_vars(self, text): + """Return a dictionary of emacs-style local variables. + + Parsing is done loosely according to this spec (and according to + some in-practice deviations from this): + http://www.gnu.org/software/emacs/manual/html_node/emacs/Specifying-File-Variables.html#Specifying-File-Variables + """ + emacs_vars = {} + SIZE = pow(2, 13) # 8kB + + # Search near the start for a '-*-'-style one-liner of variables. + head = text[:SIZE] + if "-*-" in head: + match = self._emacs_oneliner_vars_pat.search(head) + if match: + emacs_vars_str = match.group(1) + assert '\n' not in emacs_vars_str + emacs_var_strs = [s.strip() for s in emacs_vars_str.split(';') + if s.strip()] + if len(emacs_var_strs) == 1 and ':' not in emacs_var_strs[0]: + # While not in the spec, this form is allowed by emacs: + # -*- Tcl -*- + # where the implied "variable" is "mode". This form + # is only allowed if there are no other variables. + emacs_vars["mode"] = emacs_var_strs[0].strip() + else: + for emacs_var_str in emacs_var_strs: + try: + variable, value = emacs_var_str.strip().split(':', 1) + except ValueError: + log.debug("emacs variables error: malformed -*- " + "line: %r", emacs_var_str) + continue + # Lowercase the variable name because Emacs allows "Mode" + # or "mode" or "MoDe", etc. + emacs_vars[variable.lower()] = value.strip() + + tail = text[-SIZE:] + if "Local Variables" in tail: + match = self._emacs_local_vars_pat.search(tail) + if match: + prefix = match.group("prefix") + suffix = match.group("suffix") + lines = match.group("content").splitlines(0) + #print "prefix=%r, suffix=%r, content=%r, lines: %s"\ + # % (prefix, suffix, match.group("content"), lines) + + # Validate the Local Variables block: proper prefix and suffix + # usage. + for i, line in enumerate(lines): + if not line.startswith(prefix): + log.debug("emacs variables error: line '%s' " + "does not use proper prefix '%s'" + % (line, prefix)) + return {} + # Don't validate suffix on last line. Emacs doesn't care, + # neither should we. + if i != len(lines)-1 and not line.endswith(suffix): + log.debug("emacs variables error: line '%s' " + "does not use proper suffix '%s'" + % (line, suffix)) + return {} + + # Parse out one emacs var per line. + continued_for = None + for line in lines[:-1]: # no var on the last line ("PREFIX End:") + if prefix: line = line[len(prefix):] # strip prefix + if suffix: line = line[:-len(suffix)] # strip suffix + line = line.strip() + if continued_for: + variable = continued_for + if line.endswith('\\'): + line = line[:-1].rstrip() + else: + continued_for = None + emacs_vars[variable] += ' ' + line + else: + try: + variable, value = line.split(':', 1) + except ValueError: + log.debug("local variables error: missing colon " + "in local variables entry: '%s'" % line) + continue + # Do NOT lowercase the variable name, because Emacs only + # allows "mode" (and not "Mode", "MoDe", etc.) in this block. + value = value.strip() + if value.endswith('\\'): + value = value[:-1].rstrip() + continued_for = variable + else: + continued_for = None + emacs_vars[variable] = value + + # Unquote values. + for var, val in emacs_vars.items(): + if len(val) > 1 and (val.startswith('"') and val.endswith('"') + or val.startswith('"') and val.endswith('"')): + emacs_vars[var] = val[1:-1] + + return emacs_vars + + # Cribbed from a post by Bart Lateur: + # <http://www.nntp.perl.org/group/perl.macperl.anyperl/154> + _detab_re = re.compile(r'(.*?)\t', re.M) + def _detab_sub(self, match): + g1 = match.group(1) + return g1 + (' ' * (self.tab_width - len(g1) % self.tab_width)) + def _detab(self, text): + r"""Remove (leading?) tabs from a file. + + >>> m = Markdown() + >>> m._detab("\tfoo") + ' foo' + >>> m._detab(" \tfoo") + ' foo' + >>> m._detab("\t foo") + ' foo' + >>> m._detab(" foo") + ' foo' + >>> m._detab(" foo\n\tbar\tblam") + ' foo\n bar blam' + """ + if '\t' not in text: + return text + return self._detab_re.subn(self._detab_sub, text)[0] + + _block_tags_a = 'p|div|h[1-6]|blockquote|pre|table|dl|ol|ul|script|noscript|form|fieldset|iframe|math|ins|del' + _strict_tag_block_re = re.compile(r""" + ( # save in \1 + ^ # start of line (with re.M) + <(%s) # start tag = \2 + \b # word break + (.*\n)*? # any number of lines, minimally matching + </\2> # the matching end tag + [ \t]* # trailing spaces/tabs + (?=\n+|\Z) # followed by a newline or end of document + ) + """ % _block_tags_a, + re.X | re.M) + + _block_tags_b = 'p|div|h[1-6]|blockquote|pre|table|dl|ol|ul|script|noscript|form|fieldset|iframe|math' + _liberal_tag_block_re = re.compile(r""" + ( # save in \1 + ^ # start of line (with re.M) + <(%s) # start tag = \2 + \b # word break + (.*\n)*? # any number of lines, minimally matching + .*</\2> # the matching end tag + [ \t]* # trailing spaces/tabs + (?=\n+|\Z) # followed by a newline or end of document + ) + """ % _block_tags_b, + re.X | re.M) + + def _hash_html_block_sub(self, match, raw=False): + html = match.group(1) + if raw and self.safe_mode: + html = self._sanitize_html(html) + key = _hash_text(html) + self.html_blocks[key] = html + return "\n\n" + key + "\n\n" + + def _hash_html_blocks(self, text, raw=False): + """Hashify HTML blocks + + We only want to do this for block-level HTML tags, such as headers, + lists, and tables. That's because we still want to wrap <p>s around + "paragraphs" that are wrapped in non-block-level tags, such as anchors, + phrase emphasis, and spans. The list of tags we're looking for is + hard-coded. + + @param raw {boolean} indicates if these are raw HTML blocks in + the original source. It makes a difference in "safe" mode. + """ + if '<' not in text: + return text + + # Pass `raw` value into our calls to self._hash_html_block_sub. + hash_html_block_sub = _curry(self._hash_html_block_sub, raw=raw) + + # First, look for nested blocks, e.g.: + # <div> + # <div> + # tags for inner block must be indented. + # </div> + # </div> + # + # The outermost tags must start at the left margin for this to match, and + # the inner nested divs must be indented. + # We need to do this before the next, more liberal match, because the next + # match will start at the first `<div>` and stop at the first `</div>`. + text = self._strict_tag_block_re.sub(hash_html_block_sub, text) + + # Now match more liberally, simply from `\n<tag>` to `</tag>\n` + text = self._liberal_tag_block_re.sub(hash_html_block_sub, text) + + # Special case just for <hr />. It was easier to make a special + # case than to make the other regex more complicated. + if "<hr" in text: + _hr_tag_re = _hr_tag_re_from_tab_width(self.tab_width) + text = _hr_tag_re.sub(hash_html_block_sub, text) + + # Special case for standalone HTML comments: + if "<!--" in text: + start = 0 + while True: + # Delimiters for next comment block. + try: + start_idx = text.index("<!--", start) + except ValueError, ex: + break + try: + end_idx = text.index("-->", start_idx) + 3 + except ValueError, ex: + break + + # Start position for next comment block search. + start = end_idx + + # Validate whitespace before comment. + if start_idx: + # - Up to `tab_width - 1` spaces before start_idx. + for i in range(self.tab_width - 1): + if text[start_idx - 1] != ' ': + break + start_idx -= 1 + if start_idx == 0: + break + # - Must be preceded by 2 newlines or hit the start of + # the document. + if start_idx == 0: + pass + elif start_idx == 1 and text[0] == '\n': + start_idx = 0 # to match minute detail of Markdown.pl regex + elif text[start_idx-2:start_idx] == '\n\n': + pass + else: + break + + # Validate whitespace after comment. + # - Any number of spaces and tabs. + while end_idx < len(text): + if text[end_idx] not in ' \t': + break + end_idx += 1 + # - Must be following by 2 newlines or hit end of text. + if text[end_idx:end_idx+2] not in ('', '\n', '\n\n'): + continue + + # Escape and hash (must match `_hash_html_block_sub`). + html = text[start_idx:end_idx] + if raw and self.safe_mode: + html = self._sanitize_html(html) + key = _hash_text(html) + self.html_blocks[key] = html + text = text[:start_idx] + "\n\n" + key + "\n\n" + text[end_idx:] + + if "xml" in self.extras: + # Treat XML processing instructions and namespaced one-liner + # tags as if they were block HTML tags. E.g., if standalone + # (i.e. are their own paragraph), the following do not get + # wrapped in a <p> tag: + # <?foo bar?> + # + # <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="chapter_1.md"/> + _xml_oneliner_re = _xml_oneliner_re_from_tab_width(self.tab_width) + text = _xml_oneliner_re.sub(hash_html_block_sub, text) + + return text + + def _strip_link_definitions(self, text): + # Strips link definitions from text, stores the URLs and titles in + # hash references. + less_than_tab = self.tab_width - 1 + + # Link defs are in the form: + # [id]: url "optional title" + _link_def_re = re.compile(r""" + ^[ ]{0,%d}\[(.+)\]: # id = \1 + [ \t]* + \n? # maybe *one* newline + [ \t]* + <?(.+?)>? # url = \2 + [ \t]* + (?: + \n? # maybe one newline + [ \t]* + (?<=\s) # lookbehind for whitespace + ['"(] + ([^\n]*) # title = \3 + ['")] + [ \t]* + )? # title is optional + (?:\n+|\Z) + """ % less_than_tab, re.X | re.M | re.U) + return _link_def_re.sub(self._extract_link_def_sub, text) + + def _extract_link_def_sub(self, match): + id, url, title = match.groups() + key = id.lower() # Link IDs are case-insensitive + self.urls[key] = self._encode_amps_and_angles(url) + if title: + self.titles[key] = title.replace('"', '"') + return "" + + def _extract_footnote_def_sub(self, match): + id, text = match.groups() + text = _dedent(text, skip_first_line=not text.startswith('\n')).strip() + normed_id = re.sub(r'\W', '-', id) + # Ensure footnote text ends with a couple newlines (for some + # block gamut matches). + self.footnotes[normed_id] = text + "\n\n" + return "" + + def _strip_footnote_definitions(self, text): + """A footnote definition looks like this: + + [^note-id]: Text of the note. + + May include one or more indented paragraphs. + + Where, + - The 'note-id' can be pretty much anything, though typically it + is the number of the footnote. + - The first paragraph may start on the next line, like so: + + [^note-id]: + Text of the note. + """ + less_than_tab = self.tab_width - 1 + footnote_def_re = re.compile(r''' + ^[ ]{0,%d}\[\^(.+)\]: # id = \1 + [ \t]* + ( # footnote text = \2 + # First line need not start with the spaces. + (?:\s*.*\n+) + (?: + (?:[ ]{%d} | \t) # Subsequent lines must be indented. + .*\n+ + )* + ) + # Lookahead for non-space at line-start, or end of doc. + (?:(?=^[ ]{0,%d}\S)|\Z) + ''' % (less_than_tab, self.tab_width, self.tab_width), + re.X | re.M) + return footnote_def_re.sub(self._extract_footnote_def_sub, text) + + + _hr_res = [ + re.compile(r"^[ ]{0,2}([ ]?\*[ ]?){3,}[ \t]*$", re.M), + re.compile(r"^[ ]{0,2}([ ]?\-[ ]?){3,}[ \t]*$", re.M), + re.compile(r"^[ ]{0,2}([ ]?\_[ ]?){3,}[ \t]*$", re.M), + ] + + def _run_block_gamut(self, text): + # These are all the transformations that form block-level + # tags like paragraphs, headers, and list items. + + text = self._do_headers(text) + + # Do Horizontal Rules: + hr = "\n<hr"+self.empty_element_suffix+"\n" + for hr_re in self._hr_res: + text = hr_re.sub(hr, text) + + text = self._do_lists(text) + + if "pyshell" in self.extras: + text = self._prepare_pyshell_blocks(text) + + text = self._do_code_blocks(text) + + text = self._do_block_quotes(text) + + # We already ran _HashHTMLBlocks() before, in Markdown(), but that + # was to escape raw HTML in the original Markdown source. This time, + # we're escaping the markup we've just created, so that we don't wrap + # <p> tags around block-level tags. + text = self._hash_html_blocks(text) + + text = self._form_paragraphs(text) + + return text + + def _pyshell_block_sub(self, match): + lines = match.group(0).splitlines(0) + _dedentlines(lines) + indent = ' ' * self.tab_width + s = ('\n' # separate from possible cuddled paragraph + + indent + ('\n'+indent).join(lines) + + '\n\n') + return s + + def _prepare_pyshell_blocks(self, text): + """Ensure that Python interactive shell sessions are put in + code blocks -- even if not properly indented. + """ + if ">>>" not in text: + return text + + less_than_tab = self.tab_width - 1 + _pyshell_block_re = re.compile(r""" + ^([ ]{0,%d})>>>[ ].*\n # first line + ^(\1.*\S+.*\n)* # any number of subsequent lines + ^\n # ends with a blank line + """ % less_than_tab, re.M | re.X) + + return _pyshell_block_re.sub(self._pyshell_block_sub, text) + + def _run_span_gamut(self, text): + # These are all the transformations that occur *within* block-level + # tags like paragraphs, headers, and list items. + + text = self._do_code_spans(text) + + text = self._escape_special_chars(text) + + # Process anchor and image tags. + text = self._do_links(text) + + # Make links out of things like `<http://example.com/>` + # Must come after _do_links(), because you can use < and > + # delimiters in inline links like [this](<url>). + text = self._do_auto_links(text) + + if "link-patterns" in self.extras: + text = self._do_link_patterns(text) + + text = self._encode_amps_and_angles(text) + + text = self._do_italics_and_bold(text) + + # Do hard breaks: + text = re.sub(r" {2,}\n", " <br%s\n" % self.empty_element_suffix, text) + + return text + + # "Sorta" because auto-links are identified as "tag" tokens. + _sorta_html_tokenize_re = re.compile(r""" + ( + # tag + </? + (?:\w+) # tag name + (?:\s+(?:[\w-]+:)?[\w-]+=(?:".*?"|'.*?'))* # attributes + \s*/?> + | + # auto-link (e.g., <http://www.activestate.com/>) + <\w+[^>]*> + | + <!--.*?--> # comment + | + <\?.*?\?> # processing instruction + ) + """, re.X) + + def _escape_special_chars(self, text): + # Python markdown note: the HTML tokenization here differs from + # that in Markdown.pl, hence the behaviour for subtle cases can + # differ (I believe the tokenizer here does a better job because + # it isn't susceptible to unmatched '<' and '>' in HTML tags). + # Note, however, that '>' is not allowed in an auto-link URL + # here. + escaped = [] + is_html_markup = False + for token in self._sorta_html_tokenize_re.split(text): + if is_html_markup: + # Within tags/HTML-comments/auto-links, encode * and _ + # so they don't conflict with their use in Markdown for + # italics and strong. We're replacing each such + # character with its corresponding MD5 checksum value; + # this is likely overkill, but it should prevent us from + # colliding with the escape values by accident. + escaped.append(token.replace('*', g_escape_table['*']) + .replace('_', g_escape_table['_'])) + else: + escaped.append(self._encode_backslash_escapes(token)) + is_html_markup = not is_html_markup + return ''.join(escaped) + + def _hash_html_spans(self, text): + # Used for safe_mode. + + def _is_auto_link(s): + if ':' in s and self._auto_link_re.match(s): + return True + elif '@' in s and self._auto_email_link_re.match(s): + return True + return False + + tokens = [] + is_html_markup = False + for token in self._sorta_html_tokenize_re.split(text): + if is_html_markup and not _is_auto_link(token): + sanitized = self._sanitize_html(token) + key = _hash_text(sanitized) + self.html_spans[key] = sanitized + tokens.append(key) + else: + tokens.append(token) + is_html_markup = not is_html_markup + return ''.join(tokens) + + def _unhash_html_spans(self, text): + for key, sanitized in self.html_spans.items(): + text = text.replace(key, sanitized) + return text + + def _sanitize_html(self, s): + if self.safe_mode == "replace": + return self.html_removed_text + elif self.safe_mode == "escape": + replacements = [ + ('&', '&'), + ('<', '<'), + ('>', '>'), + ] + for before, after in replacements: + s = s.replace(before, after) + return s + else: + raise MarkdownError("invalid value for 'safe_mode': %r (must be " + "'escape' or 'replace')" % self.safe_mode) + + _tail_of_inline_link_re = re.compile(r''' + # Match tail of: [text](/url/) or [text](/url/ "title") + \( # literal paren + [ \t]* + (?P<url> # \1 + <.*?> + | + .*? + ) + [ \t]* + ( # \2 + (['"]) # quote char = \3 + (?P<title>.*?) + \3 # matching quote + )? # title is optional + \) + ''', re.X | re.S) + _tail_of_reference_link_re = re.compile(r''' + # Match tail of: [text][id] + [ ]? # one optional space + (?:\n[ ]*)? # one optional newline followed by spaces + \[ + (?P<id>.*?) + \] + ''', re.X | re.S) + + def _do_links(self, text): + """Turn Markdown link shortcuts into XHTML <a> and <img> tags. + + This is a combination of Markdown.pl's _DoAnchors() and + _DoImages(). They are done together because that simplified the + approach. It was necessary to use a different approach than + Markdown.pl because of the lack of atomic matching support in + Python's regex engine used in $g_nested_brackets. + """ + MAX_LINK_TEXT_SENTINEL = 3000 # markdown2 issue 24 + + # `anchor_allowed_pos` is used to support img links inside + # anchors, but not anchors inside anchors. An anchor's start + # pos must be `>= anchor_allowed_pos`. + anchor_allowed_pos = 0 + + curr_pos = 0 + while True: # Handle the next link. + # The next '[' is the start of: + # - an inline anchor: [text](url "title") + # - a reference anchor: [text][id] + # - an inline img:  + # - a reference img: ![text][id] + # - a footnote ref: [^id] + # (Only if 'footnotes' extra enabled) + # - a footnote defn: [^id]: ... + # (Only if 'footnotes' extra enabled) These have already + # been stripped in _strip_footnote_definitions() so no + # need to watch for them. + # - a link definition: [id]: url "title" + # These have already been stripped in + # _strip_link_definitions() so no need to watch for them. + # - not markup: [...anything else... + try: + start_idx = text.index('[', curr_pos) + except ValueError: + break + text_length = len(text) + + # Find the matching closing ']'. + # Markdown.pl allows *matching* brackets in link text so we + # will here too. Markdown.pl *doesn't* currently allow + # matching brackets in img alt text -- we'll differ in that + # regard. + bracket_depth = 0 + for p in range(start_idx+1, min(start_idx+MAX_LINK_TEXT_SENTINEL, + text_length)): + ch = text[p] + if ch == ']': + bracket_depth -= 1 + if bracket_depth < 0: + break + elif ch == '[': + bracket_depth += 1 + else: + # Closing bracket not found within sentinel length. + # This isn't markup. + curr_pos = start_idx + 1 + continue + link_text = text[start_idx+1:p] + + # Possibly a footnote ref? + if "footnotes" in self.extras and link_text.startswith("^"): + normed_id = re.sub(r'\W', '-', link_text[1:]) + if normed_id in self.footnotes: + self.footnote_ids.append(normed_id) + result = '<sup class="footnote-ref" id="fnref-%s">' \ + '<a href="#fn-%s">%s</a></sup>' \ + % (normed_id, normed_id, len(self.footnote_ids)) + text = text[:start_idx] + result + text[p+1:] + else: + # This id isn't defined, leave the markup alone. + curr_pos = p+1 + continue + + # Now determine what this is by the remainder. + p += 1 + if p == text_length: + return text + + # Inline anchor or img? + if text[p] == '(': # attempt at perf improvement + match = self._tail_of_inline_link_re.match(text, p) + if match: + # Handle an inline anchor or img. + is_img = start_idx > 0 and text[start_idx-1] == "!" + if is_img: + start_idx -= 1 + + url, title = match.group("url"), match.group("title") + if url and url[0] == '<': + url = url[1:-1] # '<url>' -> 'url' + # We've got to encode these to avoid conflicting + # with italics/bold. + url = url.replace('*', g_escape_table['*']) \ + .replace('_', g_escape_table['_']) + if title: + title_str = ' title="%s"' \ + % title.replace('*', g_escape_table['*']) \ + .replace('_', g_escape_table['_']) \ + .replace('"', '"') + else: + title_str = '' + if is_img: + result = '<img src="%s" alt="%s"%s%s' \ + % (url, link_text.replace('"', '"'), + title_str, self.empty_element_suffix) + curr_pos = start_idx + len(result) + text = text[:start_idx] + result + text[match.end():] + elif start_idx >= anchor_allowed_pos: + result_head = '<a href="%s"%s>' % (url, title_str) + result = '%s%s</a>' % (result_head, link_text) + # <img> allowed from curr_pos on, <a> from + # anchor_allowed_pos on. + curr_pos = start_idx + len(result_head) + anchor_allowed_pos = start_idx + len(result) + text = text[:start_idx] + result + text[match.end():] + else: + # Anchor not allowed here. + curr_pos = start_idx + 1 + continue + + # Reference anchor or img? + else: + match = self._tail_of_reference_link_re.match(text, p) + if match: + # Handle a reference-style anchor or img. + is_img = start_idx > 0 and text[start_idx-1] == "!" + if is_img: + start_idx -= 1 + link_id = match.group("id").lower() + if not link_id: + link_id = link_text.lower() # for links like [this][] + if link_id in self.urls: + url = self.urls[link_id] + # We've got to encode these to avoid conflicting + # with italics/bold. + url = url.replace('*', g_escape_table['*']) \ + .replace('_', g_escape_table['_']) + title = self.titles.get(link_id) + if title: + title = title.replace('*', g_escape_table['*']) \ + .replace('_', g_escape_table['_']) + title_str = ' title="%s"' % title + else: + title_str = '' + if is_img: + result = '<img src="%s" alt="%s"%s%s' \ + % (url, link_text.replace('"', '"'), + title_str, self.empty_element_suffix) + curr_pos = start_idx + len(result) + text = text[:start_idx] + result + text[match.end():] + elif start_idx >= anchor_allowed_pos: + result = '<a href="%s"%s>%s</a>' \ + % (url, title_str, link_text) + result_head = '<a href="%s"%s>' % (url, title_str) + result = '%s%s</a>' % (result_head, link_text) + # <img> allowed from curr_pos on, <a> from + # anchor_allowed_pos on. + curr_pos = start_idx + len(result_head) + anchor_allowed_pos = start_idx + len(result) + text = text[:start_idx] + result + text[match.end():] + else: + # Anchor not allowed here. + curr_pos = start_idx + 1 + else: + # This id isn't defined, leave the markup alone. + curr_pos = match.end() + continue + + # Otherwise, it isn't markup. + curr_pos = start_idx + 1 + + return text + + + _setext_h_re = re.compile(r'^(.+)[ \t]*\n(=+|-+)[ \t]*\n+', re.M) + def _setext_h_sub(self, match): + n = {"=": 1, "-": 2}[match.group(2)[0]] + demote_headers = self.extras.get("demote-headers") + if demote_headers: + n = min(n + demote_headers, 6) + return "<h%d>%s</h%d>\n\n" \ + % (n, self._run_span_gamut(match.group(1)), n) + + _atx_h_re = re.compile(r''' + ^(\#{1,6}) # \1 = string of #'s + [ \t]* + (.+?) # \2 = Header text + [ \t]* + (?<!\\) # ensure not an escaped trailing '#' + \#* # optional closing #'s (not counted) + \n+ + ''', re.X | re.M) + def _atx_h_sub(self, match): + n = len(match.group(1)) + demote_headers = self.extras.get("demote-headers") + if demote_headers: + n = min(n + demote_headers, 6) + return "<h%d>%s</h%d>\n\n" \ + % (n, self._run_span_gamut(match.group(2)), n) + + def _do_headers(self, text): + # Setext-style headers: + # Header 1 + # ======== + # + # Header 2 + # -------- + text = self._setext_h_re.sub(self._setext_h_sub, text) + + # atx-style headers: + # # Header 1 + # ## Header 2 + # ## Header 2 with closing hashes ## + # ... + # ###### Header 6 + text = self._atx_h_re.sub(self._atx_h_sub, text) + + return text + + + _marker_ul_chars = '*+-' + _marker_any = r'(?:[%s]|\d+\.)' % _marker_ul_chars + _marker_ul = '(?:[%s])' % _marker_ul_chars + _marker_ol = r'(?:\d+\.)' + + def _list_sub(self, match): + lst = match.group(1) + lst_type = match.group(3) in self._marker_ul_chars and "ul" or "ol" + result = self._process_list_items(lst) + if self.list_level: + return "<%s>\n%s</%s>\n" % (lst_type, result, lst_type) + else: + return "<%s>\n%s</%s>\n\n" % (lst_type, result, lst_type) + + def _do_lists(self, text): + # Form HTML ordered (numbered) and unordered (bulleted) lists. + + for marker_pat in (self._marker_ul, self._marker_ol): + # Re-usable pattern to match any entire ul or ol list: + less_than_tab = self.tab_width - 1 + whole_list = r''' + ( # \1 = whole list + ( # \2 + [ ]{0,%d} + (%s) # \3 = first list item marker + [ \t]+ + ) + (?:.+?) + ( # \4 + \Z + | + \n{2,} + (?=\S) + (?! # Negative lookahead for another list item marker + [ \t]* + %s[ \t]+ + ) + ) + ) + ''' % (less_than_tab, marker_pat, marker_pat) + + # We use a different prefix before nested lists than top-level lists. + # See extended comment in _process_list_items(). + # + # Note: There's a bit of duplication here. My original implementation + # created a scalar regex pattern as the conditional result of the test on + # $g_list_level, and then only ran the $text =~ s{...}{...}egmx + # substitution once, using the scalar as the pattern. This worked, + # everywhere except when running under MT on my hosting account at Pair + # Networks. There, this caused all rebuilds to be killed by the reaper (or + # perhaps they crashed, but that seems incredibly unlikely given that the + # same script on the same server ran fine *except* under MT. I've spent + # more time trying to figure out why this is happening than I'd like to + # admit. My only guess, backed up by the fact that this workaround works, + # is that Perl optimizes the substition when it can figure out that the + # pattern will never change, and when this optimization isn't on, we run + # afoul of the reaper. Thus, the slightly redundant code to that uses two + # static s/// patterns rather than one conditional pattern. + + if self.list_level: + sub_list_re = re.compile("^"+whole_list, re.X | re.M | re.S) + text = sub_list_re.sub(self._list_sub, text) + else: + list_re = re.compile(r"(?:(?<=\n\n)|\A\n?)"+whole_list, + re.X | re.M | re.S) + text = list_re.sub(self._list_sub, text) + + return text + + _list_item_re = re.compile(r''' + (\n)? # leading line = \1 + (^[ \t]*) # leading whitespace = \2 + (%s) [ \t]+ # list marker = \3 + ((?:.+?) # list item text = \4 + (\n{1,2})) # eols = \5 + (?= \n* (\Z | \2 (%s) [ \t]+)) + ''' % (_marker_any, _marker_any), + re.M | re.X | re.S) + + _last_li_endswith_two_eols = False + def _list_item_sub(self, match): + item = match.group(4) + leading_line = match.group(1) + leading_space = match.group(2) + if leading_line or "\n\n" in item or self._last_li_endswith_two_eols: + item = self._run_block_gamut(self._outdent(item)) + else: + # Recursion for sub-lists: + item = self._do_lists(self._outdent(item)) + if item.endswith('\n'): + item = item[:-1] + item = self._run_span_gamut(item) + self._last_li_endswith_two_eols = (len(match.group(5)) == 2) + return "<li>%s</li>\n" % item + + def _process_list_items(self, list_str): + # Process the contents of a single ordered or unordered list, + # splitting it into individual list items. + + # The $g_list_level global keeps track of when we're inside a list. + # Each time we enter a list, we increment it; when we leave a list, + # we decrement. If it's zero, we're not in a list anymore. + # + # We do this because when we're not inside a list, we want to treat + # something like this: + # + # I recommend upgrading to version + # 8. Oops, now this line is treated + # as a sub-list. + # + # As a single paragraph, despite the fact that the second line starts + # with a digit-period-space sequence. + # + # Whereas when we're inside a list (or sub-list), that line will be + # treated as the start of a sub-list. What a kludge, huh? This is + # an aspect of Markdown's syntax that's hard to parse perfectly + # without resorting to mind-reading. Perhaps the solution is to + # change the syntax rules such that sub-lists must start with a + # starting cardinal number; e.g. "1." or "a.". + self.list_level += 1 + self._last_li_endswith_two_eols = False + list_str = list_str.rstrip('\n') + '\n' + list_str = self._list_item_re.sub(self._list_item_sub, list_str) + self.list_level -= 1 + return list_str + + def _get_pygments_lexer(self, lexer_name): + try: + from pygments import lexers, util + except ImportError: + return None + try: + return lexers.get_lexer_by_name(lexer_name) + except util.ClassNotFound: + return None + + def _color_with_pygments(self, codeblock, lexer, **formatter_opts): + import pygments + import pygments.formatters + + class HtmlCodeFormatter(pygments.formatters.HtmlFormatter): + def _wrap_code(self, inner): + """A function for use in a Pygments Formatter which + wraps in <code> tags. + """ + yield 0, "<code>" + for tup in inner: + yield tup + yield 0, "</code>" + + def wrap(self, source, outfile): + """Return the source with a code, pre, and div.""" + return self._wrap_div(self._wrap_pre(self._wrap_code(source))) + + formatter = HtmlCodeFormatter(cssclass="codehilite", **formatter_opts) + return pygments.highlight(codeblock, lexer, formatter) + + def _code_block_sub(self, match): + codeblock = match.group(1) + codeblock = self._outdent(codeblock) + codeblock = self._detab(codeblock) + codeblock = codeblock.lstrip('\n') # trim leading newlines + codeblock = codeblock.rstrip() # trim trailing whitespace + + if "code-color" in self.extras and codeblock.startswith(":::"): + lexer_name, rest = codeblock.split('\n', 1) + lexer_name = lexer_name[3:].strip() + lexer = self._get_pygments_lexer(lexer_name) + codeblock = rest.lstrip("\n") # Remove lexer declaration line. + if lexer: + formatter_opts = self.extras['code-color'] or {} + colored = self._color_with_pygments(codeblock, lexer, + **formatter_opts) + return "\n\n%s\n\n" % colored + + codeblock = self._encode_code(codeblock) + return "\n\n<pre><code>%s\n</code></pre>\n\n" % codeblock + + def _do_code_blocks(self, text): + """Process Markdown `<pre><code>` blocks.""" + code_block_re = re.compile(r''' + (?:\n\n|\A) + ( # $1 = the code block -- one or more lines, starting with a space/tab + (?: + (?:[ ]{%d} | \t) # Lines must start with a tab or a tab-width of spaces + .*\n+ + )+ + ) + ((?=^[ ]{0,%d}\S)|\Z) # Lookahead for non-space at line-start, or end of doc + ''' % (self.tab_width, self.tab_width), + re.M | re.X) + + return code_block_re.sub(self._code_block_sub, text) + + + # Rules for a code span: + # - backslash escapes are not interpreted in a code span + # - to include one or or a run of more backticks the delimiters must + # be a longer run of backticks + # - cannot start or end a code span with a backtick; pad with a + # space and that space will be removed in the emitted HTML + # See `test/tm-cases/escapes.text` for a number of edge-case + # examples. + _code_span_re = re.compile(r''' + (?<!\\) + (`+) # \1 = Opening run of ` + (?!`) # See Note A test/tm-cases/escapes.text + (.+?) # \2 = The code block + (?<!`) + \1 # Matching closer + (?!`) + ''', re.X | re.S) + + def _code_span_sub(self, match): + c = match.group(2).strip(" \t") + c = self._encode_code(c) + return "<code>%s</code>" % c + + def _do_code_spans(self, text): + # * Backtick quotes are used for <code></code> spans. + # + # * You can use multiple backticks as the delimiters if you want to + # include literal backticks in the code span. So, this input: + # + # Just type ``foo `bar` baz`` at the prompt. + # + # Will translate to: + # + # <p>Just type <code>foo `bar` baz</code> at the prompt.</p> + # + # There's no arbitrary limit to the number of backticks you + # can use as delimters. If you need three consecutive backticks + # in your code, use four for delimiters, etc. + # + # * You can use spaces to get literal backticks at the edges: + # + # ... type `` `bar` `` ... + # + # Turns to: + # + # ... type <code>`bar`</code> ... + return self._code_span_re.sub(self._code_span_sub, text) + + def _encode_code(self, text): + """Encode/escape certain characters inside Markdown code runs. + The point is that in code, these characters are literals, + and lose their special Markdown meanings. + """ + replacements = [ + # Encode all ampersands; HTML entities are not + # entities within a Markdown code span. + ('&', '&'), + # Do the angle bracket song and dance: + ('<', '<'), + ('>', '>'), + # Now, escape characters that are magic in Markdown: + ('*', g_escape_table['*']), + ('_', g_escape_table['_']), + ('{', g_escape_table['{']), + ('}', g_escape_table['}']), + ('[', g_escape_table['[']), + (']', g_escape_table[']']), + ('\\', g_escape_table['\\']), + ] + for before, after in replacements: + text = text.replace(before, after) + return text + + _strong_re = re.compile(r"(\*\*|__)(?=\S)(.+?[*_]*)(?<=\S)\1", re.S) + _em_re = re.compile(r"(\*|_)(?=\S)(.+?)(?<=\S)\1", re.S) + _code_friendly_strong_re = re.compile(r"\*\*(?=\S)(.+?[*_]*)(?<=\S)\*\*", re.S) + _code_friendly_em_re = re.compile(r"\*(?=\S)(.+?)(?<=\S)\*", re.S) + def _do_italics_and_bold(self, text): + # <strong> must go first: + if "code-friendly" in self.extras: + text = self._code_friendly_strong_re.sub(r"<strong>\1</strong>", text) + text = self._code_friendly_em_re.sub(r"<em>\1</em>", text) + else: + text = self._strong_re.sub(r"<strong>\2</strong>", text) + text = self._em_re.sub(r"<em>\2</em>", text) + return text + + + _block_quote_re = re.compile(r''' + ( # Wrap whole match in \1 + ( + ^[ \t]*>[ \t]? # '>' at the start of a line + .+\n # rest of the first line + (.+\n)* # subsequent consecutive lines + \n* # blanks + )+ + ) + ''', re.M | re.X) + _bq_one_level_re = re.compile('^[ \t]*>[ \t]?', re.M); + + _html_pre_block_re = re.compile(r'(\s*<pre>.+?</pre>)', re.S) + def _dedent_two_spaces_sub(self, match): + return re.sub(r'(?m)^ ', '', match.group(1)) + + def _block_quote_sub(self, match): + bq = match.group(1) + bq = self._bq_one_level_re.sub('', bq) # trim one level of quoting + bq = self._ws_only_line_re.sub('', bq) # trim whitespace-only lines + bq = self._run_block_gamut(bq) # recurse + + bq = re.sub('(?m)^', ' ', bq) + # These leading spaces screw with <pre> content, so we need to fix that: + bq = self._html_pre_block_re.sub(self._dedent_two_spaces_sub, bq) + + return "<blockquote>\n%s\n</blockquote>\n\n" % bq + + def _do_block_quotes(self, text): + if '>' not in text: + return text + return self._block_quote_re.sub(self._block_quote_sub, text) + + def _form_paragraphs(self, text): + # Strip leading and trailing lines: + text = text.strip('\n') + + # Wrap <p> tags. + grafs = re.split(r"\n{2,}", text) + for i, graf in enumerate(grafs): + if graf in self.html_blocks: + # Unhashify HTML blocks + grafs[i] = self.html_blocks[graf] + else: + # Wrap <p> tags. + graf = self._run_span_gamut(graf) + grafs[i] = "<p>" + graf.lstrip(" \t") + "</p>" + + return "\n\n".join(grafs) + + def _add_footnotes(self, text): + if self.footnotes: + footer = [ + '<div class="footnotes">', + '<hr' + self.empty_element_suffix, + '<ol>', + ] + for i, id in enumerate(self.footnote_ids): + if i != 0: + footer.append('') + footer.append('<li id="fn-%s">' % id) + footer.append(self._run_block_gamut(self.footnotes[id])) + backlink = ('<a href="#fnref-%s" ' + 'class="footnoteBackLink" ' + 'title="Jump back to footnote %d in the text.">' + '↩</a>' % (id, i+1)) + if footer[-1].endswith("</p>"): + footer[-1] = footer[-1][:-len("</p>")] \ + + ' ' + backlink + "</p>" + else: + footer.append("\n<p>%s</p>" % backlink) + footer.append('</li>') + footer.append('</ol>') + footer.append('</div>') + return text + '\n\n' + '\n'.join(footer) + else: + return text + + # Ampersand-encoding based entirely on Nat Irons's Amputator MT plugin: + # http://bumppo.net/projects/amputator/ + _ampersand_re = re.compile(r'&(?!#?[xX]?(?:[0-9a-fA-F]+|\w+);)') + _naked_lt_re = re.compile(r'<(?![a-z/?\$!])', re.I) + _naked_gt_re = re.compile(r'''(?<![a-z?!/'"-])>''', re.I) + + def _encode_amps_and_angles(self, text): + # Smart processing for ampersands and angle brackets that need + # to be encoded. + text = self._ampersand_re.sub('&', text) + + # Encode naked <'s + text = self._naked_lt_re.sub('<', text) + + # Encode naked >'s + # Note: Other markdown implementations (e.g. Markdown.pl, PHP + # Markdown) don't do this. + text = self._naked_gt_re.sub('>', text) + return text + + def _encode_backslash_escapes(self, text): + for ch, escape in g_escape_table.items(): + text = text.replace("\\"+ch, escape) + return text + + _auto_link_re = re.compile(r'<((https?|ftp):[^\'">\s]+)>', re.I) + def _auto_link_sub(self, match): + g1 = match.group(1) + return '<a href="%s">%s</a>' % (g1, g1) + + _auto_email_link_re = re.compile(r""" + < + (?:mailto:)? + ( + [-.\w]+ + \@ + [-\w]+(\.[-\w]+)*\.[a-z]+ + ) + > + """, re.I | re.X | re.U) + def _auto_email_link_sub(self, match): + return self._encode_email_address( + self._unescape_special_chars(match.group(1))) + + def _do_auto_links(self, text): + text = self._auto_link_re.sub(self._auto_link_sub, text) + text = self._auto_email_link_re.sub(self._auto_email_link_sub, text) + return text + + def _encode_email_address(self, addr): + # Input: an email address, e.g. "foo@example.com" + # + # Output: the email address as a mailto link, with each character + # of the address encoded as either a decimal or hex entity, in + # the hopes of foiling most address harvesting spam bots. E.g.: + # + # <a href="mailto:foo@e + # xample.com">foo + # @example.com</a> + # + # Based on a filter by Matthew Wickline, posted to the BBEdit-Talk + # mailing list: <http://tinyurl.com/yu7ue> + chars = [_xml_encode_email_char_at_random(ch) + for ch in "mailto:" + addr] + # Strip the mailto: from the visible part. + addr = '<a href="%s">%s</a>' \ + % (''.join(chars), ''.join(chars[7:])) + return addr + + def _do_link_patterns(self, text): + """Caveat emptor: there isn't much guarding against link + patterns being formed inside other standard Markdown links, e.g. + inside a [link def][like this]. + + Dev Notes: *Could* consider prefixing regexes with a negative + lookbehind assertion to attempt to guard against this. + """ + link_from_hash = {} + for regex, repl in self.link_patterns: + replacements = [] + for match in regex.finditer(text): + if hasattr(repl, "__call__"): + href = repl(match) + else: + href = match.expand(repl) + replacements.append((match.span(), href)) + for (start, end), href in reversed(replacements): + escaped_href = ( + href.replace('"', '"') # b/c of attr quote + # To avoid markdown <em> and <strong>: + .replace('*', g_escape_table['*']) + .replace('_', g_escape_table['_'])) + link = '<a href="%s">%s</a>' % (escaped_href, text[start:end]) + hash = md5(link).hexdigest() + link_from_hash[hash] = link + text = text[:start] + hash + text[end:] + for hash, link in link_from_hash.items(): + text = text.replace(hash, link) + return text + + def _unescape_special_chars(self, text): + # Swap back in all the special characters we've hidden. + for ch, hash in g_escape_table.items(): + text = text.replace(hash, ch) + return text + + def _outdent(self, text): + # Remove one level of line-leading tabs or spaces + return self._outdent_re.sub('', text) + + +class MarkdownWithExtras(Markdown): + """A markdowner class that enables most extras: + + - footnotes + - code-color (only has effect if 'pygments' Python module on path) + + These are not included: + - pyshell (specific to Python-related documenting) + - code-friendly (because it *disables* part of the syntax) + - link-patterns (because you need to specify some actual + link-patterns anyway) + """ + extras = ["footnotes", "code-color"] + + +#---- internal support functions + +# From http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52549 +def _curry(*args, **kwargs): + function, args = args[0], args[1:] + def result(*rest, **kwrest): + combined = kwargs.copy() + combined.update(kwrest) + return function(*args + rest, **combined) + return result + +# Recipe: regex_from_encoded_pattern (1.0) +def _regex_from_encoded_pattern(s): + """'foo' -> re.compile(re.escape('foo')) + '/foo/' -> re.compile('foo') + '/foo/i' -> re.compile('foo', re.I) + """ + if s.startswith('/') and s.rfind('/') != 0: + # Parse it: /PATTERN/FLAGS + idx = s.rfind('/') + pattern, flags_str = s[1:idx], s[idx+1:] + flag_from_char = { + "i": re.IGNORECASE, + "l": re.LOCALE, + "s": re.DOTALL, + "m": re.MULTILINE, + "u": re.UNICODE, + } + flags = 0 + for char in flags_str: + try: + flags |= flag_from_char[char] + except KeyError: + raise ValueError("unsupported regex flag: '%s' in '%s' " + "(must be one of '%s')" + % (char, s, ''.join(flag_from_char.keys()))) + return re.compile(s[1:idx], flags) + else: # not an encoded regex + return re.compile(re.escape(s)) + +# Recipe: dedent (0.1.2) +def _dedentlines(lines, tabsize=8, skip_first_line=False): + """_dedentlines(lines, tabsize=8, skip_first_line=False) -> dedented lines + + "lines" is a list of lines to dedent. + "tabsize" is the tab width to use for indent width calculations. + "skip_first_line" is a boolean indicating if the first line should + be skipped for calculating the indent width and for dedenting. + This is sometimes useful for docstrings and similar. + + Same as dedent() except operates on a sequence of lines. Note: the + lines list is modified **in-place**. + """ + DEBUG = False + if DEBUG: + print "dedent: dedent(..., tabsize=%d, skip_first_line=%r)"\ + % (tabsize, skip_first_line) + indents = [] + margin = None + for i, line in enumerate(lines): + if i == 0 and skip_first_line: continue + indent = 0 + for ch in line: + if ch == ' ': + indent += 1 + elif ch == '\t': + indent += tabsize - (indent % tabsize) + elif ch in '\r\n': + continue # skip all-whitespace lines + else: + break + else: + continue # skip all-whitespace lines + if DEBUG: print "dedent: indent=%d: %r" % (indent, line) + if margin is None: + margin = indent + else: + margin = min(margin, indent) + if DEBUG: print "dedent: margin=%r" % margin + + if margin is not None and margin > 0: + for i, line in enumerate(lines): + if i == 0 and skip_first_line: continue + removed = 0 + for j, ch in enumerate(line): + if ch == ' ': + removed += 1 + elif ch == '\t': + removed += tabsize - (removed % tabsize) + elif ch in '\r\n': + if DEBUG: print "dedent: %r: EOL -> strip up to EOL" % line + lines[i] = lines[i][j:] + break + else: + raise ValueError("unexpected non-whitespace char %r in " + "line %r while removing %d-space margin" + % (ch, line, margin)) + if DEBUG: + print "dedent: %r: %r -> removed %d/%d"\ + % (line, ch, removed, margin) + if removed == margin: + lines[i] = lines[i][j+1:] + break + elif removed > margin: + lines[i] = ' '*(removed-margin) + lines[i][j+1:] + break + else: + if removed: + lines[i] = lines[i][removed:] + return lines + +def _dedent(text, tabsize=8, skip_first_line=False): + """_dedent(text, tabsize=8, skip_first_line=False) -> dedented text + + "text" is the text to dedent. + "tabsize" is the tab width to use for indent width calculations. + "skip_first_line" is a boolean indicating if the first line should + be skipped for calculating the indent width and for dedenting. + This is sometimes useful for docstrings and similar. + + textwrap.dedent(s), but don't expand tabs to spaces + """ + lines = text.splitlines(1) + _dedentlines(lines, tabsize=tabsize, skip_first_line=skip_first_line) + return ''.join(lines) + + +class _memoized(object): + """Decorator that caches a function's return value each time it is called. + If called later with the same arguments, the cached value is returned, and + not re-evaluated. + + http://wiki.python.org/moin/PythonDecoratorLibrary + """ + def __init__(self, func): + self.func = func + self.cache = {} + def __call__(self, *args): + try: + return self.cache[args] + except KeyError: + self.cache[args] = value = self.func(*args) + return value + except TypeError: + # uncachable -- for instance, passing a list as an argument. + # Better to not cache than to blow up entirely. + return self.func(*args) + def __repr__(self): + """Return the function's docstring.""" + return self.func.__doc__ + + +def _xml_oneliner_re_from_tab_width(tab_width): + """Standalone XML processing instruction regex.""" + return re.compile(r""" + (?: + (?<=\n\n) # Starting after a blank line + | # or + \A\n? # the beginning of the doc + ) + ( # save in $1 + [ ]{0,%d} + (?: + <\?\w+\b\s+.*?\?> # XML processing instruction + | + <\w+:\w+\b\s+.*?/> # namespaced single tag + ) + [ \t]* + (?=\n{2,}|\Z) # followed by a blank line or end of document + ) + """ % (tab_width - 1), re.X) +_xml_oneliner_re_from_tab_width = _memoized(_xml_oneliner_re_from_tab_width) + +def _hr_tag_re_from_tab_width(tab_width): + return re.compile(r""" + (?: + (?<=\n\n) # Starting after a blank line + | # or + \A\n? # the beginning of the doc + ) + ( # save in \1 + [ ]{0,%d} + <(hr) # start tag = \2 + \b # word break + ([^<>])*? # + /?> # the matching end tag + [ \t]* + (?=\n{2,}|\Z) # followed by a blank line or end of document + ) + """ % (tab_width - 1), re.X) +_hr_tag_re_from_tab_width = _memoized(_hr_tag_re_from_tab_width) + + +def _xml_encode_email_char_at_random(ch): + r = random() + # Roughly 10% raw, 45% hex, 45% dec. + # '@' *must* be encoded. I [John Gruber] insist. + # Issue 26: '_' must be encoded. + if r > 0.9 and ch not in "@_": + return ch + elif r < 0.45: + # The [1:] is to drop leading '0': 0x63 -> x63 + return '&#%s;' % hex(ord(ch))[1:] + else: + return '&#%s;' % ord(ch) + +def _hash_text(text): + return 'md5:'+md5(text.encode("utf-8")).hexdigest() + + +#---- mainline + +class _NoReflowFormatter(optparse.IndentedHelpFormatter): + """An optparse formatter that does NOT reflow the description.""" + def format_description(self, description): + return description or "" + +def _test(): + import doctest + doctest.testmod() + +def main(argv=None): + if argv is None: + argv = sys.argv + if not logging.root.handlers: + logging.basicConfig() + + usage = "usage: %prog [PATHS...]" + version = "%prog "+__version__ + parser = optparse.OptionParser(prog="markdown2", usage=usage, + version=version, description=cmdln_desc, + formatter=_NoReflowFormatter()) + parser.add_option("-v", "--verbose", dest="log_level", + action="store_const", const=logging.DEBUG, + help="more verbose output") + parser.add_option("--encoding", + help="specify encoding of text content") + parser.add_option("--html4tags", action="store_true", default=False, + help="use HTML 4 style for empty element tags") + parser.add_option("-s", "--safe", metavar="MODE", dest="safe_mode", + help="sanitize literal HTML: 'escape' escapes " + "HTML meta chars, 'replace' replaces with an " + "[HTML_REMOVED] note") + parser.add_option("-x", "--extras", action="append", + help="Turn on specific extra features (not part of " + "the core Markdown spec). Supported values: " + "'code-friendly' disables _/__ for emphasis; " + "'code-color' adds code-block syntax coloring; " + "'link-patterns' adds auto-linking based on patterns; " + "'footnotes' adds the footnotes syntax;" + "'xml' passes one-liner processing instructions and namespaced XML tags;" + "'pyshell' to put unindented Python interactive shell sessions in a <code> block.") + parser.add_option("--use-file-vars", + help="Look for and use Emacs-style 'markdown-extras' " + "file var to turn on extras. See " + "<http://code.google.com/p/python-markdown2/wiki/Extras>.") + parser.add_option("--link-patterns-file", + help="path to a link pattern file") + parser.add_option("--self-test", action="store_true", + help="run internal self-tests (some doctests)") + parser.add_option("--compare", action="store_true", + help="run against Markdown.pl as well (for testing)") + parser.set_defaults(log_level=logging.INFO, compare=False, + encoding="utf-8", safe_mode=None, use_file_vars=False) + opts, paths = parser.parse_args() + log.setLevel(opts.log_level) + + if opts.self_test: + return _test() + + if opts.extras: + extras = {} + for s in opts.extras: + splitter = re.compile("[,;: ]+") + for e in splitter.split(s): + if '=' in e: + ename, earg = e.split('=', 1) + try: + earg = int(earg) + except ValueError: + pass + else: + ename, earg = e, None + extras[ename] = earg + else: + extras = None + + if opts.link_patterns_file: + link_patterns = [] + f = open(opts.link_patterns_file) + try: + for i, line in enumerate(f.readlines()): + if not line.strip(): continue + if line.lstrip().startswith("#"): continue + try: + pat, href = line.rstrip().rsplit(None, 1) + except ValueError: + raise MarkdownError("%s:%d: invalid link pattern line: %r" + % (opts.link_patterns_file, i+1, line)) + link_patterns.append( + (_regex_from_encoded_pattern(pat), href)) + finally: + f.close() + else: + link_patterns = None + + from os.path import join, dirname, abspath, exists + markdown_pl = join(dirname(dirname(abspath(__file__))), "test", + "Markdown.pl") + for path in paths: + if opts.compare: + print "==== Markdown.pl ====" + perl_cmd = 'perl %s "%s"' % (markdown_pl, path) + o = os.popen(perl_cmd) + perl_html = o.read() + o.close() + sys.stdout.write(perl_html) + print "==== markdown2.py ====" + html = markdown_path(path, encoding=opts.encoding, + html4tags=opts.html4tags, + safe_mode=opts.safe_mode, + extras=extras, link_patterns=link_patterns, + use_file_vars=opts.use_file_vars) + sys.stdout.write( + html.encode(sys.stdout.encoding or "utf-8", 'xmlcharrefreplace')) + if opts.compare: + test_dir = join(dirname(dirname(abspath(__file__))), "test") + if exists(join(test_dir, "test_markdown2.py")): + sys.path.insert(0, test_dir) + from test_markdown2 import norm_html_from_html + norm_html = norm_html_from_html(html) + norm_perl_html = norm_html_from_html(perl_html) + else: + norm_html = html + norm_perl_html = perl_html + print "==== match? %r ====" % (norm_perl_html == norm_html) + + +if __name__ == "__main__": + sys.exit( main(sys.argv) ) + diff --git a/vendor/tornado/demos/appengine/static/blog.css b/vendor/tornado/demos/appengine/static/blog.css new file mode 100644 index 0000000000..8902ec1f22 --- /dev/null +++ b/vendor/tornado/demos/appengine/static/blog.css @@ -0,0 +1,153 @@ +/* + * Copyright 2009 Facebook + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +body { + background: white; + color: black; + margin: 15px; + margin-top: 0; +} + +body, +input, +textarea { + font-family: Georgia, serif; + font-size: 12pt; +} + +table { + border-collapse: collapse; + border: 0; +} + +td { + border: 0; + padding: 0; +} + +h1, +h2, +h3, +h4 { + font-family: "Helvetica Nue", Helvetica, Arial, sans-serif; + margin: 0; +} + +h1 { + font-size: 20pt; +} + +pre, +code { + font-family: monospace; + color: #060; +} + +pre { + margin-left: 1em; + padding-left: 1em; + border-left: 1px solid silver; + line-height: 14pt; +} + +a, +a code { + color: #00c; +} + +#body { + max-width: 800px; + margin: auto; +} + +#header { + background-color: #3b5998; + padding: 5px; + padding-left: 10px; + padding-right: 10px; + margin-bottom: 1em; +} + +#header, +#header a { + color: white; +} + +#header h1 a { + text-decoration: none; +} + +#footer, +#content { + margin-left: 10px; + margin-right: 10px; +} + +#footer { + margin-top: 3em; +} + +.entry h1 a { + color: black; + text-decoration: none; +} + +.entry { + margin-bottom: 2em; +} + +.entry .date { + margin-top: 3px; +} + +.entry p { + margin: 0; + margin-bottom: 1em; +} + +.entry .body { + margin-top: 1em; + line-height: 16pt; +} + +.compose td { + vertical-align: middle; + padding-bottom: 5px; +} + +.compose td.field { + padding-right: 10px; +} + +.compose .title, +.compose .submit { + font-family: "Helvetica Nue", Helvetica, Arial, sans-serif; + font-weight: bold; +} + +.compose .title { + font-size: 20pt; +} + +.compose .title, +.compose .markdown { + width: 100%; +} + +.compose .markdown { + height: 500px; + line-height: 16pt; +} diff --git a/vendor/tornado/demos/appengine/templates/archive.html b/vendor/tornado/demos/appengine/templates/archive.html new file mode 100644 index 0000000000..dcca9511a4 --- /dev/null +++ b/vendor/tornado/demos/appengine/templates/archive.html @@ -0,0 +1,31 @@ +{% extends "base.html" %} + +{% block head %} + <style type="text/css"> + ul.archive { + list-style-type: none; + margin: 0; + padding: 0; + } + + ul.archive li { + margin-bottom: 1em; + } + + ul.archive .title { + font-family: "Helvetica Nue", Helvetica, Arial, sans-serif; + font-size: 14pt; + } + </style> +{% end %} + +{% block body %} + <ul class="archive"> + {% for entry in entries %} + <li> + <div class="title"><a href="/entry/{{ entry.slug }}">{{ escape(entry.title) }}</a></div> + <div class="date">{{ locale.format_date(entry.published, full_format=True, shorter=True) }}</div> + </li> + {% end %} + </ul> +{% end %} diff --git a/vendor/tornado/demos/appengine/templates/base.html b/vendor/tornado/demos/appengine/templates/base.html new file mode 100644 index 0000000000..0154aea8ca --- /dev/null +++ b/vendor/tornado/demos/appengine/templates/base.html @@ -0,0 +1,29 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml"> + <head> + <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/> + <title>{{ escape(handler.settings["blog_title"]) }}</title> + <link rel="stylesheet" href="/static/blog.css" type="text/css"/> + <link rel="alternate" href="/feed" type="application/atom+xml" title="{{ escape(handler.settings["blog_title"]) }}"/> + {% block head %}{% end %} + </head> + <body> + <div id="body"> + <div id="header"> + <div style="float:right"> + {% if not current_user %} + {{ _('<a href="%(url)s">Sign in</a> to compose/edit') % {"url": escape(users.create_login_url(request.uri))} }} + {% else %} + {% if current_user.administrator %} + <a href="/compose">{{ _("New post") }}</a> - + {% end %} + <a href="{{ escape(users.create_logout_url(request.uri)) }}">{{ _("Sign out") }}</a> + {% end %} + </div> + <h1><a href="/">{{ escape(handler.settings["blog_title"]) }}</a></h1> + </div> + <div id="content">{% block body %}{% end %}</div> + </div> + {% block bottom %}{% end %} + </body> +</html> diff --git a/vendor/tornado/demos/appengine/templates/compose.html b/vendor/tornado/demos/appengine/templates/compose.html new file mode 100644 index 0000000000..5ad548307c --- /dev/null +++ b/vendor/tornado/demos/appengine/templates/compose.html @@ -0,0 +1,42 @@ +{% extends "base.html" %} + +{% block body %} + <form action="{{ request.path }}" method="post" class="compose"> + <div style="margin-bottom:5px"><input name="title" type="text" class="title" value="{{ escape(entry.title) if entry else "" }}"/></div> + <div style="margin-bottom:5px"><textarea name="markdown" rows="30" cols="40" class="markdown">{{ escape(entry.markdown) if entry else "" }}</textarea></div> + <div> + <div style="float:right"><a href="http://daringfireball.net/projects/markdown/syntax">{{ _("Syntax documentation") }}</a></div> + <input type="submit" value="{{ _("Save changes") if entry else _("Publish post") }}" class="submit"/> + <a href="{{ "/entry/" + entry.slug if entry else "/" }}">{{ _("Cancel") }}</a> + </div> + {% if entry %} + <input type="hidden" name="key" value="{{ str(entry.key()) }}"/> + {% end %} + {{ xsrf_form_html() }} + </form> +{% end %} + +{% block bottom %} + <script src="http://ajax.googleapis.com/ajax/libs/jquery/1.3.2/jquery.min.js" type="text/javascript"></script> + <script type="text/javascript"> + //<![CDATA[ + + $(function() { + $("input[name=title]").select(); + $("form.compose").submit(function() { + var required = ["title", "markdown"]; + var form = $(this).get(0); + for (var i = 0; i < required.length; i++) { + if (!form[required[i]].value) { + $(form[required[i]]).select(); + return false; + } + } + return true; + }); + }); + + //]]> + </script> +{% end %} + diff --git a/vendor/tornado/demos/appengine/templates/entry.html b/vendor/tornado/demos/appengine/templates/entry.html new file mode 100644 index 0000000000..43c835dead --- /dev/null +++ b/vendor/tornado/demos/appengine/templates/entry.html @@ -0,0 +1,5 @@ +{% extends "base.html" %} + +{% block body %} + {{ modules.Entry(entry) }} +{% end %} diff --git a/vendor/tornado/demos/appengine/templates/feed.xml b/vendor/tornado/demos/appengine/templates/feed.xml new file mode 100644 index 0000000000..c6c368656c --- /dev/null +++ b/vendor/tornado/demos/appengine/templates/feed.xml @@ -0,0 +1,26 @@ +<?xml version="1.0" encoding="utf-8"?> +<feed xmlns="http://www.w3.org/2005/Atom"> + {% set date_format = "%Y-%m-%dT%H:%M:%SZ" %} + <title>{{ escape(handler.settings["blog_title"]) }}</title> + {% if len(entries) > 0 %} + <updated>{{ max(e.updated for e in entries).strftime(date_format) }}</updated> + {% else %} + <updated>{{ datetime.datetime.utcnow().strftime(date_format) }}</updated> + {% end %} + <id>http://{{ request.host }}/</id> + <link rel="alternate" href="http://{{ request.host }}/" title="{{ escape(handler.settings["blog_title"]) }}" type="text/html"/> + <link rel="self" href="{{ request.full_url() }}" title="{{ escape(handler.settings["blog_title"]) }}" type="application/atom+xml"/> + <author><name>{{ escape(handler.settings["blog_title"]) }}</name></author> + {% for entry in entries %} + <entry> + <id>http://{{ request.host }}/entry/{{ entry.slug }}</id> + <title type="text">{{ escape(entry.title) }}</title> + <link href="http://{{ request.host }}/entry/{{ entry.slug }}" rel="alternate" type="text/html"/> + <updated>{{ entry.updated.strftime(date_format) }}</updated> + <published>{{ entry.published.strftime(date_format) }}</published> + <content type="xhtml" xml:base="http://{{ request.host }}/"> + <div xmlns="http://www.w3.org/1999/xhtml">{{ entry.html }}</div> + </content> + </entry> + {% end %} +</feed> diff --git a/vendor/tornado/demos/appengine/templates/home.html b/vendor/tornado/demos/appengine/templates/home.html new file mode 100644 index 0000000000..dd069a97f3 --- /dev/null +++ b/vendor/tornado/demos/appengine/templates/home.html @@ -0,0 +1,8 @@ +{% extends "base.html" %} + +{% block body %} + {% for entry in entries %} + {{ modules.Entry(entry) }} + {% end %} + <div><a href="/archive">{{ _("Archive") }}</a></div> +{% end %} diff --git a/vendor/tornado/demos/appengine/templates/modules/entry.html b/vendor/tornado/demos/appengine/templates/modules/entry.html new file mode 100644 index 0000000000..06237657c8 --- /dev/null +++ b/vendor/tornado/demos/appengine/templates/modules/entry.html @@ -0,0 +1,8 @@ +<div class="entry"> + <h1><a href="/entry/{{ entry.slug }}">{{ escape(entry.title) }}</a></h1> + <div class="date">{{ locale.format_date(entry.published, full_format=True, shorter=True) }}</div> + <div class="body">{{ entry.html }}</div> + {% if current_user and current_user.administrator %} + <div class="admin"><a href="/compose?key={{ str(entry.key()) }}">{{ _("Edit this post") }}</a></div> + {% end %} +</div> diff --git a/vendor/tornado/demos/auth/authdemo.py b/vendor/tornado/demos/auth/authdemo.py new file mode 100755 index 0000000000..e6136d1b53 --- /dev/null +++ b/vendor/tornado/demos/auth/authdemo.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python +# +# Copyright 2009 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import tornado.auth +import tornado.escape +import tornado.httpserver +import tornado.ioloop +import tornado.options +import tornado.web + +from tornado.options import define, options + +define("port", default=8888, help="run on the given port", type=int) + + +class Application(tornado.web.Application): + def __init__(self): + handlers = [ + (r"/", MainHandler), + (r"/auth/login", AuthHandler), + ] + settings = dict( + cookie_secret="32oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=", + login_url="/auth/login", + ) + tornado.web.Application.__init__(self, handlers, **settings) + + +class BaseHandler(tornado.web.RequestHandler): + def get_current_user(self): + user_json = self.get_secure_cookie("user") + if not user_json: return None + return tornado.escape.json_decode(user_json) + + +class MainHandler(BaseHandler): + @tornado.web.authenticated + def get(self): + name = tornado.escape.xhtml_escape(self.current_user["name"]) + self.write("Hello, " + name) + + +class AuthHandler(BaseHandler, tornado.auth.GoogleMixin): + @tornado.web.asynchronous + def get(self): + if self.get_argument("openid.mode", None): + self.get_authenticated_user(self.async_callback(self._on_auth)) + return + self.authenticate_redirect() + + def _on_auth(self, user): + if not user: + raise tornado.web.HTTPError(500, "Google auth failed") + self.set_secure_cookie("user", tornado.escape.json_encode(user)) + self.redirect("/") + + +def main(): + tornado.options.parse_command_line() + http_server = tornado.httpserver.HTTPServer(Application()) + http_server.listen(options.port) + tornado.ioloop.IOLoop.instance().start() + + +if __name__ == "__main__": + main() diff --git a/vendor/tornado/demos/blog/README b/vendor/tornado/demos/blog/README new file mode 100644 index 0000000000..a033e7a11c --- /dev/null +++ b/vendor/tornado/demos/blog/README @@ -0,0 +1,57 @@ +Running the Tornado Blog example app +==================================== +This demo is a simple blogging engine that uses MySQL to store posts and +Google Accounts for author authentication. Since it depends on MySQL, you +need to set up MySQL and the database schema for the demo to run. + +1. Install prerequisites and build tornado + + See http://www.tornadoweb.org/ for installation instructions. If you can + run the "helloworld" example application, your environment is set up + correctly. + +2. Install MySQL if needed + + Consult the documentation for your platform. Under Ubuntu Linux you + can run "apt-get install mysql". Under OS X you can download the + MySQL PKG file from http://dev.mysql.com/downloads/mysql/ + +3. Connect to MySQL and create a database and user for the blog. + + Connect to MySQL as a user that can create databases and users: + mysql -u root + + Create a database named "blog": + mysql> CREATE DATABASE blog; + + Allow the "blog" user to connect with the password "blog": + mysql> GRANT ALL PRIVILEGES ON blog.* TO 'blog'@'localhost' IDENTIFIED BY 'blog'; + +4. Create the tables in your new database. + + You can use the provided schema.sql file by running this command: + mysql --user=blog --password=blog --database=blog < schema.sql + + You can run the above command again later if you want to delete the + contents of the blog and start over after testing. + +5. Run the blog example + + With the default user, password, and database you can just run: + ./blog.py + + If you've changed anything, you can alter the default MySQL settings + with arguments on the command line, e.g.: + ./blog.py --mysql_user=casey --mysql_password=happiness --mysql_database=foodblog + +6. Visit your new blog + + Open http://localhost:8888/ in your web browser. You will be redirected to + a Google account sign-in page because the blog uses Google accounts for + authentication. + + Currently the first user to connect will automatically be given the + ability to create and edit posts. + + Once you've created one blog post, subsequent users will not be + prompted to sign in. diff --git a/vendor/tornado/demos/blog/blog.py b/vendor/tornado/demos/blog/blog.py new file mode 100755 index 0000000000..808a9afc55 --- /dev/null +++ b/vendor/tornado/demos/blog/blog.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python +# +# Copyright 2009 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import markdown +import os.path +import re +import tornado.auth +import tornado.database +import tornado.httpserver +import tornado.ioloop +import tornado.options +import tornado.web +import unicodedata + +from tornado.options import define, options + +define("port", default=8888, help="run on the given port", type=int) +define("mysql_host", default="127.0.0.1:3306", help="blog database host") +define("mysql_database", default="blog", help="blog database name") +define("mysql_user", default="blog", help="blog database user") +define("mysql_password", default="blog", help="blog database password") + + +class Application(tornado.web.Application): + def __init__(self): + handlers = [ + (r"/", HomeHandler), + (r"/archive", ArchiveHandler), + (r"/feed", FeedHandler), + (r"/entry/([^/]+)", EntryHandler), + (r"/compose", ComposeHandler), + (r"/auth/login", AuthLoginHandler), + (r"/auth/logout", AuthLogoutHandler), + ] + settings = dict( + blog_title=u"Tornado Blog", + template_path=os.path.join(os.path.dirname(__file__), "templates"), + static_path=os.path.join(os.path.dirname(__file__), "static"), + ui_modules={"Entry": EntryModule}, + xsrf_cookies=True, + cookie_secret="11oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=", + login_url="/auth/login", + ) + tornado.web.Application.__init__(self, handlers, **settings) + + # Have one global connection to the blog DB across all handlers + self.db = tornado.database.Connection( + host=options.mysql_host, database=options.mysql_database, + user=options.mysql_user, password=options.mysql_password) + + +class BaseHandler(tornado.web.RequestHandler): + @property + def db(self): + return self.application.db + + def get_current_user(self): + user_id = self.get_secure_cookie("user") + if not user_id: return None + return self.db.get("SELECT * FROM authors WHERE id = %s", int(user_id)) + + +class HomeHandler(BaseHandler): + def get(self): + entries = self.db.query("SELECT * FROM entries ORDER BY published " + "DESC LIMIT 5") + if not entries: + self.redirect("/compose") + return + self.render("home.html", entries=entries) + + +class EntryHandler(BaseHandler): + def get(self, slug): + entry = self.db.get("SELECT * FROM entries WHERE slug = %s", slug) + if not entry: raise tornado.web.HTTPError(404) + self.render("entry.html", entry=entry) + + +class ArchiveHandler(BaseHandler): + def get(self): + entries = self.db.query("SELECT * FROM entries ORDER BY published " + "DESC") + self.render("archive.html", entries=entries) + + +class FeedHandler(BaseHandler): + def get(self): + entries = self.db.query("SELECT * FROM entries ORDER BY published " + "DESC LIMIT 10") + self.set_header("Content-Type", "application/atom+xml") + self.render("feed.xml", entries=entries) + + +class ComposeHandler(BaseHandler): + @tornado.web.authenticated + def get(self): + id = self.get_argument("id", None) + entry = None + if id: + entry = self.db.get("SELECT * FROM entries WHERE id = %s", int(id)) + self.render("compose.html", entry=entry) + + @tornado.web.authenticated + def post(self): + id = self.get_argument("id", None) + title = self.get_argument("title") + text = self.get_argument("markdown") + html = markdown.markdown(text) + if id: + entry = self.db.get("SELECT * FROM entries WHERE id = %s", int(id)) + if not entry: raise tornado.web.HTTPError(404) + slug = entry.slug + self.db.execute( + "UPDATE entries SET title = %s, markdown = %s, html = %s " + "WHERE id = %s", title, text, html, int(id)) + else: + slug = unicodedata.normalize("NFKD", title).encode( + "ascii", "ignore") + slug = re.sub(r"[^\w]+", " ", slug) + slug = "-".join(slug.lower().strip().split()) + if not slug: slug = "entry" + while True: + e = self.db.get("SELECT * FROM entries WHERE slug = %s", slug) + if not e: break + slug += "-2" + self.db.execute( + "INSERT INTO entries (author_id,title,slug,markdown,html," + "published) VALUES (%s,%s,%s,%s,%s,UTC_TIMESTAMP())", + self.current_user.id, title, slug, text, html) + self.redirect("/entry/" + slug) + + +class AuthLoginHandler(BaseHandler, tornado.auth.GoogleMixin): + @tornado.web.asynchronous + def get(self): + if self.get_argument("openid.mode", None): + self.get_authenticated_user(self.async_callback(self._on_auth)) + return + self.authenticate_redirect() + + def _on_auth(self, user): + if not user: + raise tornado.web.HTTPError(500, "Google auth failed") + author = self.db.get("SELECT * FROM authors WHERE email = %s", + user["email"]) + if not author: + # Auto-create first author + any_author = self.db.get("SELECT * FROM authors LIMIT 1") + if not any_author: + author_id = self.db.execute( + "INSERT INTO authors (email,name) VALUES (%s,%s)", + user["email"], user["name"]) + else: + self.redirect("/") + return + else: + author_id = author["id"] + self.set_secure_cookie("user", str(author_id)) + self.redirect(self.get_argument("next", "/")) + + +class AuthLogoutHandler(BaseHandler): + def get(self): + self.clear_cookie("user") + self.redirect(self.get_argument("next", "/")) + + +class EntryModule(tornado.web.UIModule): + def render(self, entry): + return self.render_string("modules/entry.html", entry=entry) + + +def main(): + tornado.options.parse_command_line() + http_server = tornado.httpserver.HTTPServer(Application()) + http_server.listen(options.port) + tornado.ioloop.IOLoop.instance().start() + + +if __name__ == "__main__": + main() diff --git a/vendor/tornado/demos/blog/markdown.py b/vendor/tornado/demos/blog/markdown.py new file mode 100644 index 0000000000..59ba731bf0 --- /dev/null +++ b/vendor/tornado/demos/blog/markdown.py @@ -0,0 +1,1877 @@ +#!/usr/bin/env python +# Copyright (c) 2007-2008 ActiveState Corp. +# License: MIT (http://www.opensource.org/licenses/mit-license.php) + +r"""A fast and complete Python implementation of Markdown. + +[from http://daringfireball.net/projects/markdown/] +> Markdown is a text-to-HTML filter; it translates an easy-to-read / +> easy-to-write structured text format into HTML. Markdown's text +> format is most similar to that of plain text email, and supports +> features such as headers, *emphasis*, code blocks, blockquotes, and +> links. +> +> Markdown's syntax is designed not as a generic markup language, but +> specifically to serve as a front-end to (X)HTML. You can use span-level +> HTML tags anywhere in a Markdown document, and you can use block level +> HTML tags (like <div> and <table> as well). + +Module usage: + + >>> import markdown2 + >>> markdown2.markdown("*boo!*") # or use `html = markdown_path(PATH)` + u'<p><em>boo!</em></p>\n' + + >>> markdowner = Markdown() + >>> markdowner.convert("*boo!*") + u'<p><em>boo!</em></p>\n' + >>> markdowner.convert("**boom!**") + u'<p><strong>boom!</strong></p>\n' + +This implementation of Markdown implements the full "core" syntax plus a +number of extras (e.g., code syntax coloring, footnotes) as described on +<http://code.google.com/p/python-markdown2/wiki/Extras>. +""" + +cmdln_desc = """A fast and complete Python implementation of Markdown, a +text-to-HTML conversion tool for web writers. +""" + +# Dev Notes: +# - There is already a Python markdown processor +# (http://www.freewisdom.org/projects/python-markdown/). +# - Python's regex syntax doesn't have '\z', so I'm using '\Z'. I'm +# not yet sure if there implications with this. Compare 'pydoc sre' +# and 'perldoc perlre'. + +__version_info__ = (1, 0, 1, 14) # first three nums match Markdown.pl +__version__ = '1.0.1.14' +__author__ = "Trent Mick" + +import os +import sys +from pprint import pprint +import re +import logging +try: + from hashlib import md5 +except ImportError: + from md5 import md5 +import optparse +from random import random +import codecs + + + +#---- Python version compat + +if sys.version_info[:2] < (2,4): + from sets import Set as set + def reversed(sequence): + for i in sequence[::-1]: + yield i + def _unicode_decode(s, encoding, errors='xmlcharrefreplace'): + return unicode(s, encoding, errors) +else: + def _unicode_decode(s, encoding, errors='strict'): + return s.decode(encoding, errors) + + +#---- globals + +DEBUG = False +log = logging.getLogger("markdown") + +DEFAULT_TAB_WIDTH = 4 + +# Table of hash values for escaped characters: +def _escape_hash(s): + # Lame attempt to avoid possible collision with someone actually + # using the MD5 hexdigest of one of these chars in there text. + # Other ideas: random.random(), uuid.uuid() + #return md5(s).hexdigest() # Markdown.pl effectively does this. + return 'md5-'+md5(s).hexdigest() +g_escape_table = dict([(ch, _escape_hash(ch)) + for ch in '\\`*_{}[]()>#+-.!']) + + + +#---- exceptions + +class MarkdownError(Exception): + pass + + + +#---- public api + +def markdown_path(path, encoding="utf-8", + html4tags=False, tab_width=DEFAULT_TAB_WIDTH, + safe_mode=None, extras=None, link_patterns=None, + use_file_vars=False): + text = codecs.open(path, 'r', encoding).read() + return Markdown(html4tags=html4tags, tab_width=tab_width, + safe_mode=safe_mode, extras=extras, + link_patterns=link_patterns, + use_file_vars=use_file_vars).convert(text) + +def markdown(text, html4tags=False, tab_width=DEFAULT_TAB_WIDTH, + safe_mode=None, extras=None, link_patterns=None, + use_file_vars=False): + return Markdown(html4tags=html4tags, tab_width=tab_width, + safe_mode=safe_mode, extras=extras, + link_patterns=link_patterns, + use_file_vars=use_file_vars).convert(text) + +class Markdown(object): + # The dict of "extras" to enable in processing -- a mapping of + # extra name to argument for the extra. Most extras do not have an + # argument, in which case the value is None. + # + # This can be set via (a) subclassing and (b) the constructor + # "extras" argument. + extras = None + + urls = None + titles = None + html_blocks = None + html_spans = None + html_removed_text = "[HTML_REMOVED]" # for compat with markdown.py + + # Used to track when we're inside an ordered or unordered list + # (see _ProcessListItems() for details): + list_level = 0 + + _ws_only_line_re = re.compile(r"^[ \t]+$", re.M) + + def __init__(self, html4tags=False, tab_width=4, safe_mode=None, + extras=None, link_patterns=None, use_file_vars=False): + if html4tags: + self.empty_element_suffix = ">" + else: + self.empty_element_suffix = " />" + self.tab_width = tab_width + + # For compatibility with earlier markdown2.py and with + # markdown.py's safe_mode being a boolean, + # safe_mode == True -> "replace" + if safe_mode is True: + self.safe_mode = "replace" + else: + self.safe_mode = safe_mode + + if self.extras is None: + self.extras = {} + elif not isinstance(self.extras, dict): + self.extras = dict([(e, None) for e in self.extras]) + if extras: + if not isinstance(extras, dict): + extras = dict([(e, None) for e in extras]) + self.extras.update(extras) + assert isinstance(self.extras, dict) + self._instance_extras = self.extras.copy() + self.link_patterns = link_patterns + self.use_file_vars = use_file_vars + self._outdent_re = re.compile(r'^(\t|[ ]{1,%d})' % tab_width, re.M) + + def reset(self): + self.urls = {} + self.titles = {} + self.html_blocks = {} + self.html_spans = {} + self.list_level = 0 + self.extras = self._instance_extras.copy() + if "footnotes" in self.extras: + self.footnotes = {} + self.footnote_ids = [] + + def convert(self, text): + """Convert the given text.""" + # Main function. The order in which other subs are called here is + # essential. Link and image substitutions need to happen before + # _EscapeSpecialChars(), so that any *'s or _'s in the <a> + # and <img> tags get encoded. + + # Clear the global hashes. If we don't clear these, you get conflicts + # from other articles when generating a page which contains more than + # one article (e.g. an index page that shows the N most recent + # articles): + self.reset() + + if not isinstance(text, unicode): + #TODO: perhaps shouldn't presume UTF-8 for string input? + text = unicode(text, 'utf-8') + + if self.use_file_vars: + # Look for emacs-style file variable hints. + emacs_vars = self._get_emacs_vars(text) + if "markdown-extras" in emacs_vars: + splitter = re.compile("[ ,]+") + for e in splitter.split(emacs_vars["markdown-extras"]): + if '=' in e: + ename, earg = e.split('=', 1) + try: + earg = int(earg) + except ValueError: + pass + else: + ename, earg = e, None + self.extras[ename] = earg + + # Standardize line endings: + text = re.sub("\r\n|\r", "\n", text) + + # Make sure $text ends with a couple of newlines: + text += "\n\n" + + # Convert all tabs to spaces. + text = self._detab(text) + + # Strip any lines consisting only of spaces and tabs. + # This makes subsequent regexen easier to write, because we can + # match consecutive blank lines with /\n+/ instead of something + # contorted like /[ \t]*\n+/ . + text = self._ws_only_line_re.sub("", text) + + if self.safe_mode: + text = self._hash_html_spans(text) + + # Turn block-level HTML blocks into hash entries + text = self._hash_html_blocks(text, raw=True) + + # Strip link definitions, store in hashes. + if "footnotes" in self.extras: + # Must do footnotes first because an unlucky footnote defn + # looks like a link defn: + # [^4]: this "looks like a link defn" + text = self._strip_footnote_definitions(text) + text = self._strip_link_definitions(text) + + text = self._run_block_gamut(text) + + if "footnotes" in self.extras: + text = self._add_footnotes(text) + + text = self._unescape_special_chars(text) + + if self.safe_mode: + text = self._unhash_html_spans(text) + + text += "\n" + return text + + _emacs_oneliner_vars_pat = re.compile(r"-\*-\s*([^\r\n]*?)\s*-\*-", re.UNICODE) + # This regular expression is intended to match blocks like this: + # PREFIX Local Variables: SUFFIX + # PREFIX mode: Tcl SUFFIX + # PREFIX End: SUFFIX + # Some notes: + # - "[ \t]" is used instead of "\s" to specifically exclude newlines + # - "(\r\n|\n|\r)" is used instead of "$" because the sre engine does + # not like anything other than Unix-style line terminators. + _emacs_local_vars_pat = re.compile(r"""^ + (?P<prefix>(?:[^\r\n|\n|\r])*?) + [\ \t]*Local\ Variables:[\ \t]* + (?P<suffix>.*?)(?:\r\n|\n|\r) + (?P<content>.*?\1End:) + """, re.IGNORECASE | re.MULTILINE | re.DOTALL | re.VERBOSE) + + def _get_emacs_vars(self, text): + """Return a dictionary of emacs-style local variables. + + Parsing is done loosely according to this spec (and according to + some in-practice deviations from this): + http://www.gnu.org/software/emacs/manual/html_node/emacs/Specifying-File-Variables.html#Specifying-File-Variables + """ + emacs_vars = {} + SIZE = pow(2, 13) # 8kB + + # Search near the start for a '-*-'-style one-liner of variables. + head = text[:SIZE] + if "-*-" in head: + match = self._emacs_oneliner_vars_pat.search(head) + if match: + emacs_vars_str = match.group(1) + assert '\n' not in emacs_vars_str + emacs_var_strs = [s.strip() for s in emacs_vars_str.split(';') + if s.strip()] + if len(emacs_var_strs) == 1 and ':' not in emacs_var_strs[0]: + # While not in the spec, this form is allowed by emacs: + # -*- Tcl -*- + # where the implied "variable" is "mode". This form + # is only allowed if there are no other variables. + emacs_vars["mode"] = emacs_var_strs[0].strip() + else: + for emacs_var_str in emacs_var_strs: + try: + variable, value = emacs_var_str.strip().split(':', 1) + except ValueError: + log.debug("emacs variables error: malformed -*- " + "line: %r", emacs_var_str) + continue + # Lowercase the variable name because Emacs allows "Mode" + # or "mode" or "MoDe", etc. + emacs_vars[variable.lower()] = value.strip() + + tail = text[-SIZE:] + if "Local Variables" in tail: + match = self._emacs_local_vars_pat.search(tail) + if match: + prefix = match.group("prefix") + suffix = match.group("suffix") + lines = match.group("content").splitlines(0) + #print "prefix=%r, suffix=%r, content=%r, lines: %s"\ + # % (prefix, suffix, match.group("content"), lines) + + # Validate the Local Variables block: proper prefix and suffix + # usage. + for i, line in enumerate(lines): + if not line.startswith(prefix): + log.debug("emacs variables error: line '%s' " + "does not use proper prefix '%s'" + % (line, prefix)) + return {} + # Don't validate suffix on last line. Emacs doesn't care, + # neither should we. + if i != len(lines)-1 and not line.endswith(suffix): + log.debug("emacs variables error: line '%s' " + "does not use proper suffix '%s'" + % (line, suffix)) + return {} + + # Parse out one emacs var per line. + continued_for = None + for line in lines[:-1]: # no var on the last line ("PREFIX End:") + if prefix: line = line[len(prefix):] # strip prefix + if suffix: line = line[:-len(suffix)] # strip suffix + line = line.strip() + if continued_for: + variable = continued_for + if line.endswith('\\'): + line = line[:-1].rstrip() + else: + continued_for = None + emacs_vars[variable] += ' ' + line + else: + try: + variable, value = line.split(':', 1) + except ValueError: + log.debug("local variables error: missing colon " + "in local variables entry: '%s'" % line) + continue + # Do NOT lowercase the variable name, because Emacs only + # allows "mode" (and not "Mode", "MoDe", etc.) in this block. + value = value.strip() + if value.endswith('\\'): + value = value[:-1].rstrip() + continued_for = variable + else: + continued_for = None + emacs_vars[variable] = value + + # Unquote values. + for var, val in emacs_vars.items(): + if len(val) > 1 and (val.startswith('"') and val.endswith('"') + or val.startswith('"') and val.endswith('"')): + emacs_vars[var] = val[1:-1] + + return emacs_vars + + # Cribbed from a post by Bart Lateur: + # <http://www.nntp.perl.org/group/perl.macperl.anyperl/154> + _detab_re = re.compile(r'(.*?)\t', re.M) + def _detab_sub(self, match): + g1 = match.group(1) + return g1 + (' ' * (self.tab_width - len(g1) % self.tab_width)) + def _detab(self, text): + r"""Remove (leading?) tabs from a file. + + >>> m = Markdown() + >>> m._detab("\tfoo") + ' foo' + >>> m._detab(" \tfoo") + ' foo' + >>> m._detab("\t foo") + ' foo' + >>> m._detab(" foo") + ' foo' + >>> m._detab(" foo\n\tbar\tblam") + ' foo\n bar blam' + """ + if '\t' not in text: + return text + return self._detab_re.subn(self._detab_sub, text)[0] + + _block_tags_a = 'p|div|h[1-6]|blockquote|pre|table|dl|ol|ul|script|noscript|form|fieldset|iframe|math|ins|del' + _strict_tag_block_re = re.compile(r""" + ( # save in \1 + ^ # start of line (with re.M) + <(%s) # start tag = \2 + \b # word break + (.*\n)*? # any number of lines, minimally matching + </\2> # the matching end tag + [ \t]* # trailing spaces/tabs + (?=\n+|\Z) # followed by a newline or end of document + ) + """ % _block_tags_a, + re.X | re.M) + + _block_tags_b = 'p|div|h[1-6]|blockquote|pre|table|dl|ol|ul|script|noscript|form|fieldset|iframe|math' + _liberal_tag_block_re = re.compile(r""" + ( # save in \1 + ^ # start of line (with re.M) + <(%s) # start tag = \2 + \b # word break + (.*\n)*? # any number of lines, minimally matching + .*</\2> # the matching end tag + [ \t]* # trailing spaces/tabs + (?=\n+|\Z) # followed by a newline or end of document + ) + """ % _block_tags_b, + re.X | re.M) + + def _hash_html_block_sub(self, match, raw=False): + html = match.group(1) + if raw and self.safe_mode: + html = self._sanitize_html(html) + key = _hash_text(html) + self.html_blocks[key] = html + return "\n\n" + key + "\n\n" + + def _hash_html_blocks(self, text, raw=False): + """Hashify HTML blocks + + We only want to do this for block-level HTML tags, such as headers, + lists, and tables. That's because we still want to wrap <p>s around + "paragraphs" that are wrapped in non-block-level tags, such as anchors, + phrase emphasis, and spans. The list of tags we're looking for is + hard-coded. + + @param raw {boolean} indicates if these are raw HTML blocks in + the original source. It makes a difference in "safe" mode. + """ + if '<' not in text: + return text + + # Pass `raw` value into our calls to self._hash_html_block_sub. + hash_html_block_sub = _curry(self._hash_html_block_sub, raw=raw) + + # First, look for nested blocks, e.g.: + # <div> + # <div> + # tags for inner block must be indented. + # </div> + # </div> + # + # The outermost tags must start at the left margin for this to match, and + # the inner nested divs must be indented. + # We need to do this before the next, more liberal match, because the next + # match will start at the first `<div>` and stop at the first `</div>`. + text = self._strict_tag_block_re.sub(hash_html_block_sub, text) + + # Now match more liberally, simply from `\n<tag>` to `</tag>\n` + text = self._liberal_tag_block_re.sub(hash_html_block_sub, text) + + # Special case just for <hr />. It was easier to make a special + # case than to make the other regex more complicated. + if "<hr" in text: + _hr_tag_re = _hr_tag_re_from_tab_width(self.tab_width) + text = _hr_tag_re.sub(hash_html_block_sub, text) + + # Special case for standalone HTML comments: + if "<!--" in text: + start = 0 + while True: + # Delimiters for next comment block. + try: + start_idx = text.index("<!--", start) + except ValueError, ex: + break + try: + end_idx = text.index("-->", start_idx) + 3 + except ValueError, ex: + break + + # Start position for next comment block search. + start = end_idx + + # Validate whitespace before comment. + if start_idx: + # - Up to `tab_width - 1` spaces before start_idx. + for i in range(self.tab_width - 1): + if text[start_idx - 1] != ' ': + break + start_idx -= 1 + if start_idx == 0: + break + # - Must be preceded by 2 newlines or hit the start of + # the document. + if start_idx == 0: + pass + elif start_idx == 1 and text[0] == '\n': + start_idx = 0 # to match minute detail of Markdown.pl regex + elif text[start_idx-2:start_idx] == '\n\n': + pass + else: + break + + # Validate whitespace after comment. + # - Any number of spaces and tabs. + while end_idx < len(text): + if text[end_idx] not in ' \t': + break + end_idx += 1 + # - Must be following by 2 newlines or hit end of text. + if text[end_idx:end_idx+2] not in ('', '\n', '\n\n'): + continue + + # Escape and hash (must match `_hash_html_block_sub`). + html = text[start_idx:end_idx] + if raw and self.safe_mode: + html = self._sanitize_html(html) + key = _hash_text(html) + self.html_blocks[key] = html + text = text[:start_idx] + "\n\n" + key + "\n\n" + text[end_idx:] + + if "xml" in self.extras: + # Treat XML processing instructions and namespaced one-liner + # tags as if they were block HTML tags. E.g., if standalone + # (i.e. are their own paragraph), the following do not get + # wrapped in a <p> tag: + # <?foo bar?> + # + # <xi:include xmlns:xi="http://www.w3.org/2001/XInclude" href="chapter_1.md"/> + _xml_oneliner_re = _xml_oneliner_re_from_tab_width(self.tab_width) + text = _xml_oneliner_re.sub(hash_html_block_sub, text) + + return text + + def _strip_link_definitions(self, text): + # Strips link definitions from text, stores the URLs and titles in + # hash references. + less_than_tab = self.tab_width - 1 + + # Link defs are in the form: + # [id]: url "optional title" + _link_def_re = re.compile(r""" + ^[ ]{0,%d}\[(.+)\]: # id = \1 + [ \t]* + \n? # maybe *one* newline + [ \t]* + <?(.+?)>? # url = \2 + [ \t]* + (?: + \n? # maybe one newline + [ \t]* + (?<=\s) # lookbehind for whitespace + ['"(] + ([^\n]*) # title = \3 + ['")] + [ \t]* + )? # title is optional + (?:\n+|\Z) + """ % less_than_tab, re.X | re.M | re.U) + return _link_def_re.sub(self._extract_link_def_sub, text) + + def _extract_link_def_sub(self, match): + id, url, title = match.groups() + key = id.lower() # Link IDs are case-insensitive + self.urls[key] = self._encode_amps_and_angles(url) + if title: + self.titles[key] = title.replace('"', '"') + return "" + + def _extract_footnote_def_sub(self, match): + id, text = match.groups() + text = _dedent(text, skip_first_line=not text.startswith('\n')).strip() + normed_id = re.sub(r'\W', '-', id) + # Ensure footnote text ends with a couple newlines (for some + # block gamut matches). + self.footnotes[normed_id] = text + "\n\n" + return "" + + def _strip_footnote_definitions(self, text): + """A footnote definition looks like this: + + [^note-id]: Text of the note. + + May include one or more indented paragraphs. + + Where, + - The 'note-id' can be pretty much anything, though typically it + is the number of the footnote. + - The first paragraph may start on the next line, like so: + + [^note-id]: + Text of the note. + """ + less_than_tab = self.tab_width - 1 + footnote_def_re = re.compile(r''' + ^[ ]{0,%d}\[\^(.+)\]: # id = \1 + [ \t]* + ( # footnote text = \2 + # First line need not start with the spaces. + (?:\s*.*\n+) + (?: + (?:[ ]{%d} | \t) # Subsequent lines must be indented. + .*\n+ + )* + ) + # Lookahead for non-space at line-start, or end of doc. + (?:(?=^[ ]{0,%d}\S)|\Z) + ''' % (less_than_tab, self.tab_width, self.tab_width), + re.X | re.M) + return footnote_def_re.sub(self._extract_footnote_def_sub, text) + + + _hr_res = [ + re.compile(r"^[ ]{0,2}([ ]?\*[ ]?){3,}[ \t]*$", re.M), + re.compile(r"^[ ]{0,2}([ ]?\-[ ]?){3,}[ \t]*$", re.M), + re.compile(r"^[ ]{0,2}([ ]?\_[ ]?){3,}[ \t]*$", re.M), + ] + + def _run_block_gamut(self, text): + # These are all the transformations that form block-level + # tags like paragraphs, headers, and list items. + + text = self._do_headers(text) + + # Do Horizontal Rules: + hr = "\n<hr"+self.empty_element_suffix+"\n" + for hr_re in self._hr_res: + text = hr_re.sub(hr, text) + + text = self._do_lists(text) + + if "pyshell" in self.extras: + text = self._prepare_pyshell_blocks(text) + + text = self._do_code_blocks(text) + + text = self._do_block_quotes(text) + + # We already ran _HashHTMLBlocks() before, in Markdown(), but that + # was to escape raw HTML in the original Markdown source. This time, + # we're escaping the markup we've just created, so that we don't wrap + # <p> tags around block-level tags. + text = self._hash_html_blocks(text) + + text = self._form_paragraphs(text) + + return text + + def _pyshell_block_sub(self, match): + lines = match.group(0).splitlines(0) + _dedentlines(lines) + indent = ' ' * self.tab_width + s = ('\n' # separate from possible cuddled paragraph + + indent + ('\n'+indent).join(lines) + + '\n\n') + return s + + def _prepare_pyshell_blocks(self, text): + """Ensure that Python interactive shell sessions are put in + code blocks -- even if not properly indented. + """ + if ">>>" not in text: + return text + + less_than_tab = self.tab_width - 1 + _pyshell_block_re = re.compile(r""" + ^([ ]{0,%d})>>>[ ].*\n # first line + ^(\1.*\S+.*\n)* # any number of subsequent lines + ^\n # ends with a blank line + """ % less_than_tab, re.M | re.X) + + return _pyshell_block_re.sub(self._pyshell_block_sub, text) + + def _run_span_gamut(self, text): + # These are all the transformations that occur *within* block-level + # tags like paragraphs, headers, and list items. + + text = self._do_code_spans(text) + + text = self._escape_special_chars(text) + + # Process anchor and image tags. + text = self._do_links(text) + + # Make links out of things like `<http://example.com/>` + # Must come after _do_links(), because you can use < and > + # delimiters in inline links like [this](<url>). + text = self._do_auto_links(text) + + if "link-patterns" in self.extras: + text = self._do_link_patterns(text) + + text = self._encode_amps_and_angles(text) + + text = self._do_italics_and_bold(text) + + # Do hard breaks: + text = re.sub(r" {2,}\n", " <br%s\n" % self.empty_element_suffix, text) + + return text + + # "Sorta" because auto-links are identified as "tag" tokens. + _sorta_html_tokenize_re = re.compile(r""" + ( + # tag + </? + (?:\w+) # tag name + (?:\s+(?:[\w-]+:)?[\w-]+=(?:".*?"|'.*?'))* # attributes + \s*/?> + | + # auto-link (e.g., <http://www.activestate.com/>) + <\w+[^>]*> + | + <!--.*?--> # comment + | + <\?.*?\?> # processing instruction + ) + """, re.X) + + def _escape_special_chars(self, text): + # Python markdown note: the HTML tokenization here differs from + # that in Markdown.pl, hence the behaviour for subtle cases can + # differ (I believe the tokenizer here does a better job because + # it isn't susceptible to unmatched '<' and '>' in HTML tags). + # Note, however, that '>' is not allowed in an auto-link URL + # here. + escaped = [] + is_html_markup = False + for token in self._sorta_html_tokenize_re.split(text): + if is_html_markup: + # Within tags/HTML-comments/auto-links, encode * and _ + # so they don't conflict with their use in Markdown for + # italics and strong. We're replacing each such + # character with its corresponding MD5 checksum value; + # this is likely overkill, but it should prevent us from + # colliding with the escape values by accident. + escaped.append(token.replace('*', g_escape_table['*']) + .replace('_', g_escape_table['_'])) + else: + escaped.append(self._encode_backslash_escapes(token)) + is_html_markup = not is_html_markup + return ''.join(escaped) + + def _hash_html_spans(self, text): + # Used for safe_mode. + + def _is_auto_link(s): + if ':' in s and self._auto_link_re.match(s): + return True + elif '@' in s and self._auto_email_link_re.match(s): + return True + return False + + tokens = [] + is_html_markup = False + for token in self._sorta_html_tokenize_re.split(text): + if is_html_markup and not _is_auto_link(token): + sanitized = self._sanitize_html(token) + key = _hash_text(sanitized) + self.html_spans[key] = sanitized + tokens.append(key) + else: + tokens.append(token) + is_html_markup = not is_html_markup + return ''.join(tokens) + + def _unhash_html_spans(self, text): + for key, sanitized in self.html_spans.items(): + text = text.replace(key, sanitized) + return text + + def _sanitize_html(self, s): + if self.safe_mode == "replace": + return self.html_removed_text + elif self.safe_mode == "escape": + replacements = [ + ('&', '&'), + ('<', '<'), + ('>', '>'), + ] + for before, after in replacements: + s = s.replace(before, after) + return s + else: + raise MarkdownError("invalid value for 'safe_mode': %r (must be " + "'escape' or 'replace')" % self.safe_mode) + + _tail_of_inline_link_re = re.compile(r''' + # Match tail of: [text](/url/) or [text](/url/ "title") + \( # literal paren + [ \t]* + (?P<url> # \1 + <.*?> + | + .*? + ) + [ \t]* + ( # \2 + (['"]) # quote char = \3 + (?P<title>.*?) + \3 # matching quote + )? # title is optional + \) + ''', re.X | re.S) + _tail_of_reference_link_re = re.compile(r''' + # Match tail of: [text][id] + [ ]? # one optional space + (?:\n[ ]*)? # one optional newline followed by spaces + \[ + (?P<id>.*?) + \] + ''', re.X | re.S) + + def _do_links(self, text): + """Turn Markdown link shortcuts into XHTML <a> and <img> tags. + + This is a combination of Markdown.pl's _DoAnchors() and + _DoImages(). They are done together because that simplified the + approach. It was necessary to use a different approach than + Markdown.pl because of the lack of atomic matching support in + Python's regex engine used in $g_nested_brackets. + """ + MAX_LINK_TEXT_SENTINEL = 3000 # markdown2 issue 24 + + # `anchor_allowed_pos` is used to support img links inside + # anchors, but not anchors inside anchors. An anchor's start + # pos must be `>= anchor_allowed_pos`. + anchor_allowed_pos = 0 + + curr_pos = 0 + while True: # Handle the next link. + # The next '[' is the start of: + # - an inline anchor: [text](url "title") + # - a reference anchor: [text][id] + # - an inline img:  + # - a reference img: ![text][id] + # - a footnote ref: [^id] + # (Only if 'footnotes' extra enabled) + # - a footnote defn: [^id]: ... + # (Only if 'footnotes' extra enabled) These have already + # been stripped in _strip_footnote_definitions() so no + # need to watch for them. + # - a link definition: [id]: url "title" + # These have already been stripped in + # _strip_link_definitions() so no need to watch for them. + # - not markup: [...anything else... + try: + start_idx = text.index('[', curr_pos) + except ValueError: + break + text_length = len(text) + + # Find the matching closing ']'. + # Markdown.pl allows *matching* brackets in link text so we + # will here too. Markdown.pl *doesn't* currently allow + # matching brackets in img alt text -- we'll differ in that + # regard. + bracket_depth = 0 + for p in range(start_idx+1, min(start_idx+MAX_LINK_TEXT_SENTINEL, + text_length)): + ch = text[p] + if ch == ']': + bracket_depth -= 1 + if bracket_depth < 0: + break + elif ch == '[': + bracket_depth += 1 + else: + # Closing bracket not found within sentinel length. + # This isn't markup. + curr_pos = start_idx + 1 + continue + link_text = text[start_idx+1:p] + + # Possibly a footnote ref? + if "footnotes" in self.extras and link_text.startswith("^"): + normed_id = re.sub(r'\W', '-', link_text[1:]) + if normed_id in self.footnotes: + self.footnote_ids.append(normed_id) + result = '<sup class="footnote-ref" id="fnref-%s">' \ + '<a href="#fn-%s">%s</a></sup>' \ + % (normed_id, normed_id, len(self.footnote_ids)) + text = text[:start_idx] + result + text[p+1:] + else: + # This id isn't defined, leave the markup alone. + curr_pos = p+1 + continue + + # Now determine what this is by the remainder. + p += 1 + if p == text_length: + return text + + # Inline anchor or img? + if text[p] == '(': # attempt at perf improvement + match = self._tail_of_inline_link_re.match(text, p) + if match: + # Handle an inline anchor or img. + is_img = start_idx > 0 and text[start_idx-1] == "!" + if is_img: + start_idx -= 1 + + url, title = match.group("url"), match.group("title") + if url and url[0] == '<': + url = url[1:-1] # '<url>' -> 'url' + # We've got to encode these to avoid conflicting + # with italics/bold. + url = url.replace('*', g_escape_table['*']) \ + .replace('_', g_escape_table['_']) + if title: + title_str = ' title="%s"' \ + % title.replace('*', g_escape_table['*']) \ + .replace('_', g_escape_table['_']) \ + .replace('"', '"') + else: + title_str = '' + if is_img: + result = '<img src="%s" alt="%s"%s%s' \ + % (url, link_text.replace('"', '"'), + title_str, self.empty_element_suffix) + curr_pos = start_idx + len(result) + text = text[:start_idx] + result + text[match.end():] + elif start_idx >= anchor_allowed_pos: + result_head = '<a href="%s"%s>' % (url, title_str) + result = '%s%s</a>' % (result_head, link_text) + # <img> allowed from curr_pos on, <a> from + # anchor_allowed_pos on. + curr_pos = start_idx + len(result_head) + anchor_allowed_pos = start_idx + len(result) + text = text[:start_idx] + result + text[match.end():] + else: + # Anchor not allowed here. + curr_pos = start_idx + 1 + continue + + # Reference anchor or img? + else: + match = self._tail_of_reference_link_re.match(text, p) + if match: + # Handle a reference-style anchor or img. + is_img = start_idx > 0 and text[start_idx-1] == "!" + if is_img: + start_idx -= 1 + link_id = match.group("id").lower() + if not link_id: + link_id = link_text.lower() # for links like [this][] + if link_id in self.urls: + url = self.urls[link_id] + # We've got to encode these to avoid conflicting + # with italics/bold. + url = url.replace('*', g_escape_table['*']) \ + .replace('_', g_escape_table['_']) + title = self.titles.get(link_id) + if title: + title = title.replace('*', g_escape_table['*']) \ + .replace('_', g_escape_table['_']) + title_str = ' title="%s"' % title + else: + title_str = '' + if is_img: + result = '<img src="%s" alt="%s"%s%s' \ + % (url, link_text.replace('"', '"'), + title_str, self.empty_element_suffix) + curr_pos = start_idx + len(result) + text = text[:start_idx] + result + text[match.end():] + elif start_idx >= anchor_allowed_pos: + result = '<a href="%s"%s>%s</a>' \ + % (url, title_str, link_text) + result_head = '<a href="%s"%s>' % (url, title_str) + result = '%s%s</a>' % (result_head, link_text) + # <img> allowed from curr_pos on, <a> from + # anchor_allowed_pos on. + curr_pos = start_idx + len(result_head) + anchor_allowed_pos = start_idx + len(result) + text = text[:start_idx] + result + text[match.end():] + else: + # Anchor not allowed here. + curr_pos = start_idx + 1 + else: + # This id isn't defined, leave the markup alone. + curr_pos = match.end() + continue + + # Otherwise, it isn't markup. + curr_pos = start_idx + 1 + + return text + + + _setext_h_re = re.compile(r'^(.+)[ \t]*\n(=+|-+)[ \t]*\n+', re.M) + def _setext_h_sub(self, match): + n = {"=": 1, "-": 2}[match.group(2)[0]] + demote_headers = self.extras.get("demote-headers") + if demote_headers: + n = min(n + demote_headers, 6) + return "<h%d>%s</h%d>\n\n" \ + % (n, self._run_span_gamut(match.group(1)), n) + + _atx_h_re = re.compile(r''' + ^(\#{1,6}) # \1 = string of #'s + [ \t]* + (.+?) # \2 = Header text + [ \t]* + (?<!\\) # ensure not an escaped trailing '#' + \#* # optional closing #'s (not counted) + \n+ + ''', re.X | re.M) + def _atx_h_sub(self, match): + n = len(match.group(1)) + demote_headers = self.extras.get("demote-headers") + if demote_headers: + n = min(n + demote_headers, 6) + return "<h%d>%s</h%d>\n\n" \ + % (n, self._run_span_gamut(match.group(2)), n) + + def _do_headers(self, text): + # Setext-style headers: + # Header 1 + # ======== + # + # Header 2 + # -------- + text = self._setext_h_re.sub(self._setext_h_sub, text) + + # atx-style headers: + # # Header 1 + # ## Header 2 + # ## Header 2 with closing hashes ## + # ... + # ###### Header 6 + text = self._atx_h_re.sub(self._atx_h_sub, text) + + return text + + + _marker_ul_chars = '*+-' + _marker_any = r'(?:[%s]|\d+\.)' % _marker_ul_chars + _marker_ul = '(?:[%s])' % _marker_ul_chars + _marker_ol = r'(?:\d+\.)' + + def _list_sub(self, match): + lst = match.group(1) + lst_type = match.group(3) in self._marker_ul_chars and "ul" or "ol" + result = self._process_list_items(lst) + if self.list_level: + return "<%s>\n%s</%s>\n" % (lst_type, result, lst_type) + else: + return "<%s>\n%s</%s>\n\n" % (lst_type, result, lst_type) + + def _do_lists(self, text): + # Form HTML ordered (numbered) and unordered (bulleted) lists. + + for marker_pat in (self._marker_ul, self._marker_ol): + # Re-usable pattern to match any entire ul or ol list: + less_than_tab = self.tab_width - 1 + whole_list = r''' + ( # \1 = whole list + ( # \2 + [ ]{0,%d} + (%s) # \3 = first list item marker + [ \t]+ + ) + (?:.+?) + ( # \4 + \Z + | + \n{2,} + (?=\S) + (?! # Negative lookahead for another list item marker + [ \t]* + %s[ \t]+ + ) + ) + ) + ''' % (less_than_tab, marker_pat, marker_pat) + + # We use a different prefix before nested lists than top-level lists. + # See extended comment in _process_list_items(). + # + # Note: There's a bit of duplication here. My original implementation + # created a scalar regex pattern as the conditional result of the test on + # $g_list_level, and then only ran the $text =~ s{...}{...}egmx + # substitution once, using the scalar as the pattern. This worked, + # everywhere except when running under MT on my hosting account at Pair + # Networks. There, this caused all rebuilds to be killed by the reaper (or + # perhaps they crashed, but that seems incredibly unlikely given that the + # same script on the same server ran fine *except* under MT. I've spent + # more time trying to figure out why this is happening than I'd like to + # admit. My only guess, backed up by the fact that this workaround works, + # is that Perl optimizes the substition when it can figure out that the + # pattern will never change, and when this optimization isn't on, we run + # afoul of the reaper. Thus, the slightly redundant code to that uses two + # static s/// patterns rather than one conditional pattern. + + if self.list_level: + sub_list_re = re.compile("^"+whole_list, re.X | re.M | re.S) + text = sub_list_re.sub(self._list_sub, text) + else: + list_re = re.compile(r"(?:(?<=\n\n)|\A\n?)"+whole_list, + re.X | re.M | re.S) + text = list_re.sub(self._list_sub, text) + + return text + + _list_item_re = re.compile(r''' + (\n)? # leading line = \1 + (^[ \t]*) # leading whitespace = \2 + (%s) [ \t]+ # list marker = \3 + ((?:.+?) # list item text = \4 + (\n{1,2})) # eols = \5 + (?= \n* (\Z | \2 (%s) [ \t]+)) + ''' % (_marker_any, _marker_any), + re.M | re.X | re.S) + + _last_li_endswith_two_eols = False + def _list_item_sub(self, match): + item = match.group(4) + leading_line = match.group(1) + leading_space = match.group(2) + if leading_line or "\n\n" in item or self._last_li_endswith_two_eols: + item = self._run_block_gamut(self._outdent(item)) + else: + # Recursion for sub-lists: + item = self._do_lists(self._outdent(item)) + if item.endswith('\n'): + item = item[:-1] + item = self._run_span_gamut(item) + self._last_li_endswith_two_eols = (len(match.group(5)) == 2) + return "<li>%s</li>\n" % item + + def _process_list_items(self, list_str): + # Process the contents of a single ordered or unordered list, + # splitting it into individual list items. + + # The $g_list_level global keeps track of when we're inside a list. + # Each time we enter a list, we increment it; when we leave a list, + # we decrement. If it's zero, we're not in a list anymore. + # + # We do this because when we're not inside a list, we want to treat + # something like this: + # + # I recommend upgrading to version + # 8. Oops, now this line is treated + # as a sub-list. + # + # As a single paragraph, despite the fact that the second line starts + # with a digit-period-space sequence. + # + # Whereas when we're inside a list (or sub-list), that line will be + # treated as the start of a sub-list. What a kludge, huh? This is + # an aspect of Markdown's syntax that's hard to parse perfectly + # without resorting to mind-reading. Perhaps the solution is to + # change the syntax rules such that sub-lists must start with a + # starting cardinal number; e.g. "1." or "a.". + self.list_level += 1 + self._last_li_endswith_two_eols = False + list_str = list_str.rstrip('\n') + '\n' + list_str = self._list_item_re.sub(self._list_item_sub, list_str) + self.list_level -= 1 + return list_str + + def _get_pygments_lexer(self, lexer_name): + try: + from pygments import lexers, util + except ImportError: + return None + try: + return lexers.get_lexer_by_name(lexer_name) + except util.ClassNotFound: + return None + + def _color_with_pygments(self, codeblock, lexer, **formatter_opts): + import pygments + import pygments.formatters + + class HtmlCodeFormatter(pygments.formatters.HtmlFormatter): + def _wrap_code(self, inner): + """A function for use in a Pygments Formatter which + wraps in <code> tags. + """ + yield 0, "<code>" + for tup in inner: + yield tup + yield 0, "</code>" + + def wrap(self, source, outfile): + """Return the source with a code, pre, and div.""" + return self._wrap_div(self._wrap_pre(self._wrap_code(source))) + + formatter = HtmlCodeFormatter(cssclass="codehilite", **formatter_opts) + return pygments.highlight(codeblock, lexer, formatter) + + def _code_block_sub(self, match): + codeblock = match.group(1) + codeblock = self._outdent(codeblock) + codeblock = self._detab(codeblock) + codeblock = codeblock.lstrip('\n') # trim leading newlines + codeblock = codeblock.rstrip() # trim trailing whitespace + + if "code-color" in self.extras and codeblock.startswith(":::"): + lexer_name, rest = codeblock.split('\n', 1) + lexer_name = lexer_name[3:].strip() + lexer = self._get_pygments_lexer(lexer_name) + codeblock = rest.lstrip("\n") # Remove lexer declaration line. + if lexer: + formatter_opts = self.extras['code-color'] or {} + colored = self._color_with_pygments(codeblock, lexer, + **formatter_opts) + return "\n\n%s\n\n" % colored + + codeblock = self._encode_code(codeblock) + return "\n\n<pre><code>%s\n</code></pre>\n\n" % codeblock + + def _do_code_blocks(self, text): + """Process Markdown `<pre><code>` blocks.""" + code_block_re = re.compile(r''' + (?:\n\n|\A) + ( # $1 = the code block -- one or more lines, starting with a space/tab + (?: + (?:[ ]{%d} | \t) # Lines must start with a tab or a tab-width of spaces + .*\n+ + )+ + ) + ((?=^[ ]{0,%d}\S)|\Z) # Lookahead for non-space at line-start, or end of doc + ''' % (self.tab_width, self.tab_width), + re.M | re.X) + + return code_block_re.sub(self._code_block_sub, text) + + + # Rules for a code span: + # - backslash escapes are not interpreted in a code span + # - to include one or or a run of more backticks the delimiters must + # be a longer run of backticks + # - cannot start or end a code span with a backtick; pad with a + # space and that space will be removed in the emitted HTML + # See `test/tm-cases/escapes.text` for a number of edge-case + # examples. + _code_span_re = re.compile(r''' + (?<!\\) + (`+) # \1 = Opening run of ` + (?!`) # See Note A test/tm-cases/escapes.text + (.+?) # \2 = The code block + (?<!`) + \1 # Matching closer + (?!`) + ''', re.X | re.S) + + def _code_span_sub(self, match): + c = match.group(2).strip(" \t") + c = self._encode_code(c) + return "<code>%s</code>" % c + + def _do_code_spans(self, text): + # * Backtick quotes are used for <code></code> spans. + # + # * You can use multiple backticks as the delimiters if you want to + # include literal backticks in the code span. So, this input: + # + # Just type ``foo `bar` baz`` at the prompt. + # + # Will translate to: + # + # <p>Just type <code>foo `bar` baz</code> at the prompt.</p> + # + # There's no arbitrary limit to the number of backticks you + # can use as delimters. If you need three consecutive backticks + # in your code, use four for delimiters, etc. + # + # * You can use spaces to get literal backticks at the edges: + # + # ... type `` `bar` `` ... + # + # Turns to: + # + # ... type <code>`bar`</code> ... + return self._code_span_re.sub(self._code_span_sub, text) + + def _encode_code(self, text): + """Encode/escape certain characters inside Markdown code runs. + The point is that in code, these characters are literals, + and lose their special Markdown meanings. + """ + replacements = [ + # Encode all ampersands; HTML entities are not + # entities within a Markdown code span. + ('&', '&'), + # Do the angle bracket song and dance: + ('<', '<'), + ('>', '>'), + # Now, escape characters that are magic in Markdown: + ('*', g_escape_table['*']), + ('_', g_escape_table['_']), + ('{', g_escape_table['{']), + ('}', g_escape_table['}']), + ('[', g_escape_table['[']), + (']', g_escape_table[']']), + ('\\', g_escape_table['\\']), + ] + for before, after in replacements: + text = text.replace(before, after) + return text + + _strong_re = re.compile(r"(\*\*|__)(?=\S)(.+?[*_]*)(?<=\S)\1", re.S) + _em_re = re.compile(r"(\*|_)(?=\S)(.+?)(?<=\S)\1", re.S) + _code_friendly_strong_re = re.compile(r"\*\*(?=\S)(.+?[*_]*)(?<=\S)\*\*", re.S) + _code_friendly_em_re = re.compile(r"\*(?=\S)(.+?)(?<=\S)\*", re.S) + def _do_italics_and_bold(self, text): + # <strong> must go first: + if "code-friendly" in self.extras: + text = self._code_friendly_strong_re.sub(r"<strong>\1</strong>", text) + text = self._code_friendly_em_re.sub(r"<em>\1</em>", text) + else: + text = self._strong_re.sub(r"<strong>\2</strong>", text) + text = self._em_re.sub(r"<em>\2</em>", text) + return text + + + _block_quote_re = re.compile(r''' + ( # Wrap whole match in \1 + ( + ^[ \t]*>[ \t]? # '>' at the start of a line + .+\n # rest of the first line + (.+\n)* # subsequent consecutive lines + \n* # blanks + )+ + ) + ''', re.M | re.X) + _bq_one_level_re = re.compile('^[ \t]*>[ \t]?', re.M); + + _html_pre_block_re = re.compile(r'(\s*<pre>.+?</pre>)', re.S) + def _dedent_two_spaces_sub(self, match): + return re.sub(r'(?m)^ ', '', match.group(1)) + + def _block_quote_sub(self, match): + bq = match.group(1) + bq = self._bq_one_level_re.sub('', bq) # trim one level of quoting + bq = self._ws_only_line_re.sub('', bq) # trim whitespace-only lines + bq = self._run_block_gamut(bq) # recurse + + bq = re.sub('(?m)^', ' ', bq) + # These leading spaces screw with <pre> content, so we need to fix that: + bq = self._html_pre_block_re.sub(self._dedent_two_spaces_sub, bq) + + return "<blockquote>\n%s\n</blockquote>\n\n" % bq + + def _do_block_quotes(self, text): + if '>' not in text: + return text + return self._block_quote_re.sub(self._block_quote_sub, text) + + def _form_paragraphs(self, text): + # Strip leading and trailing lines: + text = text.strip('\n') + + # Wrap <p> tags. + grafs = re.split(r"\n{2,}", text) + for i, graf in enumerate(grafs): + if graf in self.html_blocks: + # Unhashify HTML blocks + grafs[i] = self.html_blocks[graf] + else: + # Wrap <p> tags. + graf = self._run_span_gamut(graf) + grafs[i] = "<p>" + graf.lstrip(" \t") + "</p>" + + return "\n\n".join(grafs) + + def _add_footnotes(self, text): + if self.footnotes: + footer = [ + '<div class="footnotes">', + '<hr' + self.empty_element_suffix, + '<ol>', + ] + for i, id in enumerate(self.footnote_ids): + if i != 0: + footer.append('') + footer.append('<li id="fn-%s">' % id) + footer.append(self._run_block_gamut(self.footnotes[id])) + backlink = ('<a href="#fnref-%s" ' + 'class="footnoteBackLink" ' + 'title="Jump back to footnote %d in the text.">' + '↩</a>' % (id, i+1)) + if footer[-1].endswith("</p>"): + footer[-1] = footer[-1][:-len("</p>")] \ + + ' ' + backlink + "</p>" + else: + footer.append("\n<p>%s</p>" % backlink) + footer.append('</li>') + footer.append('</ol>') + footer.append('</div>') + return text + '\n\n' + '\n'.join(footer) + else: + return text + + # Ampersand-encoding based entirely on Nat Irons's Amputator MT plugin: + # http://bumppo.net/projects/amputator/ + _ampersand_re = re.compile(r'&(?!#?[xX]?(?:[0-9a-fA-F]+|\w+);)') + _naked_lt_re = re.compile(r'<(?![a-z/?\$!])', re.I) + _naked_gt_re = re.compile(r'''(?<![a-z?!/'"-])>''', re.I) + + def _encode_amps_and_angles(self, text): + # Smart processing for ampersands and angle brackets that need + # to be encoded. + text = self._ampersand_re.sub('&', text) + + # Encode naked <'s + text = self._naked_lt_re.sub('<', text) + + # Encode naked >'s + # Note: Other markdown implementations (e.g. Markdown.pl, PHP + # Markdown) don't do this. + text = self._naked_gt_re.sub('>', text) + return text + + def _encode_backslash_escapes(self, text): + for ch, escape in g_escape_table.items(): + text = text.replace("\\"+ch, escape) + return text + + _auto_link_re = re.compile(r'<((https?|ftp):[^\'">\s]+)>', re.I) + def _auto_link_sub(self, match): + g1 = match.group(1) + return '<a href="%s">%s</a>' % (g1, g1) + + _auto_email_link_re = re.compile(r""" + < + (?:mailto:)? + ( + [-.\w]+ + \@ + [-\w]+(\.[-\w]+)*\.[a-z]+ + ) + > + """, re.I | re.X | re.U) + def _auto_email_link_sub(self, match): + return self._encode_email_address( + self._unescape_special_chars(match.group(1))) + + def _do_auto_links(self, text): + text = self._auto_link_re.sub(self._auto_link_sub, text) + text = self._auto_email_link_re.sub(self._auto_email_link_sub, text) + return text + + def _encode_email_address(self, addr): + # Input: an email address, e.g. "foo@example.com" + # + # Output: the email address as a mailto link, with each character + # of the address encoded as either a decimal or hex entity, in + # the hopes of foiling most address harvesting spam bots. E.g.: + # + # <a href="mailto:foo@e + # xample.com">foo + # @example.com</a> + # + # Based on a filter by Matthew Wickline, posted to the BBEdit-Talk + # mailing list: <http://tinyurl.com/yu7ue> + chars = [_xml_encode_email_char_at_random(ch) + for ch in "mailto:" + addr] + # Strip the mailto: from the visible part. + addr = '<a href="%s">%s</a>' \ + % (''.join(chars), ''.join(chars[7:])) + return addr + + def _do_link_patterns(self, text): + """Caveat emptor: there isn't much guarding against link + patterns being formed inside other standard Markdown links, e.g. + inside a [link def][like this]. + + Dev Notes: *Could* consider prefixing regexes with a negative + lookbehind assertion to attempt to guard against this. + """ + link_from_hash = {} + for regex, repl in self.link_patterns: + replacements = [] + for match in regex.finditer(text): + if hasattr(repl, "__call__"): + href = repl(match) + else: + href = match.expand(repl) + replacements.append((match.span(), href)) + for (start, end), href in reversed(replacements): + escaped_href = ( + href.replace('"', '"') # b/c of attr quote + # To avoid markdown <em> and <strong>: + .replace('*', g_escape_table['*']) + .replace('_', g_escape_table['_'])) + link = '<a href="%s">%s</a>' % (escaped_href, text[start:end]) + hash = md5(link).hexdigest() + link_from_hash[hash] = link + text = text[:start] + hash + text[end:] + for hash, link in link_from_hash.items(): + text = text.replace(hash, link) + return text + + def _unescape_special_chars(self, text): + # Swap back in all the special characters we've hidden. + for ch, hash in g_escape_table.items(): + text = text.replace(hash, ch) + return text + + def _outdent(self, text): + # Remove one level of line-leading tabs or spaces + return self._outdent_re.sub('', text) + + +class MarkdownWithExtras(Markdown): + """A markdowner class that enables most extras: + + - footnotes + - code-color (only has effect if 'pygments' Python module on path) + + These are not included: + - pyshell (specific to Python-related documenting) + - code-friendly (because it *disables* part of the syntax) + - link-patterns (because you need to specify some actual + link-patterns anyway) + """ + extras = ["footnotes", "code-color"] + + +#---- internal support functions + +# From http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/52549 +def _curry(*args, **kwargs): + function, args = args[0], args[1:] + def result(*rest, **kwrest): + combined = kwargs.copy() + combined.update(kwrest) + return function(*args + rest, **combined) + return result + +# Recipe: regex_from_encoded_pattern (1.0) +def _regex_from_encoded_pattern(s): + """'foo' -> re.compile(re.escape('foo')) + '/foo/' -> re.compile('foo') + '/foo/i' -> re.compile('foo', re.I) + """ + if s.startswith('/') and s.rfind('/') != 0: + # Parse it: /PATTERN/FLAGS + idx = s.rfind('/') + pattern, flags_str = s[1:idx], s[idx+1:] + flag_from_char = { + "i": re.IGNORECASE, + "l": re.LOCALE, + "s": re.DOTALL, + "m": re.MULTILINE, + "u": re.UNICODE, + } + flags = 0 + for char in flags_str: + try: + flags |= flag_from_char[char] + except KeyError: + raise ValueError("unsupported regex flag: '%s' in '%s' " + "(must be one of '%s')" + % (char, s, ''.join(flag_from_char.keys()))) + return re.compile(s[1:idx], flags) + else: # not an encoded regex + return re.compile(re.escape(s)) + +# Recipe: dedent (0.1.2) +def _dedentlines(lines, tabsize=8, skip_first_line=False): + """_dedentlines(lines, tabsize=8, skip_first_line=False) -> dedented lines + + "lines" is a list of lines to dedent. + "tabsize" is the tab width to use for indent width calculations. + "skip_first_line" is a boolean indicating if the first line should + be skipped for calculating the indent width and for dedenting. + This is sometimes useful for docstrings and similar. + + Same as dedent() except operates on a sequence of lines. Note: the + lines list is modified **in-place**. + """ + DEBUG = False + if DEBUG: + print "dedent: dedent(..., tabsize=%d, skip_first_line=%r)"\ + % (tabsize, skip_first_line) + indents = [] + margin = None + for i, line in enumerate(lines): + if i == 0 and skip_first_line: continue + indent = 0 + for ch in line: + if ch == ' ': + indent += 1 + elif ch == '\t': + indent += tabsize - (indent % tabsize) + elif ch in '\r\n': + continue # skip all-whitespace lines + else: + break + else: + continue # skip all-whitespace lines + if DEBUG: print "dedent: indent=%d: %r" % (indent, line) + if margin is None: + margin = indent + else: + margin = min(margin, indent) + if DEBUG: print "dedent: margin=%r" % margin + + if margin is not None and margin > 0: + for i, line in enumerate(lines): + if i == 0 and skip_first_line: continue + removed = 0 + for j, ch in enumerate(line): + if ch == ' ': + removed += 1 + elif ch == '\t': + removed += tabsize - (removed % tabsize) + elif ch in '\r\n': + if DEBUG: print "dedent: %r: EOL -> strip up to EOL" % line + lines[i] = lines[i][j:] + break + else: + raise ValueError("unexpected non-whitespace char %r in " + "line %r while removing %d-space margin" + % (ch, line, margin)) + if DEBUG: + print "dedent: %r: %r -> removed %d/%d"\ + % (line, ch, removed, margin) + if removed == margin: + lines[i] = lines[i][j+1:] + break + elif removed > margin: + lines[i] = ' '*(removed-margin) + lines[i][j+1:] + break + else: + if removed: + lines[i] = lines[i][removed:] + return lines + +def _dedent(text, tabsize=8, skip_first_line=False): + """_dedent(text, tabsize=8, skip_first_line=False) -> dedented text + + "text" is the text to dedent. + "tabsize" is the tab width to use for indent width calculations. + "skip_first_line" is a boolean indicating if the first line should + be skipped for calculating the indent width and for dedenting. + This is sometimes useful for docstrings and similar. + + textwrap.dedent(s), but don't expand tabs to spaces + """ + lines = text.splitlines(1) + _dedentlines(lines, tabsize=tabsize, skip_first_line=skip_first_line) + return ''.join(lines) + + +class _memoized(object): + """Decorator that caches a function's return value each time it is called. + If called later with the same arguments, the cached value is returned, and + not re-evaluated. + + http://wiki.python.org/moin/PythonDecoratorLibrary + """ + def __init__(self, func): + self.func = func + self.cache = {} + def __call__(self, *args): + try: + return self.cache[args] + except KeyError: + self.cache[args] = value = self.func(*args) + return value + except TypeError: + # uncachable -- for instance, passing a list as an argument. + # Better to not cache than to blow up entirely. + return self.func(*args) + def __repr__(self): + """Return the function's docstring.""" + return self.func.__doc__ + + +def _xml_oneliner_re_from_tab_width(tab_width): + """Standalone XML processing instruction regex.""" + return re.compile(r""" + (?: + (?<=\n\n) # Starting after a blank line + | # or + \A\n? # the beginning of the doc + ) + ( # save in $1 + [ ]{0,%d} + (?: + <\?\w+\b\s+.*?\?> # XML processing instruction + | + <\w+:\w+\b\s+.*?/> # namespaced single tag + ) + [ \t]* + (?=\n{2,}|\Z) # followed by a blank line or end of document + ) + """ % (tab_width - 1), re.X) +_xml_oneliner_re_from_tab_width = _memoized(_xml_oneliner_re_from_tab_width) + +def _hr_tag_re_from_tab_width(tab_width): + return re.compile(r""" + (?: + (?<=\n\n) # Starting after a blank line + | # or + \A\n? # the beginning of the doc + ) + ( # save in \1 + [ ]{0,%d} + <(hr) # start tag = \2 + \b # word break + ([^<>])*? # + /?> # the matching end tag + [ \t]* + (?=\n{2,}|\Z) # followed by a blank line or end of document + ) + """ % (tab_width - 1), re.X) +_hr_tag_re_from_tab_width = _memoized(_hr_tag_re_from_tab_width) + + +def _xml_encode_email_char_at_random(ch): + r = random() + # Roughly 10% raw, 45% hex, 45% dec. + # '@' *must* be encoded. I [John Gruber] insist. + # Issue 26: '_' must be encoded. + if r > 0.9 and ch not in "@_": + return ch + elif r < 0.45: + # The [1:] is to drop leading '0': 0x63 -> x63 + return '&#%s;' % hex(ord(ch))[1:] + else: + return '&#%s;' % ord(ch) + +def _hash_text(text): + return 'md5:'+md5(text.encode("utf-8")).hexdigest() + + +#---- mainline + +class _NoReflowFormatter(optparse.IndentedHelpFormatter): + """An optparse formatter that does NOT reflow the description.""" + def format_description(self, description): + return description or "" + +def _test(): + import doctest + doctest.testmod() + +def main(argv=None): + if argv is None: + argv = sys.argv + if not logging.root.handlers: + logging.basicConfig() + + usage = "usage: %prog [PATHS...]" + version = "%prog "+__version__ + parser = optparse.OptionParser(prog="markdown2", usage=usage, + version=version, description=cmdln_desc, + formatter=_NoReflowFormatter()) + parser.add_option("-v", "--verbose", dest="log_level", + action="store_const", const=logging.DEBUG, + help="more verbose output") + parser.add_option("--encoding", + help="specify encoding of text content") + parser.add_option("--html4tags", action="store_true", default=False, + help="use HTML 4 style for empty element tags") + parser.add_option("-s", "--safe", metavar="MODE", dest="safe_mode", + help="sanitize literal HTML: 'escape' escapes " + "HTML meta chars, 'replace' replaces with an " + "[HTML_REMOVED] note") + parser.add_option("-x", "--extras", action="append", + help="Turn on specific extra features (not part of " + "the core Markdown spec). Supported values: " + "'code-friendly' disables _/__ for emphasis; " + "'code-color' adds code-block syntax coloring; " + "'link-patterns' adds auto-linking based on patterns; " + "'footnotes' adds the footnotes syntax;" + "'xml' passes one-liner processing instructions and namespaced XML tags;" + "'pyshell' to put unindented Python interactive shell sessions in a <code> block.") + parser.add_option("--use-file-vars", + help="Look for and use Emacs-style 'markdown-extras' " + "file var to turn on extras. See " + "<http://code.google.com/p/python-markdown2/wiki/Extras>.") + parser.add_option("--link-patterns-file", + help="path to a link pattern file") + parser.add_option("--self-test", action="store_true", + help="run internal self-tests (some doctests)") + parser.add_option("--compare", action="store_true", + help="run against Markdown.pl as well (for testing)") + parser.set_defaults(log_level=logging.INFO, compare=False, + encoding="utf-8", safe_mode=None, use_file_vars=False) + opts, paths = parser.parse_args() + log.setLevel(opts.log_level) + + if opts.self_test: + return _test() + + if opts.extras: + extras = {} + for s in opts.extras: + splitter = re.compile("[,;: ]+") + for e in splitter.split(s): + if '=' in e: + ename, earg = e.split('=', 1) + try: + earg = int(earg) + except ValueError: + pass + else: + ename, earg = e, None + extras[ename] = earg + else: + extras = None + + if opts.link_patterns_file: + link_patterns = [] + f = open(opts.link_patterns_file) + try: + for i, line in enumerate(f.readlines()): + if not line.strip(): continue + if line.lstrip().startswith("#"): continue + try: + pat, href = line.rstrip().rsplit(None, 1) + except ValueError: + raise MarkdownError("%s:%d: invalid link pattern line: %r" + % (opts.link_patterns_file, i+1, line)) + link_patterns.append( + (_regex_from_encoded_pattern(pat), href)) + finally: + f.close() + else: + link_patterns = None + + from os.path import join, dirname, abspath, exists + markdown_pl = join(dirname(dirname(abspath(__file__))), "test", + "Markdown.pl") + for path in paths: + if opts.compare: + print "==== Markdown.pl ====" + perl_cmd = 'perl %s "%s"' % (markdown_pl, path) + o = os.popen(perl_cmd) + perl_html = o.read() + o.close() + sys.stdout.write(perl_html) + print "==== markdown2.py ====" + html = markdown_path(path, encoding=opts.encoding, + html4tags=opts.html4tags, + safe_mode=opts.safe_mode, + extras=extras, link_patterns=link_patterns, + use_file_vars=opts.use_file_vars) + sys.stdout.write( + html.encode(sys.stdout.encoding or "utf-8", 'xmlcharrefreplace')) + if opts.compare: + test_dir = join(dirname(dirname(abspath(__file__))), "test") + if exists(join(test_dir, "test_markdown2.py")): + sys.path.insert(0, test_dir) + from test_markdown2 import norm_html_from_html + norm_html = norm_html_from_html(html) + norm_perl_html = norm_html_from_html(perl_html) + else: + norm_html = html + norm_perl_html = perl_html + print "==== match? %r ====" % (norm_perl_html == norm_html) + + +if __name__ == "__main__": + sys.exit( main(sys.argv) ) + diff --git a/vendor/tornado/demos/blog/schema.sql b/vendor/tornado/demos/blog/schema.sql new file mode 100644 index 0000000000..86bff9a8ad --- /dev/null +++ b/vendor/tornado/demos/blog/schema.sql @@ -0,0 +1,44 @@ +-- Copyright 2009 FriendFeed +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); you may +-- not use this file except in compliance with the License. You may obtain +-- a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +-- WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +-- License for the specific language governing permissions and limitations +-- under the License. + +-- To create the database: +-- CREATE DATABASE blog; +-- GRANT ALL PRIVILEGES ON blog.* TO 'blog'@'localhost' IDENTIFIED BY 'blog'; +-- +-- To reload the tables: +-- mysql --user=blog --password=blog --database=blog < schema.sql + +SET SESSION storage_engine = "InnoDB"; +SET SESSION time_zone = "+0:00"; +ALTER DATABASE CHARACTER SET "utf8"; + +DROP TABLE IF EXISTS entries; +CREATE TABLE entries ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + author_id INT NOT NULL REFERENCES authors(id), + slug VARCHAR(100) NOT NULL UNIQUE, + title VARCHAR(512) NOT NULL, + markdown MEDIUMTEXT NOT NULL, + html MEDIUMTEXT NOT NULL, + published DATETIME NOT NULL, + updated TIMESTAMP NOT NULL, + KEY (published) +); + +DROP TABLE IF EXISTS authors; +CREATE TABLE authors ( + id INT NOT NULL AUTO_INCREMENT PRIMARY KEY, + email VARCHAR(100) NOT NULL UNIQUE, + name VARCHAR(100) NOT NULL +); diff --git a/vendor/tornado/demos/blog/static/blog.css b/vendor/tornado/demos/blog/static/blog.css new file mode 100644 index 0000000000..8902ec1f22 --- /dev/null +++ b/vendor/tornado/demos/blog/static/blog.css @@ -0,0 +1,153 @@ +/* + * Copyright 2009 Facebook + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +body { + background: white; + color: black; + margin: 15px; + margin-top: 0; +} + +body, +input, +textarea { + font-family: Georgia, serif; + font-size: 12pt; +} + +table { + border-collapse: collapse; + border: 0; +} + +td { + border: 0; + padding: 0; +} + +h1, +h2, +h3, +h4 { + font-family: "Helvetica Nue", Helvetica, Arial, sans-serif; + margin: 0; +} + +h1 { + font-size: 20pt; +} + +pre, +code { + font-family: monospace; + color: #060; +} + +pre { + margin-left: 1em; + padding-left: 1em; + border-left: 1px solid silver; + line-height: 14pt; +} + +a, +a code { + color: #00c; +} + +#body { + max-width: 800px; + margin: auto; +} + +#header { + background-color: #3b5998; + padding: 5px; + padding-left: 10px; + padding-right: 10px; + margin-bottom: 1em; +} + +#header, +#header a { + color: white; +} + +#header h1 a { + text-decoration: none; +} + +#footer, +#content { + margin-left: 10px; + margin-right: 10px; +} + +#footer { + margin-top: 3em; +} + +.entry h1 a { + color: black; + text-decoration: none; +} + +.entry { + margin-bottom: 2em; +} + +.entry .date { + margin-top: 3px; +} + +.entry p { + margin: 0; + margin-bottom: 1em; +} + +.entry .body { + margin-top: 1em; + line-height: 16pt; +} + +.compose td { + vertical-align: middle; + padding-bottom: 5px; +} + +.compose td.field { + padding-right: 10px; +} + +.compose .title, +.compose .submit { + font-family: "Helvetica Nue", Helvetica, Arial, sans-serif; + font-weight: bold; +} + +.compose .title { + font-size: 20pt; +} + +.compose .title, +.compose .markdown { + width: 100%; +} + +.compose .markdown { + height: 500px; + line-height: 16pt; +} diff --git a/vendor/tornado/demos/blog/templates/archive.html b/vendor/tornado/demos/blog/templates/archive.html new file mode 100644 index 0000000000..dcca9511a4 --- /dev/null +++ b/vendor/tornado/demos/blog/templates/archive.html @@ -0,0 +1,31 @@ +{% extends "base.html" %} + +{% block head %} + <style type="text/css"> + ul.archive { + list-style-type: none; + margin: 0; + padding: 0; + } + + ul.archive li { + margin-bottom: 1em; + } + + ul.archive .title { + font-family: "Helvetica Nue", Helvetica, Arial, sans-serif; + font-size: 14pt; + } + </style> +{% end %} + +{% block body %} + <ul class="archive"> + {% for entry in entries %} + <li> + <div class="title"><a href="/entry/{{ entry.slug }}">{{ escape(entry.title) }}</a></div> + <div class="date">{{ locale.format_date(entry.published, full_format=True, shorter=True) }}</div> + </li> + {% end %} + </ul> +{% end %} diff --git a/vendor/tornado/demos/blog/templates/base.html b/vendor/tornado/demos/blog/templates/base.html new file mode 100644 index 0000000000..038c5b3fff --- /dev/null +++ b/vendor/tornado/demos/blog/templates/base.html @@ -0,0 +1,27 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml"> + <head> + <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/> + <title>{{ escape(handler.settings["blog_title"]) }}</title> + <link rel="stylesheet" href="{{ static_url("blog.css") }}" type="text/css"/> + <link rel="alternate" href="/feed" type="application/atom+xml" title="{{ escape(handler.settings["blog_title"]) }}"/> + {% block head %}{% end %} + </head> + <body> + <div id="body"> + <div id="header"> + <div style="float:right"> + {% if current_user %} + <a href="/compose">{{ _("New post") }}</a> - + <a href="/auth/logout?next={{ url_escape(request.uri) }}">{{ _("Sign out") }}</a> + {% else %} + {{ _('<a href="%(url)s">Sign in</a> to compose/edit') % {"url": "/auth/login?next=" + url_escape(request.uri)} }} + {% end %} + </div> + <h1><a href="/">{{ escape(handler.settings["blog_title"]) }}</a></h1> + </div> + <div id="content">{% block body %}{% end %}</div> + </div> + {% block bottom %}{% end %} + </body> +</html> diff --git a/vendor/tornado/demos/blog/templates/compose.html b/vendor/tornado/demos/blog/templates/compose.html new file mode 100644 index 0000000000..bc054b3349 --- /dev/null +++ b/vendor/tornado/demos/blog/templates/compose.html @@ -0,0 +1,42 @@ +{% extends "base.html" %} + +{% block body %} + <form action="{{ request.path }}" method="post" class="compose"> + <div style="margin-bottom:5px"><input name="title" type="text" class="title" value="{{ escape(entry.title) if entry else "" }}"/></div> + <div style="margin-bottom:5px"><textarea name="markdown" rows="30" cols="40" class="markdown">{{ escape(entry.markdown) if entry else "" }}</textarea></div> + <div> + <div style="float:right"><a href="http://daringfireball.net/projects/markdown/syntax">{{ _("Syntax documentation") }}</a></div> + <input type="submit" value="{{ _("Save changes") if entry else _("Publish post") }}" class="submit"/> + <a href="{{ "/entry/" + entry.slug if entry else "/" }}">{{ _("Cancel") }}</a> + </div> + {% if entry %} + <input type="hidden" name="id" value="{{ entry.id }}"/> + {% end %} + {{ xsrf_form_html() }} + </form> +{% end %} + +{% block bottom %} + <script src="http://ajax.googleapis.com/ajax/libs/jquery/1.3.2/jquery.min.js" type="text/javascript"></script> + <script type="text/javascript"> + //<![CDATA[ + + $(function() { + $("input[name=title]").select(); + $("form.compose").submit(function() { + var required = ["title", "markdown"]; + var form = $(this).get(0); + for (var i = 0; i < required.length; i++) { + if (!form[required[i]].value) { + $(form[required[i]]).select(); + return false; + } + } + return true; + }); + }); + + //]]> + </script> +{% end %} + diff --git a/vendor/tornado/demos/blog/templates/entry.html b/vendor/tornado/demos/blog/templates/entry.html new file mode 100644 index 0000000000..43c835dead --- /dev/null +++ b/vendor/tornado/demos/blog/templates/entry.html @@ -0,0 +1,5 @@ +{% extends "base.html" %} + +{% block body %} + {{ modules.Entry(entry) }} +{% end %} diff --git a/vendor/tornado/demos/blog/templates/feed.xml b/vendor/tornado/demos/blog/templates/feed.xml new file mode 100644 index 0000000000..c6c368656c --- /dev/null +++ b/vendor/tornado/demos/blog/templates/feed.xml @@ -0,0 +1,26 @@ +<?xml version="1.0" encoding="utf-8"?> +<feed xmlns="http://www.w3.org/2005/Atom"> + {% set date_format = "%Y-%m-%dT%H:%M:%SZ" %} + <title>{{ escape(handler.settings["blog_title"]) }}</title> + {% if len(entries) > 0 %} + <updated>{{ max(e.updated for e in entries).strftime(date_format) }}</updated> + {% else %} + <updated>{{ datetime.datetime.utcnow().strftime(date_format) }}</updated> + {% end %} + <id>http://{{ request.host }}/</id> + <link rel="alternate" href="http://{{ request.host }}/" title="{{ escape(handler.settings["blog_title"]) }}" type="text/html"/> + <link rel="self" href="{{ request.full_url() }}" title="{{ escape(handler.settings["blog_title"]) }}" type="application/atom+xml"/> + <author><name>{{ escape(handler.settings["blog_title"]) }}</name></author> + {% for entry in entries %} + <entry> + <id>http://{{ request.host }}/entry/{{ entry.slug }}</id> + <title type="text">{{ escape(entry.title) }}</title> + <link href="http://{{ request.host }}/entry/{{ entry.slug }}" rel="alternate" type="text/html"/> + <updated>{{ entry.updated.strftime(date_format) }}</updated> + <published>{{ entry.published.strftime(date_format) }}</published> + <content type="xhtml" xml:base="http://{{ request.host }}/"> + <div xmlns="http://www.w3.org/1999/xhtml">{{ entry.html }}</div> + </content> + </entry> + {% end %} +</feed> diff --git a/vendor/tornado/demos/blog/templates/home.html b/vendor/tornado/demos/blog/templates/home.html new file mode 100644 index 0000000000..dd069a97f3 --- /dev/null +++ b/vendor/tornado/demos/blog/templates/home.html @@ -0,0 +1,8 @@ +{% extends "base.html" %} + +{% block body %} + {% for entry in entries %} + {{ modules.Entry(entry) }} + {% end %} + <div><a href="/archive">{{ _("Archive") }}</a></div> +{% end %} diff --git a/vendor/tornado/demos/blog/templates/modules/entry.html b/vendor/tornado/demos/blog/templates/modules/entry.html new file mode 100644 index 0000000000..27ea0d76c2 --- /dev/null +++ b/vendor/tornado/demos/blog/templates/modules/entry.html @@ -0,0 +1,8 @@ +<div class="entry"> + <h1><a href="/entry/{{ entry.slug }}">{{ escape(entry.title) }}</a></h1> + <div class="date">{{ locale.format_date(entry.published, full_format=True, shorter=True) }}</div> + <div class="body">{{ entry.html }}</div> + {% if current_user %} + <div class="admin"><a href="/compose?id={{ entry.id }}">{{ _("Edit this post") }}</a></div> + {% end %} +</div> diff --git a/vendor/tornado/demos/chat/chatdemo.py b/vendor/tornado/demos/chat/chatdemo.py new file mode 100755 index 0000000000..7086592ec4 --- /dev/null +++ b/vendor/tornado/demos/chat/chatdemo.py @@ -0,0 +1,156 @@ +#!/usr/bin/env python +# +# Copyright 2009 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import tornado.auth +import tornado.escape +import tornado.httpserver +import tornado.ioloop +import tornado.options +import tornado.web +import os.path +import uuid + +from tornado.options import define, options + +define("port", default=8888, help="run on the given port", type=int) + + +class Application(tornado.web.Application): + def __init__(self): + handlers = [ + (r"/", MainHandler), + (r"/auth/login", AuthLoginHandler), + (r"/auth/logout", AuthLogoutHandler), + (r"/a/message/new", MessageNewHandler), + (r"/a/message/updates", MessageUpdatesHandler), + ] + settings = dict( + cookie_secret="43oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=", + login_url="/auth/login", + template_path=os.path.join(os.path.dirname(__file__), "templates"), + static_path=os.path.join(os.path.dirname(__file__), "static"), + xsrf_cookies=True, + ) + tornado.web.Application.__init__(self, handlers, **settings) + + +class BaseHandler(tornado.web.RequestHandler): + def get_current_user(self): + user_json = self.get_secure_cookie("user") + if not user_json: return None + return tornado.escape.json_decode(user_json) + + +class MainHandler(BaseHandler): + @tornado.web.authenticated + def get(self): + self.render("index.html", messages=MessageMixin.cache) + + +class MessageMixin(object): + waiters = [] + cache = [] + cache_size = 200 + + def wait_for_messages(self, callback, cursor=None): + cls = MessageMixin + if cursor: + index = 0 + for i in xrange(len(cls.cache)): + index = len(cls.cache) - i - 1 + if cls.cache[index]["id"] == cursor: break + recent = cls.cache[index + 1:] + if recent: + callback(recent) + return + cls.waiters.append(callback) + + def new_messages(self, messages): + cls = MessageMixin + logging.info("Sending new message to %r listeners", len(cls.waiters)) + for callback in cls.waiters: + try: + callback(messages) + except: + logging.error("Error in waiter callback", exc_info=True) + cls.waiters = [] + cls.cache.extend(messages) + if len(cls.cache) > self.cache_size: + cls.cache = cls.cache[-self.cache_size:] + + +class MessageNewHandler(BaseHandler, MessageMixin): + @tornado.web.authenticated + def post(self): + message = { + "id": str(uuid.uuid4()), + "from": self.current_user["first_name"], + "body": self.get_argument("body"), + } + message["html"] = self.render_string("message.html", message=message) + if self.get_argument("next", None): + self.redirect(self.get_argument("next")) + else: + self.write(message) + self.new_messages([message]) + + +class MessageUpdatesHandler(BaseHandler, MessageMixin): + @tornado.web.authenticated + @tornado.web.asynchronous + def post(self): + cursor = self.get_argument("cursor", None) + self.wait_for_messages(self.async_callback(self.on_new_messages), + cursor=cursor) + + def on_new_messages(self, messages): + # Closed client connection + if self.request.connection.stream.closed(): + return + self.finish(dict(messages=messages)) + + +class AuthLoginHandler(BaseHandler, tornado.auth.GoogleMixin): + @tornado.web.asynchronous + def get(self): + if self.get_argument("openid.mode", None): + self.get_authenticated_user(self.async_callback(self._on_auth)) + return + self.authenticate_redirect(ax_attrs=["name"]) + + def _on_auth(self, user): + if not user: + raise tornado.web.HTTPError(500, "Google auth failed") + self.set_secure_cookie("user", tornado.escape.json_encode(user)) + self.redirect("/") + + +class AuthLogoutHandler(BaseHandler): + def get(self): + self.clear_cookie("user") + self.write("You are now logged out") + + +def main(): + tornado.options.parse_command_line() + http_server = tornado.httpserver.HTTPServer(Application()) + http_server.listen(options.port) + tornado.ioloop.IOLoop.instance().start() + + +if __name__ == "__main__": + main() diff --git a/vendor/tornado/demos/chat/static/chat.css b/vendor/tornado/demos/chat/static/chat.css new file mode 100644 index 0000000000..a400c32605 --- /dev/null +++ b/vendor/tornado/demos/chat/static/chat.css @@ -0,0 +1,56 @@ +/* + * Copyright 2009 FriendFeed + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +body { + background: white; + margin: 10px; +} + +body, +input { + font-family: sans-serif; + font-size: 10pt; + color: black; +} + +table { + border-collapse: collapse; + border: 0; +} + +td { + border: 0; + padding: 0; +} + +#body { + position: absolute; + bottom: 10px; + left: 10px; +} + +#input { + margin-top: 0.5em; +} + +#inbox .message { + padding-top: 0.25em; +} + +#nav { + float: right; + z-index: 99; +} diff --git a/vendor/tornado/demos/chat/static/chat.js b/vendor/tornado/demos/chat/static/chat.js new file mode 100644 index 0000000000..0054c710d6 --- /dev/null +++ b/vendor/tornado/demos/chat/static/chat.js @@ -0,0 +1,135 @@ +// Copyright 2009 FriendFeed +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +$(document).ready(function() { + if (!window.console) window.console = {}; + if (!window.console.log) window.console.log = function() {}; + + $("#messageform").live("submit", function() { + newMessage($(this)); + return false; + }); + $("#messageform").live("keypress", function(e) { + if (e.keyCode == 13) { + newMessage($(this)); + return false; + } + }); + $("#message").select(); + updater.poll(); +}); + +function newMessage(form) { + var message = form.formToDict(); + var disabled = form.find("input[type=submit]"); + disabled.disable(); + $.postJSON("/a/message/new", message, function(response) { + updater.showMessage(response); + if (message.id) { + form.parent().remove(); + } else { + form.find("input[type=text]").val("").select(); + disabled.enable(); + } + }); +} + +function getCookie(name) { + var r = document.cookie.match("\\b" + name + "=([^;]*)\\b"); + return r ? r[1] : undefined; +} + +jQuery.postJSON = function(url, args, callback) { + args._xsrf = getCookie("_xsrf"); + $.ajax({url: url, data: $.param(args), dataType: "text", type: "POST", + success: function(response) { + if (callback) callback(eval("(" + response + ")")); + }, error: function(response) { + console.log("ERROR:", response) + }}); +}; + +jQuery.fn.formToDict = function() { + var fields = this.serializeArray(); + var json = {} + for (var i = 0; i < fields.length; i++) { + json[fields[i].name] = fields[i].value; + } + if (json.next) delete json.next; + return json; +}; + +jQuery.fn.disable = function() { + this.enable(false); + return this; +}; + +jQuery.fn.enable = function(opt_enable) { + if (arguments.length && !opt_enable) { + this.attr("disabled", "disabled"); + } else { + this.removeAttr("disabled"); + } + return this; +}; + +var updater = { + errorSleepTime: 500, + cursor: null, + + poll: function() { + var args = {"_xsrf": getCookie("_xsrf")}; + if (updater.cursor) args.cursor = updater.cursor; + $.ajax({url: "/a/message/updates", type: "POST", dataType: "text", + data: $.param(args), success: updater.onSuccess, + error: updater.onError}); + }, + + onSuccess: function(response) { + try { + updater.newMessages(eval("(" + response + ")")); + } catch (e) { + updater.onError(); + return; + } + updater.errorSleepTime = 500; + window.setTimeout(updater.poll, 0); + }, + + onError: function(response) { + updater.errorSleepTime *= 2; + console.log("Poll error; sleeping for", updater.errorSleepTime, "ms"); + window.setTimeout(updater.poll, updater.errorSleepTime); + }, + + newMessages: function(response) { + if (!response.messages) return; + updater.cursor = response.cursor; + var messages = response.messages; + updater.cursor = messages[messages.length - 1].id; + console.log(messages.length, "new messages, cursor:", updater.cursor); + for (var i = 0; i < messages.length; i++) { + updater.showMessage(messages[i]); + } + }, + + showMessage: function(message) { + var existing = $("#m" + message.id); + if (existing.length > 0) return; + var node = $(message.html); + node.hide(); + $("#inbox").append(node); + node.slideDown(); + } +}; diff --git a/vendor/tornado/demos/chat/templates/index.html b/vendor/tornado/demos/chat/templates/index.html new file mode 100644 index 0000000000..de051d852b --- /dev/null +++ b/vendor/tornado/demos/chat/templates/index.html @@ -0,0 +1,37 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml"> + <head> + <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/> + <title>Tornado Chat Demo</title> + <link rel="stylesheet" href="{{ static_url("chat.css") }}" type="text/css"/> + </head> + <body> + <div id="nav"> + <b>{{ escape(current_user["name"]) }}</b> - + <a href="/auth/logout">{{ _("Sign out") }}</a> + </div> + <div id="body"> + <div id="inbox"> + {% for message in messages %} + {% include "message.html" %} + {% end %} + </div> + <div id="input"> + <form action="/a/message/new" method="post" id="messageform"> + <table> + <tr> + <td><input name="body" id="message" style="width:500px"/></td> + <td style="padding-left:5px"> + <input type="submit" value="{{ _("Post") }}"/> + <input type="hidden" name="next" value="{{ request.path }}"/> + {{ xsrf_form_html() }} + </td> + </tr> + </table> + </form> + </div> + </div> + <script src="http://ajax.googleapis.com/ajax/libs/jquery/1.3/jquery.min.js" type="text/javascript"></script> + <script src="{{ static_url("chat.js") }}" type="text/javascript"></script> + </body> +</html> diff --git a/vendor/tornado/demos/chat/templates/message.html b/vendor/tornado/demos/chat/templates/message.html new file mode 100644 index 0000000000..4445cbdfaf --- /dev/null +++ b/vendor/tornado/demos/chat/templates/message.html @@ -0,0 +1 @@ +<div class="message" id="m{{ message["id"] }}"><b>{{ escape(message["from"]) }}: </b>{{ escape(message["body"]) }}</div> diff --git a/vendor/tornado/demos/facebook/README b/vendor/tornado/demos/facebook/README new file mode 100644 index 0000000000..2f0dc28e84 --- /dev/null +++ b/vendor/tornado/demos/facebook/README @@ -0,0 +1,8 @@ +Running the Tornado Facebook example +===================================== +To work with the provided Facebook api key, this example must be +accessed at http://localhost:8888/ to match the Connect URL set in the +example application. + +To use any other domain, a new Facebook application must be registered +with a Connect URL set to that domain. diff --git a/vendor/tornado/demos/facebook/facebook.py b/vendor/tornado/demos/facebook/facebook.py new file mode 100755 index 0000000000..0c984ddaa0 --- /dev/null +++ b/vendor/tornado/demos/facebook/facebook.py @@ -0,0 +1,127 @@ +#!/usr/bin/env python +# +# Copyright 2009 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import logging +import os.path +import tornado.auth +import tornado.escape +import tornado.httpserver +import tornado.ioloop +import tornado.options +import tornado.web +import uimodules + +from tornado.options import define, options + +define("port", default=8888, help="run on the given port", type=int) +define("facebook_api_key", help="your Facebook application API key", + default="9e2ada1b462142c4dfcc8e894ea1e37c") +define("facebook_secret", help="your Facebook application secret", + default="32fc6114554e3c53d5952594510021e2") + + +class Application(tornado.web.Application): + def __init__(self): + handlers = [ + (r"/", MainHandler), + (r"/auth/login", AuthLoginHandler), + (r"/auth/logout", AuthLogoutHandler), + ] + settings = dict( + cookie_secret="12oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=", + login_url="/auth/login", + template_path=os.path.join(os.path.dirname(__file__), "templates"), + static_path=os.path.join(os.path.dirname(__file__), "static"), + xsrf_cookies=True, + facebook_api_key=options.facebook_api_key, + facebook_secret=options.facebook_secret, + ui_modules= {"Post": PostModule}, + debug=True, + ) + tornado.web.Application.__init__(self, handlers, **settings) + + +class BaseHandler(tornado.web.RequestHandler): + def get_current_user(self): + user_json = self.get_secure_cookie("user") + if not user_json: return None + return tornado.escape.json_decode(user_json) + + +class MainHandler(BaseHandler, tornado.auth.FacebookMixin): + @tornado.web.authenticated + @tornado.web.asynchronous + def get(self): + self.facebook_request( + method="stream.get", + callback=self.async_callback(self._on_stream), + session_key=self.current_user["session_key"]) + + def _on_stream(self, stream): + if stream is None: + # Session may have expired + self.redirect("/auth/login") + return + # Turn profiles into a dict mapping id => profile + stream["profiles"] = dict((p["id"], p) for p in stream["profiles"]) + self.render("stream.html", stream=stream) + + +class AuthLoginHandler(BaseHandler, tornado.auth.FacebookMixin): + @tornado.web.asynchronous + def get(self): + if self.get_argument("session", None): + self.get_authenticated_user(self.async_callback(self._on_auth)) + return + self.authorize_redirect("read_stream") + + def _on_auth(self, user): + if not user: + raise tornado.web.HTTPError(500, "Facebook auth failed") + self.set_secure_cookie("user", tornado.escape.json_encode(user)) + self.redirect(self.get_argument("next", "/")) + + +class AuthLogoutHandler(BaseHandler, tornado.auth.FacebookMixin): + @tornado.web.asynchronous + def get(self): + self.clear_cookie("user") + if not self.current_user: + self.redirect(self.get_argument("next", "/")) + return + self.facebook_request( + method="auth.revokeAuthorization", + callback=self.async_callback(self._on_deauthorize), + session_key=self.current_user["session_key"]) + + def _on_deauthorize(self, response): + self.redirect(self.get_argument("next", "/")) + + +class PostModule(tornado.web.UIModule): + def render(self, post, actor): + return self.render_string("modules/post.html", post=post, actor=actor) + + +def main(): + tornado.options.parse_command_line() + http_server = tornado.httpserver.HTTPServer(Application()) + http_server.listen(options.port) + tornado.ioloop.IOLoop.instance().start() + + +if __name__ == "__main__": + main() diff --git a/vendor/tornado/demos/facebook/static/facebook.css b/vendor/tornado/demos/facebook/static/facebook.css new file mode 100644 index 0000000000..4fee72678f --- /dev/null +++ b/vendor/tornado/demos/facebook/static/facebook.css @@ -0,0 +1,97 @@ +/* + * Copyright 2009 Facebook + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +body { + background: white; + color: black; + margin: 15px; +} + +body, +input, +textarea { + font-family: "Lucida Grande", Tahoma, Verdana, sans-serif; + font-size: 10pt; +} + +table { + border-collapse: collapse; + border: 0; +} + +td { + border: 0; + padding: 0; +} + +img { + border: 0; +} + +a { + text-decoration: none; + color: #3b5998; +} + +a:hover { + text-decoration: underline; +} + +.post { + border-bottom: 1px solid #eeeeee; + min-height: 50px; + padding-bottom: 10px; + margin-top: 10px; +} + +.post .picture { + float: left; +} + +.post .picture img { + height: 50px; + width: 50px; +} + +.post .body { + margin-left: 60px; +} + +.post .media img { + border: 1px solid #cccccc; + padding: 3px; +} + +.post .media:hover img { + border: 1px solid #3b5998; +} + +.post a.actor { + font-weight: bold; +} + +.post .meta { + font-size: 11px; +} + +.post a.permalink { + color: #777777; +} + +#body { + max-width: 700px; + margin: auto; +} diff --git a/vendor/tornado/demos/facebook/static/facebook.js b/vendor/tornado/demos/facebook/static/facebook.js new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/vendor/tornado/demos/facebook/static/facebook.js diff --git a/vendor/tornado/demos/facebook/templates/modules/post.html b/vendor/tornado/demos/facebook/templates/modules/post.html new file mode 100644 index 0000000000..6b50ac0f72 --- /dev/null +++ b/vendor/tornado/demos/facebook/templates/modules/post.html @@ -0,0 +1,29 @@ +<div class="post"> + <div class="picture"> + <a href="{{ actor["url"] }}"><img src="{{ actor["pic_square"] }}"/></a> + </div> + <div class="body"> + <a href="{{ actor["url"] }}" class="actor">{{ escape(actor["name"]) }}</a> + {% if post["message"] %} + <span class="message">{{ escape(post["message"]) }}</span> + {% end %} + {% if post["attachment"] %} + <div class="attachment"> + {% if post["attachment"].get("name") %} + <div class="name"><a href="{{ post["attachment"]["href"] }}">{{ escape(post["attachment"]["name"]) }}</a></div> + {% end %} + {% if post["attachment"].get("description") %} + <div class="description">{{ post["attachment"]["description"] }}</div> + {% end %} + {% for media in filter(lambda m: m.get("src") and m["type"] in ("photo", "link"), post["attachment"].get("media", [])) %} + <span class="media"> + <a href="{{ media["href"] }}"><img src="{{ media["src"] }}" alt="{{ escape(media.get("alt", "")) }}"/></a> + </span> + {% end %} + </div> + {% end %} + <div class="meta"> + <a href="{{ post["permalink"] }}" class="permalink">{{ locale.format_date(post["created_time"]) }}</a> + </div> + </div> +</div> diff --git a/vendor/tornado/demos/facebook/templates/stream.html b/vendor/tornado/demos/facebook/templates/stream.html new file mode 100644 index 0000000000..19baa28cbf --- /dev/null +++ b/vendor/tornado/demos/facebook/templates/stream.html @@ -0,0 +1,22 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml"> + <head> + <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/> + <title>Tornado Facebook Stream Demo</title> + <link rel="stylesheet" href="{{ static_url("facebook.css") }}" type="text/css"/> + </head> + <body> + <div id="body"> + <div style="float:right"> + <b>{{ escape(current_user["name"]) }}</b> - + <a href="/auth/logout">{{ _("Sign out") }}</a> + </div> + <div style="margin-bottom:1em"><a href="/">{{ _("Refresh stream") }}</a></div> + <div id="stream"> + {% for post in stream["posts"] %} + {{ modules.Post(post, stream["profiles"][post["actor_id"]]) }} + {% end %} + </div> + </div> + </body> +</html> diff --git a/vendor/tornado/demos/facebook/uimodules.py b/vendor/tornado/demos/facebook/uimodules.py new file mode 100644 index 0000000000..1173db634e --- /dev/null +++ b/vendor/tornado/demos/facebook/uimodules.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +# +# Copyright 2009 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import tornado.web + + +class Entry(tornado.web.UIModule): + def render(self): + return '<div>ENTRY</div>' diff --git a/vendor/tornado/demos/helloworld/helloworld.py b/vendor/tornado/demos/helloworld/helloworld.py new file mode 100755 index 0000000000..0f1ed61ff5 --- /dev/null +++ b/vendor/tornado/demos/helloworld/helloworld.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +# +# Copyright 2009 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import tornado.httpserver +import tornado.ioloop +import tornado.options +import tornado.web + +from tornado.options import define, options + +define("port", default=8888, help="run on the given port", type=int) + + +class MainHandler(tornado.web.RequestHandler): + def get(self): + self.write("Hello, world") + + +def main(): + tornado.options.parse_command_line() + application = tornado.web.Application([ + (r"/", MainHandler), + ]) + http_server = tornado.httpserver.HTTPServer(application) + http_server.listen(options.port) + tornado.ioloop.IOLoop.instance().start() + + +if __name__ == "__main__": + main() diff --git a/vendor/tornado/setup.py b/vendor/tornado/setup.py new file mode 100644 index 0000000000..5cb69df2da --- /dev/null +++ b/vendor/tornado/setup.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python +# +# Copyright 2009 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import distutils.core +import sys +# Importing setuptools adds some features like "setup.py develop", but +# it's optional so swallow the error if it's not there. +try: + import setuptools +except ImportError: + pass + +# Build the epoll extension for Linux systems with Python < 2.6 +extensions = [] +major, minor = sys.version_info[:2] +python_26 = (major > 2 or (major == 2 and minor >= 6)) +if "linux" in sys.platform.lower() and not python_26: + extensions.append(distutils.core.Extension( + "tornado.epoll", ["tornado/epoll.c"])) + +distutils.core.setup( + name="tornado", + version="0.2", + packages = ["tornado"], + ext_modules = extensions, + author="Facebook", + author_email="python-tornado@googlegroups.com", + url="http://www.tornadoweb.org/", + license="http://www.apache.org/licenses/LICENSE-2.0", + description="Tornado is an open source version of the scalable, non-blocking web server and and tools that power FriendFeed", +) diff --git a/vendor/tornado/tornado/__init__.py b/vendor/tornado/tornado/__init__.py new file mode 100644 index 0000000000..8f73764eb2 --- /dev/null +++ b/vendor/tornado/tornado/__init__.py @@ -0,0 +1,17 @@ +#!/usr/bin/env python +# +# Copyright 2009 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""The Tornado web server and tools.""" diff --git a/vendor/tornado/tornado/auth.py b/vendor/tornado/tornado/auth.py new file mode 100644 index 0000000000..f67d9e5482 --- /dev/null +++ b/vendor/tornado/tornado/auth.py @@ -0,0 +1,883 @@ +#!/usr/bin/env python +# +# Copyright 2009 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Implementations of various third-party authentication schemes. + +All the classes in this file are class Mixins designed to be used with +web.py RequestHandler classes. The primary methods for each service are +authenticate_redirect(), authorize_redirect(), and get_authenticated_user(). +The former should be called to redirect the user to, e.g., the OpenID +authentication page on the third party service, and the latter should +be called upon return to get the user data from the data returned by +the third party service. + +They all take slightly different arguments due to the fact all these +services implement authentication and authorization slightly differently. +See the individual service classes below for complete documentation. + +Example usage for Google OpenID: + +class GoogleHandler(tornado.web.RequestHandler, tornado.auth.GoogleMixin): + @tornado.web.asynchronous + def get(self): + if self.get_argument("openid.mode", None): + self.get_authenticated_user(self.async_callback(self._on_auth)) + return + self.authenticate_redirect() + + def _on_auth(self, user): + if not user: + raise tornado.web.HTTPError(500, "Google auth failed") + # Save the user with, e.g., set_secure_cookie() + +""" + +import base64 +import binascii +import cgi +import hashlib +import hmac +import httpclient +import escape +import logging +import time +import urllib +import urlparse +import uuid + +_log = logging.getLogger("tornado.auth") + +class OpenIdMixin(object): + """Abstract implementation of OpenID and Attribute Exchange. + + See GoogleMixin below for example implementations. + """ + def authenticate_redirect(self, callback_uri=None, + ax_attrs=["name","email","language","username"]): + """Returns the authentication URL for this service. + + After authentication, the service will redirect back to the given + callback URI. + + We request the given attributes for the authenticated user by + default (name, email, language, and username). If you don't need + all those attributes for your app, you can request fewer with + the ax_attrs keyword argument. + """ + callback_uri = callback_uri or self.request.path + args = self._openid_args(callback_uri, ax_attrs=ax_attrs) + self.redirect(self._OPENID_ENDPOINT + "?" + urllib.urlencode(args)) + + def get_authenticated_user(self, callback): + """Fetches the authenticated user data upon redirect. + + This method should be called by the handler that receives the + redirect from the authenticate_redirect() or authorize_redirect() + methods. + """ + # Verify the OpenID response via direct request to the OP + args = dict((k, v[-1]) for k, v in self.request.arguments.iteritems()) + args["openid.mode"] = u"check_authentication" + url = self._OPENID_ENDPOINT + "?" + urllib.urlencode(args) + http = httpclient.AsyncHTTPClient() + http.fetch(url, self.async_callback( + self._on_authentication_verified, callback)) + + def _openid_args(self, callback_uri, ax_attrs=[], oauth_scope=None): + url = urlparse.urljoin(self.request.full_url(), callback_uri) + args = { + "openid.ns": "http://specs.openid.net/auth/2.0", + "openid.claimed_id": + "http://specs.openid.net/auth/2.0/identifier_select", + "openid.identity": + "http://specs.openid.net/auth/2.0/identifier_select", + "openid.return_to": url, + "openid.realm": "http://" + self.request.host + "/", + "openid.mode": "checkid_setup", + } + if ax_attrs: + args.update({ + "openid.ns.ax": "http://openid.net/srv/ax/1.0", + "openid.ax.mode": "fetch_request", + }) + ax_attrs = set(ax_attrs) + required = [] + if "name" in ax_attrs: + ax_attrs -= set(["name", "firstname", "fullname", "lastname"]) + required += ["firstname", "fullname", "lastname"] + args.update({ + "openid.ax.type.firstname": + "http://axschema.org/namePerson/first", + "openid.ax.type.fullname": + "http://axschema.org/namePerson", + "openid.ax.type.lastname": + "http://axschema.org/namePerson/last", + }) + known_attrs = { + "email": "http://axschema.org/contact/email", + "language": "http://axschema.org/pref/language", + "username": "http://axschema.org/namePerson/friendly", + } + for name in ax_attrs: + args["openid.ax.type." + name] = known_attrs[name] + required.append(name) + args["openid.ax.required"] = ",".join(required) + if oauth_scope: + args.update({ + "openid.ns.oauth": + "http://specs.openid.net/extensions/oauth/1.0", + "openid.oauth.consumer": self.request.host.split(":")[0], + "openid.oauth.scope": oauth_scope, + }) + return args + + def _on_authentication_verified(self, callback, response): + if response.error or u"is_valid:true" not in response.body: + _log.warning("Invalid OpenID response: %s", response.error or + response.body) + callback(None) + return + + # Make sure we got back at least an email from attribute exchange + ax_ns = None + for name, values in self.request.arguments.iteritems(): + if name.startswith("openid.ns.") and \ + values[-1] == u"http://openid.net/srv/ax/1.0": + ax_ns = name[10:] + break + def get_ax_arg(uri): + if not ax_ns: return u"" + prefix = "openid." + ax_ns + ".type." + ax_name = None + for name, values in self.request.arguments.iteritems(): + if values[-1] == uri and name.startswith(prefix): + part = name[len(prefix):] + ax_name = "openid." + ax_ns + ".value." + part + break + if not ax_name: return u"" + return self.get_argument(ax_name, u"") + + email = get_ax_arg("http://axschema.org/contact/email") + name = get_ax_arg("http://axschema.org/namePerson") + first_name = get_ax_arg("http://axschema.org/namePerson/first") + last_name = get_ax_arg("http://axschema.org/namePerson/last") + username = get_ax_arg("http://axschema.org/namePerson/friendly") + locale = get_ax_arg("http://axschema.org/pref/language").lower() + user = dict() + name_parts = [] + if first_name: + user["first_name"] = first_name + name_parts.append(first_name) + if last_name: + user["last_name"] = last_name + name_parts.append(last_name) + if name: + user["name"] = name + elif name_parts: + user["name"] = u" ".join(name_parts) + elif email: + user["name"] = email.split("@")[0] + if email: user["email"] = email + if locale: user["locale"] = locale + if username: user["username"] = username + callback(user) + + +class OAuthMixin(object): + """Abstract implementation of OAuth. + + See TwitterMixin and FriendFeedMixin below for example implementations. + """ + def authorize_redirect(self, callback_uri=None): + """Redirects the user to obtain OAuth authorization for this service. + + Twitter and FriendFeed both require that you register a Callback + URL with your application. You should call this method to log the + user in, and then call get_authenticated_user() in the handler + you registered as your Callback URL to complete the authorization + process. + + This method sets a cookie called _oauth_request_token which is + subsequently used (and cleared) in get_authenticated_user for + security purposes. + """ + if callback_uri and getattr(self, "_OAUTH_NO_CALLBACKS", False): + raise Exception("This service does not support oauth_callback") + http = httpclient.AsyncHTTPClient() + http.fetch(self._oauth_request_token_url(), self.async_callback( + self._on_request_token, self._OAUTH_AUTHORIZE_URL, callback_uri)) + + def get_authenticated_user(self, callback): + """Gets the OAuth authorized user and access token on callback. + + This method should be called from the handler for your registered + OAuth Callback URL to complete the registration process. We call + callback with the authenticated user, which in addition to standard + attributes like 'name' includes the 'access_key' attribute, which + contains the OAuth access you can use to make authorized requests + to this service on behalf of the user. + """ + request_key = self.get_argument("oauth_token") + request_cookie = self.get_cookie("_oauth_request_token") + if not request_cookie: + _log.warning("Missing OAuth request token cookie") + callback(None) + return + cookie_key, cookie_secret = request_cookie.split("|") + if cookie_key != request_key: + _log.warning("Request token does not match cookie") + callback(None) + return + token = dict(key=cookie_key, secret=cookie_secret) + http = httpclient.AsyncHTTPClient() + http.fetch(self._oauth_access_token_url(token), self.async_callback( + self._on_access_token, callback)) + + def _oauth_request_token_url(self): + consumer_token = self._oauth_consumer_token() + url = self._OAUTH_REQUEST_TOKEN_URL + args = dict( + oauth_consumer_key=consumer_token["key"], + oauth_signature_method="HMAC-SHA1", + oauth_timestamp=str(int(time.time())), + oauth_nonce=binascii.b2a_hex(uuid.uuid4().bytes), + oauth_version="1.0", + ) + signature = _oauth_signature(consumer_token, "GET", url, args) + args["oauth_signature"] = signature + return url + "?" + urllib.urlencode(args) + + def _on_request_token(self, authorize_url, callback_uri, response): + if response.error: + raise Exception("Could not get request token") + request_token = _oauth_parse_response(response.body) + data = "|".join([request_token["key"], request_token["secret"]]) + self.set_cookie("_oauth_request_token", data) + args = dict(oauth_token=request_token["key"]) + if callback_uri: + args["oauth_callback"] = urlparse.urljoin( + self.request.full_url(), callback_uri) + self.redirect(authorize_url + "?" + urllib.urlencode(args)) + + def _oauth_access_token_url(self, request_token): + consumer_token = self._oauth_consumer_token() + url = self._OAUTH_ACCESS_TOKEN_URL + args = dict( + oauth_consumer_key=consumer_token["key"], + oauth_token=request_token["key"], + oauth_signature_method="HMAC-SHA1", + oauth_timestamp=str(int(time.time())), + oauth_nonce=binascii.b2a_hex(uuid.uuid4().bytes), + oauth_version="1.0", + ) + signature = _oauth_signature(consumer_token, "GET", url, args, + request_token) + args["oauth_signature"] = signature + return url + "?" + urllib.urlencode(args) + + def _on_access_token(self, callback, response): + if response.error: + _log.warning("Could not fetch access token") + callback(None) + return + access_token = _oauth_parse_response(response.body) + user = self._oauth_get_user(access_token, self.async_callback( + self._on_oauth_get_user, access_token, callback)) + + def _oauth_get_user(self, access_token, callback): + raise NotImplementedError() + + def _on_oauth_get_user(self, access_token, callback, user): + if not user: + callback(None) + return + user["access_token"] = access_token + callback(user) + + def _oauth_request_parameters(self, url, access_token, parameters={}, + method="GET"): + """Returns the OAuth parameters as a dict for the given request. + + parameters should include all POST arguments and query string arguments + that will be sent with the request. + """ + consumer_token = self._oauth_consumer_token() + base_args = dict( + oauth_consumer_key=consumer_token["key"], + oauth_token=access_token["key"], + oauth_signature_method="HMAC-SHA1", + oauth_timestamp=str(int(time.time())), + oauth_nonce=binascii.b2a_hex(uuid.uuid4().bytes), + oauth_version="1.0", + ) + args = {} + args.update(base_args) + args.update(parameters) + signature = _oauth_signature(consumer_token, method, url, args, + access_token) + base_args["oauth_signature"] = signature + return base_args + + +class TwitterMixin(OAuthMixin): + """Twitter OAuth authentication. + + To authenticate with Twitter, register your application with + Twitter at http://twitter.com/apps. Then copy your Consumer Key and + Consumer Secret to the application settings 'twitter_consumer_key' and + 'twitter_consumer_secret'. Use this Mixin on the handler for the URL + you registered as your application's Callback URL. + + When your application is set up, you can use this Mixin like this + to authenticate the user with Twitter and get access to their stream: + + class TwitterHandler(tornado.web.RequestHandler, + tornado.auth.TwitterMixin): + @tornado.web.asynchronous + def get(self): + if self.get_argument("oauth_token", None): + self.get_authenticated_user(self.async_callback(self._on_auth)) + return + self.authorize_redirect() + + def _on_auth(self, user): + if not user: + raise tornado.web.HTTPError(500, "Twitter auth failed") + # Save the user using, e.g., set_secure_cookie() + + The user object returned by get_authenticated_user() includes the + attributes 'username', 'name', and all of the custom Twitter user + attributes describe at + http://apiwiki.twitter.com/Twitter-REST-API-Method%3A-users%C2%A0show + in addition to 'access_token'. You should save the access token with + the user; it is required to make requests on behalf of the user later + with twitter_request(). + """ + _OAUTH_REQUEST_TOKEN_URL = "http://twitter.com/oauth/request_token" + _OAUTH_ACCESS_TOKEN_URL = "http://twitter.com/oauth/access_token" + _OAUTH_AUTHORIZE_URL = "http://twitter.com/oauth/authorize" + _OAUTH_AUTHENTICATE_URL = "http://twitter.com/oauth/authenticate" + _OAUTH_NO_CALLBACKS = True + + def authenticate_redirect(self): + """Just like authorize_redirect(), but auto-redirects if authorized. + + This is generally the right interface to use if you are using + Twitter for single-sign on. + """ + http = httpclient.AsyncHTTPClient() + http.fetch(self._oauth_request_token_url(), self.async_callback( + self._on_request_token, self._OAUTH_AUTHENTICATE_URL, None)) + + def twitter_request(self, path, callback, access_token=None, + post_args=None, **args): + """Fetches the given API path, e.g., "/statuses/user_timeline/btaylor" + + The path should not include the format (we automatically append + ".json" and parse the JSON output). + + If the request is a POST, post_args should be provided. Query + string arguments should be given as keyword arguments. + + All the Twitter methods are documented at + http://apiwiki.twitter.com/Twitter-API-Documentation. + + Many methods require an OAuth access token which you can obtain + through authorize_redirect() and get_authenticated_user(). The + user returned through that process includes an 'access_token' + attribute that can be used to make authenticated requests via + this method. Example usage: + + class MainHandler(tornado.web.RequestHandler, + tornado.auth.TwitterMixin): + @tornado.web.authenticated + @tornado.web.asynchronous + def get(self): + self.twitter_request( + "/statuses/update", + post_args={"status": "Testing Tornado Web Server"}, + access_token=user["access_token"], + callback=self.async_callback(self._on_post)) + + def _on_post(self, new_entry): + if not new_entry: + # Call failed; perhaps missing permission? + self.authorize_redirect() + return + self.finish("Posted a message!") + + """ + # Add the OAuth resource request signature if we have credentials + url = "http://twitter.com" + path + ".json" + if access_token: + all_args = {} + all_args.update(args) + all_args.update(post_args or {}) + consumer_token = self._oauth_consumer_token() + method = "POST" if post_args is not None else "GET" + oauth = self._oauth_request_parameters( + url, access_token, all_args, method=method) + args.update(oauth) + if args: url += "?" + urllib.urlencode(args) + callback = self.async_callback(self._on_twitter_request, callback) + http = httpclient.AsyncHTTPClient() + if post_args is not None: + http.fetch(url, method="POST", body=urllib.urlencode(post_args), + callback=callback) + else: + http.fetch(url, callback=callback) + + def _on_twitter_request(self, callback, response): + if response.error: + _log.warning("Error response %s fetching %s", response.error, + response.request.url) + callback(None) + return + callback(escape.json_decode(response.body)) + + def _oauth_consumer_token(self): + self.require_setting("twitter_consumer_key", "Twitter OAuth") + self.require_setting("twitter_consumer_secret", "Twitter OAuth") + return dict( + key=self.settings["twitter_consumer_key"], + secret=self.settings["twitter_consumer_secret"]) + + def _oauth_get_user(self, access_token, callback): + callback = self.async_callback(self._parse_user_response, callback) + self.twitter_request( + "/users/show/" + access_token["screen_name"], + access_token=access_token, callback=callback) + + def _parse_user_response(self, callback, user): + if user: + user["username"] = user["screen_name"] + callback(user) + + +class FriendFeedMixin(OAuthMixin): + """FriendFeed OAuth authentication. + + To authenticate with FriendFeed, register your application with + FriendFeed at http://friendfeed.com/api/applications. Then + copy your Consumer Key and Consumer Secret to the application settings + 'friendfeed_consumer_key' and 'friendfeed_consumer_secret'. Use + this Mixin on the handler for the URL you registered as your + application's Callback URL. + + When your application is set up, you can use this Mixin like this + to authenticate the user with FriendFeed and get access to their feed: + + class FriendFeedHandler(tornado.web.RequestHandler, + tornado.auth.FriendFeedMixin): + @tornado.web.asynchronous + def get(self): + if self.get_argument("oauth_token", None): + self.get_authenticated_user(self.async_callback(self._on_auth)) + return + self.authorize_redirect() + + def _on_auth(self, user): + if not user: + raise tornado.web.HTTPError(500, "FriendFeed auth failed") + # Save the user using, e.g., set_secure_cookie() + + The user object returned by get_authenticated_user() includes the + attributes 'username', 'name', and 'description' in addition to + 'access_token'. You should save the access token with the user; + it is required to make requests on behalf of the user later with + friendfeed_request(). + """ + _OAUTH_REQUEST_TOKEN_URL = "https://friendfeed.com/account/oauth/request_token" + _OAUTH_ACCESS_TOKEN_URL = "https://friendfeed.com/account/oauth/access_token" + _OAUTH_AUTHORIZE_URL = "https://friendfeed.com/account/oauth/authorize" + _OAUTH_NO_CALLBACKS = True + + def friendfeed_request(self, path, callback, access_token=None, + post_args=None, **args): + """Fetches the given relative API path, e.g., "/bret/friends" + + If the request is a POST, post_args should be provided. Query + string arguments should be given as keyword arguments. + + All the FriendFeed methods are documented at + http://friendfeed.com/api/documentation. + + Many methods require an OAuth access token which you can obtain + through authorize_redirect() and get_authenticated_user(). The + user returned through that process includes an 'access_token' + attribute that can be used to make authenticated requests via + this method. Example usage: + + class MainHandler(tornado.web.RequestHandler, + tornado.auth.FriendFeedMixin): + @tornado.web.authenticated + @tornado.web.asynchronous + def get(self): + self.friendfeed_request( + "/entry", + post_args={"body": "Testing Tornado Web Server"}, + access_token=self.current_user["access_token"], + callback=self.async_callback(self._on_post)) + + def _on_post(self, new_entry): + if not new_entry: + # Call failed; perhaps missing permission? + self.authorize_redirect() + return + self.finish("Posted a message!") + + """ + # Add the OAuth resource request signature if we have credentials + url = "http://friendfeed-api.com/v2" + path + if access_token: + all_args = {} + all_args.update(args) + all_args.update(post_args or {}) + consumer_token = self._oauth_consumer_token() + method = "POST" if post_args is not None else "GET" + oauth = self._oauth_request_parameters( + url, access_token, all_args, method=method) + args.update(oauth) + if args: url += "?" + urllib.urlencode(args) + callback = self.async_callback(self._on_friendfeed_request, callback) + http = httpclient.AsyncHTTPClient() + if post_args is not None: + http.fetch(url, method="POST", body=urllib.urlencode(post_args), + callback=callback) + else: + http.fetch(url, callback=callback) + + def _on_friendfeed_request(self, callback, response): + if response.error: + _log.warning("Error response %s fetching %s", response.error, + response.request.url) + callback(None) + return + callback(escape.json_decode(response.body)) + + def _oauth_consumer_token(self): + self.require_setting("friendfeed_consumer_key", "FriendFeed OAuth") + self.require_setting("friendfeed_consumer_secret", "FriendFeed OAuth") + return dict( + key=self.settings["friendfeed_consumer_key"], + secret=self.settings["friendfeed_consumer_secret"]) + + def _oauth_get_user(self, access_token, callback): + callback = self.async_callback(self._parse_user_response, callback) + self.friendfeed_request( + "/feedinfo/" + access_token["username"], + include="id,name,description", access_token=access_token, + callback=callback) + + def _parse_user_response(self, callback, user): + if user: + user["username"] = user["id"] + callback(user) + + +class GoogleMixin(OpenIdMixin, OAuthMixin): + """Google Open ID / OAuth authentication. + + No application registration is necessary to use Google for authentication + or to access Google resources on behalf of a user. To authenticate with + Google, redirect with authenticate_redirect(). On return, parse the + response with get_authenticated_user(). We send a dict containing the + values for the user, including 'email', 'name', and 'locale'. + Example usage: + + class GoogleHandler(tornado.web.RequestHandler, tornado.auth.GoogleMixin): + @tornado.web.asynchronous + def get(self): + if self.get_argument("openid.mode", None): + self.get_authenticated_user(self.async_callback(self._on_auth)) + return + self.authenticate_redirect() + + def _on_auth(self, user): + if not user: + raise tornado.web.HTTPError(500, "Google auth failed") + # Save the user with, e.g., set_secure_cookie() + + """ + _OPENID_ENDPOINT = "https://www.google.com/accounts/o8/ud" + _OAUTH_ACCESS_TOKEN_URL = "https://www.google.com/accounts/OAuthGetAccessToken" + + def authorize_redirect(self, oauth_scope, callback_uri=None, + ax_attrs=["name","email","language","username"]): + """Authenticates and authorizes for the given Google resource. + + Some of the available resources are: + + Gmail Contacts - http://www.google.com/m8/feeds/ + Calendar - http://www.google.com/calendar/feeds/ + Finance - http://finance.google.com/finance/feeds/ + + You can authorize multiple resources by separating the resource + URLs with a space. + """ + callback_uri = callback_uri or self.request.path + args = self._openid_args(callback_uri, ax_attrs=ax_attrs, + oauth_scope=oauth_scope) + self.redirect(self._OPENID_ENDPOINT + "?" + urllib.urlencode(args)) + + def get_authenticated_user(self, callback): + """Fetches the authenticated user data upon redirect.""" + # Look to see if we are doing combined OpenID/OAuth + oauth_ns = "" + for name, values in self.request.arguments.iteritems(): + if name.startswith("openid.ns.") and \ + values[-1] == u"http://specs.openid.net/extensions/oauth/1.0": + oauth_ns = name[10:] + break + token = self.get_argument("openid." + oauth_ns + ".request_token", "") + if token: + http = httpclient.AsyncHTTPClient() + token = dict(key=token, secret="") + http.fetch(self._oauth_access_token_url(token), + self.async_callback(self._on_access_token, callback)) + else: + OpenIdMixin.get_authenticated_user(self, callback) + + def _oauth_consumer_token(self): + self.require_setting("google_consumer_key", "Google OAuth") + self.require_setting("google_consumer_secret", "Google OAuth") + return dict( + key=self.settings["google_consumer_key"], + secret=self.settings["google_consumer_secret"]) + + def _oauth_get_user(self, access_token, callback): + OpenIdMixin.get_authenticated_user(self, callback) + + +class FacebookMixin(object): + """Facebook Connect authentication. + + To authenticate with Facebook, register your application with + Facebook at http://www.facebook.com/developers/apps.php. Then + copy your API Key and Application Secret to the application settings + 'facebook_api_key' and 'facebook_secret'. + + When your application is set up, you can use this Mixin like this + to authenticate the user with Facebook: + + class FacebookHandler(tornado.web.RequestHandler, + tornado.auth.FacebookMixin): + @tornado.web.asynchronous + def get(self): + if self.get_argument("session", None): + self.get_authenticated_user(self.async_callback(self._on_auth)) + return + self.authenticate_redirect() + + def _on_auth(self, user): + if not user: + raise tornado.web.HTTPError(500, "Facebook auth failed") + # Save the user using, e.g., set_secure_cookie() + + The user object returned by get_authenticated_user() includes the + attributes 'facebook_uid' and 'name' in addition to session attributes + like 'session_key'. You should save the session key with the user; it is + required to make requests on behalf of the user later with + facebook_request(). + """ + def authenticate_redirect(self, callback_uri=None, cancel_uri=None, + extended_permissions=None): + """Authenticates/installs this app for the current user.""" + self.require_setting("facebook_api_key", "Facebook Connect") + callback_uri = callback_uri or self.request.path + args = { + "api_key": self.settings["facebook_api_key"], + "v": "1.0", + "fbconnect": "true", + "display": "page", + "next": urlparse.urljoin(self.request.full_url(), callback_uri), + "return_session": "true", + } + if cancel_uri: + args["cancel_url"] = urlparse.urljoin( + self.request.full_url(), cancel_uri) + if extended_permissions: + if isinstance(extended_permissions, basestring): + extended_permissions = [extended_permissions] + args["req_perms"] = ",".join(extended_permissions) + self.redirect("http://www.facebook.com/login.php?" + + urllib.urlencode(args)) + + def authorize_redirect(self, extended_permissions, callback_uri=None, + cancel_uri=None): + """Redirects to an authorization request for the given FB resource. + + The available resource names are listed at + http://wiki.developers.facebook.com/index.php/Extended_permission. + The most common resource types include: + + publish_stream + read_stream + email + sms + + extended_permissions can be a single permission name or a list of + names. To get the session secret and session key, call + get_authenticated_user() just as you would with + authenticate_redirect(). + """ + self.authenticate_redirect(callback_uri, cancel_uri, + extended_permissions) + + def get_authenticated_user(self, callback): + """Fetches the authenticated Facebook user. + + The authenticated user includes the special Facebook attributes + 'session_key' and 'facebook_uid' in addition to the standard + user attributes like 'name'. + """ + self.require_setting("facebook_api_key", "Facebook Connect") + session = escape.json_decode(self.get_argument("session")) + self.facebook_request( + method="facebook.users.getInfo", + callback=self.async_callback( + self._on_get_user_info, callback, session), + session_key=session["session_key"], + uids=session["uid"], + fields="uid,first_name,last_name,name,locale,pic_square," \ + "profile_url,username") + + def facebook_request(self, method, callback, **args): + """Makes a Facebook API REST request. + + We automatically include the Facebook API key and signature, but + it is the callers responsibility to include 'session_key' and any + other required arguments to the method. + + The available Facebook methods are documented here: + http://wiki.developers.facebook.com/index.php/API + + Here is an example for the stream.get() method: + + class MainHandler(tornado.web.RequestHandler, + tornado.auth.FacebookMixin): + @tornado.web.authenticated + @tornado.web.asynchronous + def get(self): + self.facebook_request( + method="stream.get", + callback=self.async_callback(self._on_stream), + session_key=self.current_user["session_key"]) + + def _on_stream(self, stream): + if stream is None: + # Not authorized to read the stream yet? + self.redirect(self.authorize_redirect("read_stream")) + return + self.render("stream.html", stream=stream) + + """ + self.require_setting("facebook_api_key", "Facebook Connect") + self.require_setting("facebook_secret", "Facebook Connect") + if not method.startswith("facebook."): + method = "facebook." + method + args["api_key"] = self.settings["facebook_api_key"] + args["v"] = "1.0" + args["method"] = method + args["call_id"] = str(long(time.time() * 1e6)) + args["format"] = "json" + args["sig"] = self._signature(args) + url = "http://api.facebook.com/restserver.php?" + \ + urllib.urlencode(args) + http = httpclient.AsyncHTTPClient() + http.fetch(url, callback=self.async_callback( + self._parse_response, callback)) + + def _on_get_user_info(self, callback, session, users): + if users is None: + callback(None) + return + callback({ + "name": users[0]["name"], + "first_name": users[0]["first_name"], + "last_name": users[0]["last_name"], + "uid": users[0]["uid"], + "locale": users[0]["locale"], + "pic_square": users[0]["pic_square"], + "profile_url": users[0]["profile_url"], + "username": users[0].get("username"), + "session_key": session["session_key"], + "session_expires": session["expires"], + }) + + def _parse_response(self, callback, response): + if response.error: + _log.warning("HTTP error from Facebook: %s", response.error) + callback(None) + return + try: + json = escape.json_decode(response.body) + except: + _log.warning("Invalid JSON from Facebook: %r", response.body) + callback(None) + return + if isinstance(json, dict) and json.get("error_code"): + _log.warning("Facebook error: %d: %r", json["error_code"], + json.get("error_msg")) + callback(None) + return + callback(json) + + def _signature(self, args): + parts = ["%s=%s" % (n, args[n]) for n in sorted(args.keys())] + body = "".join(parts) + self.settings["facebook_secret"] + if isinstance(body, unicode): body = body.encode("utf-8") + return hashlib.md5(body).hexdigest() + + +def _oauth_signature(consumer_token, method, url, parameters={}, token=None): + """Calculates the HMAC-SHA1 OAuth signature for the given request. + + See http://oauth.net/core/1.0/#signing_process + """ + parts = urlparse.urlparse(url) + scheme, netloc, path = parts[:3] + normalized_url = scheme.lower() + "://" + netloc.lower() + path + + base_elems = [] + base_elems.append(method.upper()) + base_elems.append(normalized_url) + base_elems.append("&".join("%s=%s" % (k, _oauth_escape(str(v))) + for k, v in sorted(parameters.items()))) + base_string = "&".join(_oauth_escape(e) for e in base_elems) + + key_elems = [consumer_token["secret"]] + key_elems.append(token["secret"] if token else "") + key = "&".join(key_elems) + + hash = hmac.new(key, base_string, hashlib.sha1) + return binascii.b2a_base64(hash.digest())[:-1] + + +def _oauth_escape(val): + if isinstance(val, unicode): + val = val.encode("utf-8") + return urllib.quote(val, safe="~") + + +def _oauth_parse_response(body): + p = cgi.parse_qs(body, keep_blank_values=False) + token = dict(key=p["oauth_token"][0], secret=p["oauth_token_secret"][0]) + + # Add the extra parameters the Provider included to the token + special = ("oauth_token", "oauth_token_secret") + token.update((k, p[k][0]) for k in p if k not in special) + return token diff --git a/vendor/tornado/tornado/autoreload.py b/vendor/tornado/tornado/autoreload.py new file mode 100644 index 0000000000..231cfe892c --- /dev/null +++ b/vendor/tornado/tornado/autoreload.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python +# +# Copyright 2009 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""A module to automatically restart the server when a module is modified. + +This module depends on IOLoop, so it will not work in WSGI applications +and Google AppEngine. +""" + +import functools +import errno +import ioloop +import logging +import os +import os.path +import sys +import types + +_log = logging.getLogger('tornado.autoreload') + +def start(io_loop=None, check_time=500): + """Restarts the process automatically when a module is modified. + + We run on the I/O loop, and restarting is a destructive operation, + so will terminate any pending requests. + """ + io_loop = io_loop or ioloop.IOLoop.instance() + modify_times = {} + callback = functools.partial(_reload_on_update, io_loop, modify_times) + scheduler = ioloop.PeriodicCallback(callback, check_time, io_loop=io_loop) + scheduler.start() + + +_reload_attempted = False + +def _reload_on_update(io_loop, modify_times): + global _reload_attempted + if _reload_attempted: + # We already tried to reload and it didn't work, so don't try again. + return + for module in sys.modules.values(): + # Some modules play games with sys.modules (e.g. email/__init__.py + # in the standard library), and occasionally this can cause strange + # failures in getattr. Just ignore anything that's not an ordinary + # module. + if not isinstance(module, types.ModuleType): continue + path = getattr(module, "__file__", None) + if not path: continue + if path.endswith(".pyc") or path.endswith(".pyo"): + path = path[:-1] + try: + modified = os.stat(path).st_mtime + except: + continue + if path not in modify_times: + modify_times[path] = modified + continue + if modify_times[path] != modified: + _log.info("%s modified; restarting server", path) + _reload_attempted = True + for fd in io_loop._handlers.keys(): + try: + os.close(fd) + except: + pass + try: + os.execv(sys.executable, [sys.executable] + sys.argv) + except OSError, e: + # Mac OS X versions prior to 10.6 do not support execv in + # a process that contains multiple threads. Instead of + # re-executing in the current process, start a new one + # and cause the current process to exit. This isn't + # ideal since the new process is detached from the parent + # terminal and thus cannot easily be killed with ctrl-C, + # but it's better than not being able to autoreload at + # all. + # Unfortunately the errno returned in this case does not + # appear to be consistent, so we can't easily check for + # this error specifically. + os.spawnv(os.P_NOWAIT, sys.executable, + [sys.executable] + sys.argv) + sys.exit(0) diff --git a/vendor/tornado/tornado/database.py b/vendor/tornado/tornado/database.py new file mode 100644 index 0000000000..3f78e00b94 --- /dev/null +++ b/vendor/tornado/tornado/database.py @@ -0,0 +1,180 @@ +#!/usr/bin/env python +# +# Copyright 2009 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""A lightweight wrapper around MySQLdb.""" + +import copy +import MySQLdb +import MySQLdb.constants +import MySQLdb.converters +import MySQLdb.cursors +import itertools +import logging + +_log = logging.getLogger('tornado.database') + +class Connection(object): + """A lightweight wrapper around MySQLdb DB-API connections. + + The main value we provide is wrapping rows in a dict/object so that + columns can be accessed by name. Typical usage: + + db = database.Connection("localhost", "mydatabase") + for article in db.query("SELECT * FROM articles"): + print article.title + + Cursors are hidden by the implementation, but other than that, the methods + are very similar to the DB-API. + + We explicitly set the timezone to UTC and the character encoding to + UTF-8 on all connections to avoid time zone and encoding errors. + """ + def __init__(self, host, database, user=None, password=None): + self.host = host + self.database = database + + args = dict(conv=CONVERSIONS, use_unicode=True, charset="utf8", + db=database, init_command='SET time_zone = "+0:00"', + sql_mode="TRADITIONAL") + if user is not None: + args["user"] = user + if password is not None: + args["passwd"] = password + + # We accept a path to a MySQL socket file or a host(:port) string + if "/" in host: + args["unix_socket"] = host + else: + self.socket = None + pair = host.split(":") + if len(pair) == 2: + args["host"] = pair[0] + args["port"] = int(pair[1]) + else: + args["host"] = host + args["port"] = 3306 + + self._db = None + self._db_args = args + try: + self.reconnect() + except: + _log.error("Cannot connect to MySQL on %s", self.host, + exc_info=True) + + def __del__(self): + self.close() + + def close(self): + """Closes this database connection.""" + if getattr(self, "_db", None) is not None: + self._db.close() + self._db = None + + def reconnect(self): + """Closes the existing database connection and re-opens it.""" + self.close() + self._db = MySQLdb.connect(**self._db_args) + self._db.autocommit(True) + + def iter(self, query, *parameters): + """Returns an iterator for the given query and parameters.""" + if self._db is None: self.reconnect() + cursor = MySQLdb.cursors.SSCursor(self._db) + try: + self._execute(cursor, query, parameters) + column_names = [d[0] for d in cursor.description] + for row in cursor: + yield Row(zip(column_names, row)) + finally: + cursor.close() + + def query(self, query, *parameters): + """Returns a row list for the given query and parameters.""" + cursor = self._cursor() + try: + self._execute(cursor, query, parameters) + column_names = [d[0] for d in cursor.description] + return [Row(itertools.izip(column_names, row)) for row in cursor] + finally: + cursor.close() + + def get(self, query, *parameters): + """Returns the first row returned for the given query.""" + rows = self.query(query, *parameters) + if not rows: + return None + elif len(rows) > 1: + raise Exception("Multiple rows returned for Database.get() query") + else: + return rows[0] + + def execute(self, query, *parameters): + """Executes the given query, returning the lastrowid from the query.""" + cursor = self._cursor() + try: + self._execute(cursor, query, parameters) + return cursor.lastrowid + finally: + cursor.close() + + def executemany(self, query, parameters): + """Executes the given query against all the given param sequences. + + We return the lastrowid from the query. + """ + cursor = self._cursor() + try: + cursor.executemany(query, parameters) + return cursor.lastrowid + finally: + cursor.close() + + def _cursor(self): + if self._db is None: self.reconnect() + return self._db.cursor() + + def _execute(self, cursor, query, parameters): + try: + return cursor.execute(query, parameters) + except OperationalError: + _log.error("Error connecting to MySQL on %s", self.host) + self.close() + raise + + +class Row(dict): + """A dict that allows for object-like property access syntax.""" + def __getattr__(self, name): + try: + return self[name] + except KeyError: + raise AttributeError(name) + + +# Fix the access conversions to properly recognize unicode/binary +FIELD_TYPE = MySQLdb.constants.FIELD_TYPE +FLAG = MySQLdb.constants.FLAG +CONVERSIONS = copy.deepcopy(MySQLdb.converters.conversions) +for field_type in \ + [FIELD_TYPE.BLOB, FIELD_TYPE.STRING, FIELD_TYPE.VAR_STRING] + \ + ([FIELD_TYPE.VARCHAR] if 'VARCHAR' in vars(FIELD_TYPE) else []): + CONVERSIONS[field_type].insert(0, (FLAG.BINARY, str)) + + +# Alias some common MySQL exceptions +IntegrityError = MySQLdb.IntegrityError +OperationalError = MySQLdb.OperationalError diff --git a/vendor/tornado/tornado/epoll.c b/vendor/tornado/tornado/epoll.c new file mode 100644 index 0000000000..9a2e3a3747 --- /dev/null +++ b/vendor/tornado/tornado/epoll.c @@ -0,0 +1,112 @@ +/* + * Copyright 2009 Facebook + * + * Licensed under the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. You may obtain + * a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + * License for the specific language governing permissions and limitations + * under the License. + */ + +#include "Python.h" +#include <string.h> +#include <sys/epoll.h> + +#define MAX_EVENTS 24 + +/* + * Simple wrapper around epoll_create. + */ +static PyObject* _epoll_create(void) { + int fd = epoll_create(MAX_EVENTS); + if (fd == -1) { + PyErr_SetFromErrno(PyExc_Exception); + return NULL; + } + + return PyInt_FromLong(fd); +} + +/* + * Simple wrapper around epoll_ctl. We throw an exception if the call fails + * rather than returning the error code since it is an infrequent (and likely + * catastrophic) event when it does happen. + */ +static PyObject* _epoll_ctl(PyObject* self, PyObject* args) { + int epfd, op, fd, events; + struct epoll_event event; + + if (!PyArg_ParseTuple(args, "iiiI", &epfd, &op, &fd, &events)) { + return NULL; + } + + memset(&event, 0, sizeof(event)); + event.events = events; + event.data.fd = fd; + if (epoll_ctl(epfd, op, fd, &event) == -1) { + PyErr_SetFromErrno(PyExc_OSError); + return NULL; + } + + Py_INCREF(Py_None); + return Py_None; +} + +/* + * Simple wrapper around epoll_wait. We return None if the call times out and + * throw an exception if an error occurs. Otherwise, we return a list of + * (fd, event) tuples. + */ +static PyObject* _epoll_wait(PyObject* self, PyObject* args) { + struct epoll_event events[MAX_EVENTS]; + int epfd, timeout, num_events, i; + PyObject* list; + PyObject* tuple; + + if (!PyArg_ParseTuple(args, "ii", &epfd, &timeout)) { + return NULL; + } + + Py_BEGIN_ALLOW_THREADS + num_events = epoll_wait(epfd, events, MAX_EVENTS, timeout); + Py_END_ALLOW_THREADS + if (num_events == -1) { + PyErr_SetFromErrno(PyExc_Exception); + return NULL; + } + + list = PyList_New(num_events); + for (i = 0; i < num_events; i++) { + tuple = PyTuple_New(2); + PyTuple_SET_ITEM(tuple, 0, PyInt_FromLong(events[i].data.fd)); + PyTuple_SET_ITEM(tuple, 1, PyInt_FromLong(events[i].events)); + PyList_SET_ITEM(list, i, tuple); + } + return list; +} + +/* + * Our method declararations + */ +static PyMethodDef kEpollMethods[] = { + {"epoll_create", (PyCFunction)_epoll_create, METH_NOARGS, + "Create an epoll file descriptor"}, + {"epoll_ctl", _epoll_ctl, METH_VARARGS, + "Control an epoll file descriptor"}, + {"epoll_wait", _epoll_wait, METH_VARARGS, + "Wait for events on an epoll file descriptor"}, + {NULL, NULL, 0, NULL} +}; + +/* + * Module initialization + */ +PyMODINIT_FUNC initepoll(void) { + Py_InitModule("epoll", kEpollMethods); +} diff --git a/vendor/tornado/tornado/escape.py b/vendor/tornado/tornado/escape.py new file mode 100644 index 0000000000..bacb1c51d0 --- /dev/null +++ b/vendor/tornado/tornado/escape.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python +# +# Copyright 2009 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Escaping/unescaping methods for HTML, JSON, URLs, and others.""" + +import htmlentitydefs +import re +import xml.sax.saxutils +import urllib + +try: + import json + assert hasattr(json, "loads") and hasattr(json, "dumps") + _json_decode = lambda s: json.loads(s) + _json_encode = lambda v: json.dumps(v) +except: + try: + import simplejson + _json_decode = lambda s: simplejson.loads(_unicode(s)) + _json_encode = lambda v: simplejson.dumps(v) + except ImportError: + try: + # For Google AppEngine + from django.utils import simplejson + _json_decode = lambda s: simplejson.loads(_unicode(s)) + _json_encode = lambda v: simplejson.dumps(v) + except ImportError: + raise Exception("A JSON parser is required, e.g., simplejson at " + "http://pypi.python.org/pypi/simplejson/") + + +def xhtml_escape(value): + """Escapes a string so it is valid within XML or XHTML.""" + return utf8(xml.sax.saxutils.escape(value, {'"': """})) + + +def xhtml_unescape(value): + """Un-escapes an XML-escaped string.""" + return re.sub(r"&(#?)(\w+?);", _convert_entity, _unicode(value)) + + +def json_encode(value): + """JSON-encodes the given Python object.""" + return _json_encode(value) + + +def json_decode(value): + """Returns Python objects for the given JSON string.""" + return _json_decode(value) + + +def squeeze(value): + """Replace all sequences of whitespace chars with a single space.""" + return re.sub(r"[\x00-\x20]+", " ", value).strip() + + +def url_escape(value): + """Returns a valid URL-encoded version of the given value.""" + return urllib.quote_plus(utf8(value)) + + +def url_unescape(value): + """Decodes the given value from a URL.""" + return _unicode(urllib.unquote_plus(value)) + + +def utf8(value): + if isinstance(value, unicode): + return value.encode("utf-8") + assert isinstance(value, str) + return value + + +def _unicode(value): + if isinstance(value, str): + return value.decode("utf-8") + assert isinstance(value, unicode) + return value + + +def _convert_entity(m): + if m.group(1) == "#": + try: + return unichr(int(m.group(2))) + except ValueError: + return "&#%s;" % m.group(2) + try: + return _HTML_UNICODE_MAP[m.group(2)] + except KeyError: + return "&%s;" % m.group(2) + + +def _build_unicode_map(): + unicode_map = {} + for name, value in htmlentitydefs.name2codepoint.iteritems(): + unicode_map[name] = unichr(value) + return unicode_map + +_HTML_UNICODE_MAP = _build_unicode_map() diff --git a/vendor/tornado/tornado/httpclient.py b/vendor/tornado/tornado/httpclient.py new file mode 100644 index 0000000000..2c9155eb9e --- /dev/null +++ b/vendor/tornado/tornado/httpclient.py @@ -0,0 +1,465 @@ +#!/usr/bin/env python +# +# Copyright 2009 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Blocking and non-blocking HTTP client implementations using pycurl.""" + +import calendar +import collections +import cStringIO +import email.utils +import errno +import functools +import httplib +import ioloop +import logging +import pycurl +import time +import weakref + +_log = logging.getLogger('tornado.httpclient') + +class HTTPClient(object): + """A blocking HTTP client backed with pycurl. + + Typical usage looks like this: + + http_client = httpclient.HTTPClient() + try: + response = http_client.fetch("http://www.google.com/") + print response.body + except httpclient.HTTPError, e: + print "Error:", e + + fetch() can take a string URL or an HTTPRequest instance, which offers + more options, like executing POST/PUT/DELETE requests. + """ + def __init__(self, max_simultaneous_connections=None): + self._curl = _curl_create(max_simultaneous_connections) + + def __del__(self): + self._curl.close() + + def fetch(self, request, **kwargs): + """Executes an HTTPRequest, returning an HTTPResponse. + + If an error occurs during the fetch, we raise an HTTPError. + """ + if not isinstance(request, HTTPRequest): + request = HTTPRequest(url=request, **kwargs) + buffer = cStringIO.StringIO() + headers = {} + try: + _curl_setup_request(self._curl, request, buffer, headers) + self._curl.perform() + code = self._curl.getinfo(pycurl.HTTP_CODE) + if code < 200 or code >= 300: + raise HTTPError(code) + effective_url = self._curl.getinfo(pycurl.EFFECTIVE_URL) + return HTTPResponse( + request=request, code=code, headers=headers, + body=buffer.getvalue(), effective_url=effective_url) + except pycurl.error, e: + raise CurlError(*e) + finally: + buffer.close() + + +class AsyncHTTPClient(object): + """An non-blocking HTTP client backed with pycurl. + + Example usage: + + import ioloop + + def handle_request(response): + if response.error: + print "Error:", response.error + else: + print response.body + ioloop.IOLoop.instance().stop() + + http_client = httpclient.AsyncHTTPClient() + http_client.fetch("http://www.google.com/", handle_request) + ioloop.IOLoop.instance().start() + + fetch() can take a string URL or an HTTPRequest instance, which offers + more options, like executing POST/PUT/DELETE requests. + + The keyword argument max_clients to the AsyncHTTPClient constructor + determines the maximum number of simultaneous fetch() operations that + can execute in parallel on each IOLoop. + """ + _ASYNC_CLIENTS = weakref.WeakKeyDictionary() + + def __new__(cls, io_loop=None, max_clients=10, + max_simultaneous_connections=None): + # There is one client per IOLoop since they share curl instances + io_loop = io_loop or ioloop.IOLoop.instance() + if io_loop in cls._ASYNC_CLIENTS: + return cls._ASYNC_CLIENTS[io_loop] + else: + instance = super(AsyncHTTPClient, cls).__new__(cls) + instance.io_loop = io_loop + instance._multi = pycurl.CurlMulti() + instance._curls = [_curl_create(max_simultaneous_connections) + for i in xrange(max_clients)] + instance._free_list = instance._curls[:] + instance._requests = collections.deque() + instance._fds = {} + instance._events = {} + instance._added_perform_callback = False + instance._timeout = None + cls._ASYNC_CLIENTS[io_loop] = instance + return instance + + def close(self): + """Destroys this http client, freeing any file descriptors used. + Not needed in normal use, but may be helpful in unittests that + create and destroy http clients. No other methods may be called + on the AsyncHTTPClient after close(). + """ + del AsyncHTTPClient._ASYNC_CLIENTS[self.io_loop] + for curl in self._curls: + curl.close() + self._multi.close() + + def fetch(self, request, callback, **kwargs): + """Executes an HTTPRequest, calling callback with an HTTPResponse. + + If an error occurs during the fetch, the HTTPResponse given to the + callback has a non-None error attribute that contains the exception + encountered during the request. You can call response.reraise() to + throw the exception (if any) in the callback. + """ + if not isinstance(request, HTTPRequest): + request = HTTPRequest(url=request, **kwargs) + self._requests.append((request, callback)) + self._add_perform_callback() + + def _add_perform_callback(self): + if not self._added_perform_callback: + self.io_loop.add_callback(self._perform) + self._added_perform_callback = True + + def _handle_events(self, fd, events): + self._events[fd] = events + self._add_perform_callback() + + def _handle_timeout(self): + self._timeout = None + self._perform() + + def _perform(self): + self._added_perform_callback = False + + while True: + while True: + ret, num_handles = self._multi.perform() + if ret != pycurl.E_CALL_MULTI_PERFORM: + break + + # Handle completed fetches + completed = 0 + while True: + num_q, ok_list, err_list = self._multi.info_read() + for curl in ok_list: + self._finish(curl) + completed += 1 + for curl, errnum, errmsg in err_list: + self._finish(curl, errnum, errmsg) + completed += 1 + if num_q == 0: + break + + # Start fetching new URLs + started = 0 + while self._free_list and self._requests: + started += 1 + curl = self._free_list.pop() + (request, callback) = self._requests.popleft() + curl.info = { + "headers": {}, + "buffer": cStringIO.StringIO(), + "request": request, + "callback": callback, + "start_time": time.time(), + } + _curl_setup_request(curl, request, curl.info["buffer"], + curl.info["headers"]) + self._multi.add_handle(curl) + + if not started and not completed: + break + + if self._timeout is not None: + self.io_loop.remove_timeout(self._timeout) + self._timeout = None + + if num_handles: + self._timeout = self.io_loop.add_timeout( + time.time() + 0.2, self._handle_timeout) + + # Wait for more I/O + fds = {} + (readable, writable, exceptable) = self._multi.fdset() + for fd in readable: + fds[fd] = fds.get(fd, 0) | 0x1 | 0x2 + for fd in writable: + fds[fd] = fds.get(fd, 0) | 0x4 + for fd in exceptable: + fds[fd] = fds.get(fd, 0) | 0x8 | 0x10 + + for fd in self._fds: + if fd not in fds: + try: + self.io_loop.remove_handler(fd) + except (OSError, IOError), e: + if e[0] != errno.ENOENT: + raise + + for fd, events in fds.iteritems(): + old_events = self._fds.get(fd, None) + if old_events is None: + self.io_loop.add_handler(fd, self._handle_events, events) + elif old_events != events: + try: + self.io_loop.update_handler(fd, events) + except (OSError, IOError), e: + if e[0] == errno.ENOENT: + self.io_loop.add_handler(fd, self._handle_events, + events) + else: + raise + self._fds = fds + + def _finish(self, curl, curl_error=None, curl_message=None): + info = curl.info + curl.info = None + self._multi.remove_handle(curl) + self._free_list.append(curl) + if curl_error: + error = CurlError(curl_error, curl_message) + code = error.code + body = None + effective_url = None + else: + error = None + code = curl.getinfo(pycurl.HTTP_CODE) + body = info["buffer"].getvalue() + effective_url = curl.getinfo(pycurl.EFFECTIVE_URL) + info["buffer"].close() + info["callback"](HTTPResponse( + request=info["request"], code=code, headers=info["headers"], + body=body, effective_url=effective_url, error=error, + request_time=time.time() - info["start_time"])) + + +class HTTPRequest(object): + def __init__(self, url, method="GET", headers={}, body=None, + auth_username=None, auth_password=None, + connect_timeout=20.0, request_timeout=20.0, + if_modified_since=None, follow_redirects=True, + max_redirects=5, user_agent=None, use_gzip=True, + network_interface=None, streaming_callback=None, + header_callback=None, prepare_curl_callback=None): + if if_modified_since: + timestamp = calendar.timegm(if_modified_since.utctimetuple()) + headers["If-Modified-Since"] = email.utils.formatdate( + timestamp, localtime=False, usegmt=True) + if "Pragma" not in headers: + headers["Pragma"] = "" + self.url = _utf8(url) + self.method = method + self.headers = headers + self.body = body + self.auth_username = _utf8(auth_username) + self.auth_password = _utf8(auth_password) + self.connect_timeout = connect_timeout + self.request_timeout = request_timeout + self.follow_redirects = follow_redirects + self.max_redirects = max_redirects + self.user_agent = user_agent + self.use_gzip = use_gzip + self.network_interface = network_interface + self.streaming_callback = streaming_callback + self.header_callback = header_callback + self.prepare_curl_callback = prepare_curl_callback + + +class HTTPResponse(object): + def __init__(self, request, code, headers={}, body="", effective_url=None, + error=None, request_time=None): + self.request = request + self.code = code + self.headers = headers + self.body = body + if effective_url is None: + self.effective_url = request.url + else: + self.effective_url = effective_url + if error is None: + if self.code < 200 or self.code >= 300: + self.error = HTTPError(self.code) + else: + self.error = None + else: + self.error = error + self.request_time = request_time + + def rethrow(self): + if self.error: + raise self.error + + def __repr__(self): + args = ",".join("%s=%r" % i for i in self.__dict__.iteritems()) + return "%s(%s)" % (self.__class__.__name__, args) + + +class HTTPError(Exception): + def __init__(self, code, message=None): + self.code = code + message = message or httplib.responses.get(code, "Unknown") + Exception.__init__(self, "HTTP %d: %s" % (self.code, message)) + + +class CurlError(HTTPError): + def __init__(self, errno, message): + HTTPError.__init__(self, 599, message) + self.errno = errno + + +def _curl_create(max_simultaneous_connections=None): + curl = pycurl.Curl() + if _log.isEnabledFor(logging.DEBUG): + curl.setopt(pycurl.VERBOSE, 1) + curl.setopt(pycurl.DEBUGFUNCTION, _curl_debug) + curl.setopt(pycurl.MAXCONNECTS, max_simultaneous_connections or 5) + return curl + + +def _curl_setup_request(curl, request, buffer, headers): + curl.setopt(pycurl.URL, request.url) + curl.setopt(pycurl.HTTPHEADER, + ["%s: %s" % i for i in request.headers.iteritems()]) + try: + if request.header_callback: + curl.setopt(pycurl.HEADERFUNCTION, request.header_callback) + else: + curl.setopt(pycurl.HEADERFUNCTION, + functools.partial(_curl_header_callback, headers)) + except: + # Old version of curl; response will not include headers + pass + if request.streaming_callback: + curl.setopt(pycurl.WRITEFUNCTION, request.streaming_callback) + else: + curl.setopt(pycurl.WRITEFUNCTION, buffer.write) + curl.setopt(pycurl.FOLLOWLOCATION, request.follow_redirects) + curl.setopt(pycurl.MAXREDIRS, request.max_redirects) + curl.setopt(pycurl.CONNECTTIMEOUT, int(request.connect_timeout)) + curl.setopt(pycurl.TIMEOUT, int(request.request_timeout)) + if request.user_agent: + curl.setopt(pycurl.USERAGENT, request.user_agent) + else: + curl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (compatible; pycurl)") + if request.network_interface: + curl.setopt(pycurl.INTERFACE, request.network_interface) + if request.use_gzip: + curl.setopt(pycurl.ENCODING, "gzip,deflate") + else: + curl.setopt(pycurl.ENCODING, "none") + + # Set the request method through curl's retarded interface which makes + # up names for almost every single method + curl_options = { + "GET": pycurl.HTTPGET, + "POST": pycurl.POST, + "PUT": pycurl.UPLOAD, + "HEAD": pycurl.NOBODY, + } + custom_methods = set(["DELETE"]) + for o in curl_options.values(): + curl.setopt(o, False) + if request.method in curl_options: + curl.unsetopt(pycurl.CUSTOMREQUEST) + curl.setopt(curl_options[request.method], True) + elif request.method in custom_methods: + curl.setopt(pycurl.CUSTOMREQUEST, request.method) + else: + raise KeyError('unknown method ' + request.method) + + # Handle curl's cryptic options for every individual HTTP method + if request.method in ("POST", "PUT"): + request_buffer = cStringIO.StringIO(request.body) + curl.setopt(pycurl.READFUNCTION, request_buffer.read) + if request.method == "POST": + def ioctl(cmd): + if cmd == curl.IOCMD_RESTARTREAD: + request_buffer.seek(0) + curl.setopt(pycurl.IOCTLFUNCTION, ioctl) + curl.setopt(pycurl.POSTFIELDSIZE, len(request.body)) + else: + curl.setopt(pycurl.INFILESIZE, len(request.body)) + + if request.auth_username and request.auth_password: + userpwd = "%s:%s" % (request.auth_username, request.auth_password) + curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC) + curl.setopt(pycurl.USERPWD, userpwd) + _log.info("%s %s (username: %r)", request.method, request.url, + request.auth_username) + else: + curl.unsetopt(pycurl.USERPWD) + _log.info("%s %s", request.method, request.url) + if request.prepare_curl_callback is not None: + request.prepare_curl_callback(curl) + + +def _curl_header_callback(headers, header_line): + if header_line.startswith("HTTP/"): + headers.clear() + return + if header_line == "\r\n": + return + parts = header_line.split(": ") + if len(parts) != 2: + _log.warning("Invalid HTTP response header line %r", header_line) + return + name = parts[0].strip() + value = parts[1].strip() + if name in headers: + headers[name] = headers[name] + ',' + value + else: + headers[name] = value + + +def _curl_debug(debug_type, debug_msg): + debug_types = ('I', '<', '>', '<', '>') + if debug_type == 0: + _log.debug('%s', debug_msg.strip()) + elif debug_type in (1, 2): + for line in debug_msg.splitlines(): + _log.debug('%s %s', debug_types[debug_type], line) + elif debug_type == 4: + _log.debug('%s %r', debug_types[debug_type], debug_msg) + + +def _utf8(value): + if value is None: + return value + if isinstance(value, unicode): + return value.encode("utf-8") + assert isinstance(value, str) + return value diff --git a/vendor/tornado/tornado/httpserver.py b/vendor/tornado/tornado/httpserver.py new file mode 100644 index 0000000000..a7ec57eec4 --- /dev/null +++ b/vendor/tornado/tornado/httpserver.py @@ -0,0 +1,450 @@ +#!/usr/bin/env python +# +# Copyright 2009 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""A non-blocking, single-threaded HTTP server.""" + +import cgi +import errno +import functools +import ioloop +import iostream +import logging +import os +import socket +import time +import urlparse + +try: + import fcntl +except ImportError: + if os.name == 'nt': + import win32_support as fcntl + else: + raise + +try: + import ssl # Python 2.6+ +except ImportError: + ssl = None + +_log = logging.getLogger('tornado.httpserver') + +class HTTPServer(object): + """A non-blocking, single-threaded HTTP server. + + A server is defined by a request callback that takes an HTTPRequest + instance as an argument and writes a valid HTTP response with + request.write(). request.finish() finishes the request (but does not + necessarily close the connection in the case of HTTP/1.1 keep-alive + requests). A simple example server that echoes back the URI you + requested: + + import httpserver + import ioloop + + def handle_request(request): + message = "You requested %s\n" % request.uri + request.write("HTTP/1.1 200 OK\r\nContent-Length: %d\r\n\r\n%s" % ( + len(message), message)) + request.finish() + + http_server = httpserver.HTTPServer(handle_request) + http_server.listen(8888) + ioloop.IOLoop.instance().start() + + HTTPServer is a very basic connection handler. Beyond parsing the + HTTP request body and headers, the only HTTP semantics implemented + in HTTPServer is HTTP/1.1 keep-alive connections. We do not, however, + implement chunked encoding, so the request callback must provide a + Content-Length header or implement chunked encoding for HTTP/1.1 + requests for the server to run correctly for HTTP/1.1 clients. If + the request handler is unable to do this, you can provide the + no_keep_alive argument to the HTTPServer constructor, which will + ensure the connection is closed on every request no matter what HTTP + version the client is using. + + If xheaders is True, we support the X-Real-Ip and X-Scheme headers, + which override the remote IP and HTTP scheme for all requests. These + headers are useful when running Tornado behind a reverse proxy or + load balancer. + + HTTPServer can serve HTTPS (SSL) traffic with Python 2.6+ and OpenSSL. + To make this server serve SSL traffic, send the ssl_options dictionary + argument with the arguments required for the ssl.wrap_socket() method, + including "certfile" and "keyfile": + + HTTPServer(applicaton, ssl_options={ + "certfile": os.path.join(data_dir, "mydomain.crt"), + "keyfile": os.path.join(data_dir, "mydomain.key"), + }) + + By default, listen() runs in a single thread in a single process. You + can utilize all available CPUs on this machine by calling bind() and + start() instead of listen(): + + http_server = httpserver.HTTPServer(handle_request) + http_server.bind(8888) + http_server.start() # Forks multiple sub-processes + ioloop.IOLoop.instance().start() + + start() detects the number of CPUs on this machine and "pre-forks" that + number of child processes so that we have one Tornado process per CPU, + all with their own IOLoop. You can also pass in the specific number of + child processes you want to run with if you want to override this + auto-detection. + """ + def __init__(self, request_callback, no_keep_alive=False, io_loop=None, + xheaders=False, ssl_options=None): + """Initializes the server with the given request callback. + + If you use pre-forking/start() instead of the listen() method to + start your server, you should not pass an IOLoop instance to this + constructor. Each pre-forked child process will create its own + IOLoop instance after the forking process. + """ + self.request_callback = request_callback + self.no_keep_alive = no_keep_alive + self.io_loop = io_loop + self.xheaders = xheaders + self.ssl_options = ssl_options + self._socket = None + self._started = False + + def listen(self, port, address=""): + """Binds to the given port and starts the server in a single process. + + This method is a shortcut for: + + server.bind(port, address) + server.start(1) + + """ + self.bind(port, address) + self.start(1) + + def bind(self, port, address=""): + """Binds this server to the given port on the given IP address. + + To start the server, call start(). If you want to run this server + in a single process, you can call listen() as a shortcut to the + sequence of bind() and start() calls. + """ + assert not self._socket + self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) + flags = fcntl.fcntl(self._socket.fileno(), fcntl.F_GETFD) + flags |= fcntl.FD_CLOEXEC + fcntl.fcntl(self._socket.fileno(), fcntl.F_SETFD, flags) + self._socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + self._socket.setblocking(0) + self._socket.bind((address, port)) + self._socket.listen(128) + + def start(self, num_processes=None): + """Starts this server in the IOLoop. + + By default, we detect the number of cores available on this machine + and fork that number of child processes. If num_processes is given, we + fork that specific number of sub-processes. + + If num_processes is 1 or we detect only 1 CPU core, we run the server + in this process and do not fork any additional child process. + + Since we run use processes and not threads, there is no shared memory + between any server code. + """ + assert not self._started + self._started = True + if num_processes is None: + # Use sysconf to detect the number of CPUs (cores) + try: + num_processes = os.sysconf("SC_NPROCESSORS_CONF") + except ValueError: + _log.error("Could not get num processors from sysconf; " + "running with one process") + num_processes = 1 + if num_processes > 1 and ioloop.IOLoop.initialized(): + _log.error("Cannot run in multiple processes: IOLoop instance " + "has already been initialized. You cannot call " + "IOLoop.instance() before calling start()") + num_processes = 1 + if num_processes > 1: + _log.info("Pre-forking %d server processes", num_processes) + for i in range(num_processes): + if os.fork() == 0: + self.io_loop = ioloop.IOLoop.instance() + self.io_loop.add_handler( + self._socket.fileno(), self._handle_events, + ioloop.IOLoop.READ) + return + os.waitpid(-1, 0) + else: + if not self.io_loop: + self.io_loop = ioloop.IOLoop.instance() + self.io_loop.add_handler(self._socket.fileno(), + self._handle_events, + ioloop.IOLoop.READ) + + def stop(self): + self.io_loop.remove_handler(self._socket.fileno()) + self._socket.close() + + def _handle_events(self, fd, events): + while True: + try: + connection, address = self._socket.accept() + except socket.error, e: + if e[0] in (errno.EWOULDBLOCK, errno.EAGAIN): + return + raise + if self.ssl_options is not None: + assert ssl, "Python 2.6+ and OpenSSL required for SSL" + connection = ssl.wrap_socket( + connection, server_side=True, **self.ssl_options) + try: + stream = iostream.IOStream(connection, io_loop=self.io_loop) + HTTPConnection(stream, address, self.request_callback, + self.no_keep_alive, self.xheaders) + except: + _log.error("Error in connection callback", exc_info=True) + + +class HTTPConnection(object): + """Handles a connection to an HTTP client, executing HTTP requests. + + We parse HTTP headers and bodies, and execute the request callback + until the HTTP conection is closed. + """ + def __init__(self, stream, address, request_callback, no_keep_alive=False, + xheaders=False): + self.stream = stream + self.address = address + self.request_callback = request_callback + self.no_keep_alive = no_keep_alive + self.xheaders = xheaders + self._request = None + self._request_finished = False + self.stream.read_until("\r\n\r\n", self._on_headers) + + def write(self, chunk): + assert self._request, "Request closed" + if not self.stream.closed(): + self.stream.write(chunk, self._on_write_complete) + + def finish(self): + assert self._request, "Request closed" + self._request_finished = True + if not self.stream.writing(): + self._finish_request() + + def _on_write_complete(self): + if self._request_finished: + self._finish_request() + + def _finish_request(self): + if self.no_keep_alive: + disconnect = True + else: + connection_header = self._request.headers.get("Connection") + if self._request.supports_http_1_1(): + disconnect = connection_header == "close" + elif ("Content-Length" in self._request.headers + or self._request.method in ("HEAD", "GET")): + disconnect = connection_header != "Keep-Alive" + else: + disconnect = True + self._request = None + self._request_finished = False + if disconnect: + self.stream.close() + return + self.stream.read_until("\r\n\r\n", self._on_headers) + + def _on_headers(self, data): + eol = data.find("\r\n") + start_line = data[:eol] + method, uri, version = start_line.split(" ") + if not version.startswith("HTTP/"): + raise Exception("Malformed HTTP version in HTTP Request-Line") + headers = HTTPHeaders.parse(data[eol:]) + self._request = HTTPRequest( + connection=self, method=method, uri=uri, version=version, + headers=headers, remote_ip=self.address[0]) + + content_length = headers.get("Content-Length") + if content_length: + content_length = int(content_length) + if content_length > self.stream.max_buffer_size: + raise Exception("Content-Length too long") + if headers.get("Expect") == "100-continue": + self.stream.write("HTTP/1.1 100 (Continue)\r\n\r\n") + self.stream.read_bytes(content_length, self._on_request_body) + return + + self.request_callback(self._request) + + def _on_request_body(self, data): + self._request.body = data + content_type = self._request.headers.get("Content-Type", "") + if self._request.method == "POST": + if content_type.startswith("application/x-www-form-urlencoded"): + arguments = cgi.parse_qs(self._request.body) + for name, values in arguments.iteritems(): + values = [v for v in values if v] + if values: + self._request.arguments.setdefault(name, []).extend( + values) + elif content_type.startswith("multipart/form-data"): + boundary = content_type[30:] + if boundary: self._parse_mime_body(boundary, data) + self.request_callback(self._request) + + def _parse_mime_body(self, boundary, data): + if data.endswith("\r\n"): + footer_length = len(boundary) + 6 + else: + footer_length = len(boundary) + 4 + parts = data[:-footer_length].split("--" + boundary + "\r\n") + for part in parts: + if not part: continue + eoh = part.find("\r\n\r\n") + if eoh == -1: + _log.warning("multipart/form-data missing headers") + continue + headers = HTTPHeaders.parse(part[:eoh]) + name_header = headers.get("Content-Disposition", "") + if not name_header.startswith("form-data;") or \ + not part.endswith("\r\n"): + _log.warning("Invalid multipart/form-data") + continue + value = part[eoh + 4:-2] + name_values = {} + for name_part in name_header[10:].split(";"): + name, name_value = name_part.strip().split("=", 1) + name_values[name] = name_value.strip('"').decode("utf-8") + if not name_values.get("name"): + _log.warning("multipart/form-data value missing name") + continue + name = name_values["name"] + if name_values.get("filename"): + ctype = headers.get("Content-Type", "application/unknown") + self._request.files.setdefault(name, []).append(dict( + filename=name_values["filename"], body=value, + content_type=ctype)) + else: + self._request.arguments.setdefault(name, []).append(value) + + +class HTTPRequest(object): + """A single HTTP request. + + GET/POST arguments are available in the arguments property, which + maps arguments names to lists of values (to support multiple values + for individual names). Names and values are both unicode always. + + File uploads are available in the files property, which maps file + names to list of files. Each file is a dictionary of the form + {"filename":..., "content_type":..., "body":...}. The content_type + comes from the provided HTTP header and should not be trusted + outright given that it can be easily forged. + + An HTTP request is attached to a single HTTP connection, which can + be accessed through the "connection" attribute. Since connections + are typically kept open in HTTP/1.1, multiple requests can be handled + sequentially on a single connection. + """ + def __init__(self, method, uri, version="HTTP/1.0", headers=None, + body=None, remote_ip=None, protocol=None, host=None, + files=None, connection=None): + self.method = method + self.uri = uri + self.version = version + self.headers = headers or HTTPHeaders() + self.body = body or "" + if connection and connection.xheaders: + # Squid uses X-Forwarded-For, others use X-Real-Ip + self.remote_ip = self.headers.get( + "X-Real-Ip", self.headers.get("X-Forwarded-For", remote_ip)) + self.protocol = self.headers.get("X-Scheme", protocol) or "http" + else: + self.remote_ip = remote_ip + self.protocol = protocol or "http" + self.host = host or self.headers.get("Host") or "127.0.0.1" + self.files = files or {} + self.connection = connection + self._start_time = time.time() + self._finish_time = None + + scheme, netloc, path, query, fragment = urlparse.urlsplit(uri) + self.path = path + self.query = query + arguments = cgi.parse_qs(query) + self.arguments = {} + for name, values in arguments.iteritems(): + values = [v for v in values if v] + if values: self.arguments[name] = values + + def supports_http_1_1(self): + """Returns True if this request supports HTTP/1.1 semantics""" + return self.version == "HTTP/1.1" + + def write(self, chunk): + """Writes the given chunk to the response stream.""" + assert isinstance(chunk, str) + self.connection.write(chunk) + + def finish(self): + """Finishes this HTTP request on the open connection.""" + self.connection.finish() + self._finish_time = time.time() + + def full_url(self): + """Reconstructs the full URL for this request.""" + return self.protocol + "://" + self.host + self.uri + + def request_time(self): + """Returns the amount of time it took for this request to execute.""" + if self._finish_time is None: + return time.time() - self._start_time + else: + return self._finish_time - self._start_time + + def __repr__(self): + attrs = ("protocol", "host", "method", "uri", "version", "remote_ip", + "remote_ip", "body") + args = ", ".join(["%s=%r" % (n, getattr(self, n)) for n in attrs]) + return "%s(%s, headers=%s)" % ( + self.__class__.__name__, args, dict(self.headers)) + + +class HTTPHeaders(dict): + """A dictionary that maintains Http-Header-Case for all keys.""" + def __setitem__(self, name, value): + dict.__setitem__(self, self._normalize_name(name), value) + + def __getitem__(self, name): + return dict.__getitem__(self, self._normalize_name(name)) + + def _normalize_name(self, name): + return "-".join([w.capitalize() for w in name.split("-")]) + + @classmethod + def parse(cls, headers_string): + headers = cls() + for line in headers_string.splitlines(): + if line: + name, value = line.split(": ", 1) + headers[name] = value + return headers diff --git a/vendor/tornado/tornado/ioloop.py b/vendor/tornado/tornado/ioloop.py new file mode 100644 index 0000000000..e94c17372e --- /dev/null +++ b/vendor/tornado/tornado/ioloop.py @@ -0,0 +1,483 @@ +#!/usr/bin/env python +# +# Copyright 2009 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""A level-triggered I/O loop for non-blocking sockets.""" + +import bisect +import errno +import os +import logging +import select +import time + +try: + import fcntl +except ImportError: + if os.name == 'nt': + import win32_support + import win32_support as fcntl + else: + raise + +_log = logging.getLogger("tornado.ioloop") + +class IOLoop(object): + """A level-triggered I/O loop. + + We use epoll if it is available, or else we fall back on select(). If + you are implementing a system that needs to handle 1000s of simultaneous + connections, you should use Linux and either compile our epoll module or + use Python 2.6+ to get epoll support. + + Example usage for a simple TCP server: + + import errno + import functools + import ioloop + import socket + + def connection_ready(sock, fd, events): + while True: + try: + connection, address = sock.accept() + except socket.error, e: + if e[0] not in (errno.EWOULDBLOCK, errno.EAGAIN): + raise + return + connection.setblocking(0) + handle_connection(connection, address) + + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) + sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + sock.setblocking(0) + sock.bind(("", port)) + sock.listen(128) + + io_loop = ioloop.IOLoop.instance() + callback = functools.partial(connection_ready, sock) + io_loop.add_handler(sock.fileno(), callback, io_loop.READ) + io_loop.start() + + """ + # Constants from the epoll module + _EPOLLIN = 0x001 + _EPOLLPRI = 0x002 + _EPOLLOUT = 0x004 + _EPOLLERR = 0x008 + _EPOLLHUP = 0x010 + _EPOLLRDHUP = 0x2000 + _EPOLLONESHOT = (1 << 30) + _EPOLLET = (1 << 31) + + # Our events map exactly to the epoll events + NONE = 0 + READ = _EPOLLIN + WRITE = _EPOLLOUT + ERROR = _EPOLLERR | _EPOLLHUP | _EPOLLRDHUP + + def __init__(self, impl=None): + self._impl = impl or _poll() + if hasattr(self._impl, 'fileno'): + self._set_close_exec(self._impl.fileno()) + self._handlers = {} + self._events = {} + self._callbacks = set() + self._timeouts = [] + self._running = False + self._stopped = False + + # Create a pipe that we send bogus data to when we want to wake + # the I/O loop when it is idle + if os.name != 'nt': + r, w = os.pipe() + self._set_nonblocking(r) + self._set_nonblocking(w) + self._set_close_exec(r) + self._set_close_exec(w) + self._waker_reader = os.fdopen(r, "r", 0) + self._waker_writer = os.fdopen(w, "w", 0) + else: + self._waker_reader = self._waker_writer = win32_support.Pipe() + r = self._waker_writer.reader_fd + self.add_handler(r, self._read_waker, self.READ) + + @classmethod + def instance(cls): + """Returns a global IOLoop instance. + + Most single-threaded applications have a single, global IOLoop. + Use this method instead of passing around IOLoop instances + throughout your code. + + A common pattern for classes that depend on IOLoops is to use + a default argument to enable programs with multiple IOLoops + but not require the argument for simpler applications: + + class MyClass(object): + def __init__(self, io_loop=None): + self.io_loop = io_loop or IOLoop.instance() + """ + if not hasattr(cls, "_instance"): + cls._instance = cls() + return cls._instance + + @classmethod + def initialized(cls): + return hasattr(cls, "_instance") + + def add_handler(self, fd, handler, events): + """Registers the given handler to receive the given events for fd.""" + self._handlers[fd] = handler + self._impl.register(fd, events | self.ERROR) + + def update_handler(self, fd, events): + """Changes the events we listen for fd.""" + self._impl.modify(fd, events | self.ERROR) + + def remove_handler(self, fd): + """Stop listening for events on fd.""" + self._handlers.pop(fd, None) + self._events.pop(fd, None) + try: + self._impl.unregister(fd) + except (OSError, IOError): + _log.debug("Error deleting fd from IOLoop", exc_info=True) + + def start(self): + """Starts the I/O loop. + + The loop will run until one of the I/O handlers calls stop(), which + will make the loop stop after the current event iteration completes. + """ + if self._stopped: + self._stopped = False + return + self._running = True + while True: + # Never use an infinite timeout here - it can stall epoll + poll_timeout = 0.2 + + # Prevent IO event starvation by delaying new callbacks + # to the next iteration of the event loop. + callbacks = list(self._callbacks) + for callback in callbacks: + # A callback can add or remove other callbacks + if callback in self._callbacks: + self._callbacks.remove(callback) + self._run_callback(callback) + + if self._callbacks: + poll_timeout = 0.0 + + if self._timeouts: + now = time.time() + while self._timeouts and self._timeouts[0].deadline <= now: + timeout = self._timeouts.pop(0) + self._run_callback(timeout.callback) + if self._timeouts: + milliseconds = self._timeouts[0].deadline - now + poll_timeout = min(milliseconds, poll_timeout) + + if not self._running: + break + + try: + event_pairs = self._impl.poll(poll_timeout) + except Exception, e: + if hasattr(e, 'errno') and e.errno == errno.EINTR: + _log.warning("Interrupted system call", exc_info=1) + continue + else: + raise + + # Pop one fd at a time from the set of pending fds and run + # its handler. Since that handler may perform actions on + # other file descriptors, there may be reentrant calls to + # this IOLoop that update self._events + self._events.update(event_pairs) + while self._events: + fd, events = self._events.popitem() + try: + self._handlers[fd](fd, events) + except (KeyboardInterrupt, SystemExit): + raise + except (OSError, IOError), e: + if e[0] == errno.EPIPE: + # Happens when the client closes the connection + pass + else: + _log.error("Exception in I/O handler for fd %d", + fd, exc_info=True) + except: + _log.error("Exception in I/O handler for fd %d", + fd, exc_info=True) + # reset the stopped flag so another start/stop pair can be issued + self._stopped = False + + def stop(self): + """Stop the loop after the current event loop iteration is complete. + If the event loop is not currently running, the next call to start() + will return immediately. + + To use asynchronous methods from otherwise-synchronous code (such as + unit tests), you can start and stop the event loop like this: + ioloop = IOLoop() + async_method(ioloop=ioloop, callback=ioloop.stop) + ioloop.start() + ioloop.start() will return after async_method has run its callback, + whether that callback was invoked before or after ioloop.start. + """ + self._running = False + self._stopped = True + self._wake() + + def running(self): + """Returns true if this IOLoop is currently running.""" + return self._running + + def add_timeout(self, deadline, callback): + """Calls the given callback at the time deadline from the I/O loop.""" + timeout = _Timeout(deadline, callback) + bisect.insort(self._timeouts, timeout) + return timeout + + def remove_timeout(self, timeout): + self._timeouts.remove(timeout) + + def add_callback(self, callback): + """Calls the given callback on the next I/O loop iteration.""" + self._callbacks.add(callback) + self._wake() + + def remove_callback(self, callback): + """Removes the given callback from the next I/O loop iteration.""" + self._callbacks.remove(callback) + + def _wake(self): + try: + self._waker_writer.write("x") + except IOError: + pass + + def _run_callback(self, callback): + try: + callback() + except (KeyboardInterrupt, SystemExit): + raise + except: + self.handle_callback_exception(callback) + + def handle_callback_exception(self, callback): + """This method is called whenever a callback run by the IOLoop + throws an exception. + + By default simply logs the exception as an error. Subclasses + may override this method to customize reporting of exceptions. + + The exception itself is not passed explicitly, but is available + in sys.exc_info. + """ + _log.error("Exception in callback %r", callback, exc_info=True) + + def _read_waker(self, fd, events): + try: + while True: + self._waker_reader.read() + except IOError: + pass + + def _set_nonblocking(self, fd): + flags = fcntl.fcntl(fd, fcntl.F_GETFL) + fcntl.fcntl(fd, fcntl.F_SETFL, flags | os.O_NONBLOCK) + + def _set_close_exec(self, fd): + flags = fcntl.fcntl(fd, fcntl.F_GETFD) + fcntl.fcntl(fd, fcntl.F_SETFD, flags | fcntl.FD_CLOEXEC) + + +class _Timeout(object): + """An IOLoop timeout, a UNIX timestamp and a callback""" + + # Reduce memory overhead when there are lots of pending callbacks + __slots__ = ['deadline', 'callback'] + + def __init__(self, deadline, callback): + self.deadline = deadline + self.callback = callback + + def __cmp__(self, other): + return cmp((self.deadline, id(self.callback)), + (other.deadline, id(other.callback))) + + +class PeriodicCallback(object): + """Schedules the given callback to be called periodically. + + The callback is called every callback_time milliseconds. + """ + def __init__(self, callback, callback_time, io_loop=None): + self.callback = callback + self.callback_time = callback_time + self.io_loop = io_loop or IOLoop.instance() + self._running = True + + def start(self): + timeout = time.time() + self.callback_time / 1000.0 + self.io_loop.add_timeout(timeout, self._run) + + def stop(self): + self._running = False + + def _run(self): + if not self._running: return + try: + self.callback() + except (KeyboardInterrupt, SystemExit): + raise + except: + _log.error("Error in periodic callback", exc_info=True) + self.start() + + +class _EPoll(object): + """An epoll-based event loop using our C module for Python 2.5 systems""" + _EPOLL_CTL_ADD = 1 + _EPOLL_CTL_DEL = 2 + _EPOLL_CTL_MOD = 3 + + def __init__(self): + self._epoll_fd = epoll.epoll_create() + + def fileno(self): + return self._epoll_fd + + def register(self, fd, events): + epoll.epoll_ctl(self._epoll_fd, self._EPOLL_CTL_ADD, fd, events) + + def modify(self, fd, events): + epoll.epoll_ctl(self._epoll_fd, self._EPOLL_CTL_MOD, fd, events) + + def unregister(self, fd): + epoll.epoll_ctl(self._epoll_fd, self._EPOLL_CTL_DEL, fd, 0) + + def poll(self, timeout): + return epoll.epoll_wait(self._epoll_fd, int(timeout * 1000)) + + +class _KQueue(object): + """A kqueue-based event loop for BSD/Mac systems.""" + def __init__(self): + self._kqueue = select.kqueue() + self._active = {} + + def fileno(self): + return self._kqueue.fileno() + + def register(self, fd, events): + self._control(fd, events, select.KQ_EV_ADD) + self._active[fd] = events + + def modify(self, fd, events): + self.unregister(fd) + self.register(fd, events) + + def unregister(self, fd): + events = self._active.pop(fd) + self._control(fd, events, select.KQ_EV_DELETE) + + def _control(self, fd, events, flags): + kevents = [] + if events & IOLoop.WRITE: + kevents.append(select.kevent( + fd, filter=select.KQ_FILTER_WRITE, flags=flags)) + if events & IOLoop.READ or not kevents: + # Always read when there is not a write + kevents.append(select.kevent( + fd, filter=select.KQ_FILTER_READ, flags=flags)) + # Even though control() takes a list, it seems to return EINVAL + # on Mac OS X (10.6) when there is more than one event in the list. + for kevent in kevents: + self._kqueue.control([kevent], 0) + + def poll(self, timeout): + kevents = self._kqueue.control(None, 1000, timeout) + events = {} + for kevent in kevents: + fd = kevent.ident + flags = 0 + if kevent.filter == select.KQ_FILTER_READ: + events[fd] = events.get(fd, 0) | IOLoop.READ + if kevent.filter == select.KQ_FILTER_WRITE: + events[fd] = events.get(fd, 0) | IOLoop.WRITE + if kevent.flags & select.KQ_EV_ERROR: + events[fd] = events.get(fd, 0) | IOLoop.ERROR + return events.items() + + +class _Select(object): + """A simple, select()-based IOLoop implementation for non-Linux systems""" + def __init__(self): + self.read_fds = set() + self.write_fds = set() + self.error_fds = set() + self.fd_sets = (self.read_fds, self.write_fds, self.error_fds) + + def register(self, fd, events): + if events & IOLoop.READ: self.read_fds.add(fd) + if events & IOLoop.WRITE: self.write_fds.add(fd) + if events & IOLoop.ERROR: self.error_fds.add(fd) + + def modify(self, fd, events): + self.unregister(fd) + self.register(fd, events) + + def unregister(self, fd): + self.read_fds.discard(fd) + self.write_fds.discard(fd) + self.error_fds.discard(fd) + + def poll(self, timeout): + readable, writeable, errors = select.select( + self.read_fds, self.write_fds, self.error_fds, timeout) + events = {} + for fd in readable: + events[fd] = events.get(fd, 0) | IOLoop.READ + for fd in writeable: + events[fd] = events.get(fd, 0) | IOLoop.WRITE + for fd in errors: + events[fd] = events.get(fd, 0) | IOLoop.ERROR + return events.items() + + +# Choose a poll implementation. Use epoll if it is available, fall back to +# select() for non-Linux platforms +if hasattr(select, "epoll"): + # Python 2.6+ on Linux + _poll = select.epoll +elif hasattr(select, "kqueue"): + # Python 2.6+ on BSD or Mac + _poll = _KQueue +else: + try: + # Linux systems with our C module installed + import epoll + _poll = _EPoll + except: + # All other systems + import sys + if "linux" in sys.platform: + _log.warning("epoll module not found; using select()") + _poll = _Select diff --git a/vendor/tornado/tornado/iostream.py b/vendor/tornado/tornado/iostream.py new file mode 100644 index 0000000000..af7c6edbfe --- /dev/null +++ b/vendor/tornado/tornado/iostream.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python +# +# Copyright 2009 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""A utility class to write to and read from a non-blocking socket.""" + +import errno +import ioloop +import logging +import socket + +_log = logging.getLogger('tornado.iostream') + +class IOStream(object): + """A utility class to write to and read from a non-blocking socket. + + We support three methods: write(), read_until(), and read_bytes(). + All of the methods take callbacks (since writing and reading are + non-blocking and asynchronous). read_until() reads the socket until + a given delimiter, and read_bytes() reads until a specified number + of bytes have been read from the socket. + + A very simple (and broken) HTTP client using this class: + + import ioloop + import iostream + import socket + + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) + s.connect(("friendfeed.com", 80)) + stream = IOStream(s) + + def on_headers(data): + headers = {} + for line in data.split("\r\n"): + parts = line.split(":") + if len(parts) == 2: + headers[parts[0].strip()] = parts[1].strip() + stream.read_bytes(int(headers["Content-Length"]), on_body) + + def on_body(data): + print data + stream.close() + ioloop.IOLoop.instance().stop() + + stream.write("GET / HTTP/1.0\r\n\r\n") + stream.read_until("\r\n\r\n", on_headers) + ioloop.IOLoop.instance().start() + + """ + def __init__(self, socket, io_loop=None, max_buffer_size=104857600, + read_chunk_size=4096): + self.socket = socket + self.socket.setblocking(False) + self.io_loop = io_loop or ioloop.IOLoop.instance() + self.max_buffer_size = max_buffer_size + self.read_chunk_size = read_chunk_size + self._read_buffer = "" + self._write_buffer = "" + self._read_delimiter = None + self._read_bytes = None + self._read_callback = None + self._write_callback = None + self._close_callback = None + self._state = self.io_loop.ERROR + self.io_loop.add_handler( + self.socket.fileno(), self._handle_events, self._state) + + def read_until(self, delimiter, callback): + """Call callback when we read the given delimiter.""" + assert not self._read_callback, "Already reading" + loc = self._read_buffer.find(delimiter) + if loc != -1: + callback(self._consume(loc + len(delimiter))) + return + self._check_closed() + self._read_delimiter = delimiter + self._read_callback = callback + self._add_io_state(self.io_loop.READ) + + def read_bytes(self, num_bytes, callback): + """Call callback when we read the given number of bytes.""" + assert not self._read_callback, "Already reading" + if len(self._read_buffer) >= num_bytes: + callback(self._consume(num_bytes)) + return + self._check_closed() + self._read_bytes = num_bytes + self._read_callback = callback + self._add_io_state(self.io_loop.READ) + + def write(self, data, callback=None): + """Write the given data to this stream. + + If callback is given, we call it when all of the buffered write + data has been successfully written to the stream. If there was + previously buffered write data and an old write callback, that + callback is simply overwritten with this new callback. + """ + self._check_closed() + self._write_buffer += data + self._add_io_state(self.io_loop.WRITE) + self._write_callback = callback + + def set_close_callback(self, callback): + """Call the given callback when the stream is closed.""" + self._close_callback = callback + + def close(self): + """Close this stream.""" + if self.socket is not None: + self.io_loop.remove_handler(self.socket.fileno()) + self.socket.close() + self.socket = None + if self._close_callback: self._close_callback() + + def reading(self): + """Returns true if we are currently reading from the stream.""" + return self._read_callback is not None + + def writing(self): + """Returns true if we are currently writing to the stream.""" + return len(self._write_buffer) > 0 + + def closed(self): + return self.socket is None + + def _handle_events(self, fd, events): + if not self.socket: + _log.warning("Got events for closed stream %d", fd) + return + if events & self.io_loop.READ: + self._handle_read() + if not self.socket: + return + if events & self.io_loop.WRITE: + self._handle_write() + if not self.socket: + return + if events & self.io_loop.ERROR: + self.close() + return + state = self.io_loop.ERROR + if self._read_delimiter or self._read_bytes: + state |= self.io_loop.READ + if self._write_buffer: + state |= self.io_loop.WRITE + if state != self._state: + self._state = state + self.io_loop.update_handler(self.socket.fileno(), self._state) + + def _handle_read(self): + try: + chunk = self.socket.recv(self.read_chunk_size) + except socket.error, e: + if e[0] in (errno.EWOULDBLOCK, errno.EAGAIN): + return + else: + _log.warning("Read error on %d: %s", + self.socket.fileno(), e) + self.close() + return + if not chunk: + self.close() + return + self._read_buffer += chunk + if len(self._read_buffer) >= self.max_buffer_size: + _log.error("Reached maximum read buffer size") + self.close() + return + if self._read_bytes: + if len(self._read_buffer) >= self._read_bytes: + num_bytes = self._read_bytes + callback = self._read_callback + self._read_callback = None + self._read_bytes = None + callback(self._consume(num_bytes)) + elif self._read_delimiter: + loc = self._read_buffer.find(self._read_delimiter) + if loc != -1: + callback = self._read_callback + delimiter_len = len(self._read_delimiter) + self._read_callback = None + self._read_delimiter = None + callback(self._consume(loc + delimiter_len)) + + def _handle_write(self): + while self._write_buffer: + try: + num_bytes = self.socket.send(self._write_buffer) + self._write_buffer = self._write_buffer[num_bytes:] + except socket.error, e: + if e[0] in (errno.EWOULDBLOCK, errno.EAGAIN): + break + else: + _log.warning("Write error on %d: %s", + self.socket.fileno(), e) + self.close() + return + if not self._write_buffer and self._write_callback: + callback = self._write_callback + self._write_callback = None + callback() + + def _consume(self, loc): + result = self._read_buffer[:loc] + self._read_buffer = self._read_buffer[loc:] + return result + + def _check_closed(self): + if not self.socket: + raise IOError("Stream is closed") + + def _add_io_state(self, state): + if not self._state & state: + self._state = self._state | state + self.io_loop.update_handler(self.socket.fileno(), self._state) diff --git a/vendor/tornado/tornado/locale.py b/vendor/tornado/tornado/locale.py new file mode 100644 index 0000000000..6a8537d750 --- /dev/null +++ b/vendor/tornado/tornado/locale.py @@ -0,0 +1,457 @@ +#!/usr/bin/env python +# +# Copyright 2009 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Translation methods for generating localized strings. + +To load a locale and generate a translated string: + + user_locale = locale.get("es_LA") + print user_locale.translate("Sign out") + +locale.get() returns the closest matching locale, not necessarily the +specific locale you requested. You can support pluralization with +additional arguments to translate(), e.g.: + + people = [...] + message = user_locale.translate( + "%(list)s is online", "%(list)s are online", len(people)) + print message % {"list": user_locale.list(people)} + +The first string is chosen if len(people) == 1, otherwise the second +string is chosen. + +Applications should call one of load_translations (which uses a simple +CSV format) or load_gettext_translations (which uses the .mo format +supported by gettext and related tools). If neither method is called, +the locale.translate method will simply return the original string. +""" + +import csv +import datetime +import logging +import os +import os.path +import re + +_default_locale = "en_US" +_translations = {} +_supported_locales = frozenset([_default_locale]) +_use_gettext = False + +_log = logging.getLogger('tornado.locale') + +def get(*locale_codes): + """Returns the closest match for the given locale codes. + + We iterate over all given locale codes in order. If we have a tight + or a loose match for the code (e.g., "en" for "en_US"), we return + the locale. Otherwise we move to the next code in the list. + + By default we return en_US if no translations are found for any of + the specified locales. You can change the default locale with + set_default_locale() below. + """ + return Locale.get_closest(*locale_codes) + + +def set_default_locale(code): + """Sets the default locale, used in get_closest_locale(). + + The default locale is assumed to be the language used for all strings + in the system. The translations loaded from disk are mappings from + the default locale to the destination locale. Consequently, you don't + need to create a translation file for the default locale. + """ + global _default_locale + global _supported_locales + _default_locale = code + _supported_locales = frozenset(_translations.keys() + [_default_locale]) + + +def load_translations(directory): + """Loads translations from CSV files in a directory. + + Translations are strings with optional Python-style named placeholders + (e.g., "My name is %(name)s") and their associated translations. + + The directory should have translation files of the form LOCALE.csv, + e.g. es_GT.csv. The CSV files should have two or three columns: string, + translation, and an optional plural indicator. Plural indicators should + be one of "plural" or "singular". A given string can have both singular + and plural forms. For example "%(name)s liked this" may have a + different verb conjugation depending on whether %(name)s is one + name or a list of names. There should be two rows in the CSV file for + that string, one with plural indicator "singular", and one "plural". + For strings with no verbs that would change on translation, simply + use "unknown" or the empty string (or don't include the column at all). + + Example translation es_LA.csv: + + "I love you","Te amo" + "%(name)s liked this","A %(name)s les gust\xf3 esto","plural" + "%(name)s liked this","A %(name)s le gust\xf3 esto","singular" + + """ + global _translations + global _supported_locales + _translations = {} + for path in os.listdir(directory): + if not path.endswith(".csv"): continue + locale, extension = path.split(".") + if locale not in LOCALE_NAMES: + _log.error("Unrecognized locale %r (path: %s)", locale, + os.path.join(directory, path)) + continue + f = open(os.path.join(directory, path), "r") + _translations[locale] = {} + for i, row in enumerate(csv.reader(f)): + if not row or len(row) < 2: continue + row = [c.decode("utf-8").strip() for c in row] + english, translation = row[:2] + if len(row) > 2: + plural = row[2] or "unknown" + else: + plural = "unknown" + if plural not in ("plural", "singular", "unknown"): + _log.error("Unrecognized plural indicator %r in %s line %d", + plural, path, i + 1) + continue + _translations[locale].setdefault(plural, {})[english] = translation + f.close() + _supported_locales = frozenset(_translations.keys() + [_default_locale]) + _log.info("Supported locales: %s", sorted(_supported_locales)) + +def load_gettext_translations(directory, domain): + """Loads translations from gettext's locale tree + + Locale tree is similar to system's /usr/share/locale, like: + + {directory}/{lang}/LC_MESSAGES/{domain}.mo + + Three steps are required to have you app translated: + + 1. Generate POT translation file + xgettext --language=Python --keyword=_:1,2 -d cyclone file1.py file2.html etc + + 2. Merge against existing POT file: + msgmerge old.po cyclone.po > new.po + + 3. Compile: + msgfmt cyclone.po -o {directory}/pt_BR/LC_MESSAGES/cyclone.mo + """ + import gettext + global _translations + global _supported_locales + global _use_gettext + _translations = {} + for lang in os.listdir(directory): + if os.path.isfile(os.path.join(directory, lang)): continue + try: + os.stat(os.path.join(directory, lang, "LC_MESSAGES", domain+".mo")) + _translations[lang] = gettext.translation(domain, directory, + languages=[lang]) + except Exception, e: + logging.error("Cannot load translation for '%s': %s", lang, str(e)) + continue + _supported_locales = frozenset(_translations.keys() + [_default_locale]) + _use_gettext = True + _log.info("Supported locales: %s", sorted(_supported_locales)) + + +def get_supported_locales(cls): + """Returns a list of all the supported locale codes.""" + return _supported_locales + + +class Locale(object): + @classmethod + def get_closest(cls, *locale_codes): + """Returns the closest match for the given locale code.""" + for code in locale_codes: + if not code: continue + code = code.replace("-", "_") + parts = code.split("_") + if len(parts) > 2: + continue + elif len(parts) == 2: + code = parts[0].lower() + "_" + parts[1].upper() + if code in _supported_locales: + return cls.get(code) + if parts[0].lower() in _supported_locales: + return cls.get(parts[0].lower()) + return cls.get(_default_locale) + + @classmethod + def get(cls, code): + """Returns the Locale for the given locale code. + + If it is not supported, we raise an exception. + """ + if not hasattr(cls, "_cache"): + cls._cache = {} + if code not in cls._cache: + assert code in _supported_locales + translations = _translations.get(code, None) + if translations is None: + locale = CSVLocale(code, {}) + elif _use_gettext: + locale = GettextLocale(code, translations) + else: + locale = CSVLocale(code, translations) + cls._cache[code] = locale + return cls._cache[code] + + def __init__(self, code, translations): + self.code = code + self.name = LOCALE_NAMES.get(code, {}).get("name", u"Unknown") + self.rtl = False + for prefix in ["fa", "ar", "he"]: + if self.code.startswith(prefix): + self.rtl = True + break + self.translations = translations + + # Initialize strings for date formatting + _ = self.translate + self._months = [ + _("January"), _("February"), _("March"), _("April"), + _("May"), _("June"), _("July"), _("August"), + _("September"), _("October"), _("November"), _("December")] + self._weekdays = [ + _("Monday"), _("Tuesday"), _("Wednesday"), _("Thursday"), + _("Friday"), _("Saturday"), _("Sunday")] + + def translate(self, message, plural_message=None, count=None): + raise NotImplementedError() + + def format_date(self, date, gmt_offset=0, relative=True, shorter=False, + full_format=False): + """Formats the given date (which should be GMT). + + By default, we return a relative time (e.g., "2 minutes ago"). You + can return an absolute date string with relative=False. + + You can force a full format date ("July 10, 1980") with + full_format=True. + """ + if self.code.startswith("ru"): + relative = False + if type(date) in (int, long, float): + date = datetime.datetime.utcfromtimestamp(date) + now = datetime.datetime.utcnow() + # Round down to now. Due to click skew, things are somethings + # slightly in the future. + if date > now: date = now + local_date = date - datetime.timedelta(minutes=gmt_offset) + local_now = now - datetime.timedelta(minutes=gmt_offset) + local_yesterday = local_now - datetime.timedelta(hours=24) + difference = now - date + seconds = difference.seconds + days = difference.days + + _ = self.translate + format = None + if not full_format: + if relative and days == 0: + if seconds < 50: + return _("1 second ago", "%(seconds)d seconds ago", + seconds) % { "seconds": seconds } + + if seconds < 50 * 60: + minutes = round(seconds / 60.0) + return _("1 minute ago", "%(minutes)d minutes ago", + minutes) % { "minutes": minutes } + + hours = round(seconds / (60.0 * 60)) + return _("1 hour ago", "%(hours)d hours ago", + hours) % { "hours": hours } + + if days == 0: + format = _("%(time)s") + elif days == 1 and local_date.day == local_yesterday.day and \ + relative: + format = _("yesterday") if shorter else \ + _("yesterday at %(time)s") + elif days < 5: + format = _("%(weekday)s") if shorter else \ + _("%(weekday)s at %(time)s") + elif days < 334: # 11mo, since confusing for same month last year + format = _("%(month_name)s %(day)s") if shorter else \ + _("%(month_name)s %(day)s at %(time)s") + + if format is None: + format = _("%(month_name)s %(day)s, %(year)s") if shorter else \ + _("%(month_name)s %(day)s, %(year)s at %(time)s") + + tfhour_clock = self.code not in ("en", "en_US", "zh_CN") + if tfhour_clock: + str_time = "%d:%02d" % (local_date.hour, local_date.minute) + elif self.code == "zh_CN": + str_time = "%s%d:%02d" % ( + (u'\u4e0a\u5348', u'\u4e0b\u5348')[local_date.hour >= 12], + local_date.hour % 12 or 12, local_date.minute) + else: + str_time = "%d:%02d %s" % ( + local_date.hour % 12 or 12, local_date.minute, + ("am", "pm")[local_date.hour >= 12]) + + return format % { + "month_name": self._months[local_date.month - 1], + "weekday": self._weekdays[local_date.weekday()], + "day": str(local_date.day), + "year": str(local_date.year), + "time": str_time + } + + def format_day(self, date, gmt_offset=0, dow=True): + """Formats the given date as a day of week. + + Example: "Monday, January 22". You can remove the day of week with + dow=False. + """ + local_date = date - datetime.timedelta(minutes=gmt_offset) + _ = self.translate + if dow: + return _("%(weekday)s, %(month_name)s %(day)s") % { + "month_name": self._months[local_date.month - 1], + "weekday": self._weekdays[local_date.weekday()], + "day": str(local_date.day), + } + else: + return _("%(month_name)s %(day)s") % { + "month_name": self._months[local_date.month - 1], + "day": str(local_date.day), + } + + def list(self, parts): + """Returns a comma-separated list for the given list of parts. + + The format is, e.g., "A, B and C", "A and B" or just "A" for lists + of size 1. + """ + _ = self.translate + if len(parts) == 0: return "" + if len(parts) == 1: return parts[0] + comma = u' \u0648 ' if self.code.startswith("fa") else u", " + return _("%(commas)s and %(last)s") % { + "commas": comma.join(parts[:-1]), + "last": parts[len(parts) - 1], + } + + def friendly_number(self, value): + """Returns a comma-separated number for the given integer.""" + if self.code not in ("en", "en_US"): + return str(value) + value = str(value) + parts = [] + while value: + parts.append(value[-3:]) + value = value[:-3] + return ",".join(reversed(parts)) + +class CSVLocale(Locale): + """Locale implementation using tornado's CSV translation format.""" + def translate(self, message, plural_message=None, count=None): + """Returns the translation for the given message for this locale. + + If plural_message is given, you must also provide count. We return + plural_message when count != 1, and we return the singular form + for the given message when count == 1. + """ + if plural_message is not None: + assert count is not None + if count != 1: + message = plural_message + message_dict = self.translations.get("plural", {}) + else: + message_dict = self.translations.get("singular", {}) + else: + message_dict = self.translations.get("unknown", {}) + return message_dict.get(message, message) + +class GettextLocale(Locale): + """Locale implementation using the gettext module.""" + def translate(self, message, plural_message=None, count=None): + if plural_message is not None: + assert count is not None + return self.translations.ungettext(message, plural_message, count) + else: + return self.translations.ugettext(message) + +LOCALE_NAMES = { + "af_ZA": {"name_en": u"Afrikaans", "name": u"Afrikaans"}, + "ar_AR": {"name_en": u"Arabic", "name": u"\u0627\u0644\u0639\u0631\u0628\u064a\u0629"}, + "bg_BG": {"name_en": u"Bulgarian", "name": u"\u0411\u044a\u043b\u0433\u0430\u0440\u0441\u043a\u0438"}, + "bn_IN": {"name_en": u"Bengali", "name": u"\u09ac\u09be\u0982\u09b2\u09be"}, + "bs_BA": {"name_en": u"Bosnian", "name": u"Bosanski"}, + "ca_ES": {"name_en": u"Catalan", "name": u"Catal\xe0"}, + "cs_CZ": {"name_en": u"Czech", "name": u"\u010ce\u0161tina"}, + "cy_GB": {"name_en": u"Welsh", "name": u"Cymraeg"}, + "da_DK": {"name_en": u"Danish", "name": u"Dansk"}, + "de_DE": {"name_en": u"German", "name": u"Deutsch"}, + "el_GR": {"name_en": u"Greek", "name": u"\u0395\u03bb\u03bb\u03b7\u03bd\u03b9\u03ba\u03ac"}, + "en_GB": {"name_en": u"English (UK)", "name": u"English (UK)"}, + "en_US": {"name_en": u"English (US)", "name": u"English (US)"}, + "es_ES": {"name_en": u"Spanish (Spain)", "name": u"Espa\xf1ol (Espa\xf1a)"}, + "es_LA": {"name_en": u"Spanish", "name": u"Espa\xf1ol"}, + "et_EE": {"name_en": u"Estonian", "name": u"Eesti"}, + "eu_ES": {"name_en": u"Basque", "name": u"Euskara"}, + "fa_IR": {"name_en": u"Persian", "name": u"\u0641\u0627\u0631\u0633\u06cc"}, + "fi_FI": {"name_en": u"Finnish", "name": u"Suomi"}, + "fr_CA": {"name_en": u"French (Canada)", "name": u"Fran\xe7ais (Canada)"}, + "fr_FR": {"name_en": u"French", "name": u"Fran\xe7ais"}, + "ga_IE": {"name_en": u"Irish", "name": u"Gaeilge"}, + "gl_ES": {"name_en": u"Galician", "name": u"Galego"}, + "he_IL": {"name_en": u"Hebrew", "name": u"\u05e2\u05d1\u05e8\u05d9\u05ea"}, + "hi_IN": {"name_en": u"Hindi", "name": u"\u0939\u093f\u0928\u094d\u0926\u0940"}, + "hr_HR": {"name_en": u"Croatian", "name": u"Hrvatski"}, + "hu_HU": {"name_en": u"Hungarian", "name": u"Magyar"}, + "id_ID": {"name_en": u"Indonesian", "name": u"Bahasa Indonesia"}, + "is_IS": {"name_en": u"Icelandic", "name": u"\xcdslenska"}, + "it_IT": {"name_en": u"Italian", "name": u"Italiano"}, + "ja_JP": {"name_en": u"Japanese", "name": u"\xe6\xe6\xe8"}, + "ko_KR": {"name_en": u"Korean", "name": u"\xed\xea\xec"}, + "lt_LT": {"name_en": u"Lithuanian", "name": u"Lietuvi\u0173"}, + "lv_LV": {"name_en": u"Latvian", "name": u"Latvie\u0161u"}, + "mk_MK": {"name_en": u"Macedonian", "name": u"\u041c\u0430\u043a\u0435\u0434\u043e\u043d\u0441\u043a\u0438"}, + "ml_IN": {"name_en": u"Malayalam", "name": u"\u0d2e\u0d32\u0d2f\u0d3e\u0d33\u0d02"}, + "ms_MY": {"name_en": u"Malay", "name": u"Bahasa Melayu"}, + "nb_NO": {"name_en": u"Norwegian (bokmal)", "name": u"Norsk (bokm\xe5l)"}, + "nl_NL": {"name_en": u"Dutch", "name": u"Nederlands"}, + "nn_NO": {"name_en": u"Norwegian (nynorsk)", "name": u"Norsk (nynorsk)"}, + "pa_IN": {"name_en": u"Punjabi", "name": u"\u0a2a\u0a70\u0a1c\u0a3e\u0a2c\u0a40"}, + "pl_PL": {"name_en": u"Polish", "name": u"Polski"}, + "pt_BR": {"name_en": u"Portuguese (Brazil)", "name": u"Portugu\xeas (Brasil)"}, + "pt_PT": {"name_en": u"Portuguese (Portugal)", "name": u"Portugu\xeas (Portugal)"}, + "ro_RO": {"name_en": u"Romanian", "name": u"Rom\xe2n\u0103"}, + "ru_RU": {"name_en": u"Russian", "name": u"\u0420\u0443\u0441\u0441\u043a\u0438\u0439"}, + "sk_SK": {"name_en": u"Slovak", "name": u"Sloven\u010dina"}, + "sl_SI": {"name_en": u"Slovenian", "name": u"Sloven\u0161\u010dina"}, + "sq_AL": {"name_en": u"Albanian", "name": u"Shqip"}, + "sr_RS": {"name_en": u"Serbian", "name": u"\u0421\u0440\u043f\u0441\u043a\u0438"}, + "sv_SE": {"name_en": u"Swedish", "name": u"Svenska"}, + "sw_KE": {"name_en": u"Swahili", "name": u"Kiswahili"}, + "ta_IN": {"name_en": u"Tamil", "name": u"\u0ba4\u0bae\u0bbf\u0bb4\u0bcd"}, + "te_IN": {"name_en": u"Telugu", "name": u"\u0c24\u0c46\u0c32\u0c41\u0c17\u0c41"}, + "th_TH": {"name_en": u"Thai", "name": u"\u0e20\u0e32\u0e29\u0e32\u0e44\u0e17\u0e22"}, + "tl_PH": {"name_en": u"Filipino", "name": u"Filipino"}, + "tr_TR": {"name_en": u"Turkish", "name": u"T\xfcrk\xe7e"}, + "uk_UA": {"name_en": u"Ukraini ", "name": u"\u0423\u043a\u0440\u0430\u0457\u043d\u0441\u044c\u043a\u0430"}, + "vi_VN": {"name_en": u"Vietnamese", "name": u"Ti\u1ebfng Vi\u1ec7t"}, + "zh_CN": {"name_en": u"Chinese (Simplified)", "name": u"\xe4\xe6(\xe7\xe4)"}, + "zh_HK": {"name_en": u"Chinese (Hong Kong)", "name": u"\xe4\xe6(\xe9\xe6)"}, + "zh_TW": {"name_en": u"Chinese (Taiwan)", "name": u"\xe4\xe6(\xe5\xe7)"}, +} diff --git a/vendor/tornado/tornado/options.py b/vendor/tornado/tornado/options.py new file mode 100644 index 0000000000..66bce091e7 --- /dev/null +++ b/vendor/tornado/tornado/options.py @@ -0,0 +1,386 @@ +#!/usr/bin/env python +# +# Copyright 2009 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""A command line parsing module that lets modules define their own options. + +Each module defines its own options, e.g., + + from tornado.options import define, options + + define("mysql_host", default="127.0.0.1:3306", help="Main user DB") + define("memcache_hosts", default="127.0.0.1:11011", multiple=True, + help="Main user memcache servers") + + def connect(): + db = database.Connection(options.mysql_host) + ... + +The main() method of your application does not need to be aware of all of +the options used throughout your program; they are all automatically loaded +when the modules are loaded. Your main() method can parse the command line +or parse a config file with: + + import tornado.options + tornado.options.parse_config_file("/etc/server.conf") + tornado.options.parse_command_line() + +Command line formats are what you would expect ("--myoption=myvalue"). +Config files are just Python files. Global names become options, e.g., + + myoption = "myvalue" + myotheroption = "myothervalue" + +We support datetimes, timedeltas, ints, and floats (just pass a 'type' +kwarg to define). We also accept multi-value options. See the documentation +for define() below. +""" + +import datetime +import logging +import logging.handlers +import re +import sys +import time + +# For pretty log messages, if available +try: + import curses +except: + curses = None + + +def define(name, default=None, type=str, help=None, metavar=None, + multiple=False): + """Defines a new command line option. + + If type is given (one of str, float, int, datetime, or timedelta), + we parse the command line arguments based on the given type. If + multiple is True, we accept comma-separated values, and the option + value is always a list. + + For multi-value integers, we also accept the syntax x:y, which + turns into range(x, y) - very useful for long integer ranges. + + help and metavar are used to construct the automatically generated + command line help string. The help message is formatted like: + + --name=METAVAR help string + + Command line option names must be unique globally. They can be parsed + from the command line with parse_command_line() or parsed from a + config file with parse_config_file. + """ + if name in options: + raise Error("Option %r already defined in %s", name, + options[name].file_name) + frame = sys._getframe(0) + options_file = frame.f_code.co_filename + file_name = frame.f_back.f_code.co_filename + if file_name == options_file: file_name = "" + options[name] = _Option(name, file_name=file_name, default=default, + type=type, help=help, metavar=metavar, + multiple=multiple) + + +def parse_command_line(args=None): + """Parses all options given on the command line. + + We return all command line arguments that are not options as a list. + """ + if args is None: args = sys.argv + remaining = [] + for i in xrange(1, len(args)): + # All things after the last option are command line arguments + if not args[i].startswith("-"): + remaining = args[i:] + break + if args[i] == "--": + remaining = args[i+1:] + break + arg = args[i].lstrip("-") + name, equals, value = arg.partition("=") + name = name.replace('-', '_') + if not name in options: + print_help() + raise Error('Unrecognized command line option: %r' % name) + option = options[name] + if not equals: + if option.type == bool: + value = "true" + else: + raise Error('Option %r requires a value' % name) + option.parse(value) + if options.help: + print_help() + sys.exit(0) + + # Set up log level and pretty console logging by default + if options.logging != 'none': + logging.getLogger().setLevel(getattr(logging, options.logging.upper())) + enable_pretty_logging() + + return remaining + + +def parse_config_file(path, overwrite=True): + """Parses and loads the Python config file at the given path.""" + config = {} + execfile(path, config, config) + for name in config: + if name in options: + options[name].set(config[name]) + + +def print_help(file=sys.stdout): + """Prints all the command line options to stdout.""" + print >> file, "Usage: %s [OPTIONS]" % sys.argv[0] + print >> file, "" + print >> file, "Options:" + by_file = {} + for option in options.itervalues(): + by_file.setdefault(option.file_name, []).append(option) + + for filename, o in sorted(by_file.items()): + if filename: print >> file, filename + o.sort(key=lambda option: option.name) + for option in o: + prefix = option.name + if option.metavar: + prefix += "=" + option.metavar + print >> file, " --%-30s %s" % (prefix, option.help or "") + print >> file + + +class _Options(dict): + """Our global program options, an dictionary with object-like access.""" + @classmethod + def instance(cls): + if not hasattr(cls, "_instance"): + cls._instance = cls() + return cls._instance + + def __getattr__(self, name): + if isinstance(self.get(name), _Option): + return self[name].value() + raise AttributeError("Unrecognized option %r" % name) + + +class _Option(object): + def __init__(self, name, default=None, type=str, help=None, metavar=None, + multiple=False, file_name=None): + if default is None and multiple: + default = [] + self.name = name + self.type = type + self.help = help + self.metavar = metavar + self.multiple = multiple + self.file_name = file_name + self.default = default + self._value = None + + def value(self): + return self.default if self._value is None else self._value + + def parse(self, value): + _parse = { + datetime.datetime: self._parse_datetime, + datetime.timedelta: self._parse_timedelta, + bool: self._parse_bool, + str: self._parse_string, + }.get(self.type, self.type) + if self.multiple: + if self._value is None: + self._value = [] + for part in value.split(","): + if self.type in (int, long): + # allow ranges of the form X:Y (inclusive at both ends) + lo, _, hi = part.partition(":") + lo = _parse(lo) + hi = _parse(hi) if hi else lo + self._value.extend(range(lo, hi+1)) + else: + self._value.append(_parse(part)) + else: + self._value = _parse(value) + return self.value() + + def set(self, value): + if self.multiple: + if not isinstance(value, list): + raise Error("Option %r is required to be a list of %s" % + (self.name, self.type.__name__)) + for item in value: + if item != None and not isinstance(item, self.type): + raise Error("Option %r is required to be a list of %s" % + (self.name, self.type.__name__)) + else: + if value != None and not isinstance(value, self.type): + raise Error("Option %r is required to be a %s" % + (self.name, self.type.__name__)) + self._value = value + + # Supported date/time formats in our options + _DATETIME_FORMATS = [ + "%a %b %d %H:%M:%S %Y", + "%Y-%m-%d %H:%M:%S", + "%Y-%m-%d %H:%M", + "%Y-%m-%dT%H:%M", + "%Y%m%d %H:%M:%S", + "%Y%m%d %H:%M", + "%Y-%m-%d", + "%Y%m%d", + "%H:%M:%S", + "%H:%M", + ] + + def _parse_datetime(self, value): + for format in self._DATETIME_FORMATS: + try: + return datetime.datetime.strptime(value, format) + except ValueError: + pass + raise Error('Unrecognized date/time format: %r' % value) + + _TIMEDELTA_ABBREVS = [ + ('hours', ['h']), + ('minutes', ['m', 'min']), + ('seconds', ['s', 'sec']), + ('milliseconds', ['ms']), + ('microseconds', ['us']), + ('days', ['d']), + ('weeks', ['w']), + ] + + _TIMEDELTA_ABBREV_DICT = dict( + (abbrev, full) for full, abbrevs in _TIMEDELTA_ABBREVS + for abbrev in abbrevs) + + _FLOAT_PATTERN = r'[-+]?(?:\d+(?:\.\d*)?|\.\d+)(?:[eE][-+]?\d+)?' + + _TIMEDELTA_PATTERN = re.compile( + r'\s*(%s)\s*(\w*)\s*' % _FLOAT_PATTERN, re.IGNORECASE) + + def _parse_timedelta(self, value): + try: + sum = datetime.timedelta() + start = 0 + while start < len(value): + m = self._TIMEDELTA_PATTERN.match(value, start) + if not m: + raise Exception() + num = float(m.group(1)) + units = m.group(2) or 'seconds' + units = self._TIMEDELTA_ABBREV_DICT.get(units, units) + sum += datetime.timedelta(**{units: num}) + start = m.end() + return sum + except: + raise + + def _parse_bool(self, value): + return value.lower() not in ("false", "0", "f") + + def _parse_string(self, value): + return value.decode("utf-8") + + +class Error(Exception): + pass + + +def enable_pretty_logging(): + """Turns on formatted logging output as configured.""" + if (options.log_to_stderr or + (options.log_to_stderr is None and not options.log_file_prefix)): + # Set up color if we are in a tty and curses is installed + color = False + if curses and sys.stderr.isatty(): + try: + curses.setupterm() + if curses.tigetnum("colors") > 0: + color = True + except: + pass + channel = logging.StreamHandler() + channel.setFormatter(_LogFormatter(color=color)) + logging.getLogger().addHandler(channel) + + if options.log_file_prefix: + channel = logging.handlers.RotatingFileHandler( + filename=options.log_file_prefix, + maxBytes=options.log_file_max_size, + backupCount=options.log_file_num_backups) + channel.setFormatter(_LogFormatter(color=False)) + logging.getLogger().addHandler(channel) + + +class _LogFormatter(logging.Formatter): + def __init__(self, color, *args, **kwargs): + logging.Formatter.__init__(self, *args, **kwargs) + self._color = color + if color: + fg_color = curses.tigetstr("setaf") or curses.tigetstr("setf") or "" + self._colors = { + logging.DEBUG: curses.tparm(fg_color, 4), # Blue + logging.INFO: curses.tparm(fg_color, 2), # Green + logging.WARNING: curses.tparm(fg_color, 3), # Yellow + logging.ERROR: curses.tparm(fg_color, 1), # Red + } + self._normal = curses.tigetstr("sgr0") + + def format(self, record): + try: + record.message = record.getMessage() + except Exception, e: + record.message = "Bad message (%r): %r" % (e, record.__dict__) + record.asctime = time.strftime( + "%y%m%d %H:%M:%S", self.converter(record.created)) + prefix = '[%(levelname)1.1s %(asctime)s %(module)s:%(lineno)d]' % \ + record.__dict__ + if self._color: + prefix = (self._colors.get(record.levelno, self._normal) + + prefix + self._normal) + formatted = prefix + " " + record.message + if record.exc_info: + if not record.exc_text: + record.exc_text = self.formatException(record.exc_info) + if record.exc_text: + formatted = formatted.rstrip() + "\n" + record.exc_text + return formatted.replace("\n", "\n ") + + +options = _Options.instance() + + +# Default options +define("help", type=bool, help="show this help information") +define("logging", default="info", + help=("Set the Python log level. If 'none', tornado won't touch the " + "logging configuration."), + metavar="info|warning|error|none") +define("log_to_stderr", type=bool, default=None, + help=("Send log output to stderr (colorized if possible). " + "By default use stderr if --log_file_prefix is not set.")) +define("log_file_prefix", type=str, default=None, metavar="PATH", + help=("Path prefix for log files. " + "Note that if you are running multiple tornado processes, " + "log_file_prefix must be different for each of them (e.g. " + "include the port number)")) +define("log_file_max_size", type=int, default=100 * 1000 * 1000, + help="max size of log files before rollover") +define("log_file_num_backups", type=int, default=10, + help="number of log files to keep") diff --git a/vendor/tornado/tornado/s3server.py b/vendor/tornado/tornado/s3server.py new file mode 100644 index 0000000000..2e8a97de20 --- /dev/null +++ b/vendor/tornado/tornado/s3server.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python +# +# Copyright 2009 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Implementation of an S3-like storage server based on local files. + +Useful to test features that will eventually run on S3, or if you want to +run something locally that was once running on S3. + +We don't support all the features of S3, but it does work with the +standard S3 client for the most basic semantics. To use the standard +S3 client with this module: + + c = S3.AWSAuthConnection("", "", server="localhost", port=8888, + is_secure=False) + c.create_bucket("mybucket") + c.put("mybucket", "mykey", "a value") + print c.get("mybucket", "mykey").body + +""" + +import bisect +import datetime +import escape +import hashlib +import httpserver +import ioloop +import os +import os.path +import urllib +import web + + +def start(port, root_directory="/tmp/s3", bucket_depth=0): + """Starts the mock S3 server on the given port at the given path.""" + application = S3Application(root_directory, bucket_depth) + http_server = httpserver.HTTPServer(application) + http_server.listen(port) + ioloop.IOLoop.instance().start() + + +class S3Application(web.Application): + """Implementation of an S3-like storage server based on local files. + + If bucket depth is given, we break files up into multiple directories + to prevent hitting file system limits for number of files in each + directories. 1 means one level of directories, 2 means 2, etc. + """ + def __init__(self, root_directory, bucket_depth=0): + web.Application.__init__(self, [ + (r"/", RootHandler), + (r"/([^/]+)/(.+)", ObjectHandler), + (r"/([^/]+)/", BucketHandler), + ]) + self.directory = os.path.abspath(root_directory) + if not os.path.exists(self.directory): + os.makedirs(self.directory) + self.bucket_depth = bucket_depth + + +class BaseRequestHandler(web.RequestHandler): + SUPPORTED_METHODS = ("PUT", "GET", "DELETE") + + def render_xml(self, value): + assert isinstance(value, dict) and len(value) == 1 + self.set_header("Content-Type", "application/xml; charset=UTF-8") + name = value.keys()[0] + parts = [] + parts.append('<' + escape.utf8(name) + + ' xmlns="http://doc.s3.amazonaws.com/2006-03-01">') + self._render_parts(value.values()[0], parts) + parts.append('</' + escape.utf8(name) + '>') + self.finish('<?xml version="1.0" encoding="UTF-8"?>\n' + + ''.join(parts)) + + def _render_parts(self, value, parts=[]): + if isinstance(value, basestring): + parts.append(escape.xhtml_escape(value)) + elif isinstance(value, int) or isinstance(value, long): + parts.append(str(value)) + elif isinstance(value, datetime.datetime): + parts.append(value.strftime("%Y-%m-%dT%H:%M:%S.000Z")) + elif isinstance(value, dict): + for name, subvalue in value.iteritems(): + if not isinstance(subvalue, list): + subvalue = [subvalue] + for subsubvalue in subvalue: + parts.append('<' + escape.utf8(name) + '>') + self._render_parts(subsubvalue, parts) + parts.append('</' + escape.utf8(name) + '>') + else: + raise Exception("Unknown S3 value type %r", value) + + def _object_path(self, bucket, object_name): + if self.application.bucket_depth < 1: + return os.path.abspath(os.path.join( + self.application.directory, bucket, object_name)) + hash = hashlib.md5(object_name).hexdigest() + path = os.path.abspath(os.path.join( + self.application.directory, bucket)) + for i in range(self.application.bucket_depth): + path = os.path.join(path, hash[:2 * (i + 1)]) + return os.path.join(path, object_name) + + +class RootHandler(BaseRequestHandler): + def get(self): + names = os.listdir(self.application.directory) + buckets = [] + for name in names: + path = os.path.join(self.application.directory, name) + info = os.stat(path) + buckets.append({ + "Name": name, + "CreationDate": datetime.datetime.utcfromtimestamp( + info.st_ctime), + }) + self.render_xml({"ListAllMyBucketsResult": { + "Buckets": {"Bucket": buckets}, + }}) + + +class BucketHandler(BaseRequestHandler): + def get(self, bucket_name): + prefix = self.get_argument("prefix", u"") + marker = self.get_argument("marker", u"") + max_keys = int(self.get_argument("max-keys", 50000)) + path = os.path.abspath(os.path.join(self.application.directory, + bucket_name)) + terse = int(self.get_argument("terse", 0)) + if not path.startswith(self.application.directory) or \ + not os.path.isdir(path): + raise web.HTTPError(404) + object_names = [] + for root, dirs, files in os.walk(path): + for file_name in files: + object_names.append(os.path.join(root, file_name)) + skip = len(path) + 1 + for i in range(self.application.bucket_depth): + skip += 2 * (i + 1) + 1 + object_names = [n[skip:] for n in object_names] + object_names.sort() + contents = [] + + start_pos = 0 + if marker: + start_pos = bisect.bisect_right(object_names, marker, start_pos) + if prefix: + start_pos = bisect.bisect_left(object_names, prefix, start_pos) + + truncated = False + for object_name in object_names[start_pos:]: + if not object_name.startswith(prefix): + break + if len(contents) >= max_keys: + truncated = True + break + object_path = self._object_path(bucket_name, object_name) + c = {"Key": object_name} + if not terse: + info = os.stat(object_path) + c.update({ + "LastModified": datetime.datetime.utcfromtimestamp( + info.st_mtime), + "Size": info.st_size, + }) + contents.append(c) + marker = object_name + self.render_xml({"ListBucketResult": { + "Name": bucket_name, + "Prefix": prefix, + "Marker": marker, + "MaxKeys": max_keys, + "IsTruncated": truncated, + "Contents": contents, + }}) + + def put(self, bucket_name): + path = os.path.abspath(os.path.join( + self.application.directory, bucket_name)) + if not path.startswith(self.application.directory) or \ + os.path.exists(path): + raise web.HTTPError(403) + os.makedirs(path) + self.finish() + + def delete(self, bucket_name): + path = os.path.abspath(os.path.join( + self.application.directory, bucket_name)) + if not path.startswith(self.application.directory) or \ + not os.path.isdir(path): + raise web.HTTPError(404) + if len(os.listdir(path)) > 0: + raise web.HTTPError(403) + os.rmdir(path) + self.set_status(204) + self.finish() + + +class ObjectHandler(BaseRequestHandler): + def get(self, bucket, object_name): + object_name = urllib.unquote(object_name) + path = self._object_path(bucket, object_name) + if not path.startswith(self.application.directory) or \ + not os.path.isfile(path): + raise web.HTTPError(404) + info = os.stat(path) + self.set_header("Content-Type", "application/unknown") + self.set_header("Last-Modified", datetime.datetime.utcfromtimestamp( + info.st_mtime)) + object_file = open(path, "r") + try: + self.finish(object_file.read()) + finally: + object_file.close() + + def put(self, bucket, object_name): + object_name = urllib.unquote(object_name) + bucket_dir = os.path.abspath(os.path.join( + self.application.directory, bucket)) + if not bucket_dir.startswith(self.application.directory) or \ + not os.path.isdir(bucket_dir): + raise web.HTTPError(404) + path = self._object_path(bucket, object_name) + if not path.startswith(bucket_dir) or os.path.isdir(path): + raise web.HTTPError(403) + directory = os.path.dirname(path) + if not os.path.exists(directory): + os.makedirs(directory) + object_file = open(path, "w") + object_file.write(self.request.body) + object_file.close() + self.finish() + + def delete(self, bucket, object_name): + object_name = urllib.unquote(object_name) + path = self._object_path(bucket, object_name) + if not path.startswith(self.application.directory) or \ + not os.path.isfile(path): + raise web.HTTPError(404) + os.unlink(path) + self.set_status(204) + self.finish() diff --git a/vendor/tornado/tornado/template.py b/vendor/tornado/tornado/template.py new file mode 100644 index 0000000000..7ed56cfa69 --- /dev/null +++ b/vendor/tornado/tornado/template.py @@ -0,0 +1,576 @@ +#!/usr/bin/env python +# +# Copyright 2009 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""A simple template system that compiles templates to Python code. + +Basic usage looks like: + + t = template.Template("<html>{{ myvalue }}</html>") + print t.generate(myvalue="XXX") + +Loader is a class that loads templates from a root directory and caches +the compiled templates: + + loader = template.Loader("/home/btaylor") + print loader.load("test.html").generate(myvalue="XXX") + +We compile all templates to raw Python. Error-reporting is currently... uh, +interesting. Syntax for the templates + + ### base.html + <html> + <head> + <title>{% block title %}Default title{% end %}</title> + </head> + <body> + <ul> + {% for student in students %} + {% block student %} + <li>{{ escape(student.name) }}</li> + {% end %} + {% end %} + </ul> + </body> + </html> + + ### bold.html + {% extends "base.html" %} + + {% block title %}A bolder title{% end %} + + {% block student %} + <li><span style="bold">{{ escape(student.name) }}</span></li> + {% block %} + +Unlike most other template systems, we do not put any restrictions on the +expressions you can include in your statements. if and for blocks get +translated exactly into Python, do you can do complex expressions like: + + {% for student in [p for p in people if p.student and p.age > 23] %} + <li>{{ escape(student.name) }}</li> + {% end %} + +Translating directly to Python means you can apply functions to expressions +easily, like the escape() function in the examples above. You can pass +functions in to your template just like any other variable: + + ### Python code + def add(x, y): + return x + y + template.execute(add=add) + + ### The template + {{ add(1, 2) }} + +We provide the functions escape(), url_escape(), json_encode(), and squeeze() +to all templates by default. +""" + +from __future__ import with_statement + +import cStringIO +import datetime +import escape +import logging +import os.path +import re + +_log = logging.getLogger('tornado.template') + +class Template(object): + """A compiled template. + + We compile into Python from the given template_string. You can generate + the template from variables with generate(). + """ + def __init__(self, template_string, name="<string>", loader=None, + compress_whitespace=None): + self.name = name + if compress_whitespace is None: + compress_whitespace = name.endswith(".html") or \ + name.endswith(".js") + reader = _TemplateReader(name, template_string) + self.file = _File(_parse(reader)) + self.code = self._generate_python(loader, compress_whitespace) + try: + self.compiled = compile(self.code, self.name, "exec") + except: + formatted_code = _format_code(self.code).rstrip() + _log.error("%s code:\n%s", self.name, formatted_code) + raise + + def generate(self, **kwargs): + """Generate this template with the given arguments.""" + namespace = { + "escape": escape.xhtml_escape, + "url_escape": escape.url_escape, + "json_encode": escape.json_encode, + "squeeze": escape.squeeze, + "datetime": datetime, + } + namespace.update(kwargs) + exec self.compiled in namespace + execute = namespace["_execute"] + try: + return execute() + except: + formatted_code = _format_code(self.code).rstrip() + _log.error("%s code:\n%s", self.name, formatted_code) + raise + + def _generate_python(self, loader, compress_whitespace): + buffer = cStringIO.StringIO() + try: + named_blocks = {} + ancestors = self._get_ancestors(loader) + ancestors.reverse() + for ancestor in ancestors: + ancestor.find_named_blocks(loader, named_blocks) + self.file.find_named_blocks(loader, named_blocks) + writer = _CodeWriter(buffer, named_blocks, loader, self, + compress_whitespace) + ancestors[0].generate(writer) + return buffer.getvalue() + finally: + buffer.close() + + def _get_ancestors(self, loader): + ancestors = [self.file] + for chunk in self.file.body.chunks: + if isinstance(chunk, _ExtendsBlock): + if not loader: + raise ParseError("{% extends %} block found, but no " + "template loader") + template = loader.load(chunk.name, self.name) + ancestors.extend(template._get_ancestors(loader)) + return ancestors + + +class Loader(object): + """A template loader that loads from a single root directory. + + You must use a template loader to use template constructs like + {% extends %} and {% include %}. Loader caches all templates after + they are loaded the first time. + """ + def __init__(self, root_directory): + self.root = os.path.abspath(root_directory) + self.templates = {} + + def reset(self): + self.templates = {} + + def resolve_path(self, name, parent_path=None): + if parent_path and not parent_path.startswith("<") and \ + not parent_path.startswith("/") and \ + not name.startswith("/"): + current_path = os.path.join(self.root, parent_path) + file_dir = os.path.dirname(os.path.abspath(current_path)) + relative_path = os.path.abspath(os.path.join(file_dir, name)) + if relative_path.startswith(self.root): + name = relative_path[len(self.root) + 1:] + return name + + def load(self, name, parent_path=None): + name = self.resolve_path(name, parent_path=parent_path) + if name not in self.templates: + path = os.path.join(self.root, name) + f = open(path, "r") + self.templates[name] = Template(f.read(), name=name, loader=self) + f.close() + return self.templates[name] + + +class _Node(object): + def each_child(self): + return () + + def generate(self, writer): + raise NotImplementedError() + + def find_named_blocks(self, loader, named_blocks): + for child in self.each_child(): + child.find_named_blocks(loader, named_blocks) + + +class _File(_Node): + def __init__(self, body): + self.body = body + + def generate(self, writer): + writer.write_line("def _execute():") + with writer.indent(): + writer.write_line("_buffer = []") + self.body.generate(writer) + writer.write_line("return ''.join(_buffer)") + + def each_child(self): + return (self.body,) + + + +class _ChunkList(_Node): + def __init__(self, chunks): + self.chunks = chunks + + def generate(self, writer): + for chunk in self.chunks: + chunk.generate(writer) + + def each_child(self): + return self.chunks + + +class _NamedBlock(_Node): + def __init__(self, name, body=None): + self.name = name + self.body = body + + def each_child(self): + return (self.body,) + + def generate(self, writer): + writer.named_blocks[self.name].generate(writer) + + def find_named_blocks(self, loader, named_blocks): + named_blocks[self.name] = self.body + _Node.find_named_blocks(self, loader, named_blocks) + + +class _ExtendsBlock(_Node): + def __init__(self, name): + self.name = name + + +class _IncludeBlock(_Node): + def __init__(self, name, reader): + self.name = name + self.template_name = reader.name + + def find_named_blocks(self, loader, named_blocks): + included = loader.load(self.name, self.template_name) + included.file.find_named_blocks(loader, named_blocks) + + def generate(self, writer): + included = writer.loader.load(self.name, self.template_name) + old = writer.current_template + writer.current_template = included + included.file.body.generate(writer) + writer.current_template = old + + +class _ApplyBlock(_Node): + def __init__(self, method, body=None): + self.method = method + self.body = body + + def each_child(self): + return (self.body,) + + def generate(self, writer): + method_name = "apply%d" % writer.apply_counter + writer.apply_counter += 1 + writer.write_line("def %s():" % method_name) + with writer.indent(): + writer.write_line("_buffer = []") + self.body.generate(writer) + writer.write_line("return ''.join(_buffer)") + writer.write_line("_buffer.append(%s(%s()))" % ( + self.method, method_name)) + + +class _ControlBlock(_Node): + def __init__(self, statement, body=None): + self.statement = statement + self.body = body + + def each_child(self): + return (self.body,) + + def generate(self, writer): + writer.write_line("%s:" % self.statement) + with writer.indent(): + self.body.generate(writer) + + +class _IntermediateControlBlock(_Node): + def __init__(self, statement): + self.statement = statement + + def generate(self, writer): + writer.write_line("%s:" % self.statement, writer.indent_size() - 1) + + +class _Statement(_Node): + def __init__(self, statement): + self.statement = statement + + def generate(self, writer): + writer.write_line(self.statement) + + +class _Expression(_Node): + def __init__(self, expression): + self.expression = expression + + def generate(self, writer): + writer.write_line("_tmp = %s" % self.expression) + writer.write_line("if isinstance(_tmp, str): _buffer.append(_tmp)") + writer.write_line("elif isinstance(_tmp, unicode): " + "_buffer.append(_tmp.encode('utf-8'))") + writer.write_line("else: _buffer.append(str(_tmp))") + + +class _Text(_Node): + def __init__(self, value): + self.value = value + + def generate(self, writer): + value = self.value + + # Compress lots of white space to a single character. If the whitespace + # breaks a line, have it continue to break a line, but just with a + # single \n character + if writer.compress_whitespace and "<pre>" not in value: + value = re.sub(r"([\t ]+)", " ", value) + value = re.sub(r"(\s*\n\s*)", "\n", value) + + if value: + writer.write_line('_buffer.append(%r)' % value) + + +class ParseError(Exception): + """Raised for template syntax errors.""" + pass + + +class _CodeWriter(object): + def __init__(self, file, named_blocks, loader, current_template, + compress_whitespace): + self.file = file + self.named_blocks = named_blocks + self.loader = loader + self.current_template = current_template + self.compress_whitespace = compress_whitespace + self.apply_counter = 0 + self._indent = 0 + + def indent(self): + return self + + def indent_size(self): + return self._indent + + def __enter__(self): + self._indent += 1 + return self + + def __exit__(self, *args): + assert self._indent > 0 + self._indent -= 1 + + def write_line(self, line, indent=None): + if indent == None: + indent = self._indent + for i in xrange(indent): + self.file.write(" ") + print >> self.file, line + + +class _TemplateReader(object): + def __init__(self, name, text): + self.name = name + self.text = text + self.line = 0 + self.pos = 0 + + def find(self, needle, start=0, end=None): + assert start >= 0, start + pos = self.pos + start += pos + if end is None: + index = self.text.find(needle, start) + else: + end += pos + assert end >= start + index = self.text.find(needle, start, end) + if index != -1: + index -= pos + return index + + def consume(self, count=None): + if count is None: + count = len(self.text) - self.pos + newpos = self.pos + count + self.line += self.text.count("\n", self.pos, newpos) + s = self.text[self.pos:newpos] + self.pos = newpos + return s + + def remaining(self): + return len(self.text) - self.pos + + def __len__(self): + return self.remaining() + + def __getitem__(self, key): + if type(key) is slice: + size = len(self) + start, stop, step = slice.indices(size) + if start is None: start = self.pos + else: start += self.pos + if stop is not None: stop += self.pos + return self.text[slice(start, stop, step)] + elif key < 0: + return self.text[key] + else: + return self.text[self.pos + key] + + def __str__(self): + return self.text[self.pos:] + + +def _format_code(code): + lines = code.splitlines() + format = "%%%dd %%s\n" % len(repr(len(lines) + 1)) + return "".join([format % (i + 1, line) for (i, line) in enumerate(lines)]) + + +def _parse(reader, in_block=None): + body = _ChunkList([]) + while True: + # Find next template directive + curly = 0 + while True: + curly = reader.find("{", curly) + if curly == -1 or curly + 1 == reader.remaining(): + # EOF + if in_block: + raise ParseError("Missing {%% end %%} block for %s" % + in_block) + body.chunks.append(_Text(reader.consume())) + return body + # If the first curly brace is not the start of a special token, + # start searching from the character after it + if reader[curly + 1] not in ("{", "%"): + curly += 1 + continue + # When there are more than 2 curlies in a row, use the + # innermost ones. This is useful when generating languages + # like latex where curlies are also meaningful + if (curly + 2 < reader.remaining() and + reader[curly + 1] == '{' and reader[curly + 2] == '{'): + curly += 1 + continue + break + + # Append any text before the special token + if curly > 0: + body.chunks.append(_Text(reader.consume(curly))) + + start_brace = reader.consume(2) + line = reader.line + + # Expression + if start_brace == "{{": + end = reader.find("}}") + if end == -1 or reader.find("\n", 0, end) != -1: + raise ParseError("Missing end expression }} on line %d" % line) + contents = reader.consume(end).strip() + reader.consume(2) + if not contents: + raise ParseError("Empty expression on line %d" % line) + body.chunks.append(_Expression(contents)) + continue + + # Block + assert start_brace == "{%", start_brace + end = reader.find("%}") + if end == -1 or reader.find("\n", 0, end) != -1: + raise ParseError("Missing end block %%} on line %d" % line) + contents = reader.consume(end).strip() + reader.consume(2) + if not contents: + raise ParseError("Empty block tag ({%% %%}) on line %d" % line) + + operator, space, suffix = contents.partition(" ") + suffix = suffix.strip() + + # Intermediate ("else", "elif", etc) blocks + intermediate_blocks = { + "else": set(["if", "for", "while"]), + "elif": set(["if"]), + "except": set(["try"]), + "finally": set(["try"]), + } + allowed_parents = intermediate_blocks.get(operator) + if allowed_parents is not None: + if not in_block: + raise ParseError("%s outside %s block" % + (operator, allowed_parents)) + if in_block not in allowed_parents: + raise ParseError("%s block cannot be attached to %s block" % (operator, in_block)) + body.chunks.append(_IntermediateControlBlock(contents)) + continue + + # End tag + elif operator == "end": + if not in_block: + raise ParseError("Extra {%% end %%} block on line %d" % line) + return body + + elif operator in ("extends", "include", "set", "import", "comment"): + if operator == "comment": + continue + if operator == "extends": + suffix = suffix.strip('"').strip("'") + if not suffix: + raise ParseError("extends missing file path on line %d" % line) + block = _ExtendsBlock(suffix) + elif operator == "import": + if not suffix: + raise ParseError("import missing statement on line %d" % line) + block = _Statement(contents) + elif operator == "include": + suffix = suffix.strip('"').strip("'") + if not suffix: + raise ParseError("include missing file path on line %d" % line) + block = _IncludeBlock(suffix, reader) + elif operator == "set": + if not suffix: + raise ParseError("set missing statement on line %d" % line) + block = _Statement(suffix) + body.chunks.append(block) + continue + + elif operator in ("apply", "block", "try", "if", "for", "while"): + # parse inner body recursively + block_body = _parse(reader, operator) + if operator == "apply": + if not suffix: + raise ParseError("apply missing method name on line %d" % line) + block = _ApplyBlock(suffix, block_body) + elif operator == "block": + if not suffix: + raise ParseError("block missing name on line %d" % line) + block = _NamedBlock(suffix, block_body) + else: + block = _ControlBlock(contents, block_body) + body.chunks.append(block) + continue + + else: + raise ParseError("unknown operator: %r" % operator) diff --git a/vendor/tornado/tornado/test/README b/vendor/tornado/tornado/test/README new file mode 100644 index 0000000000..2d6195d807 --- /dev/null +++ b/vendor/tornado/tornado/test/README @@ -0,0 +1,4 @@ +Test coverage is almost non-existent, but it's a start. Be sure to +set PYTHONPATH apprioriately (generally to the root directory of your +tornado checkout) when running tests to make sure you're getting the +version of the tornado package that you expect.
\ No newline at end of file diff --git a/vendor/tornado/tornado/test/test_ioloop.py b/vendor/tornado/tornado/test/test_ioloop.py new file mode 100755 index 0000000000..2541fa87e1 --- /dev/null +++ b/vendor/tornado/tornado/test/test_ioloop.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python + +import unittest +import time + +from tornado import ioloop + + +class TestIOLoop(unittest.TestCase): + def setUp(self): + self.loop = ioloop.IOLoop() + + def tearDown(self): + pass + + def _callback(self): + self.called = True + self.loop.stop() + + def _schedule_callback(self): + self.loop.add_callback(self._callback) + # Scroll away the time so we can check if we woke up immediately + self._start_time = time.time() + self.called = False + + def test_add_callback(self): + self.loop.add_timeout(time.time(), self._schedule_callback) + self.loop.start() # Set some long poll timeout so we can check wakeup + self.assertAlmostEqual(time.time(), self._start_time, places=2) + self.assertTrue(self.called) + + +if __name__ == "__main__": + import logging + + logging.basicConfig(level=logging.DEBUG, format='%(asctime)s:%(msecs)03d %(levelname)-8s %(name)-8s %(message)s', datefmt='%H:%M:%S') + + unittest.main() diff --git a/vendor/tornado/tornado/web.py b/vendor/tornado/tornado/web.py new file mode 100644 index 0000000000..7559fae8a5 --- /dev/null +++ b/vendor/tornado/tornado/web.py @@ -0,0 +1,1445 @@ +#!/usr/bin/env python +# +# Copyright 2009 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""The Tornado web framework. + +The Tornado web framework looks a bit like web.py (http://webpy.org/) or +Google's webapp (http://code.google.com/appengine/docs/python/tools/webapp/), +but with additional tools and optimizations to take advantage of the +Tornado non-blocking web server and tools. + +Here is the canonical "Hello, world" example app: + + import tornado.httpserver + import tornado.ioloop + import tornado.web + + class MainHandler(tornado.web.RequestHandler): + def get(self): + self.write("Hello, world") + + if __name__ == "__main__": + application = tornado.web.Application([ + (r"/", MainHandler), + ]) + http_server = tornado.httpserver.HTTPServer(application) + http_server.listen(8888) + tornado.ioloop.IOLoop.instance().start() + +See the Tornado walkthrough on GitHub for more details and a good +getting started guide. +""" + +import base64 +import binascii +import calendar +import Cookie +import cStringIO +import datetime +import email.utils +import escape +import functools +import gzip +import hashlib +import hmac +import httplib +import locale +import logging +import mimetypes +import os.path +import re +import stat +import sys +import template +import time +import types +import urllib +import urlparse +import uuid + +_log = logging.getLogger('tornado.web') + +class RequestHandler(object): + """Subclass this class and define get() or post() to make a handler. + + If you want to support more methods than the standard GET/HEAD/POST, you + should override the class variable SUPPORTED_METHODS in your + RequestHandler class. + """ + SUPPORTED_METHODS = ("GET", "HEAD", "POST", "DELETE", "PUT") + + def __init__(self, application, request, transforms=None): + self.application = application + self.request = request + self._headers_written = False + self._finished = False + self._auto_finish = True + self._transforms = transforms or [] + self.ui = _O((n, self._ui_method(m)) for n, m in + application.ui_methods.iteritems()) + self.ui["modules"] = _O((n, self._ui_module(n, m)) for n, m in + application.ui_modules.iteritems()) + self.clear() + # Check since connection is not available in WSGI + if hasattr(self.request, "connection"): + self.request.connection.stream.set_close_callback( + self.on_connection_close) + + @property + def settings(self): + return self.application.settings + + def head(self, *args, **kwargs): + raise HTTPError(405) + + def get(self, *args, **kwargs): + raise HTTPError(405) + + def post(self, *args, **kwargs): + raise HTTPError(405) + + def delete(self, *args, **kwargs): + raise HTTPError(405) + + def put(self, *args, **kwargs): + raise HTTPError(405) + + def prepare(self): + """Called before the actual handler method. + + Useful to override in a handler if you want a common bottleneck for + all of your requests. + """ + pass + + def on_connection_close(self): + """Called in async handlers if the client closed the connection. + + You may override this to clean up resources associated with + long-lived connections. + + Note that the select()-based implementation of IOLoop does not detect + closed connections and so this method will not be called until + you try (and fail) to produce some output. The epoll- and kqueue- + based implementations should detect closed connections even while + the request is idle. + """ + pass + + def clear(self): + """Resets all headers and content for this response.""" + self._headers = { + "Server": "TornadoServer/0.1", + "Content-Type": "text/html; charset=UTF-8", + } + if not self.request.supports_http_1_1(): + if self.request.headers.get("Connection") == "Keep-Alive": + self.set_header("Connection", "Keep-Alive") + self._write_buffer = [] + self._status_code = 200 + + def set_status(self, status_code): + """Sets the status code for our response.""" + assert status_code in httplib.responses + self._status_code = status_code + + def set_header(self, name, value): + """Sets the given response header name and value. + + If a datetime is given, we automatically format it according to the + HTTP specification. If the value is not a string, we convert it to + a string. All header values are then encoded as UTF-8. + """ + if isinstance(value, datetime.datetime): + t = calendar.timegm(value.utctimetuple()) + value = email.utils.formatdate(t, localtime=False, usegmt=True) + elif isinstance(value, int) or isinstance(value, long): + value = str(value) + else: + value = _utf8(value) + # If \n is allowed into the header, it is possible to inject + # additional headers or split the request. Also cap length to + # prevent obviously erroneous values. + safe_value = re.sub(r"[\x00-\x1f]", " ", value)[:4000] + if safe_value != value: + raise ValueError("Unsafe header value %r", value) + self._headers[name] = value + + _ARG_DEFAULT = [] + def get_argument(self, name, default=_ARG_DEFAULT, strip=True): + """Returns the value of the argument with the given name. + + If default is not provided, the argument is considered to be + required, and we throw an HTTP 404 exception if it is missing. + + The returned value is always unicode. + """ + values = self.request.arguments.get(name, None) + if values is None: + if default is self._ARG_DEFAULT: + raise HTTPError(404, "Missing argument %s" % name) + return default + # Get rid of any weird control chars + value = re.sub(r"[\x00-\x08\x0e-\x1f]", " ", values[-1]) + value = _unicode(value) + if strip: value = value.strip() + return value + + @property + def cookies(self): + """A dictionary of Cookie.Morsel objects.""" + if not hasattr(self, "_cookies"): + self._cookies = Cookie.BaseCookie() + if "Cookie" in self.request.headers: + try: + self._cookies.load(self.request.headers["Cookie"]) + except: + self.clear_all_cookies() + return self._cookies + + def get_cookie(self, name, default=None): + """Gets the value of the cookie with the given name, else default.""" + if name in self.cookies: + return self.cookies[name].value + return default + + def set_cookie(self, name, value, domain=None, expires=None, path="/", + expires_days=None): + """Sets the given cookie name/value with the given options.""" + name = _utf8(name) + value = _utf8(value) + if re.search(r"[\x00-\x20]", name + value): + # Don't let us accidentally inject bad stuff + raise ValueError("Invalid cookie %r: %r" % (name, value)) + if not hasattr(self, "_new_cookies"): + self._new_cookies = [] + new_cookie = Cookie.BaseCookie() + self._new_cookies.append(new_cookie) + new_cookie[name] = value + if domain: + new_cookie[name]["domain"] = domain + if expires_days is not None and not expires: + expires = datetime.datetime.utcnow() + datetime.timedelta( + days=expires_days) + if expires: + timestamp = calendar.timegm(expires.utctimetuple()) + new_cookie[name]["expires"] = email.utils.formatdate( + timestamp, localtime=False, usegmt=True) + if path: + new_cookie[name]["path"] = path + + def clear_cookie(self, name, path="/", domain=None): + """Deletes the cookie with the given name.""" + expires = datetime.datetime.utcnow() - datetime.timedelta(days=365) + self.set_cookie(name, value="", path=path, expires=expires, + domain=domain) + + def clear_all_cookies(self): + """Deletes all the cookies the user sent with this request.""" + for name in self.cookies.iterkeys(): + self.clear_cookie(name) + + def set_secure_cookie(self, name, value, expires_days=30, **kwargs): + """Signs and timestamps a cookie so it cannot be forged. + + You must specify the 'cookie_secret' setting in your Application + to use this method. It should be a long, random sequence of bytes + to be used as the HMAC secret for the signature. + + To read a cookie set with this method, use get_secure_cookie(). + """ + timestamp = str(int(time.time())) + value = base64.b64encode(value) + signature = self._cookie_signature(name, value, timestamp) + value = "|".join([value, timestamp, signature]) + self.set_cookie(name, value, expires_days=expires_days, **kwargs) + + def get_secure_cookie(self, name, include_name=True, value=None): + """Returns the given signed cookie if it validates, or None. + + In older versions of Tornado (0.1 and 0.2), we did not include the + name of the cookie in the cookie signature. To read these old-style + cookies, pass include_name=False to this method. Otherwise, all + attempts to read old-style cookies will fail (and you may log all + your users out whose cookies were written with a previous Tornado + version). + """ + if value is None: value = self.get_cookie(name) + if not value: return None + parts = value.split("|") + if len(parts) != 3: return None + if include_name: + signature = self._cookie_signature(name, parts[0], parts[1]) + else: + signature = self._cookie_signature(parts[0], parts[1]) + if not _time_independent_equals(parts[2], signature): + _log.warning("Invalid cookie signature %r", value) + return None + timestamp = int(parts[1]) + if timestamp < time.time() - 31 * 86400: + _log.warning("Expired cookie %r", value) + return None + try: + return base64.b64decode(parts[0]) + except: + return None + + def _cookie_signature(self, *parts): + self.require_setting("cookie_secret", "secure cookies") + hash = hmac.new(self.application.settings["cookie_secret"], + digestmod=hashlib.sha1) + for part in parts: hash.update(part) + return hash.hexdigest() + + def redirect(self, url, permanent=False): + """Sends a redirect to the given (optionally relative) URL.""" + if self._headers_written: + raise Exception("Cannot redirect after headers have been written") + self.set_status(301 if permanent else 302) + # Remove whitespace + url = re.sub(r"[\x00-\x20]+", "", _utf8(url)) + self.set_header("Location", urlparse.urljoin(self.request.uri, url)) + self.finish() + + def write(self, chunk): + """Writes the given chunk to the output buffer. + + To write the output to the network, use the flush() method below. + + If the given chunk is a dictionary, we write it as JSON and set + the Content-Type of the response to be text/javascript. + """ + assert not self._finished + if isinstance(chunk, dict): + chunk = escape.json_encode(chunk) + self.set_header("Content-Type", "text/javascript; charset=UTF-8") + chunk = _utf8(chunk) + self._write_buffer.append(chunk) + + def render(self, template_name, **kwargs): + """Renders the template with the given arguments as the response.""" + html = self.render_string(template_name, **kwargs) + + # Insert the additional JS and CSS added by the modules on the page + js_embed = [] + js_files = [] + css_embed = [] + css_files = [] + html_heads = [] + html_bodies = [] + for module in getattr(self, "_active_modules", {}).itervalues(): + embed_part = module.embedded_javascript() + if embed_part: js_embed.append(_utf8(embed_part)) + file_part = module.javascript_files() + if file_part: + if isinstance(file_part, basestring): + js_files.append(file_part) + else: + js_files.extend(file_part) + embed_part = module.embedded_css() + if embed_part: css_embed.append(_utf8(embed_part)) + file_part = module.css_files() + if file_part: + if isinstance(file_part, basestring): + css_files.append(file_part) + else: + css_files.extend(file_part) + head_part = module.html_head() + if head_part: html_heads.append(_utf8(head_part)) + body_part = module.html_body() + if body_part: html_bodies.append(_utf8(body_part)) + if js_files: + # Maintain order of JavaScript files given by modules + paths = [] + unique_paths = set() + for path in js_files: + if not path.startswith("/") and not path.startswith("http:"): + path = self.static_url(path) + if path not in unique_paths: + paths.append(path) + unique_paths.add(path) + js = ''.join('<script src="' + escape.xhtml_escape(p) + + '" type="text/javascript"></script>' + for p in paths) + sloc = html.rindex('</body>') + html = html[:sloc] + js + '\n' + html[sloc:] + if js_embed: + js = '<script type="text/javascript">\n//<![CDATA[\n' + \ + '\n'.join(js_embed) + '\n//]]>\n</script>' + sloc = html.rindex('</body>') + html = html[:sloc] + js + '\n' + html[sloc:] + if css_files: + paths = set() + for path in css_files: + if not path.startswith("/") and not path.startswith("http:"): + paths.add(self.static_url(path)) + else: + paths.add(path) + css = ''.join('<link href="' + escape.xhtml_escape(p) + '" ' + 'type="text/css" rel="stylesheet"/>' + for p in paths) + hloc = html.index('</head>') + html = html[:hloc] + css + '\n' + html[hloc:] + if css_embed: + css = '<style type="text/css">\n' + '\n'.join(css_embed) + \ + '\n</style>' + hloc = html.index('</head>') + html = html[:hloc] + css + '\n' + html[hloc:] + if html_heads: + hloc = html.index('</head>') + html = html[:hloc] + ''.join(html_heads) + '\n' + html[hloc:] + if html_bodies: + hloc = html.index('</body>') + html = html[:hloc] + ''.join(html_bodies) + '\n' + html[hloc:] + self.finish(html) + + def render_string(self, template_name, **kwargs): + """Generate the given template with the given arguments. + + We return the generated string. To generate and write a template + as a response, use render() above. + """ + # If no template_path is specified, use the path of the calling file + template_path = self.application.settings.get("template_path") + if not template_path: + frame = sys._getframe(0) + web_file = frame.f_code.co_filename + while frame.f_code.co_filename == web_file: + frame = frame.f_back + template_path = os.path.dirname(frame.f_code.co_filename) + if not getattr(RequestHandler, "_templates", None): + RequestHandler._templates = {} + if template_path not in RequestHandler._templates: + loader = self.application.settings.get("template_loader") or\ + template.Loader(template_path) + RequestHandler._templates[template_path] = loader + t = RequestHandler._templates[template_path].load(template_name) + args = dict( + handler=self, + request=self.request, + current_user=self.current_user, + locale=self.locale, + _=self.locale.translate, + static_url=self.static_url, + xsrf_form_html=self.xsrf_form_html, + reverse_url=self.application.reverse_url + ) + args.update(self.ui) + args.update(kwargs) + return t.generate(**args) + + def flush(self, include_footers=False): + """Flushes the current output buffer to the nextwork.""" + if self.application._wsgi: + raise Exception("WSGI applications do not support flush()") + + chunk = "".join(self._write_buffer) + self._write_buffer = [] + if not self._headers_written: + self._headers_written = True + for transform in self._transforms: + self._headers, chunk = transform.transform_first_chunk( + self._headers, chunk, include_footers) + headers = self._generate_headers() + else: + for transform in self._transforms: + chunk = transform.transform_chunk(chunk, include_footers) + headers = "" + + # Ignore the chunk and only write the headers for HEAD requests + if self.request.method == "HEAD": + if headers: self.request.write(headers) + return + + if headers or chunk: + self.request.write(headers + chunk) + + def finish(self, chunk=None): + """Finishes this response, ending the HTTP request.""" + assert not self._finished + if chunk is not None: self.write(chunk) + + # Automatically support ETags and add the Content-Length header if + # we have not flushed any content yet. + if not self._headers_written: + if (self._status_code == 200 and self.request.method == "GET" and + "Etag" not in self._headers): + hasher = hashlib.sha1() + for part in self._write_buffer: + hasher.update(part) + etag = '"%s"' % hasher.hexdigest() + inm = self.request.headers.get("If-None-Match") + if inm and inm.find(etag) != -1: + self._write_buffer = [] + self.set_status(304) + else: + self.set_header("Etag", etag) + if "Content-Length" not in self._headers: + content_length = sum(len(part) for part in self._write_buffer) + self.set_header("Content-Length", content_length) + + if not self.application._wsgi: + self.flush(include_footers=True) + self.request.finish() + self._log() + self._finished = True + + def send_error(self, status_code=500, **kwargs): + """Sends the given HTTP error code to the browser. + + We also send the error HTML for the given error code as returned by + get_error_html. Override that method if you want custom error pages + for your application. + """ + if self._headers_written: + _log.error("Cannot send error response after headers written") + if not self._finished: + self.finish() + return + self.clear() + self.set_status(status_code) + message = self.get_error_html(status_code, **kwargs) + self.finish(message) + + def get_error_html(self, status_code, **kwargs): + """Override to implement custom error pages. + + If this error was caused by an uncaught exception, the + exception object can be found in kwargs e.g. kwargs['exception'] + """ + return "<html><title>%(code)d: %(message)s</title>" \ + "<body>%(code)d: %(message)s</body></html>" % { + "code": status_code, + "message": httplib.responses[status_code], + } + + @property + def locale(self): + """The local for the current session. + + Determined by either get_user_locale, which you can override to + set the locale based on, e.g., a user preference stored in a + database, or get_browser_locale, which uses the Accept-Language + header. + """ + if not hasattr(self, "_locale"): + self._locale = self.get_user_locale() + if not self._locale: + self._locale = self.get_browser_locale() + assert self._locale + return self._locale + + def get_user_locale(self): + """Override to determine the locale from the authenticated user. + + If None is returned, we use the Accept-Language header. + """ + return None + + def get_browser_locale(self, default="en_US"): + """Determines the user's locale from Accept-Language header. + + See http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.4 + """ + if "Accept-Language" in self.request.headers: + languages = self.request.headers["Accept-Language"].split(",") + locales = [] + for language in languages: + parts = language.strip().split(";") + if len(parts) > 1 and parts[1].startswith("q="): + try: + score = float(parts[1][2:]) + except (ValueError, TypeError): + score = 0.0 + else: + score = 1.0 + locales.append((parts[0], score)) + if locales: + locales.sort(key=lambda (l, s): s, reverse=True) + codes = [l[0] for l in locales] + return locale.get(*codes) + return locale.get(default) + + @property + def current_user(self): + """The authenticated user for this request. + + Determined by either get_current_user, which you can override to + set the user based on, e.g., a cookie. If that method is not + overridden, this method always returns None. + + We lazy-load the current user the first time this method is called + and cache the result after that. + """ + if not hasattr(self, "_current_user"): + self._current_user = self.get_current_user() + return self._current_user + + def get_current_user(self): + """Override to determine the current user from, e.g., a cookie.""" + return None + + def get_login_url(self): + """Override to customize the login URL based on the request. + + By default, we use the 'login_url' application setting. + """ + self.require_setting("login_url", "@tornado.web.authenticated") + return self.application.settings["login_url"] + + @property + def xsrf_token(self): + """The XSRF-prevention token for the current user/session. + + To prevent cross-site request forgery, we set an '_xsrf' cookie + and include the same '_xsrf' value as an argument with all POST + requests. If the two do not match, we reject the form submission + as a potential forgery. + + See http://en.wikipedia.org/wiki/Cross-site_request_forgery + """ + if not hasattr(self, "_xsrf_token"): + token = self.get_cookie("_xsrf") + if not token: + token = binascii.b2a_hex(uuid.uuid4().bytes) + expires_days = 30 if self.current_user else None + self.set_cookie("_xsrf", token, expires_days=expires_days) + self._xsrf_token = token + return self._xsrf_token + + def check_xsrf_cookie(self): + """Verifies that the '_xsrf' cookie matches the '_xsrf' argument. + + To prevent cross-site request forgery, we set an '_xsrf' cookie + and include the same '_xsrf' value as an argument with all POST + requests. If the two do not match, we reject the form submission + as a potential forgery. + + See http://en.wikipedia.org/wiki/Cross-site_request_forgery + """ + if self.request.headers.get("X-Requested-With") == "XMLHttpRequest": + return + token = self.get_argument("_xsrf", None) + if not token: + raise HTTPError(403, "'_xsrf' argument missing from POST") + if self.xsrf_token != token: + raise HTTPError(403, "XSRF cookie does not match POST argument") + + def xsrf_form_html(self): + """An HTML <input/> element to be included with all POST forms. + + It defines the _xsrf input value, which we check on all POST + requests to prevent cross-site request forgery. If you have set + the 'xsrf_cookies' application setting, you must include this + HTML within all of your HTML forms. + + See check_xsrf_cookie() above for more information. + """ + return '<input type="hidden" name="_xsrf" value="' + \ + escape.xhtml_escape(self.xsrf_token) + '"/>' + + def static_url(self, path): + """Returns a static URL for the given relative static file path. + + This method requires you set the 'static_path' setting in your + application (which specifies the root directory of your static + files). + + We append ?v=<signature> to the returned URL, which makes our + static file handler set an infinite expiration header on the + returned content. The signature is based on the content of the + file. + + If this handler has a "include_host" attribute, we include the + full host for every static URL, including the "http://". Set + this attribute for handlers whose output needs non-relative static + path names. + """ + self.require_setting("static_path", "static_url") + if not hasattr(RequestHandler, "_static_hashes"): + RequestHandler._static_hashes = {} + hashes = RequestHandler._static_hashes + if path not in hashes: + try: + f = open(os.path.join( + self.application.settings["static_path"], path)) + hashes[path] = hashlib.md5(f.read()).hexdigest() + f.close() + except: + _log.error("Could not open static file %r", path) + hashes[path] = None + base = self.request.protocol + "://" + self.request.host \ + if getattr(self, "include_host", False) else "" + static_url_prefix = self.settings.get('static_url_prefix', '/static/') + if hashes.get(path): + return base + static_url_prefix + path + "?v=" + hashes[path][:5] + else: + return base + static_url_prefix + path + + def async_callback(self, callback, *args, **kwargs): + """Wrap callbacks with this if they are used on asynchronous requests. + + Catches exceptions and properly finishes the request. + """ + if callback is None: + return None + if args or kwargs: + callback = functools.partial(callback, *args, **kwargs) + def wrapper(*args, **kwargs): + try: + return callback(*args, **kwargs) + except Exception, e: + if self._headers_written: + _log.error("Exception after headers written", + exc_info=True) + else: + self._handle_request_exception(e) + return wrapper + + def require_setting(self, name, feature="this feature"): + """Raises an exception if the given app setting is not defined.""" + if not self.application.settings.get(name): + raise Exception("You must define the '%s' setting in your " + "application to use %s" % (name, feature)) + + def reverse_url(self, name, *args): + return self.application.reverse_url(name, *args) + + def _execute(self, transforms, *args, **kwargs): + """Executes this request with the given output transforms.""" + self._transforms = transforms + try: + if self.request.method not in self.SUPPORTED_METHODS: + raise HTTPError(405) + # If XSRF cookies are turned on, reject form submissions without + # the proper cookie + if self.request.method == "POST" and \ + self.application.settings.get("xsrf_cookies"): + self.check_xsrf_cookie() + self.prepare() + if not self._finished: + getattr(self, self.request.method.lower())(*args, **kwargs) + if self._auto_finish and not self._finished: + self.finish() + except Exception, e: + self._handle_request_exception(e) + + def _generate_headers(self): + lines = [self.request.version + " " + str(self._status_code) + " " + + httplib.responses[self._status_code]] + lines.extend(["%s: %s" % (n, v) for n, v in self._headers.iteritems()]) + for cookie_dict in getattr(self, "_new_cookies", []): + for cookie in cookie_dict.values(): + lines.append("Set-Cookie: " + cookie.OutputString(None)) + return "\r\n".join(lines) + "\r\n\r\n" + + def _log(self): + if self._status_code < 400: + log_method = _log.info + elif self._status_code < 500: + log_method = _log.warning + else: + log_method = _log.error + request_time = 1000.0 * self.request.request_time() + log_method("%d %s %.2fms", self._status_code, + self._request_summary(), request_time) + + def _request_summary(self): + return self.request.method + " " + self.request.uri + " (" + \ + self.request.remote_ip + ")" + + def _handle_request_exception(self, e): + if isinstance(e, HTTPError): + if e.log_message: + format = "%d %s: " + e.log_message + args = [e.status_code, self._request_summary()] + list(e.args) + _log.warning(format, *args) + if e.status_code not in httplib.responses: + _log.error("Bad HTTP status code: %d", e.status_code) + self.send_error(500, exception=e) + else: + self.send_error(e.status_code, exception=e) + else: + _log.error("Uncaught exception %s\n%r", self._request_summary(), + self.request, exc_info=e) + self.send_error(500, exception=e) + + def _ui_module(self, name, module): + def render(*args, **kwargs): + if not hasattr(self, "_active_modules"): + self._active_modules = {} + if name not in self._active_modules: + self._active_modules[name] = module(self) + rendered = self._active_modules[name].render(*args, **kwargs) + return rendered + return render + + def _ui_method(self, method): + return lambda *args, **kwargs: method(self, *args, **kwargs) + + +def asynchronous(method): + """Wrap request handler methods with this if they are asynchronous. + + If this decorator is given, the response is not finished when the + method returns. It is up to the request handler to call self.finish() + to finish the HTTP request. Without this decorator, the request is + automatically finished when the get() or post() method returns. + + class MyRequestHandler(web.RequestHandler): + @web.asynchronous + def get(self): + http = httpclient.AsyncHTTPClient() + http.fetch("http://friendfeed.com/", self._on_download) + + def _on_download(self, response): + self.write("Downloaded!") + self.finish() + + """ + @functools.wraps(method) + def wrapper(self, *args, **kwargs): + if self.application._wsgi: + raise Exception("@asynchronous is not supported for WSGI apps") + self._auto_finish = False + return method(self, *args, **kwargs) + return wrapper + + +def removeslash(method): + """Use this decorator to remove trailing slashes from the request path. + + For example, a request to '/foo/' would redirect to '/foo' with this + decorator. Your request handler mapping should use a regular expression + like r'/foo/*' in conjunction with using the decorator. + """ + @functools.wraps(method) + def wrapper(self, *args, **kwargs): + if self.request.path.endswith("/"): + if self.request.method == "GET": + uri = self.request.path.rstrip("/") + if self.request.query: uri += "?" + self.request.query + self.redirect(uri) + return + raise HTTPError(404) + return method(self, *args, **kwargs) + return wrapper + + +def addslash(method): + """Use this decorator to add a missing trailing slash to the request path. + + For example, a request to '/foo' would redirect to '/foo/' with this + decorator. Your request handler mapping should use a regular expression + like r'/foo/?' in conjunction with using the decorator. + """ + @functools.wraps(method) + def wrapper(self, *args, **kwargs): + if not self.request.path.endswith("/"): + if self.request.method == "GET": + uri = self.request.path + "/" + if self.request.query: uri += "?" + self.request.query + self.redirect(uri) + return + raise HTTPError(404) + return method(self, *args, **kwargs) + return wrapper + + +class Application(object): + """A collection of request handlers that make up a web application. + + Instances of this class are callable and can be passed directly to + HTTPServer to serve the application: + + application = web.Application([ + (r"/", MainPageHandler), + ]) + http_server = httpserver.HTTPServer(application) + http_server.listen(8080) + ioloop.IOLoop.instance().start() + + The constructor for this class takes in a list of URLSpec objects + or (regexp, request_class) tuples. When we receive requests, we + iterate over the list in order and instantiate an instance of the + first request class whose regexp matches the request path. + + Each tuple can contain an optional third element, which should be a + dictionary if it is present. That dictionary is passed as keyword + arguments to the contructor of the handler. This pattern is used + for the StaticFileHandler below: + + application = web.Application([ + (r"/static/(.*)", web.StaticFileHandler, {"path": "/var/www"}), + ]) + + We support virtual hosts with the add_handlers method, which takes in + a host regular expression as the first argument: + + application.add_handlers(r"www\.myhost\.com", [ + (r"/article/([0-9]+)", ArticleHandler), + ]) + + You can serve static files by sending the static_path setting as a + keyword argument. We will serve those files from the /static/ URI + (this is configurable with the static_url_prefix setting), + and we will serve /favicon.ico and /robots.txt from the same directory. + """ + def __init__(self, handlers=None, default_host="", transforms=None, + wsgi=False, **settings): + if transforms is None: + self.transforms = [] + if settings.get("gzip"): + self.transforms.append(GZipContentEncoding) + self.transforms.append(ChunkedTransferEncoding) + else: + self.transforms = transforms + self.handlers = [] + self.named_handlers = {} + self.default_host = default_host + self.settings = settings + self.ui_modules = {} + self.ui_methods = {} + self._wsgi = wsgi + self._load_ui_modules(settings.get("ui_modules", {})) + self._load_ui_methods(settings.get("ui_methods", {})) + if self.settings.get("static_path"): + path = self.settings["static_path"] + handlers = list(handlers or []) + static_url_prefix = settings.get("static_url_prefix", + "/static/") + handlers = [ + (re.escape(static_url_prefix) + r"(.*)", StaticFileHandler, + dict(path=path)), + (r"/(favicon\.ico)", StaticFileHandler, dict(path=path)), + (r"/(robots\.txt)", StaticFileHandler, dict(path=path)), + ] + handlers + if handlers: self.add_handlers(".*$", handlers) + + # Automatically reload modified modules + if self.settings.get("debug") and not wsgi: + import autoreload + autoreload.start() + + def add_handlers(self, host_pattern, host_handlers): + """Appends the given handlers to our handler list.""" + if not host_pattern.endswith("$"): + host_pattern += "$" + handlers = [] + # The handlers with the wildcard host_pattern are a special + # case - they're added in the constructor but should have lower + # precedence than the more-precise handlers added later. + # If a wildcard handler group exists, it should always be last + # in the list, so insert new groups just before it. + if self.handlers and self.handlers[-1][0].pattern == '.*$': + self.handlers.insert(-1, (re.compile(host_pattern), handlers)) + else: + self.handlers.append((re.compile(host_pattern), handlers)) + + for spec in host_handlers: + if type(spec) is type(()): + assert len(spec) in (2, 3) + pattern = spec[0] + handler = spec[1] + if len(spec) == 3: + kwargs = spec[2] + else: + kwargs = {} + spec = URLSpec(pattern, handler, kwargs) + handlers.append(spec) + if spec.name: + if spec.name in self.named_handlers: + _log.warning( + "Multiple handlers named %s; replacing previous value", + spec.name) + self.named_handlers[spec.name] = spec + + def add_transform(self, transform_class): + """Adds the given OutputTransform to our transform list.""" + self.transforms.append(transform_class) + + def _get_host_handlers(self, request): + host = request.host.lower().split(':')[0] + for pattern, handlers in self.handlers: + if pattern.match(host): + return handlers + # Look for default host if not behind load balancer (for debugging) + if "X-Real-Ip" not in request.headers: + for pattern, handlers in self.handlers: + if pattern.match(self.default_host): + return handlers + return None + + def _load_ui_methods(self, methods): + if type(methods) is types.ModuleType: + self._load_ui_methods(dict((n, getattr(methods, n)) + for n in dir(methods))) + elif isinstance(methods, list): + for m in list: self._load_ui_methods(m) + else: + for name, fn in methods.iteritems(): + if not name.startswith("_") and hasattr(fn, "__call__") \ + and name[0].lower() == name[0]: + self.ui_methods[name] = fn + + def _load_ui_modules(self, modules): + if type(modules) is types.ModuleType: + self._load_ui_modules(dict((n, getattr(modules, n)) + for n in dir(modules))) + elif isinstance(modules, list): + for m in list: self._load_ui_modules(m) + else: + assert isinstance(modules, dict) + for name, cls in modules.iteritems(): + try: + if issubclass(cls, UIModule): + self.ui_modules[name] = cls + except TypeError: + pass + + def __call__(self, request): + """Called by HTTPServer to execute the request.""" + transforms = [t(request) for t in self.transforms] + handler = None + args = [] + kwargs = {} + handlers = self._get_host_handlers(request) + if not handlers: + handler = RedirectHandler( + request, "http://" + self.default_host + "/") + else: + for spec in handlers: + match = spec.regex.match(request.path) + if match: + handler = spec.handler_class(self, request, **spec.kwargs) + # Pass matched groups to the handler. Since + # match.groups() includes both named and unnamed groups, + # we want to use either groups or groupdict but not both. + kwargs = match.groupdict() + if kwargs: + args = [] + else: + args = match.groups() + break + if not handler: + handler = ErrorHandler(self, request, 404) + + # In debug mode, re-compile templates and reload static files on every + # request so you don't need to restart to see changes + if self.settings.get("debug"): + if getattr(RequestHandler, "_templates", None): + map(lambda loader: loader.reset(), + RequestHandler._templates.values()) + RequestHandler._static_hashes = {} + + handler._execute(transforms, *args, **kwargs) + return handler + + def reverse_url(self, name, *args): + """Returns a URL path for handler named `name` + + The handler must be added to the application as a named URLSpec + """ + if name in self.named_handlers: + return self.named_handlers[name].reverse(*args) + raise KeyError("%s not found in named urls" % name) + + +class HTTPError(Exception): + """An exception that will turn into an HTTP error response.""" + def __init__(self, status_code, log_message=None, *args): + self.status_code = status_code + self.log_message = log_message + self.args = args + + def __str__(self): + message = "HTTP %d: %s" % ( + self.status_code, httplib.responses[self.status_code]) + if self.log_message: + return message + " (" + (self.log_message % self.args) + ")" + else: + return message + + +class ErrorHandler(RequestHandler): + """Generates an error response with status_code for all requests.""" + def __init__(self, application, request, status_code): + RequestHandler.__init__(self, application, request) + self.set_status(status_code) + + def prepare(self): + raise HTTPError(self._status_code) + + +class RedirectHandler(RequestHandler): + """Redirects the client to the given URL for all GET requests. + + You should provide the keyword argument "url" to the handler, e.g.: + + application = web.Application([ + (r"/oldpath", web.RedirectHandler, {"url": "/newpath"}), + ]) + """ + def __init__(self, application, request, url, permanent=True): + RequestHandler.__init__(self, application, request) + self._url = url + self._permanent = permanent + + def get(self): + self.redirect(self._url, permanent=self._permanent) + + +class StaticFileHandler(RequestHandler): + """A simple handler that can serve static content from a directory. + + To map a path to this handler for a static data directory /var/www, + you would add a line to your application like: + + application = web.Application([ + (r"/static/(.*)", web.StaticFileHandler, {"path": "/var/www"}), + ]) + + The local root directory of the content should be passed as the "path" + argument to the handler. + + To support aggressive browser caching, if the argument "v" is given + with the path, we set an infinite HTTP expiration header. So, if you + want browsers to cache a file indefinitely, send them to, e.g., + /static/images/myimage.png?v=xxx. + """ + def __init__(self, application, request, path): + RequestHandler.__init__(self, application, request) + self.root = os.path.abspath(path) + os.path.sep + + def head(self, path): + self.get(path, include_body=False) + + def get(self, path, include_body=True): + abspath = os.path.abspath(os.path.join(self.root, path)) + if not abspath.startswith(self.root): + raise HTTPError(403, "%s is not in root static directory", path) + if not os.path.exists(abspath): + raise HTTPError(404) + if not os.path.isfile(abspath): + raise HTTPError(403, "%s is not a file", path) + + stat_result = os.stat(abspath) + modified = datetime.datetime.fromtimestamp(stat_result[stat.ST_MTIME]) + + self.set_header("Last-Modified", modified) + if "v" in self.request.arguments: + self.set_header("Expires", datetime.datetime.utcnow() + \ + datetime.timedelta(days=365*10)) + self.set_header("Cache-Control", "max-age=" + str(86400*365*10)) + else: + self.set_header("Cache-Control", "public") + mime_type, encoding = mimetypes.guess_type(abspath) + if mime_type: + self.set_header("Content-Type", mime_type) + + # Check the If-Modified-Since, and don't send the result if the + # content has not been modified + ims_value = self.request.headers.get("If-Modified-Since") + if ims_value is not None: + date_tuple = email.utils.parsedate(ims_value) + if_since = datetime.datetime.fromtimestamp(time.mktime(date_tuple)) + if if_since >= modified: + self.set_status(304) + return + + if not include_body: + return + self.set_header("Content-Length", stat_result[stat.ST_SIZE]) + file = open(abspath, "rb") + try: + self.write(file.read()) + finally: + file.close() + + +class FallbackHandler(RequestHandler): + """A RequestHandler that wraps another HTTP server callback. + + The fallback is a callable object that accepts an HTTPRequest, + such as an Application or tornado.wsgi.WSGIContainer. This is most + useful to use both tornado RequestHandlers and WSGI in the same server. + Typical usage: + wsgi_app = tornado.wsgi.WSGIContainer( + django.core.handlers.wsgi.WSGIHandler()) + application = tornado.web.Application([ + (r"/foo", FooHandler), + (r".*", FallbackHandler, dict(fallback=wsgi_app), + ]) + """ + def __init__(self, app, request, fallback): + RequestHandler.__init__(self, app, request) + self.fallback = fallback + + def prepare(self): + self.fallback(self.request) + self._finished = True + + +class OutputTransform(object): + """A transform modifies the result of an HTTP request (e.g., GZip encoding) + + A new transform instance is created for every request. See the + ChunkedTransferEncoding example below if you want to implement a + new Transform. + """ + def __init__(self, request): + pass + + def transform_first_chunk(self, headers, chunk, finishing): + return headers, chunk + + def transform_chunk(self, chunk, finishing): + return chunk + + +class GZipContentEncoding(OutputTransform): + """Applies the gzip content encoding to the response. + + See http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.11 + """ + CONTENT_TYPES = set([ + "text/plain", "text/html", "text/css", "text/xml", + "application/x-javascript", "application/xml", "application/atom+xml", + "text/javascript", "application/json", "application/xhtml+xml"]) + MIN_LENGTH = 5 + + def __init__(self, request): + self._gzipping = request.supports_http_1_1() and \ + "gzip" in request.headers.get("Accept-Encoding", "") + + def transform_first_chunk(self, headers, chunk, finishing): + if self._gzipping: + ctype = headers.get("Content-Type", "").split(";")[0] + self._gzipping = (ctype in self.CONTENT_TYPES) and \ + (not finishing or len(chunk) >= self.MIN_LENGTH) and \ + (finishing or "Content-Length" not in headers) and \ + ("Content-Encoding" not in headers) + if self._gzipping: + headers["Content-Encoding"] = "gzip" + self._gzip_value = cStringIO.StringIO() + self._gzip_file = gzip.GzipFile(mode="w", fileobj=self._gzip_value) + self._gzip_pos = 0 + chunk = self.transform_chunk(chunk, finishing) + if "Content-Length" in headers: + headers["Content-Length"] = str(len(chunk)) + return headers, chunk + + def transform_chunk(self, chunk, finishing): + if self._gzipping: + self._gzip_file.write(chunk) + if finishing: + self._gzip_file.close() + else: + self._gzip_file.flush() + chunk = self._gzip_value.getvalue() + if self._gzip_pos > 0: + chunk = chunk[self._gzip_pos:] + self._gzip_pos += len(chunk) + return chunk + + +class ChunkedTransferEncoding(OutputTransform): + """Applies the chunked transfer encoding to the response. + + See http://www.w3.org/Protocols/rfc2616/rfc2616-sec3.html#sec3.6.1 + """ + def __init__(self, request): + self._chunking = request.supports_http_1_1() + + def transform_first_chunk(self, headers, chunk, finishing): + if self._chunking: + # No need to chunk the output if a Content-Length is specified + if "Content-Length" in headers or "Transfer-Encoding" in headers: + self._chunking = False + else: + headers["Transfer-Encoding"] = "chunked" + chunk = self.transform_chunk(chunk, finishing) + return headers, chunk + + def transform_chunk(self, block, finishing): + if self._chunking: + # Don't write out empty chunks because that means END-OF-STREAM + # with chunked encoding + if block: + block = ("%x" % len(block)) + "\r\n" + block + "\r\n" + if finishing: + block += "0\r\n\r\n" + return block + + +def authenticated(method): + """Decorate methods with this to require that the user be logged in.""" + @functools.wraps(method) + def wrapper(self, *args, **kwargs): + if not self.current_user: + if self.request.method == "GET": + url = self.get_login_url() + if "?" not in url: + url += "?" + urllib.urlencode(dict(next=self.request.uri)) + self.redirect(url) + return + raise HTTPError(403) + return method(self, *args, **kwargs) + return wrapper + + +class UIModule(object): + """A UI re-usable, modular unit on a page. + + UI modules often execute additional queries, and they can include + additional CSS and JavaScript that will be included in the output + page, which is automatically inserted on page render. + """ + def __init__(self, handler): + self.handler = handler + self.request = handler.request + self.ui = handler.ui + self.current_user = handler.current_user + self.locale = handler.locale + + def render(self, *args, **kwargs): + raise NotImplementedError() + + def embedded_javascript(self): + """Returns a JavaScript string that will be embedded in the page.""" + return None + + def javascript_files(self): + """Returns a list of JavaScript files required by this module.""" + return None + + def embedded_css(self): + """Returns a CSS string that will be embedded in the page.""" + return None + + def css_files(self): + """Returns a list of JavaScript files required by this module.""" + return None + + def html_head(self): + """Returns a CSS string that will be put in the <head/> element""" + return None + + def html_body(self): + """Returns an HTML string that will be put in the <body/> element""" + return None + + def render_string(self, path, **kwargs): + return self.handler.render_string(path, **kwargs) + +class URLSpec(object): + """Specifies mappings between URLs and handlers.""" + def __init__(self, pattern, handler_class, kwargs={}, name=None): + """Creates a URLSpec. + + Parameters: + pattern: Regular expression to be matched. Any groups in the regex + will be passed in to the handler's get/post/etc methods as + arguments. + handler_class: RequestHandler subclass to be invoked. + kwargs (optional): A dictionary of additional arguments to be passed + to the handler's constructor. + name (optional): A name for this handler. Used by + Application.reverse_url. + """ + if not pattern.endswith('$'): + pattern += '$' + self.regex = re.compile(pattern) + self.handler_class = handler_class + self.kwargs = kwargs + self.name = name + self._path, self._group_count = self._find_groups() + + def _find_groups(self): + """Returns a tuple (reverse string, group count) for a url. + + For example: Given the url pattern /([0-9]{4})/([a-z-]+)/, this method + would return ('/%s/%s/', 2). + """ + pattern = self.regex.pattern + if pattern.startswith('^'): + pattern = pattern[1:] + if pattern.endswith('$'): + pattern = pattern[:-1] + + if self.regex.groups != pattern.count('('): + # The pattern is too complicated for our simplistic matching, + # so we can't support reversing it. + return (None, None) + + pieces = [] + for fragment in pattern.split('('): + if ')' in fragment: + paren_loc = fragment.index(')') + if paren_loc >= 0: + pieces.append('%s' + fragment[paren_loc + 1:]) + else: + pieces.append(fragment) + + return (''.join(pieces), self.regex.groups) + + def reverse(self, *args): + assert self._path is not None, \ + "Cannot reverse url regex " + self.regex.pattern + assert len(args) == self._group_count, "required number of arguments "\ + "not found" + if not len(args): + return self._path + return self._path % tuple([str(a) for a in args]) + +url = URLSpec + +def _utf8(s): + if isinstance(s, unicode): + return s.encode("utf-8") + assert isinstance(s, str) + return s + + +def _unicode(s): + if isinstance(s, str): + try: + return s.decode("utf-8") + except UnicodeDecodeError: + raise HTTPError(400, "Non-utf8 argument") + assert isinstance(s, unicode) + return s + + +def _time_independent_equals(a, b): + if len(a) != len(b): + return False + result = 0 + for x, y in zip(a, b): + result |= ord(x) ^ ord(y) + return result == 0 + + +class _O(dict): + """Makes a dictionary behave like an object.""" + def __getattr__(self, name): + try: + return self[name] + except KeyError: + raise AttributeError(name) + + def __setattr__(self, name, value): + self[name] = value diff --git a/vendor/tornado/tornado/websocket.py b/vendor/tornado/tornado/websocket.py new file mode 100644 index 0000000000..38a58012cc --- /dev/null +++ b/vendor/tornado/tornado/websocket.py @@ -0,0 +1,138 @@ +#!/usr/bin/env python +# +# Copyright 2009 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import functools +import logging +import tornado.escape +import tornado.web + +_log = logging.getLogger('tornado.websocket') + +class WebSocketHandler(tornado.web.RequestHandler): + """A request handler for HTML 5 Web Sockets. + + See http://www.w3.org/TR/2009/WD-websockets-20091222/ for details on the + JavaScript interface. We implement the protocol as specified at + http://tools.ietf.org/html/draft-hixie-thewebsocketprotocol-55. + + Here is an example Web Socket handler that echos back all received messages + back to the client: + + class EchoWebSocket(websocket.WebSocketHandler): + def open(self): + self.receive_message(self.on_message) + + def on_message(self, message): + self.write_message(u"You said: " + message) + + Web Sockets are not standard HTTP connections. The "handshake" is HTTP, + but after the handshake, the protocol is message-based. Consequently, + most of the Tornado HTTP facilities are not available in handlers of this + type. The only communication methods available to you are send_message() + and receive_message(). Likewise, your request handler class should + implement open() method rather than get() or post(). + + If you map the handler above to "/websocket" in your application, you can + invoke it in JavaScript with: + + var ws = new WebSocket("ws://localhost:8888/websocket"); + ws.onopen = function() { + ws.send("Hello, world"); + }; + ws.onmessage = function (evt) { + alert(evt.data); + }; + + This script pops up an alert box that says "You said: Hello, world". + """ + def __init__(self, application, request): + tornado.web.RequestHandler.__init__(self, application, request) + self.stream = request.connection.stream + + def _execute(self, transforms, *args, **kwargs): + if self.request.headers.get("Upgrade") != "WebSocket" or \ + self.request.headers.get("Connection") != "Upgrade" or \ + not self.request.headers.get("Origin"): + message = "Expected WebSocket headers" + self.stream.write( + "HTTP/1.1 403 Forbidden\r\nContent-Length: " + + str(len(message)) + "\r\n\r\n" + message) + return + self.stream.write( + "HTTP/1.1 101 Web Socket Protocol Handshake\r\n" + "Upgrade: WebSocket\r\n" + "Connection: Upgrade\r\n" + "Server: TornadoServer/0.1\r\n" + "WebSocket-Origin: " + self.request.headers["Origin"] + "\r\n" + "WebSocket-Location: ws://" + self.request.host + + self.request.path + "\r\n\r\n") + self.async_callback(self.open)(*args, **kwargs) + + def write_message(self, message): + """Sends the given message to the client of this Web Socket.""" + if isinstance(message, dict): + message = tornado.escape.json_encode(message) + if isinstance(message, unicode): + message = message.encode("utf-8") + assert isinstance(message, str) + self.stream.write("\x00" + message + "\xff") + + def receive_message(self, callback): + """Calls callback when the browser calls send() on this Web Socket.""" + callback = self.async_callback(callback) + self.stream.read_bytes( + 1, functools.partial(self._on_frame_type, callback)) + + def close(self): + """Closes this Web Socket. + + The browser will receive the onclose event for the open web socket + when this method is called. + """ + self.stream.close() + + def async_callback(self, callback, *args, **kwargs): + """Wrap callbacks with this if they are used on asynchronous requests. + + Catches exceptions properly and closes this Web Socket if an exception + is uncaught. + """ + if args or kwargs: + callback = functools.partial(callback, *args, **kwargs) + def wrapper(*args, **kwargs): + try: + return callback(*args, **kwargs) + except Exception, e: + _log.error("Uncaught exception in %s", + self.request.path, exc_info=True) + self.stream.close() + return wrapper + + def _on_frame_type(self, callback, byte): + if ord(byte) & 0x80 == 0x80: + raise Exception("Length-encoded format not yet supported") + self.stream.read_until( + "\xff", functools.partial(self._on_end_delimiter, callback)) + + def _on_end_delimiter(self, callback, frame): + callback(frame[:-1].decode("utf-8", "replace")) + + def _not_supported(self, *args, **kwargs): + raise Exception("Method not supported for Web Sockets") + +for method in ["write", "redirect", "set_header", "send_error", "set_cookie", + "set_status", "flush", "finish"]: + setattr(WebSocketHandler, method, WebSocketHandler._not_supported) diff --git a/vendor/tornado/tornado/win32_support.py b/vendor/tornado/tornado/win32_support.py new file mode 100644 index 0000000000..f3efa8e892 --- /dev/null +++ b/vendor/tornado/tornado/win32_support.py @@ -0,0 +1,123 @@ +# NOTE: win32 support is currently experimental, and not recommended +# for production use. + +import ctypes +import ctypes.wintypes +import os +import socket +import errno + + +# See: http://msdn.microsoft.com/en-us/library/ms738573(VS.85).aspx +ioctlsocket = ctypes.windll.ws2_32.ioctlsocket +ioctlsocket.argtypes = (ctypes.wintypes.HANDLE, ctypes.wintypes.LONG, ctypes.wintypes.ULONG) +ioctlsocket.restype = ctypes.c_int + +# See: http://msdn.microsoft.com/en-us/library/ms724935(VS.85).aspx +SetHandleInformation = ctypes.windll.kernel32.SetHandleInformation +SetHandleInformation.argtypes = (ctypes.wintypes.HANDLE, ctypes.wintypes.DWORD, ctypes.wintypes.DWORD) +SetHandleInformation.restype = ctypes.wintypes.BOOL + +HANDLE_FLAG_INHERIT = 0x00000001 + + +F_GETFD = 1 +F_SETFD = 2 +F_GETFL = 3 +F_SETFL = 4 + +FD_CLOEXEC = 1 + +os.O_NONBLOCK = 2048 + +FIONBIO = 126 + + +def fcntl(fd, op, arg=0): + if op == F_GETFD or op == F_GETFL: + return 0 + elif op == F_SETFD: + # Check that the flag is CLOEXEC and translate + if arg == FD_CLOEXEC: + success = SetHandleInformation(fd, HANDLE_FLAG_INHERIT, arg) + if not success: + raise ctypes.GetLastError() + else: + raise ValueError("Unsupported arg") + #elif op == F_SETFL: + ## Check that the flag is NONBLOCK and translate + #if arg == os.O_NONBLOCK: + ##pass + #result = ioctlsocket(fd, FIONBIO, 1) + #if result != 0: + #raise ctypes.GetLastError() + #else: + #raise ValueError("Unsupported arg") + else: + raise ValueError("Unsupported op") + + +class Pipe(object): + """Create an OS independent asynchronous pipe""" + def __init__(self): + # Based on Zope async.py: http://svn.zope.org/zc.ngi/trunk/src/zc/ngi/async.py + + self.writer = socket.socket() + # Disable buffering -- pulling the trigger sends 1 byte, + # and we want that sent immediately, to wake up ASAP. + self.writer.setsockopt(socket.IPPROTO_TCP, socket.TCP_NODELAY, 1) + + count = 0 + while 1: + count += 1 + # Bind to a local port; for efficiency, let the OS pick + # a free port for us. + # Unfortunately, stress tests showed that we may not + # be able to connect to that port ("Address already in + # use") despite that the OS picked it. This appears + # to be a race bug in the Windows socket implementation. + # So we loop until a connect() succeeds (almost always + # on the first try). See the long thread at + # http://mail.zope.org/pipermail/zope/2005-July/160433.html + # for hideous details. + a = socket.socket() + a.bind(("127.0.0.1", 0)) + connect_address = a.getsockname() # assigned (host, port) pair + a.listen(1) + try: + self.writer.connect(connect_address) + break # success + except socket.error, detail: + if detail[0] != errno.WSAEADDRINUSE: + # "Address already in use" is the only error + # I've seen on two WinXP Pro SP2 boxes, under + # Pythons 2.3.5 and 2.4.1. + raise + # (10048, 'Address already in use') + # assert count <= 2 # never triggered in Tim's tests + if count >= 10: # I've never seen it go above 2 + a.close() + self.writer.close() + raise socket.error("Cannot bind trigger!") + # Close `a` and try again. Note: I originally put a short + # sleep() here, but it didn't appear to help or hurt. + a.close() + + self.reader, addr = a.accept() + self.reader.setblocking(0) + self.writer.setblocking(0) + a.close() + self.reader_fd = self.reader.fileno() + + def read(self): + """Emulate a file descriptors read method""" + try: + return self.reader.recv(1) + except socket.error, ex: + if ex.args[0] == errno.EWOULDBLOCK: + raise IOError + raise + + def write(self, data): + """Emulate a file descriptors write method""" + return self.writer.send(data) diff --git a/vendor/tornado/tornado/wsgi.py b/vendor/tornado/tornado/wsgi.py new file mode 100644 index 0000000000..69fa0988eb --- /dev/null +++ b/vendor/tornado/tornado/wsgi.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python +# +# Copyright 2009 Facebook +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""WSGI support for the Tornado web framework. + +We export WSGIApplication, which is very similar to web.Application, except +no asynchronous methods are supported (since WSGI does not support +non-blocking requests properly). If you call self.flush() or other +asynchronous methods in your request handlers running in a WSGIApplication, +we throw an exception. + +Example usage: + + import tornado.web + import tornado.wsgi + import wsgiref.simple_server + + class MainHandler(tornado.web.RequestHandler): + def get(self): + self.write("Hello, world") + + if __name__ == "__main__": + application = tornado.wsgi.WSGIApplication([ + (r"/", MainHandler), + ]) + server = wsgiref.simple_server.make_server('', 8888, application) + server.serve_forever() + +See the 'appengine' demo for an example of using this module to run +a Tornado app on Google AppEngine. + +Since no asynchronous methods are available for WSGI applications, the +httpclient and auth modules are both not available for WSGI applications. + +We also export WSGIContainer, which lets you run other WSGI-compatible +frameworks on the Tornado HTTP server and I/O loop. See WSGIContainer for +details and documentation. +""" + +import cgi +import cStringIO +import escape +import httplib +import logging +import sys +import time +import urllib +import web + +_log = logging.getLogger('tornado.wsgi') + +class WSGIApplication(web.Application): + """A WSGI-equivalent of web.Application. + + We support the same interface, but handlers running in a WSGIApplication + do not support flush() or asynchronous methods. + """ + def __init__(self, handlers=None, default_host="", **settings): + web.Application.__init__(self, handlers, default_host, transforms=[], + wsgi=True, **settings) + + def __call__(self, environ, start_response): + handler = web.Application.__call__(self, HTTPRequest(environ)) + assert handler._finished + status = str(handler._status_code) + " " + \ + httplib.responses[handler._status_code] + headers = handler._headers.items() + for cookie_dict in getattr(handler, "_new_cookies", []): + for cookie in cookie_dict.values(): + headers.append(("Set-Cookie", cookie.OutputString(None))) + start_response(status, headers) + return handler._write_buffer + + +class HTTPRequest(object): + """Mimics httpserver.HTTPRequest for WSGI applications.""" + def __init__(self, environ): + """Parses the given WSGI environ to construct the request.""" + self.method = environ["REQUEST_METHOD"] + self.path = urllib.quote(environ.get("SCRIPT_NAME", "")) + self.path += urllib.quote(environ.get("PATH_INFO", "")) + self.uri = self.path + self.arguments = {} + self.query = environ.get("QUERY_STRING", "") + if self.query: + self.uri += "?" + self.query + arguments = cgi.parse_qs(self.query) + for name, values in arguments.iteritems(): + values = [v for v in values if v] + if values: self.arguments[name] = values + self.version = "HTTP/1.1" + self.headers = HTTPHeaders() + if environ.get("CONTENT_TYPE"): + self.headers["Content-Type"] = environ["CONTENT_TYPE"] + if environ.get("CONTENT_LENGTH"): + self.headers["Content-Length"] = int(environ["CONTENT_LENGTH"]) + for key in environ: + if key.startswith("HTTP_"): + self.headers[key[5:].replace("_", "-")] = environ[key] + if self.headers.get("Content-Length"): + self.body = environ["wsgi.input"].read() + else: + self.body = "" + self.protocol = environ["wsgi.url_scheme"] + self.remote_ip = environ.get("REMOTE_ADDR", "") + if environ.get("HTTP_HOST"): + self.host = environ["HTTP_HOST"] + else: + self.host = environ["SERVER_NAME"] + + # Parse request body + self.files = {} + content_type = self.headers.get("Content-Type", "") + if content_type.startswith("application/x-www-form-urlencoded"): + for name, values in cgi.parse_qs(self.body).iteritems(): + self.arguments.setdefault(name, []).extend(values) + elif content_type.startswith("multipart/form-data"): + boundary = content_type[30:] + if boundary: self._parse_mime_body(boundary) + + self._start_time = time.time() + self._finish_time = None + + def supports_http_1_1(self): + """Returns True if this request supports HTTP/1.1 semantics""" + return self.version == "HTTP/1.1" + + def full_url(self): + """Reconstructs the full URL for this request.""" + return self.protocol + "://" + self.host + self.uri + + def request_time(self): + """Returns the amount of time it took for this request to execute.""" + if self._finish_time is None: + return time.time() - self._start_time + else: + return self._finish_time - self._start_time + + def _parse_mime_body(self, boundary): + if self.body.endswith("\r\n"): + footer_length = len(boundary) + 6 + else: + footer_length = len(boundary) + 4 + parts = self.body[:-footer_length].split("--" + boundary + "\r\n") + for part in parts: + if not part: continue + eoh = part.find("\r\n\r\n") + if eoh == -1: + _log.warning("multipart/form-data missing headers") + continue + headers = HTTPHeaders.parse(part[:eoh]) + name_header = headers.get("Content-Disposition", "") + if not name_header.startswith("form-data;") or \ + not part.endswith("\r\n"): + _log.warning("Invalid multipart/form-data") + continue + value = part[eoh + 4:-2] + name_values = {} + for name_part in name_header[10:].split(";"): + name, name_value = name_part.strip().split("=", 1) + name_values[name] = name_value.strip('"').decode("utf-8") + if not name_values.get("name"): + _log.warning("multipart/form-data value missing name") + continue + name = name_values["name"] + if name_values.get("filename"): + ctype = headers.get("Content-Type", "application/unknown") + self.files.setdefault(name, []).append(dict( + filename=name_values["filename"], body=value, + content_type=ctype)) + else: + self.arguments.setdefault(name, []).append(value) + + +class WSGIContainer(object): + """Makes a WSGI-compatible function runnable on Tornado's HTTP server. + + Wrap a WSGI function in a WSGIContainer and pass it to HTTPServer to + run it. For example: + + def simple_app(environ, start_response): + status = "200 OK" + response_headers = [("Content-type", "text/plain")] + start_response(status, response_headers) + return ["Hello world!\n"] + + container = tornado.wsgi.WSGIContainer(simple_app) + http_server = tornado.httpserver.HTTPServer(container) + http_server.listen(8888) + tornado.ioloop.IOLoop.instance().start() + + This class is intended to let other frameworks (Django, web.py, etc) + run on the Tornado HTTP server and I/O loop. It has not yet been + thoroughly tested in production. + """ + def __init__(self, wsgi_application): + self.wsgi_application = wsgi_application + + def __call__(self, request): + data = {} + response = [] + def start_response(status, response_headers, exc_info=None): + data["status"] = status + data["headers"] = response_headers + return response.append + response.extend(self.wsgi_application( + WSGIContainer.environ(request), start_response)) + body = "".join(response) + if hasattr(response, "close"): + response.close() + if not data: raise Exception("WSGI app did not call start_response") + + status_code = int(data["status"].split()[0]) + headers = data["headers"] + header_set = set(k.lower() for (k,v) in headers) + body = escape.utf8(body) + if "content-length" not in header_set: + headers.append(("Content-Length", str(len(body)))) + if "content-type" not in header_set: + headers.append(("Content-Type", "text/html; charset=UTF-8")) + if "server" not in header_set: + headers.append(("Server", "TornadoServer/0.1")) + + parts = ["HTTP/1.1 " + data["status"] + "\r\n"] + for key, value in headers: + parts.append(escape.utf8(key) + ": " + escape.utf8(value) + "\r\n") + parts.append("\r\n") + parts.append(body) + request.write("".join(parts)) + request.finish() + self._log(status_code, request) + + @staticmethod + def environ(request): + hostport = request.host.split(":") + if len(hostport) == 2: + host = hostport[0] + port = int(hostport[1]) + else: + host = request.host + port = 443 if request.protocol == "https" else 80 + environ = { + "REQUEST_METHOD": request.method, + "SCRIPT_NAME": "", + "PATH_INFO": request.path, + "QUERY_STRING": request.query, + "REMOTE_ADDR": request.remote_ip, + "SERVER_NAME": host, + "SERVER_PORT": port, + "SERVER_PROTOCOL": request.version, + "wsgi.version": (1, 0), + "wsgi.url_scheme": request.protocol, + "wsgi.input": cStringIO.StringIO(request.body), + "wsgi.errors": sys.stderr, + "wsgi.multithread": False, + "wsgi.multiprocess": True, + "wsgi.run_once": False, + } + if "Content-Type" in request.headers: + environ["CONTENT_TYPE"] = request.headers["Content-Type"] + if "Content-Length" in request.headers: + environ["CONTENT_LENGTH"] = request.headers["Content-Length"] + for key, value in request.headers.iteritems(): + environ["HTTP_" + key.replace("-", "_").upper()] = value + return environ + + def _log(self, status_code, request): + if status_code < 400: + log_method = _log.info + elif status_code < 500: + log_method = _log.warning + else: + log_method = _log.error + request_time = 1000.0 * request.request_time() + summary = request.method + " " + request.uri + " (" + \ + request.remote_ip + ")" + log_method("%d %s %.2fms", status_code, summary, request_time) + + +class HTTPHeaders(dict): + """A dictionary that maintains Http-Header-Case for all keys.""" + def __setitem__(self, name, value): + dict.__setitem__(self, self._normalize_name(name), value) + + def __getitem__(self, name): + return dict.__getitem__(self, self._normalize_name(name)) + + def _normalize_name(self, name): + return "-".join([w.capitalize() for w in name.split("-")]) + + @classmethod + def parse(cls, headers_string): + headers = cls() + for line in headers_string.splitlines(): + if line: + name, value = line.split(": ", 1) + headers[name] = value + return headers diff --git a/vendor/tornado/website/app.yaml b/vendor/tornado/website/app.yaml new file mode 100644 index 0000000000..8a1ff06648 --- /dev/null +++ b/vendor/tornado/website/app.yaml @@ -0,0 +1,15 @@ +application: python-tornado +version: 1 +runtime: python +api_version: 1 + +handlers: +- url: /static/ + static_dir: static + +- url: /robots\.txt + static_files: static/robots.txt + upload: static/robots.txt + +- url: /.* + script: website.py diff --git a/vendor/tornado/website/index.yaml b/vendor/tornado/website/index.yaml new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/vendor/tornado/website/index.yaml diff --git a/vendor/tornado/website/markdown/__init__.py b/vendor/tornado/website/markdown/__init__.py new file mode 100644 index 0000000000..0d1c504979 --- /dev/null +++ b/vendor/tornado/website/markdown/__init__.py @@ -0,0 +1,603 @@ +""" +Python Markdown +=============== + +Python Markdown converts Markdown to HTML and can be used as a library or +called from the command line. + +## Basic usage as a module: + + import markdown + md = Markdown() + html = md.convert(your_text_string) + +## Basic use from the command line: + + python markdown.py source.txt > destination.html + +Run "python markdown.py --help" to see more options. + +## Extensions + +See <http://www.freewisdom.org/projects/python-markdown/> for more +information and instructions on how to extend the functionality of +Python Markdown. Read that before you try modifying this file. + +## Authors and License + +Started by [Manfred Stienstra](http://www.dwerg.net/). Continued and +maintained by [Yuri Takhteyev](http://www.freewisdom.org), [Waylan +Limberg](http://achinghead.com/) and [Artem Yunusov](http://blog.splyer.com). + +Contact: markdown@freewisdom.org + +Copyright 2007, 2008 The Python Markdown Project (v. 1.7 and later) +Copyright 200? Django Software Foundation (OrderedDict implementation) +Copyright 2004, 2005, 2006 Yuri Takhteyev (v. 0.2-1.6b) +Copyright 2004 Manfred Stienstra (the original version) + +License: BSD (see docs/LICENSE for details). +""" + +version = "2.0" +version_info = (2,0,0, "Final") + +import re +import codecs +import sys +import warnings +import logging +from logging import DEBUG, INFO, WARN, ERROR, CRITICAL + + +""" +CONSTANTS +============================================================================= +""" + +""" +Constants you might want to modify +----------------------------------------------------------------------------- +""" + +# default logging level for command-line use +COMMAND_LINE_LOGGING_LEVEL = CRITICAL +TAB_LENGTH = 4 # expand tabs to this many spaces +ENABLE_ATTRIBUTES = True # @id = xyz -> <... id="xyz"> +SMART_EMPHASIS = True # this_or_that does not become this<i>or</i>that +DEFAULT_OUTPUT_FORMAT = 'xhtml1' # xhtml or html4 output +HTML_REMOVED_TEXT = "[HTML_REMOVED]" # text used instead of HTML in safe mode +BLOCK_LEVEL_ELEMENTS = re.compile("p|div|h[1-6]|blockquote|pre|table|dl|ol|ul" + "|script|noscript|form|fieldset|iframe|math" + "|ins|del|hr|hr/|style|li|dt|dd|thead|tbody" + "|tr|th|td") +DOC_TAG = "div" # Element used to wrap document - later removed + +# Placeholders +STX = u'\u0002' # Use STX ("Start of text") for start-of-placeholder +ETX = u'\u0003' # Use ETX ("End of text") for end-of-placeholder +INLINE_PLACEHOLDER_PREFIX = STX+"klzzwxh:" +INLINE_PLACEHOLDER = INLINE_PLACEHOLDER_PREFIX + "%s" + ETX +AMP_SUBSTITUTE = STX+"amp"+ETX + + +""" +Constants you probably do not need to change +----------------------------------------------------------------------------- +""" + +RTL_BIDI_RANGES = ( (u'\u0590', u'\u07FF'), + # Hebrew (0590-05FF), Arabic (0600-06FF), + # Syriac (0700-074F), Arabic supplement (0750-077F), + # Thaana (0780-07BF), Nko (07C0-07FF). + (u'\u2D30', u'\u2D7F'), # Tifinagh + ) + + +""" +AUXILIARY GLOBAL FUNCTIONS +============================================================================= +""" + + +def message(level, text): + """ A wrapper method for logging debug messages. """ + logger = logging.getLogger('MARKDOWN') + if logger.handlers: + # The logger is configured + logger.log(level, text) + if level > WARN: + sys.exit(0) + elif level > WARN: + raise MarkdownException, text + else: + warnings.warn(text, MarkdownWarning) + + +def isBlockLevel(tag): + """Check if the tag is a block level HTML tag.""" + return BLOCK_LEVEL_ELEMENTS.match(tag) + +""" +MISC AUXILIARY CLASSES +============================================================================= +""" + +class AtomicString(unicode): + """A string which should not be further processed.""" + pass + + +class MarkdownException(Exception): + """ A Markdown Exception. """ + pass + + +class MarkdownWarning(Warning): + """ A Markdown Warning. """ + pass + + +""" +OVERALL DESIGN +============================================================================= + +Markdown processing takes place in four steps: + +1. A bunch of "preprocessors" munge the input text. +2. BlockParser() parses the high-level structural elements of the + pre-processed text into an ElementTree. +3. A bunch of "treeprocessors" are run against the ElementTree. One such + treeprocessor runs InlinePatterns against the ElementTree, detecting inline + markup. +4. Some post-processors are run against the text after the ElementTree has + been serialized into text. +5. The output is written to a string. + +Those steps are put together by the Markdown() class. + +""" + +import preprocessors +import blockprocessors +import treeprocessors +import inlinepatterns +import postprocessors +import blockparser +import etree_loader +import odict + +# Extensions should use "markdown.etree" instead of "etree" (or do `from +# markdown import etree`). Do not import it by yourself. + +etree = etree_loader.importETree() + +# Adds the ability to output html4 +import html4 + + +class Markdown: + """Convert Markdown to HTML.""" + + def __init__(self, + extensions=[], + extension_configs={}, + safe_mode = False, + output_format=DEFAULT_OUTPUT_FORMAT): + """ + Creates a new Markdown instance. + + Keyword arguments: + + * extensions: A list of extensions. + If they are of type string, the module mdx_name.py will be loaded. + If they are a subclass of markdown.Extension, they will be used + as-is. + * extension-configs: Configuration setting for extensions. + * safe_mode: Disallow raw html. One of "remove", "replace" or "escape". + * output_format: Format of output. Supported formats are: + * "xhtml1": Outputs XHTML 1.x. Default. + * "xhtml": Outputs latest supported version of XHTML (currently XHTML 1.1). + * "html4": Outputs HTML 4 + * "html": Outputs latest supported version of HTML (currently HTML 4). + Note that it is suggested that the more specific formats ("xhtml1" + and "html4") be used as "xhtml" or "html" may change in the future + if it makes sense at that time. + + """ + + self.safeMode = safe_mode + self.registeredExtensions = [] + self.docType = "" + self.stripTopLevelTags = True + + # Preprocessors + self.preprocessors = odict.OrderedDict() + self.preprocessors["html_block"] = \ + preprocessors.HtmlBlockPreprocessor(self) + self.preprocessors["reference"] = \ + preprocessors.ReferencePreprocessor(self) + # footnote preprocessor will be inserted with "<reference" + + # Block processors - ran by the parser + self.parser = blockparser.BlockParser() + self.parser.blockprocessors['empty'] = \ + blockprocessors.EmptyBlockProcessor(self.parser) + self.parser.blockprocessors['indent'] = \ + blockprocessors.ListIndentProcessor(self.parser) + self.parser.blockprocessors['code'] = \ + blockprocessors.CodeBlockProcessor(self.parser) + self.parser.blockprocessors['hashheader'] = \ + blockprocessors.HashHeaderProcessor(self.parser) + self.parser.blockprocessors['setextheader'] = \ + blockprocessors.SetextHeaderProcessor(self.parser) + self.parser.blockprocessors['hr'] = \ + blockprocessors.HRProcessor(self.parser) + self.parser.blockprocessors['olist'] = \ + blockprocessors.OListProcessor(self.parser) + self.parser.blockprocessors['ulist'] = \ + blockprocessors.UListProcessor(self.parser) + self.parser.blockprocessors['quote'] = \ + blockprocessors.BlockQuoteProcessor(self.parser) + self.parser.blockprocessors['paragraph'] = \ + blockprocessors.ParagraphProcessor(self.parser) + + + #self.prePatterns = [] + + # Inline patterns - Run on the tree + self.inlinePatterns = odict.OrderedDict() + self.inlinePatterns["backtick"] = \ + inlinepatterns.BacktickPattern(inlinepatterns.BACKTICK_RE) + self.inlinePatterns["escape"] = \ + inlinepatterns.SimpleTextPattern(inlinepatterns.ESCAPE_RE) + self.inlinePatterns["reference"] = \ + inlinepatterns.ReferencePattern(inlinepatterns.REFERENCE_RE, self) + self.inlinePatterns["link"] = \ + inlinepatterns.LinkPattern(inlinepatterns.LINK_RE, self) + self.inlinePatterns["image_link"] = \ + inlinepatterns.ImagePattern(inlinepatterns.IMAGE_LINK_RE, self) + self.inlinePatterns["image_reference"] = \ + inlinepatterns.ImageReferencePattern(inlinepatterns.IMAGE_REFERENCE_RE, self) + self.inlinePatterns["autolink"] = \ + inlinepatterns.AutolinkPattern(inlinepatterns.AUTOLINK_RE, self) + self.inlinePatterns["automail"] = \ + inlinepatterns.AutomailPattern(inlinepatterns.AUTOMAIL_RE, self) + self.inlinePatterns["linebreak2"] = \ + inlinepatterns.SubstituteTagPattern(inlinepatterns.LINE_BREAK_2_RE, 'br') + self.inlinePatterns["linebreak"] = \ + inlinepatterns.SubstituteTagPattern(inlinepatterns.LINE_BREAK_RE, 'br') + self.inlinePatterns["html"] = \ + inlinepatterns.HtmlPattern(inlinepatterns.HTML_RE, self) + self.inlinePatterns["entity"] = \ + inlinepatterns.HtmlPattern(inlinepatterns.ENTITY_RE, self) + self.inlinePatterns["not_strong"] = \ + inlinepatterns.SimpleTextPattern(inlinepatterns.NOT_STRONG_RE) + self.inlinePatterns["strong_em"] = \ + inlinepatterns.DoubleTagPattern(inlinepatterns.STRONG_EM_RE, 'strong,em') + self.inlinePatterns["strong"] = \ + inlinepatterns.SimpleTagPattern(inlinepatterns.STRONG_RE, 'strong') + self.inlinePatterns["emphasis"] = \ + inlinepatterns.SimpleTagPattern(inlinepatterns.EMPHASIS_RE, 'em') + self.inlinePatterns["emphasis2"] = \ + inlinepatterns.SimpleTagPattern(inlinepatterns.EMPHASIS_2_RE, 'em') + # The order of the handlers matters!!! + + + # Tree processors - run once we have a basic parse. + self.treeprocessors = odict.OrderedDict() + self.treeprocessors["inline"] = treeprocessors.InlineProcessor(self) + self.treeprocessors["prettify"] = \ + treeprocessors.PrettifyTreeprocessor(self) + + # Postprocessors - finishing touches. + self.postprocessors = odict.OrderedDict() + self.postprocessors["raw_html"] = \ + postprocessors.RawHtmlPostprocessor(self) + self.postprocessors["amp_substitute"] = \ + postprocessors.AndSubstitutePostprocessor() + # footnote postprocessor will be inserted with ">amp_substitute" + + # Map format keys to serializers + self.output_formats = { + 'html' : html4.to_html_string, + 'html4' : html4.to_html_string, + 'xhtml' : etree.tostring, + 'xhtml1': etree.tostring, + } + + self.references = {} + self.htmlStash = preprocessors.HtmlStash() + self.registerExtensions(extensions = extensions, + configs = extension_configs) + self.set_output_format(output_format) + self.reset() + + def registerExtensions(self, extensions, configs): + """ + Register extensions with this instance of Markdown. + + Keyword aurguments: + + * extensions: A list of extensions, which can either + be strings or objects. See the docstring on Markdown. + * configs: A dictionary mapping module names to config options. + + """ + for ext in extensions: + if isinstance(ext, basestring): + ext = load_extension(ext, configs.get(ext, [])) + try: + ext.extendMarkdown(self, globals()) + except AttributeError: + message(ERROR, "Incorrect type! Extension '%s' is " + "neither a string or an Extension." %(repr(ext))) + + + def registerExtension(self, extension): + """ This gets called by the extension """ + self.registeredExtensions.append(extension) + + def reset(self): + """ + Resets all state variables so that we can start with a new text. + """ + self.htmlStash.reset() + self.references.clear() + + for extension in self.registeredExtensions: + extension.reset() + + def set_output_format(self, format): + """ Set the output format for the class instance. """ + try: + self.serializer = self.output_formats[format.lower()] + except KeyError: + message(CRITICAL, 'Invalid Output Format: "%s". Use one of %s.' \ + % (format, self.output_formats.keys())) + + def convert(self, source): + """ + Convert markdown to serialized XHTML or HTML. + + Keyword arguments: + + * source: Source text as a Unicode string. + + """ + + # Fixup the source text + if not source.strip(): + return u"" # a blank unicode string + try: + source = unicode(source) + except UnicodeDecodeError: + message(CRITICAL, 'UnicodeDecodeError: Markdown only accepts unicode or ascii input.') + return u"" + + source = source.replace(STX, "").replace(ETX, "") + source = source.replace("\r\n", "\n").replace("\r", "\n") + "\n\n" + source = re.sub(r'\n\s+\n', '\n\n', source) + source = source.expandtabs(TAB_LENGTH) + + # Split into lines and run the line preprocessors. + self.lines = source.split("\n") + for prep in self.preprocessors.values(): + self.lines = prep.run(self.lines) + + # Parse the high-level elements. + root = self.parser.parseDocument(self.lines).getroot() + + # Run the tree-processors + for treeprocessor in self.treeprocessors.values(): + newRoot = treeprocessor.run(root) + if newRoot: + root = newRoot + + # Serialize _properly_. Strip top-level tags. + output, length = codecs.utf_8_decode(self.serializer(root, encoding="utf8")) + if self.stripTopLevelTags: + start = output.index('<%s>'%DOC_TAG)+len(DOC_TAG)+2 + end = output.rindex('</%s>'%DOC_TAG) + output = output[start:end].strip() + + # Run the text post-processors + for pp in self.postprocessors.values(): + output = pp.run(output) + + return output.strip() + + def convertFile(self, input=None, output=None, encoding=None): + """Converts a markdown file and returns the HTML as a unicode string. + + Decodes the file using the provided encoding (defaults to utf-8), + passes the file content to markdown, and outputs the html to either + the provided stream or the file with provided name, using the same + encoding as the source file. + + **Note:** This is the only place that decoding and encoding of unicode + takes place in Python-Markdown. (All other code is unicode-in / + unicode-out.) + + Keyword arguments: + + * input: Name of source text file. + * output: Name of output file. Writes to stdout if `None`. + * encoding: Encoding of input and output files. Defaults to utf-8. + + """ + + encoding = encoding or "utf-8" + + # Read the source + input_file = codecs.open(input, mode="r", encoding=encoding) + text = input_file.read() + input_file.close() + text = text.lstrip(u'\ufeff') # remove the byte-order mark + + # Convert + html = self.convert(text) + + # Write to file or stdout + if isinstance(output, (str, unicode)): + output_file = codecs.open(output, "w", encoding=encoding) + output_file.write(html) + output_file.close() + else: + output.write(html.encode(encoding)) + + +""" +Extensions +----------------------------------------------------------------------------- +""" + +class Extension: + """ Base class for extensions to subclass. """ + def __init__(self, configs = {}): + """Create an instance of an Extention. + + Keyword arguments: + + * configs: A dict of configuration setting used by an Extension. + """ + self.config = configs + + def getConfig(self, key): + """ Return a setting for the given key or an empty string. """ + if key in self.config: + return self.config[key][0] + else: + return "" + + def getConfigInfo(self): + """ Return all config settings as a list of tuples. """ + return [(key, self.config[key][1]) for key in self.config.keys()] + + def setConfig(self, key, value): + """ Set a config setting for `key` with the given `value`. """ + self.config[key][0] = value + + def extendMarkdown(self, md, md_globals): + """ + Add the various proccesors and patterns to the Markdown Instance. + + This method must be overriden by every extension. + + Keyword arguments: + + * md: The Markdown instance. + + * md_globals: Global variables in the markdown module namespace. + + """ + pass + + +def load_extension(ext_name, configs = []): + """Load extension by name, then return the module. + + The extension name may contain arguments as part of the string in the + following format: "extname(key1=value1,key2=value2)" + + """ + + # Parse extensions config params (ignore the order) + configs = dict(configs) + pos = ext_name.find("(") # find the first "(" + if pos > 0: + ext_args = ext_name[pos+1:-1] + ext_name = ext_name[:pos] + pairs = [x.split("=") for x in ext_args.split(",")] + configs.update([(x.strip(), y.strip()) for (x, y) in pairs]) + + # Setup the module names + ext_module = 'markdown.extensions' + module_name_new_style = '.'.join([ext_module, ext_name]) + module_name_old_style = '_'.join(['mdx', ext_name]) + + # Try loading the extention first from one place, then another + try: # New style (markdown.extensons.<extension>) + module = __import__(module_name_new_style, {}, {}, [ext_module]) + except ImportError: + try: # Old style (mdx.<extension>) + module = __import__(module_name_old_style) + except ImportError: + message(WARN, "Failed loading extension '%s' from '%s' or '%s'" + % (ext_name, module_name_new_style, module_name_old_style)) + # Return None so we don't try to initiate none-existant extension + return None + + # If the module is loaded successfully, we expect it to define a + # function called makeExtension() + try: + return module.makeExtension(configs.items()) + except AttributeError: + message(CRITICAL, "Failed to initiate extension '%s'" % ext_name) + + +def load_extensions(ext_names): + """Loads multiple extensions""" + extensions = [] + for ext_name in ext_names: + extension = load_extension(ext_name) + if extension: + extensions.append(extension) + return extensions + + +""" +EXPORTED FUNCTIONS +============================================================================= + +Those are the two functions we really mean to export: markdown() and +markdownFromFile(). +""" + +def markdown(text, + extensions = [], + safe_mode = False, + output_format = DEFAULT_OUTPUT_FORMAT): + """Convert a markdown string to HTML and return HTML as a unicode string. + + This is a shortcut function for `Markdown` class to cover the most + basic use case. It initializes an instance of Markdown, loads the + necessary extensions and runs the parser on the given text. + + Keyword arguments: + + * text: Markdown formatted text as Unicode or ASCII string. + * extensions: A list of extensions or extension names (may contain config args). + * safe_mode: Disallow raw html. One of "remove", "replace" or "escape". + * output_format: Format of output. Supported formats are: + * "xhtml1": Outputs XHTML 1.x. Default. + * "xhtml": Outputs latest supported version of XHTML (currently XHTML 1.1). + * "html4": Outputs HTML 4 + * "html": Outputs latest supported version of HTML (currently HTML 4). + Note that it is suggested that the more specific formats ("xhtml1" + and "html4") be used as "xhtml" or "html" may change in the future + if it makes sense at that time. + + Returns: An HTML document as a string. + + """ + md = Markdown(extensions=load_extensions(extensions), + safe_mode=safe_mode, + output_format=output_format) + return md.convert(text) + + +def markdownFromFile(input = None, + output = None, + extensions = [], + encoding = None, + safe_mode = False, + output_format = DEFAULT_OUTPUT_FORMAT): + """Read markdown code from a file and write it to a file or a stream.""" + md = Markdown(extensions=load_extensions(extensions), + safe_mode=safe_mode, + output_format=output_format) + md.convertFile(input, output, encoding) + + + diff --git a/vendor/tornado/website/markdown/blockparser.py b/vendor/tornado/website/markdown/blockparser.py new file mode 100644 index 0000000000..e18b338487 --- /dev/null +++ b/vendor/tornado/website/markdown/blockparser.py @@ -0,0 +1,95 @@ + +import markdown + +class State(list): + """ Track the current and nested state of the parser. + + This utility class is used to track the state of the BlockParser and + support multiple levels if nesting. It's just a simple API wrapped around + a list. Each time a state is set, that state is appended to the end of the + list. Each time a state is reset, that state is removed from the end of + the list. + + Therefore, each time a state is set for a nested block, that state must be + reset when we back out of that level of nesting or the state could be + corrupted. + + While all the methods of a list object are available, only the three + defined below need be used. + + """ + + def set(self, state): + """ Set a new state. """ + self.append(state) + + def reset(self): + """ Step back one step in nested state. """ + self.pop() + + def isstate(self, state): + """ Test that top (current) level is of given state. """ + if len(self): + return self[-1] == state + else: + return False + +class BlockParser: + """ Parse Markdown blocks into an ElementTree object. + + A wrapper class that stitches the various BlockProcessors together, + looping through them and creating an ElementTree object. + """ + + def __init__(self): + self.blockprocessors = markdown.odict.OrderedDict() + self.state = State() + + def parseDocument(self, lines): + """ Parse a markdown document into an ElementTree. + + Given a list of lines, an ElementTree object (not just a parent Element) + is created and the root element is passed to the parser as the parent. + The ElementTree object is returned. + + This should only be called on an entire document, not pieces. + + """ + # Create a ElementTree from the lines + self.root = markdown.etree.Element(markdown.DOC_TAG) + self.parseChunk(self.root, '\n'.join(lines)) + return markdown.etree.ElementTree(self.root) + + def parseChunk(self, parent, text): + """ Parse a chunk of markdown text and attach to given etree node. + + While the ``text`` argument is generally assumed to contain multiple + blocks which will be split on blank lines, it could contain only one + block. Generally, this method would be called by extensions when + block parsing is required. + + The ``parent`` etree Element passed in is altered in place. + Nothing is returned. + + """ + self.parseBlocks(parent, text.split('\n\n')) + + def parseBlocks(self, parent, blocks): + """ Process blocks of markdown text and attach to given etree node. + + Given a list of ``blocks``, each blockprocessor is stepped through + until there are no blocks left. While an extension could potentially + call this method directly, it's generally expected to be used internally. + + This is a public method as an extension may need to add/alter additional + BlockProcessors which call this method to recursively parse a nested + block. + + """ + while blocks: + for processor in self.blockprocessors.values(): + if processor.test(parent, blocks[0]): + processor.run(parent, blocks) + break + + diff --git a/vendor/tornado/website/markdown/blockprocessors.py b/vendor/tornado/website/markdown/blockprocessors.py new file mode 100644 index 0000000000..79f4db93bc --- /dev/null +++ b/vendor/tornado/website/markdown/blockprocessors.py @@ -0,0 +1,460 @@ +""" +CORE MARKDOWN BLOCKPARSER +============================================================================= + +This parser handles basic parsing of Markdown blocks. It doesn't concern itself +with inline elements such as **bold** or *italics*, but rather just catches +blocks, lists, quotes, etc. + +The BlockParser is made up of a bunch of BlockProssors, each handling a +different type of block. Extensions may add/replace/remove BlockProcessors +as they need to alter how markdown blocks are parsed. + +""" + +import re +import markdown + +class BlockProcessor: + """ Base class for block processors. + + Each subclass will provide the methods below to work with the source and + tree. Each processor will need to define it's own ``test`` and ``run`` + methods. The ``test`` method should return True or False, to indicate + whether the current block should be processed by this processor. If the + test passes, the parser will call the processors ``run`` method. + + """ + + def __init__(self, parser=None): + self.parser = parser + + def lastChild(self, parent): + """ Return the last child of an etree element. """ + if len(parent): + return parent[-1] + else: + return None + + def detab(self, text): + """ Remove a tab from the front of each line of the given text. """ + newtext = [] + lines = text.split('\n') + for line in lines: + if line.startswith(' '*markdown.TAB_LENGTH): + newtext.append(line[markdown.TAB_LENGTH:]) + elif not line.strip(): + newtext.append('') + else: + break + return '\n'.join(newtext), '\n'.join(lines[len(newtext):]) + + def looseDetab(self, text, level=1): + """ Remove a tab from front of lines but allowing dedented lines. """ + lines = text.split('\n') + for i in range(len(lines)): + if lines[i].startswith(' '*markdown.TAB_LENGTH*level): + lines[i] = lines[i][markdown.TAB_LENGTH*level:] + return '\n'.join(lines) + + def test(self, parent, block): + """ Test for block type. Must be overridden by subclasses. + + As the parser loops through processors, it will call the ``test`` method + on each to determine if the given block of text is of that type. This + method must return a boolean ``True`` or ``False``. The actual method of + testing is left to the needs of that particular block type. It could + be as simple as ``block.startswith(some_string)`` or a complex regular + expression. As the block type may be different depending on the parent + of the block (i.e. inside a list), the parent etree element is also + provided and may be used as part of the test. + + Keywords: + + * ``parent``: A etree element which will be the parent of the block. + * ``block``: A block of text from the source which has been split at + blank lines. + """ + pass + + def run(self, parent, blocks): + """ Run processor. Must be overridden by subclasses. + + When the parser determines the appropriate type of a block, the parser + will call the corresponding processor's ``run`` method. This method + should parse the individual lines of the block and append them to + the etree. + + Note that both the ``parent`` and ``etree`` keywords are pointers + to instances of the objects which should be edited in place. Each + processor must make changes to the existing objects as there is no + mechanism to return new/different objects to replace them. + + This means that this method should be adding SubElements or adding text + to the parent, and should remove (``pop``) or add (``insert``) items to + the list of blocks. + + Keywords: + + * ``parent``: A etree element which is the parent of the current block. + * ``blocks``: A list of all remaining blocks of the document. + """ + pass + + +class ListIndentProcessor(BlockProcessor): + """ Process children of list items. + + Example: + * a list item + process this part + + or this part + + """ + + INDENT_RE = re.compile(r'^(([ ]{%s})+)'% markdown.TAB_LENGTH) + ITEM_TYPES = ['li'] + LIST_TYPES = ['ul', 'ol'] + + def test(self, parent, block): + return block.startswith(' '*markdown.TAB_LENGTH) and \ + not self.parser.state.isstate('detabbed') and \ + (parent.tag in self.ITEM_TYPES or \ + (len(parent) and parent[-1] and \ + (parent[-1].tag in self.LIST_TYPES) + ) + ) + + def run(self, parent, blocks): + block = blocks.pop(0) + level, sibling = self.get_level(parent, block) + block = self.looseDetab(block, level) + + self.parser.state.set('detabbed') + if parent.tag in self.ITEM_TYPES: + # The parent is already a li. Just parse the child block. + self.parser.parseBlocks(parent, [block]) + elif sibling.tag in self.ITEM_TYPES: + # The sibling is a li. Use it as parent. + self.parser.parseBlocks(sibling, [block]) + elif len(sibling) and sibling[-1].tag in self.ITEM_TYPES: + # The parent is a list (``ol`` or ``ul``) which has children. + # Assume the last child li is the parent of this block. + if sibling[-1].text: + # If the parent li has text, that text needs to be moved to a p + block = '%s\n\n%s' % (sibling[-1].text, block) + sibling[-1].text = '' + self.parser.parseChunk(sibling[-1], block) + else: + self.create_item(sibling, block) + self.parser.state.reset() + + def create_item(self, parent, block): + """ Create a new li and parse the block with it as the parent. """ + li = markdown.etree.SubElement(parent, 'li') + self.parser.parseBlocks(li, [block]) + + def get_level(self, parent, block): + """ Get level of indent based on list level. """ + # Get indent level + m = self.INDENT_RE.match(block) + if m: + indent_level = len(m.group(1))/markdown.TAB_LENGTH + else: + indent_level = 0 + if self.parser.state.isstate('list'): + # We're in a tightlist - so we already are at correct parent. + level = 1 + else: + # We're in a looselist - so we need to find parent. + level = 0 + # Step through children of tree to find matching indent level. + while indent_level > level: + child = self.lastChild(parent) + if child and (child.tag in self.LIST_TYPES or child.tag in self.ITEM_TYPES): + if child.tag in self.LIST_TYPES: + level += 1 + parent = child + else: + # No more child levels. If we're short of indent_level, + # we have a code block. So we stop here. + break + return level, parent + + +class CodeBlockProcessor(BlockProcessor): + """ Process code blocks. """ + + def test(self, parent, block): + return block.startswith(' '*markdown.TAB_LENGTH) + + def run(self, parent, blocks): + sibling = self.lastChild(parent) + block = blocks.pop(0) + theRest = '' + if sibling and sibling.tag == "pre" and len(sibling) \ + and sibling[0].tag == "code": + # The previous block was a code block. As blank lines do not start + # new code blocks, append this block to the previous, adding back + # linebreaks removed from the split into a list. + code = sibling[0] + block, theRest = self.detab(block) + code.text = markdown.AtomicString('%s\n%s\n' % (code.text, block.rstrip())) + else: + # This is a new codeblock. Create the elements and insert text. + pre = markdown.etree.SubElement(parent, 'pre') + code = markdown.etree.SubElement(pre, 'code') + block, theRest = self.detab(block) + code.text = markdown.AtomicString('%s\n' % block.rstrip()) + if theRest: + # This block contained unindented line(s) after the first indented + # line. Insert these lines as the first block of the master blocks + # list for future processing. + blocks.insert(0, theRest) + + +class BlockQuoteProcessor(BlockProcessor): + + RE = re.compile(r'(^|\n)[ ]{0,3}>[ ]?(.*)') + + def test(self, parent, block): + return bool(self.RE.search(block)) + + def run(self, parent, blocks): + block = blocks.pop(0) + m = self.RE.search(block) + if m: + before = block[:m.start()] # Lines before blockquote + # Pass lines before blockquote in recursively for parsing forst. + self.parser.parseBlocks(parent, [before]) + # Remove ``> `` from begining of each line. + block = '\n'.join([self.clean(line) for line in + block[m.start():].split('\n')]) + sibling = self.lastChild(parent) + if sibling and sibling.tag == "blockquote": + # Previous block was a blockquote so set that as this blocks parent + quote = sibling + else: + # This is a new blockquote. Create a new parent element. + quote = markdown.etree.SubElement(parent, 'blockquote') + # Recursively parse block with blockquote as parent. + self.parser.parseChunk(quote, block) + + def clean(self, line): + """ Remove ``>`` from beginning of a line. """ + m = self.RE.match(line) + if line.strip() == ">": + return "" + elif m: + return m.group(2) + else: + return line + +class OListProcessor(BlockProcessor): + """ Process ordered list blocks. """ + + TAG = 'ol' + # Detect an item (``1. item``). ``group(1)`` contains contents of item. + RE = re.compile(r'^[ ]{0,3}\d+\.[ ](.*)') + # Detect items on secondary lines. they can be of either list type. + CHILD_RE = re.compile(r'^[ ]{0,3}((\d+\.)|[*+-])[ ](.*)') + # Detect indented (nested) items of either type + INDENT_RE = re.compile(r'^[ ]{4,7}((\d+\.)|[*+-])[ ].*') + + def test(self, parent, block): + return bool(self.RE.match(block)) + + def run(self, parent, blocks): + # Check fr multiple items in one block. + items = self.get_items(blocks.pop(0)) + sibling = self.lastChild(parent) + if sibling and sibling.tag in ['ol', 'ul']: + # Previous block was a list item, so set that as parent + lst = sibling + # make sure previous item is in a p. + if len(lst) and lst[-1].text and not len(lst[-1]): + p = markdown.etree.SubElement(lst[-1], 'p') + p.text = lst[-1].text + lst[-1].text = '' + # parse first block differently as it gets wrapped in a p. + li = markdown.etree.SubElement(lst, 'li') + self.parser.state.set('looselist') + firstitem = items.pop(0) + self.parser.parseBlocks(li, [firstitem]) + self.parser.state.reset() + else: + # This is a new list so create parent with appropriate tag. + lst = markdown.etree.SubElement(parent, self.TAG) + self.parser.state.set('list') + # Loop through items in block, recursively parsing each with the + # appropriate parent. + for item in items: + if item.startswith(' '*markdown.TAB_LENGTH): + # Item is indented. Parse with last item as parent + self.parser.parseBlocks(lst[-1], [item]) + else: + # New item. Create li and parse with it as parent + li = markdown.etree.SubElement(lst, 'li') + self.parser.parseBlocks(li, [item]) + self.parser.state.reset() + + def get_items(self, block): + """ Break a block into list items. """ + items = [] + for line in block.split('\n'): + m = self.CHILD_RE.match(line) + if m: + # This is a new item. Append + items.append(m.group(3)) + elif self.INDENT_RE.match(line): + # This is an indented (possibly nested) item. + if items[-1].startswith(' '*markdown.TAB_LENGTH): + # Previous item was indented. Append to that item. + items[-1] = '%s\n%s' % (items[-1], line) + else: + items.append(line) + else: + # This is another line of previous item. Append to that item. + items[-1] = '%s\n%s' % (items[-1], line) + return items + + +class UListProcessor(OListProcessor): + """ Process unordered list blocks. """ + + TAG = 'ul' + RE = re.compile(r'^[ ]{0,3}[*+-][ ](.*)') + + +class HashHeaderProcessor(BlockProcessor): + """ Process Hash Headers. """ + + # Detect a header at start of any line in block + RE = re.compile(r'(^|\n)(?P<level>#{1,6})(?P<header>.*?)#*(\n|$)') + + def test(self, parent, block): + return bool(self.RE.search(block)) + + def run(self, parent, blocks): + block = blocks.pop(0) + m = self.RE.search(block) + if m: + before = block[:m.start()] # All lines before header + after = block[m.end():] # All lines after header + if before: + # As the header was not the first line of the block and the + # lines before the header must be parsed first, + # recursively parse this lines as a block. + self.parser.parseBlocks(parent, [before]) + # Create header using named groups from RE + h = markdown.etree.SubElement(parent, 'h%d' % len(m.group('level'))) + h.text = m.group('header').strip() + if after: + # Insert remaining lines as first block for future parsing. + blocks.insert(0, after) + else: + # This should never happen, but just in case... + message(CRITICAL, "We've got a problem header!") + + +class SetextHeaderProcessor(BlockProcessor): + """ Process Setext-style Headers. """ + + # Detect Setext-style header. Must be first 2 lines of block. + RE = re.compile(r'^.*?\n[=-]{3,}', re.MULTILINE) + + def test(self, parent, block): + return bool(self.RE.match(block)) + + def run(self, parent, blocks): + lines = blocks.pop(0).split('\n') + # Determine level. ``=`` is 1 and ``-`` is 2. + if lines[1].startswith('='): + level = 1 + else: + level = 2 + h = markdown.etree.SubElement(parent, 'h%d' % level) + h.text = lines[0].strip() + if len(lines) > 2: + # Block contains additional lines. Add to master blocks for later. + blocks.insert(0, '\n'.join(lines[2:])) + + +class HRProcessor(BlockProcessor): + """ Process Horizontal Rules. """ + + RE = r'[ ]{0,3}(?P<ch>[*_-])[ ]?((?P=ch)[ ]?){2,}[ ]*' + # Detect hr on any line of a block. + SEARCH_RE = re.compile(r'(^|\n)%s(\n|$)' % RE) + # Match a hr on a single line of text. + MATCH_RE = re.compile(r'^%s$' % RE) + + def test(self, parent, block): + return bool(self.SEARCH_RE.search(block)) + + def run(self, parent, blocks): + lines = blocks.pop(0).split('\n') + prelines = [] + # Check for lines in block before hr. + for line in lines: + m = self.MATCH_RE.match(line) + if m: + break + else: + prelines.append(line) + if len(prelines): + # Recursively parse lines before hr so they get parsed first. + self.parser.parseBlocks(parent, ['\n'.join(prelines)]) + # create hr + hr = markdown.etree.SubElement(parent, 'hr') + # check for lines in block after hr. + lines = lines[len(prelines)+1:] + if len(lines): + # Add lines after hr to master blocks for later parsing. + blocks.insert(0, '\n'.join(lines)) + + +class EmptyBlockProcessor(BlockProcessor): + """ Process blocks and start with an empty line. """ + + # Detect a block that only contains whitespace + # or only whitespace on the first line. + RE = re.compile(r'^\s*\n') + + def test(self, parent, block): + return bool(self.RE.match(block)) + + def run(self, parent, blocks): + block = blocks.pop(0) + m = self.RE.match(block) + if m: + # Add remaining line to master blocks for later. + blocks.insert(0, block[m.end():]) + sibling = self.lastChild(parent) + if sibling and sibling.tag == 'pre' and sibling[0] and \ + sibling[0].tag == 'code': + # Last block is a codeblock. Append to preserve whitespace. + sibling[0].text = markdown.AtomicString('%s/n/n/n' % sibling[0].text ) + + +class ParagraphProcessor(BlockProcessor): + """ Process Paragraph blocks. """ + + def test(self, parent, block): + return True + + def run(self, parent, blocks): + block = blocks.pop(0) + if block.strip(): + # Not a blank block. Add to parent, otherwise throw it away. + if self.parser.state.isstate('list'): + # The parent is a tight-list. Append to parent.text + if parent.text: + parent.text = '%s\n%s' % (parent.text, block) + else: + parent.text = block.lstrip() + else: + # Create a regular paragraph + p = markdown.etree.SubElement(parent, 'p') + p.text = block.lstrip() diff --git a/vendor/tornado/website/markdown/commandline.py b/vendor/tornado/website/markdown/commandline.py new file mode 100644 index 0000000000..1eedc6dbb1 --- /dev/null +++ b/vendor/tornado/website/markdown/commandline.py @@ -0,0 +1,96 @@ +""" +COMMAND-LINE SPECIFIC STUFF +============================================================================= + +The rest of the code is specifically for handling the case where Python +Markdown is called from the command line. +""" + +import markdown +import sys +import logging +from logging import DEBUG, INFO, WARN, ERROR, CRITICAL + +EXECUTABLE_NAME_FOR_USAGE = "python markdown.py" +""" The name used in the usage statement displayed for python versions < 2.3. +(With python 2.3 and higher the usage statement is generated by optparse +and uses the actual name of the executable called.) """ + +OPTPARSE_WARNING = """ +Python 2.3 or higher required for advanced command line options. +For lower versions of Python use: + + %s INPUT_FILE > OUTPUT_FILE + +""" % EXECUTABLE_NAME_FOR_USAGE + +def parse_options(): + """ + Define and parse `optparse` options for command-line usage. + """ + + try: + optparse = __import__("optparse") + except: + if len(sys.argv) == 2: + return {'input': sys.argv[1], + 'output': None, + 'safe': False, + 'extensions': [], + 'encoding': None }, CRITICAL + else: + print OPTPARSE_WARNING + return None, None + + parser = optparse.OptionParser(usage="%prog INPUTFILE [options]") + parser.add_option("-f", "--file", dest="filename", default=sys.stdout, + help="write output to OUTPUT_FILE", + metavar="OUTPUT_FILE") + parser.add_option("-e", "--encoding", dest="encoding", + help="encoding for input and output files",) + parser.add_option("-q", "--quiet", default = CRITICAL, + action="store_const", const=CRITICAL+10, dest="verbose", + help="suppress all messages") + parser.add_option("-v", "--verbose", + action="store_const", const=INFO, dest="verbose", + help="print info messages") + parser.add_option("-s", "--safe", dest="safe", default=False, + metavar="SAFE_MODE", + help="safe mode ('replace', 'remove' or 'escape' user's HTML tag)") + parser.add_option("-o", "--output_format", dest="output_format", + default='xhtml1', metavar="OUTPUT_FORMAT", + help="Format of output. One of 'xhtml1' (default) or 'html4'.") + parser.add_option("--noisy", + action="store_const", const=DEBUG, dest="verbose", + help="print debug messages") + parser.add_option("-x", "--extension", action="append", dest="extensions", + help = "load extension EXTENSION", metavar="EXTENSION") + + (options, args) = parser.parse_args() + + if not len(args) == 1: + parser.print_help() + return None, None + else: + input_file = args[0] + + if not options.extensions: + options.extensions = [] + + return {'input': input_file, + 'output': options.filename, + 'safe_mode': options.safe, + 'extensions': options.extensions, + 'encoding': options.encoding, + 'output_format': options.output_format}, options.verbose + +def run(): + """Run Markdown from the command line.""" + + # Parse options and adjust logging level if necessary + options, logging_level = parse_options() + if not options: sys.exit(0) + if logging_level: logging.getLogger('MARKDOWN').setLevel(logging_level) + + # Run + markdown.markdownFromFile(**options) diff --git a/vendor/tornado/website/markdown/etree_loader.py b/vendor/tornado/website/markdown/etree_loader.py new file mode 100644 index 0000000000..e2599b2cb9 --- /dev/null +++ b/vendor/tornado/website/markdown/etree_loader.py @@ -0,0 +1,33 @@ + +from markdown import message, CRITICAL +import sys + +## Import +def importETree(): + """Import the best implementation of ElementTree, return a module object.""" + etree_in_c = None + try: # Is it Python 2.5+ with C implemenation of ElementTree installed? + import xml.etree.cElementTree as etree_in_c + except ImportError: + try: # Is it Python 2.5+ with Python implementation of ElementTree? + import xml.etree.ElementTree as etree + except ImportError: + try: # An earlier version of Python with cElementTree installed? + import cElementTree as etree_in_c + except ImportError: + try: # An earlier version of Python with Python ElementTree? + import elementtree.ElementTree as etree + except ImportError: + message(CRITICAL, "Failed to import ElementTree") + sys.exit(1) + if etree_in_c and etree_in_c.VERSION < "1.0": + message(CRITICAL, "For cElementTree version 1.0 or higher is required.") + sys.exit(1) + elif etree_in_c : + return etree_in_c + elif etree.VERSION < "1.1": + message(CRITICAL, "For ElementTree version 1.1 or higher is required") + sys.exit(1) + else : + return etree + diff --git a/vendor/tornado/website/markdown/extensions/__init__.py b/vendor/tornado/website/markdown/extensions/__init__.py new file mode 100644 index 0000000000..e69de29bb2 --- /dev/null +++ b/vendor/tornado/website/markdown/extensions/__init__.py diff --git a/vendor/tornado/website/markdown/extensions/toc.py b/vendor/tornado/website/markdown/extensions/toc.py new file mode 100644 index 0000000000..3afaea0488 --- /dev/null +++ b/vendor/tornado/website/markdown/extensions/toc.py @@ -0,0 +1,140 @@ +""" +Table of Contents Extension for Python-Markdown +* * * + +(c) 2008 [Jack Miller](http://codezen.org) + +Dependencies: +* [Markdown 2.0+](http://www.freewisdom.org/projects/python-markdown/) + +""" +import markdown +from markdown import etree +import re + +class TocTreeprocessor(markdown.treeprocessors.Treeprocessor): + # Iterator wrapper to get parent and child all at once + def iterparent(self, root): + for parent in root.getiterator(): + for child in parent: + yield parent, child + + def run(self, doc): + div = etree.Element("div") + div.attrib["class"] = "toc" + last_li = None + + # Add title to the div + if self.config["title"][0]: + header = etree.SubElement(div, "span") + header.attrib["class"] = "toctitle" + header.text = self.config["title"][0] + + level = 0 + list_stack=[div] + header_rgx = re.compile("[Hh][123456]") + + # Get a list of id attributes + used_ids = [] + for c in doc.getiterator(): + if "id" in c.attrib: + used_ids.append(c.attrib["id"]) + + for (p, c) in self.iterparent(doc): + if not c.text: + continue + + # To keep the output from screwing up the + # validation by putting a <div> inside of a <p> + # we actually replace the <p> in its entirety. + # We do not allow the marker inside a header as that + # would causes an enless loop of placing a new TOC + # inside previously generated TOC. + + if c.text.find(self.config["marker"][0]) > -1 and not header_rgx.match(c.tag): + for i in range(len(p)): + if p[i] == c: + p[i] = div + break + + if header_rgx.match(c.tag): + tag_level = int(c.tag[-1]) + + # Regardless of how many levels we jumped + # only one list should be created, since + # empty lists containing lists are illegal. + + if tag_level < level: + list_stack.pop() + level = tag_level + + if tag_level > level: + newlist = etree.Element("ul") + if last_li: + last_li.append(newlist) + else: + list_stack[-1].append(newlist) + list_stack.append(newlist) + level = tag_level + + # Do not override pre-existing ids + if not "id" in c.attrib: + id = self.config["slugify"][0](c.text) + if id in used_ids: + ctr = 1 + while "%s_%d" % (id, ctr) in used_ids: + ctr += 1 + id = "%s_%d" % (id, ctr) + used_ids.append(id) + c.attrib["id"] = id + else: + id = c.attrib["id"] + + # List item link, to be inserted into the toc div + last_li = etree.Element("li") + link = etree.SubElement(last_li, "a") + link.text = c.text + link.attrib["href"] = '#' + id + + if int(self.config["anchorlink"][0]): + anchor = etree.SubElement(c, "a") + anchor.text = c.text + anchor.attrib["href"] = "#" + id + anchor.attrib["class"] = "toclink" + c.text = "" + + list_stack[-1].append(last_li) + +class TocExtension(markdown.Extension): + def __init__(self, configs): + self.config = { "marker" : ["[TOC]", + "Text to find and replace with Table of Contents -" + "Defaults to \"[TOC]\""], + "slugify" : [self.slugify, + "Function to generate anchors based on header text-" + "Defaults to a built in slugify function."], + "title" : [None, + "Title to insert into TOC <div> - " + "Defaults to None"], + "anchorlink" : [0, + "1 if header should be a self link" + "Defaults to 0"]} + + for key, value in configs: + self.setConfig(key, value) + + # This is exactly the same as Django's slugify + def slugify(self, value): + """ Slugify a string, to make it URL friendly. """ + import unicodedata + value = unicodedata.normalize('NFKD', value).encode('ascii', 'ignore') + value = unicode(re.sub('[^\w\s-]', '', value).strip().lower()) + return re.sub('[-\s]+','-',value) + + def extendMarkdown(self, md, md_globals): + tocext = TocTreeprocessor(md) + tocext.config = self.config + md.treeprocessors.add("toc", tocext, "_begin") + +def makeExtension(configs={}): + return TocExtension(configs=configs) diff --git a/vendor/tornado/website/markdown/html4.py b/vendor/tornado/website/markdown/html4.py new file mode 100644 index 0000000000..08f241d57a --- /dev/null +++ b/vendor/tornado/website/markdown/html4.py @@ -0,0 +1,274 @@ +# markdown/html4.py +# +# Add html4 serialization to older versions of Elementree +# Taken from ElementTree 1.3 preview with slight modifications +# +# Copyright (c) 1999-2007 by Fredrik Lundh. All rights reserved. +# +# fredrik@pythonware.com +# http://www.pythonware.com +# +# -------------------------------------------------------------------- +# The ElementTree toolkit is +# +# Copyright (c) 1999-2007 by Fredrik Lundh +# +# By obtaining, using, and/or copying this software and/or its +# associated documentation, you agree that you have read, understood, +# and will comply with the following terms and conditions: +# +# Permission to use, copy, modify, and distribute this software and +# its associated documentation for any purpose and without fee is +# hereby granted, provided that the above copyright notice appears in +# all copies, and that both that copyright notice and this permission +# notice appear in supporting documentation, and that the name of +# Secret Labs AB or the author not be used in advertising or publicity +# pertaining to distribution of the software without specific, written +# prior permission. +# +# SECRET LABS AB AND THE AUTHOR DISCLAIMS ALL WARRANTIES WITH REGARD +# TO THIS SOFTWARE, INCLUDING ALL IMPLIED WARRANTIES OF MERCHANT- +# ABILITY AND FITNESS. IN NO EVENT SHALL SECRET LABS AB OR THE AUTHOR +# BE LIABLE FOR ANY SPECIAL, INDIRECT OR CONSEQUENTIAL DAMAGES OR ANY +# DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, +# WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS +# ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE +# OF THIS SOFTWARE. +# -------------------------------------------------------------------- + + +import markdown +ElementTree = markdown.etree.ElementTree +QName = markdown.etree.QName +Comment = markdown.etree.Comment +PI = markdown.etree.PI +ProcessingInstruction = markdown.etree.ProcessingInstruction + +HTML_EMPTY = ("area", "base", "basefont", "br", "col", "frame", "hr", + "img", "input", "isindex", "link", "meta" "param") + +try: + HTML_EMPTY = set(HTML_EMPTY) +except NameError: + pass + +_namespace_map = { + # "well-known" namespace prefixes + "http://www.w3.org/XML/1998/namespace": "xml", + "http://www.w3.org/1999/xhtml": "html", + "http://www.w3.org/1999/02/22-rdf-syntax-ns#": "rdf", + "http://schemas.xmlsoap.org/wsdl/": "wsdl", + # xml schema + "http://www.w3.org/2001/XMLSchema": "xs", + "http://www.w3.org/2001/XMLSchema-instance": "xsi", + # dublic core + "http://purl.org/dc/elements/1.1/": "dc", +} + + +def _raise_serialization_error(text): + raise TypeError( + "cannot serialize %r (type %s)" % (text, type(text).__name__) + ) + +def _encode(text, encoding): + try: + return text.encode(encoding, "xmlcharrefreplace") + except (TypeError, AttributeError): + _raise_serialization_error(text) + +def _escape_cdata(text, encoding): + # escape character data + try: + # it's worth avoiding do-nothing calls for strings that are + # shorter than 500 character, or so. assume that's, by far, + # the most common case in most applications. + if "&" in text: + text = text.replace("&", "&") + if "<" in text: + text = text.replace("<", "<") + if ">" in text: + text = text.replace(">", ">") + return text.encode(encoding, "xmlcharrefreplace") + except (TypeError, AttributeError): + _raise_serialization_error(text) + + +def _escape_attrib(text, encoding): + # escape attribute value + try: + if "&" in text: + text = text.replace("&", "&") + if "<" in text: + text = text.replace("<", "<") + if ">" in text: + text = text.replace(">", ">") + if "\"" in text: + text = text.replace("\"", """) + if "\n" in text: + text = text.replace("\n", " ") + return text.encode(encoding, "xmlcharrefreplace") + except (TypeError, AttributeError): + _raise_serialization_error(text) + +def _escape_attrib_html(text, encoding): + # escape attribute value + try: + if "&" in text: + text = text.replace("&", "&") + if ">" in text: + text = text.replace(">", ">") + if "\"" in text: + text = text.replace("\"", """) + return text.encode(encoding, "xmlcharrefreplace") + except (TypeError, AttributeError): + _raise_serialization_error(text) + + +def _serialize_html(write, elem, encoding, qnames, namespaces): + tag = elem.tag + text = elem.text + if tag is Comment: + write("<!--%s-->" % _escape_cdata(text, encoding)) + elif tag is ProcessingInstruction: + write("<?%s?>" % _escape_cdata(text, encoding)) + else: + tag = qnames[tag] + if tag is None: + if text: + write(_escape_cdata(text, encoding)) + for e in elem: + _serialize_html(write, e, encoding, qnames, None) + else: + write("<" + tag) + items = elem.items() + if items or namespaces: + items.sort() # lexical order + for k, v in items: + if isinstance(k, QName): + k = k.text + if isinstance(v, QName): + v = qnames[v.text] + else: + v = _escape_attrib_html(v, encoding) + # FIXME: handle boolean attributes + write(" %s=\"%s\"" % (qnames[k], v)) + if namespaces: + items = namespaces.items() + items.sort(key=lambda x: x[1]) # sort on prefix + for v, k in items: + if k: + k = ":" + k + write(" xmlns%s=\"%s\"" % ( + k.encode(encoding), + _escape_attrib(v, encoding) + )) + write(">") + tag = tag.lower() + if text: + if tag == "script" or tag == "style": + write(_encode(text, encoding)) + else: + write(_escape_cdata(text, encoding)) + for e in elem: + _serialize_html(write, e, encoding, qnames, None) + if tag not in HTML_EMPTY: + write("</" + tag + ">") + if elem.tail: + write(_escape_cdata(elem.tail, encoding)) + +def write_html(root, f, + # keyword arguments + encoding="us-ascii", + default_namespace=None): + assert root is not None + if not hasattr(f, "write"): + f = open(f, "wb") + write = f.write + if not encoding: + encoding = "us-ascii" + qnames, namespaces = _namespaces( + root, encoding, default_namespace + ) + _serialize_html( + write, root, encoding, qnames, namespaces + ) + +# -------------------------------------------------------------------- +# serialization support + +def _namespaces(elem, encoding, default_namespace=None): + # identify namespaces used in this tree + + # maps qnames to *encoded* prefix:local names + qnames = {None: None} + + # maps uri:s to prefixes + namespaces = {} + if default_namespace: + namespaces[default_namespace] = "" + + def encode(text): + return text.encode(encoding) + + def add_qname(qname): + # calculate serialized qname representation + try: + if qname[:1] == "{": + uri, tag = qname[1:].split("}", 1) + prefix = namespaces.get(uri) + if prefix is None: + prefix = _namespace_map.get(uri) + if prefix is None: + prefix = "ns%d" % len(namespaces) + if prefix != "xml": + namespaces[uri] = prefix + if prefix: + qnames[qname] = encode("%s:%s" % (prefix, tag)) + else: + qnames[qname] = encode(tag) # default element + else: + if default_namespace: + # FIXME: can this be handled in XML 1.0? + raise ValueError( + "cannot use non-qualified names with " + "default_namespace option" + ) + qnames[qname] = encode(qname) + except TypeError: + _raise_serialization_error(qname) + + # populate qname and namespaces table + try: + iterate = elem.iter + except AttributeError: + iterate = elem.getiterator # cET compatibility + for elem in iterate(): + tag = elem.tag + if isinstance(tag, QName) and tag.text not in qnames: + add_qname(tag.text) + elif isinstance(tag, basestring): + if tag not in qnames: + add_qname(tag) + elif tag is not None and tag is not Comment and tag is not PI: + _raise_serialization_error(tag) + for key, value in elem.items(): + if isinstance(key, QName): + key = key.text + if key not in qnames: + add_qname(key) + if isinstance(value, QName) and value.text not in qnames: + add_qname(value.text) + text = elem.text + if isinstance(text, QName) and text.text not in qnames: + add_qname(text.text) + return qnames, namespaces + +def to_html_string(element, encoding=None): + class dummy: + pass + data = [] + file = dummy() + file.write = data.append + write_html(ElementTree(element).getroot(),file,encoding) + return "".join(data) diff --git a/vendor/tornado/website/markdown/inlinepatterns.py b/vendor/tornado/website/markdown/inlinepatterns.py new file mode 100644 index 0000000000..89fa3b2ef4 --- /dev/null +++ b/vendor/tornado/website/markdown/inlinepatterns.py @@ -0,0 +1,371 @@ +""" +INLINE PATTERNS +============================================================================= + +Inline patterns such as *emphasis* are handled by means of auxiliary +objects, one per pattern. Pattern objects must be instances of classes +that extend markdown.Pattern. Each pattern object uses a single regular +expression and needs support the following methods: + + pattern.getCompiledRegExp() # returns a regular expression + + pattern.handleMatch(m) # takes a match object and returns + # an ElementTree element or just plain text + +All of python markdown's built-in patterns subclass from Pattern, +but you can add additional patterns that don't. + +Also note that all the regular expressions used by inline must +capture the whole block. For this reason, they all start with +'^(.*)' and end with '(.*)!'. In case with built-in expression +Pattern takes care of adding the "^(.*)" and "(.*)!". + +Finally, the order in which regular expressions are applied is very +important - e.g. if we first replace http://.../ links with <a> tags +and _then_ try to replace inline html, we would end up with a mess. +So, we apply the expressions in the following order: + +* escape and backticks have to go before everything else, so + that we can preempt any markdown patterns by escaping them. + +* then we handle auto-links (must be done before inline html) + +* then we handle inline HTML. At this point we will simply + replace all inline HTML strings with a placeholder and add + the actual HTML to a hash. + +* then inline images (must be done before links) + +* then bracketed links, first regular then reference-style + +* finally we apply strong and emphasis +""" + +import markdown +import re +from urlparse import urlparse, urlunparse +import sys +if sys.version >= "3.0": + from html import entities as htmlentitydefs +else: + import htmlentitydefs + +""" +The actual regular expressions for patterns +----------------------------------------------------------------------------- +""" + +NOBRACKET = r'[^\]\[]*' +BRK = ( r'\[(' + + (NOBRACKET + r'(\[')*6 + + (NOBRACKET+ r'\])*')*6 + + NOBRACKET + r')\]' ) +NOIMG = r'(?<!\!)' + +BACKTICK_RE = r'(?<!\\)(`+)(.+?)(?<!`)\2(?!`)' # `e=f()` or ``e=f("`")`` +ESCAPE_RE = r'\\(.)' # \< +EMPHASIS_RE = r'(\*)([^\*]*)\2' # *emphasis* +STRONG_RE = r'(\*{2}|_{2})(.*?)\2' # **strong** +STRONG_EM_RE = r'(\*{3}|_{3})(.*?)\2' # ***strong*** + +if markdown.SMART_EMPHASIS: + EMPHASIS_2_RE = r'(?<!\S)(_)(\S.*?)\2' # _emphasis_ +else: + EMPHASIS_2_RE = r'(_)(.*?)\2' # _emphasis_ + +LINK_RE = NOIMG + BRK + \ +r'''\(\s*(<.*?>|((?:(?:\(.*?\))|[^\(\)]))*?)\s*((['"])(.*)\12)?\)''' +# [text](url) or [text](<url>) + +IMAGE_LINK_RE = r'\!' + BRK + r'\s*\((<.*?>|([^\)]*))\)' +#  or  +REFERENCE_RE = NOIMG + BRK+ r'\s*\[([^\]]*)\]' # [Google][3] +IMAGE_REFERENCE_RE = r'\!' + BRK + '\s*\[([^\]]*)\]' # ![alt text][2] +NOT_STRONG_RE = r'( \* )' # stand-alone * or _ +AUTOLINK_RE = r'<((?:f|ht)tps?://[^>]*)>' # <http://www.123.com> +AUTOMAIL_RE = r'<([^> \!]*@[^> ]*)>' # <me@example.com> + +HTML_RE = r'(\<([a-zA-Z/][^\>]*?|\!--.*?--)\>)' # <...> +ENTITY_RE = r'(&[\#a-zA-Z0-9]*;)' # & +LINE_BREAK_RE = r' \n' # two spaces at end of line +LINE_BREAK_2_RE = r' $' # two spaces at end of text + + +def dequote(string): + """Remove quotes from around a string.""" + if ( ( string.startswith('"') and string.endswith('"')) + or (string.startswith("'") and string.endswith("'")) ): + return string[1:-1] + else: + return string + +ATTR_RE = re.compile("\{@([^\}]*)=([^\}]*)}") # {@id=123} + +def handleAttributes(text, parent): + """Set values of an element based on attribute definitions ({@id=123}).""" + def attributeCallback(match): + parent.set(match.group(1), match.group(2).replace('\n', ' ')) + return ATTR_RE.sub(attributeCallback, text) + + +""" +The pattern classes +----------------------------------------------------------------------------- +""" + +class Pattern: + """Base class that inline patterns subclass. """ + + def __init__ (self, pattern, markdown_instance=None): + """ + Create an instant of an inline pattern. + + Keyword arguments: + + * pattern: A regular expression that matches a pattern + + """ + self.pattern = pattern + self.compiled_re = re.compile("^(.*?)%s(.*?)$" % pattern, re.DOTALL) + + # Api for Markdown to pass safe_mode into instance + self.safe_mode = False + if markdown_instance: + self.markdown = markdown_instance + + def getCompiledRegExp (self): + """ Return a compiled regular expression. """ + return self.compiled_re + + def handleMatch(self, m): + """Return a ElementTree element from the given match. + + Subclasses should override this method. + + Keyword arguments: + + * m: A re match object containing a match of the pattern. + + """ + pass + + def type(self): + """ Return class name, to define pattern type """ + return self.__class__.__name__ + +BasePattern = Pattern # for backward compatibility + +class SimpleTextPattern (Pattern): + """ Return a simple text of group(2) of a Pattern. """ + def handleMatch(self, m): + text = m.group(2) + if text == markdown.INLINE_PLACEHOLDER_PREFIX: + return None + return text + +class SimpleTagPattern (Pattern): + """ + Return element of type `tag` with a text attribute of group(3) + of a Pattern. + + """ + def __init__ (self, pattern, tag): + Pattern.__init__(self, pattern) + self.tag = tag + + def handleMatch(self, m): + el = markdown.etree.Element(self.tag) + el.text = m.group(3) + return el + + +class SubstituteTagPattern (SimpleTagPattern): + """ Return a eLement of type `tag` with no children. """ + def handleMatch (self, m): + return markdown.etree.Element(self.tag) + + +class BacktickPattern (Pattern): + """ Return a `<code>` element containing the matching text. """ + def __init__ (self, pattern): + Pattern.__init__(self, pattern) + self.tag = "code" + + def handleMatch(self, m): + el = markdown.etree.Element(self.tag) + el.text = markdown.AtomicString(m.group(3).strip()) + return el + + +class DoubleTagPattern (SimpleTagPattern): + """Return a ElementTree element nested in tag2 nested in tag1. + + Useful for strong emphasis etc. + + """ + def handleMatch(self, m): + tag1, tag2 = self.tag.split(",") + el1 = markdown.etree.Element(tag1) + el2 = markdown.etree.SubElement(el1, tag2) + el2.text = m.group(3) + return el1 + + +class HtmlPattern (Pattern): + """ Store raw inline html and return a placeholder. """ + def handleMatch (self, m): + rawhtml = m.group(2) + inline = True + place_holder = self.markdown.htmlStash.store(rawhtml) + return place_holder + + +class LinkPattern (Pattern): + """ Return a link element from the given match. """ + def handleMatch(self, m): + el = markdown.etree.Element("a") + el.text = m.group(2) + title = m.group(11) + href = m.group(9) + + if href: + if href[0] == "<": + href = href[1:-1] + el.set("href", self.sanitize_url(href.strip())) + else: + el.set("href", "") + + if title: + title = dequote(title) #.replace('"', """) + el.set("title", title) + return el + + def sanitize_url(self, url): + """ + Sanitize a url against xss attacks in "safe_mode". + + Rather than specifically blacklisting `javascript:alert("XSS")` and all + its aliases (see <http://ha.ckers.org/xss.html>), we whitelist known + safe url formats. Most urls contain a network location, however some + are known not to (i.e.: mailto links). Script urls do not contain a + location. Additionally, for `javascript:...`, the scheme would be + "javascript" but some aliases will appear to `urlparse()` to have no + scheme. On top of that relative links (i.e.: "foo/bar.html") have no + scheme. Therefore we must check "path", "parameters", "query" and + "fragment" for any literal colons. We don't check "scheme" for colons + because it *should* never have any and "netloc" must allow the form: + `username:password@host:port`. + + """ + locless_schemes = ['', 'mailto', 'news'] + scheme, netloc, path, params, query, fragment = url = urlparse(url) + safe_url = False + if netloc != '' or scheme in locless_schemes: + safe_url = True + + for part in url[2:]: + if ":" in part: + safe_url = False + + if self.markdown.safeMode and not safe_url: + return '' + else: + return urlunparse(url) + +class ImagePattern(LinkPattern): + """ Return a img element from the given match. """ + def handleMatch(self, m): + el = markdown.etree.Element("img") + src_parts = m.group(9).split() + if src_parts: + src = src_parts[0] + if src[0] == "<" and src[-1] == ">": + src = src[1:-1] + el.set('src', self.sanitize_url(src)) + else: + el.set('src', "") + if len(src_parts) > 1: + el.set('title', dequote(" ".join(src_parts[1:]))) + + if markdown.ENABLE_ATTRIBUTES: + truealt = handleAttributes(m.group(2), el) + else: + truealt = m.group(2) + + el.set('alt', truealt) + return el + +class ReferencePattern(LinkPattern): + """ Match to a stored reference and return link element. """ + def handleMatch(self, m): + if m.group(9): + id = m.group(9).lower() + else: + # if we got something like "[Google][]" + # we'll use "google" as the id + id = m.group(2).lower() + + if not id in self.markdown.references: # ignore undefined refs + return None + href, title = self.markdown.references[id] + + text = m.group(2) + return self.makeTag(href, title, text) + + def makeTag(self, href, title, text): + el = markdown.etree.Element('a') + + el.set('href', self.sanitize_url(href)) + if title: + el.set('title', title) + + el.text = text + return el + + +class ImageReferencePattern (ReferencePattern): + """ Match to a stored reference and return img element. """ + def makeTag(self, href, title, text): + el = markdown.etree.Element("img") + el.set("src", self.sanitize_url(href)) + if title: + el.set("title", title) + el.set("alt", text) + return el + + +class AutolinkPattern (Pattern): + """ Return a link Element given an autolink (`<http://example/com>`). """ + def handleMatch(self, m): + el = markdown.etree.Element("a") + el.set('href', m.group(2)) + el.text = markdown.AtomicString(m.group(2)) + return el + +class AutomailPattern (Pattern): + """ + Return a mailto link Element given an automail link (`<foo@example.com>`). + """ + def handleMatch(self, m): + el = markdown.etree.Element('a') + email = m.group(2) + if email.startswith("mailto:"): + email = email[len("mailto:"):] + + def codepoint2name(code): + """Return entity definition by code, or the code if not defined.""" + entity = htmlentitydefs.codepoint2name.get(code) + if entity: + return "%s%s;" % (markdown.AMP_SUBSTITUTE, entity) + else: + return "%s#%d;" % (markdown.AMP_SUBSTITUTE, code) + + letters = [codepoint2name(ord(letter)) for letter in email] + el.text = markdown.AtomicString(''.join(letters)) + + mailto = "mailto:" + email + mailto = "".join([markdown.AMP_SUBSTITUTE + '#%d;' % + ord(letter) for letter in mailto]) + el.set('href', mailto) + return el + diff --git a/vendor/tornado/website/markdown/odict.py b/vendor/tornado/website/markdown/odict.py new file mode 100644 index 0000000000..bf3ef07182 --- /dev/null +++ b/vendor/tornado/website/markdown/odict.py @@ -0,0 +1,162 @@ +class OrderedDict(dict): + """ + A dictionary that keeps its keys in the order in which they're inserted. + + Copied from Django's SortedDict with some modifications. + + """ + def __new__(cls, *args, **kwargs): + instance = super(OrderedDict, cls).__new__(cls, *args, **kwargs) + instance.keyOrder = [] + return instance + + def __init__(self, data=None): + if data is None: + data = {} + super(OrderedDict, self).__init__(data) + if isinstance(data, dict): + self.keyOrder = data.keys() + else: + self.keyOrder = [] + for key, value in data: + if key not in self.keyOrder: + self.keyOrder.append(key) + + def __deepcopy__(self, memo): + from copy import deepcopy + return self.__class__([(key, deepcopy(value, memo)) + for key, value in self.iteritems()]) + + def __setitem__(self, key, value): + super(OrderedDict, self).__setitem__(key, value) + if key not in self.keyOrder: + self.keyOrder.append(key) + + def __delitem__(self, key): + super(OrderedDict, self).__delitem__(key) + self.keyOrder.remove(key) + + def __iter__(self): + for k in self.keyOrder: + yield k + + def pop(self, k, *args): + result = super(OrderedDict, self).pop(k, *args) + try: + self.keyOrder.remove(k) + except ValueError: + # Key wasn't in the dictionary in the first place. No problem. + pass + return result + + def popitem(self): + result = super(OrderedDict, self).popitem() + self.keyOrder.remove(result[0]) + return result + + def items(self): + return zip(self.keyOrder, self.values()) + + def iteritems(self): + for key in self.keyOrder: + yield key, super(OrderedDict, self).__getitem__(key) + + def keys(self): + return self.keyOrder[:] + + def iterkeys(self): + return iter(self.keyOrder) + + def values(self): + return [super(OrderedDict, self).__getitem__(k) for k in self.keyOrder] + + def itervalues(self): + for key in self.keyOrder: + yield super(OrderedDict, self).__getitem__(key) + + def update(self, dict_): + for k, v in dict_.items(): + self.__setitem__(k, v) + + def setdefault(self, key, default): + if key not in self.keyOrder: + self.keyOrder.append(key) + return super(OrderedDict, self).setdefault(key, default) + + def value_for_index(self, index): + """Return the value of the item at the given zero-based index.""" + return self[self.keyOrder[index]] + + def insert(self, index, key, value): + """Insert the key, value pair before the item with the given index.""" + if key in self.keyOrder: + n = self.keyOrder.index(key) + del self.keyOrder[n] + if n < index: + index -= 1 + self.keyOrder.insert(index, key) + super(OrderedDict, self).__setitem__(key, value) + + def copy(self): + """Return a copy of this object.""" + # This way of initializing the copy means it works for subclasses, too. + obj = self.__class__(self) + obj.keyOrder = self.keyOrder[:] + return obj + + def __repr__(self): + """ + Replace the normal dict.__repr__ with a version that returns the keys + in their sorted order. + """ + return '{%s}' % ', '.join(['%r: %r' % (k, v) for k, v in self.items()]) + + def clear(self): + super(OrderedDict, self).clear() + self.keyOrder = [] + + def index(self, key): + """ Return the index of a given key. """ + return self.keyOrder.index(key) + + def index_for_location(self, location): + """ Return index or None for a given location. """ + if location == '_begin': + i = 0 + elif location == '_end': + i = None + elif location.startswith('<') or location.startswith('>'): + i = self.index(location[1:]) + if location.startswith('>'): + if i >= len(self): + # last item + i = None + else: + i += 1 + else: + raise ValueError('Not a valid location: "%s". Location key ' + 'must start with a ">" or "<".' % location) + return i + + def add(self, key, value, location): + """ Insert by key location. """ + i = self.index_for_location(location) + if i is not None: + self.insert(i, key, value) + else: + self.__setitem__(key, value) + + def link(self, key, location): + """ Change location of an existing item. """ + n = self.keyOrder.index(key) + del self.keyOrder[n] + i = self.index_for_location(location) + try: + if i is not None: + self.keyOrder.insert(i, key) + else: + self.keyOrder.append(key) + except Error: + # restore to prevent data loss and reraise + self.keyOrder.insert(n, key) + raise Error diff --git a/vendor/tornado/website/markdown/postprocessors.py b/vendor/tornado/website/markdown/postprocessors.py new file mode 100644 index 0000000000..80227bb909 --- /dev/null +++ b/vendor/tornado/website/markdown/postprocessors.py @@ -0,0 +1,77 @@ +""" +POST-PROCESSORS +============================================================================= + +Markdown also allows post-processors, which are similar to preprocessors in +that they need to implement a "run" method. However, they are run after core +processing. + +""" + + +import markdown + +class Processor: + def __init__(self, markdown_instance=None): + if markdown_instance: + self.markdown = markdown_instance + +class Postprocessor(Processor): + """ + Postprocessors are run after the ElementTree it converted back into text. + + Each Postprocessor implements a "run" method that takes a pointer to a + text string, modifies it as necessary and returns a text string. + + Postprocessors must extend markdown.Postprocessor. + + """ + + def run(self, text): + """ + Subclasses of Postprocessor should implement a `run` method, which + takes the html document as a single text string and returns a + (possibly modified) string. + + """ + pass + + +class RawHtmlPostprocessor(Postprocessor): + """ Restore raw html to the document. """ + + def run(self, text): + """ Iterate over html stash and restore "safe" html. """ + for i in range(self.markdown.htmlStash.html_counter): + html, safe = self.markdown.htmlStash.rawHtmlBlocks[i] + if self.markdown.safeMode and not safe: + if str(self.markdown.safeMode).lower() == 'escape': + html = self.escape(html) + elif str(self.markdown.safeMode).lower() == 'remove': + html = '' + else: + html = markdown.HTML_REMOVED_TEXT + if safe or not self.markdown.safeMode: + text = text.replace("<p>%s</p>" % + (markdown.preprocessors.HTML_PLACEHOLDER % i), + html + "\n") + text = text.replace(markdown.preprocessors.HTML_PLACEHOLDER % i, + html) + return text + + def escape(self, html): + """ Basic html escaping """ + html = html.replace('&', '&') + html = html.replace('<', '<') + html = html.replace('>', '>') + return html.replace('"', '"') + + +class AndSubstitutePostprocessor(Postprocessor): + """ Restore valid entities """ + def __init__(self): + pass + + def run(self, text): + text = text.replace(markdown.AMP_SUBSTITUTE, "&") + return text diff --git a/vendor/tornado/website/markdown/preprocessors.py b/vendor/tornado/website/markdown/preprocessors.py new file mode 100644 index 0000000000..712a1e8755 --- /dev/null +++ b/vendor/tornado/website/markdown/preprocessors.py @@ -0,0 +1,214 @@ + +""" +PRE-PROCESSORS +============================================================================= + +Preprocessors work on source text before we start doing anything too +complicated. +""" + +import re +import markdown + +HTML_PLACEHOLDER_PREFIX = markdown.STX+"wzxhzdk:" +HTML_PLACEHOLDER = HTML_PLACEHOLDER_PREFIX + "%d" + markdown.ETX + +class Processor: + def __init__(self, markdown_instance=None): + if markdown_instance: + self.markdown = markdown_instance + +class Preprocessor (Processor): + """ + Preprocessors are run after the text is broken into lines. + + Each preprocessor implements a "run" method that takes a pointer to a + list of lines of the document, modifies it as necessary and returns + either the same pointer or a pointer to a new list. + + Preprocessors must extend markdown.Preprocessor. + + """ + def run(self, lines): + """ + Each subclass of Preprocessor should override the `run` method, which + takes the document as a list of strings split by newlines and returns + the (possibly modified) list of lines. + + """ + pass + +class HtmlStash: + """ + This class is used for stashing HTML objects that we extract + in the beginning and replace with place-holders. + """ + + def __init__ (self): + """ Create a HtmlStash. """ + self.html_counter = 0 # for counting inline html segments + self.rawHtmlBlocks=[] + + def store(self, html, safe=False): + """ + Saves an HTML segment for later reinsertion. Returns a + placeholder string that needs to be inserted into the + document. + + Keyword arguments: + + * html: an html segment + * safe: label an html segment as safe for safemode + + Returns : a placeholder string + + """ + self.rawHtmlBlocks.append((html, safe)) + placeholder = HTML_PLACEHOLDER % self.html_counter + self.html_counter += 1 + return placeholder + + def reset(self): + self.html_counter = 0 + self.rawHtmlBlocks = [] + + +class HtmlBlockPreprocessor(Preprocessor): + """Remove html blocks from the text and store them for later retrieval.""" + + right_tag_patterns = ["</%s>", "%s>"] + + def _get_left_tag(self, block): + return block[1:].replace(">", " ", 1).split()[0].lower() + + def _get_right_tag(self, left_tag, block): + for p in self.right_tag_patterns: + tag = p % left_tag + i = block.rfind(tag) + if i > 2: + return tag.lstrip("<").rstrip(">"), i + len(p)-2 + len(left_tag) + return block.rstrip()[-len(left_tag)-2:-1].lower(), len(block) + + def _equal_tags(self, left_tag, right_tag): + if left_tag == 'div' or left_tag[0] in ['?', '@', '%']: # handle PHP, etc. + return True + if ("/" + left_tag) == right_tag: + return True + if (right_tag == "--" and left_tag == "--"): + return True + elif left_tag == right_tag[1:] \ + and right_tag[0] != "<": + return True + else: + return False + + def _is_oneliner(self, tag): + return (tag in ['hr', 'hr/']) + + def run(self, lines): + text = "\n".join(lines) + new_blocks = [] + text = text.split("\n\n") + items = [] + left_tag = '' + right_tag = '' + in_tag = False # flag + + while text: + block = text[0] + if block.startswith("\n"): + block = block[1:] + text = text[1:] + + if block.startswith("\n"): + block = block[1:] + + if not in_tag: + if block.startswith("<"): + left_tag = self._get_left_tag(block) + right_tag, data_index = self._get_right_tag(left_tag, block) + + if data_index < len(block): + text.insert(0, block[data_index:]) + block = block[:data_index] + + if not (markdown.isBlockLevel(left_tag) \ + or block[1] in ["!", "?", "@", "%"]): + new_blocks.append(block) + continue + + if self._is_oneliner(left_tag): + new_blocks.append(block.strip()) + continue + + if block[1] == "!": + # is a comment block + left_tag = "--" + right_tag, data_index = self._get_right_tag(left_tag, block) + # keep checking conditions below and maybe just append + + if block.rstrip().endswith(">") \ + and self._equal_tags(left_tag, right_tag): + new_blocks.append( + self.markdown.htmlStash.store(block.strip())) + continue + else: #if not block[1] == "!": + # if is block level tag and is not complete + + if markdown.isBlockLevel(left_tag) or left_tag == "--" \ + and not block.rstrip().endswith(">"): + items.append(block.strip()) + in_tag = True + else: + new_blocks.append( + self.markdown.htmlStash.store(block.strip())) + + continue + + new_blocks.append(block) + + else: + items.append(block.strip()) + + right_tag, data_index = self._get_right_tag(left_tag, block) + + if self._equal_tags(left_tag, right_tag): + # if find closing tag + in_tag = False + new_blocks.append( + self.markdown.htmlStash.store('\n\n'.join(items))) + items = [] + + if items: + new_blocks.append(self.markdown.htmlStash.store('\n\n'.join(items))) + new_blocks.append('\n') + + new_text = "\n\n".join(new_blocks) + return new_text.split("\n") + + +class ReferencePreprocessor(Preprocessor): + """ Remove reference definitions from text and store for later use. """ + + RE = re.compile(r'^(\ ?\ ?\ ?)\[([^\]]*)\]:\s*([^ ]*)(.*)$', re.DOTALL) + + def run (self, lines): + new_text = []; + for line in lines: + m = self.RE.match(line) + if m: + id = m.group(2).strip().lower() + t = m.group(4).strip() # potential title + if not t: + self.markdown.references[id] = (m.group(3), t) + elif (len(t) >= 2 + and (t[0] == t[-1] == "\"" + or t[0] == t[-1] == "\'" + or (t[0] == "(" and t[-1] == ")") ) ): + self.markdown.references[id] = (m.group(3), t[1:-1]) + else: + new_text.append(line) + else: + new_text.append(line) + + return new_text #+ "\n" diff --git a/vendor/tornado/website/markdown/treeprocessors.py b/vendor/tornado/website/markdown/treeprocessors.py new file mode 100644 index 0000000000..1dc612a95e --- /dev/null +++ b/vendor/tornado/website/markdown/treeprocessors.py @@ -0,0 +1,329 @@ +import markdown +import re + +def isString(s): + """ Check if it's string """ + return isinstance(s, unicode) or isinstance(s, str) + +class Processor: + def __init__(self, markdown_instance=None): + if markdown_instance: + self.markdown = markdown_instance + +class Treeprocessor(Processor): + """ + Treeprocessors are run on the ElementTree object before serialization. + + Each Treeprocessor implements a "run" method that takes a pointer to an + ElementTree, modifies it as necessary and returns an ElementTree + object. + + Treeprocessors must extend markdown.Treeprocessor. + + """ + def run(self, root): + """ + Subclasses of Treeprocessor should implement a `run` method, which + takes a root ElementTree. This method can return another ElementTree + object, and the existing root ElementTree will be replaced, or it can + modify the current tree and return None. + """ + pass + + +class InlineProcessor(Treeprocessor): + """ + A Treeprocessor that traverses a tree, applying inline patterns. + """ + + def __init__ (self, md): + self.__placeholder_prefix = markdown.INLINE_PLACEHOLDER_PREFIX + self.__placeholder_suffix = markdown.ETX + self.__placeholder_length = 4 + len(self.__placeholder_prefix) \ + + len(self.__placeholder_suffix) + self.__placeholder_re = re.compile(markdown.INLINE_PLACEHOLDER % r'([0-9]{4})') + self.markdown = md + + def __makePlaceholder(self, type): + """ Generate a placeholder """ + id = "%04d" % len(self.stashed_nodes) + hash = markdown.INLINE_PLACEHOLDER % id + return hash, id + + def __findPlaceholder(self, data, index): + """ + Extract id from data string, start from index + + Keyword arguments: + + * data: string + * index: index, from which we start search + + Returns: placeholder id and string index, after the found placeholder. + """ + + m = self.__placeholder_re.search(data, index) + if m: + return m.group(1), m.end() + else: + return None, index + 1 + + def __stashNode(self, node, type): + """ Add node to stash """ + placeholder, id = self.__makePlaceholder(type) + self.stashed_nodes[id] = node + return placeholder + + def __handleInline(self, data, patternIndex=0): + """ + Process string with inline patterns and replace it + with placeholders + + Keyword arguments: + + * data: A line of Markdown text + * patternIndex: The index of the inlinePattern to start with + + Returns: String with placeholders. + + """ + if not isinstance(data, markdown.AtomicString): + startIndex = 0 + while patternIndex < len(self.markdown.inlinePatterns): + data, matched, startIndex = self.__applyPattern( + self.markdown.inlinePatterns.value_for_index(patternIndex), + data, patternIndex, startIndex) + if not matched: + patternIndex += 1 + return data + + def __processElementText(self, node, subnode, isText=True): + """ + Process placeholders in Element.text or Element.tail + of Elements popped from self.stashed_nodes. + + Keywords arguments: + + * node: parent node + * subnode: processing node + * isText: bool variable, True - it's text, False - it's tail + + Returns: None + + """ + if isText: + text = subnode.text + subnode.text = None + else: + text = subnode.tail + subnode.tail = None + + childResult = self.__processPlaceholders(text, subnode) + + if not isText and node is not subnode: + pos = node.getchildren().index(subnode) + node.remove(subnode) + else: + pos = 0 + + childResult.reverse() + for newChild in childResult: + node.insert(pos, newChild) + + def __processPlaceholders(self, data, parent): + """ + Process string with placeholders and generate ElementTree tree. + + Keyword arguments: + + * data: string with placeholders instead of ElementTree elements. + * parent: Element, which contains processing inline data + + Returns: list with ElementTree elements with applied inline patterns. + """ + def linkText(text): + if text: + if result: + if result[-1].tail: + result[-1].tail += text + else: + result[-1].tail = text + else: + if parent.text: + parent.text += text + else: + parent.text = text + + result = [] + strartIndex = 0 + while data: + index = data.find(self.__placeholder_prefix, strartIndex) + if index != -1: + id, phEndIndex = self.__findPlaceholder(data, index) + + if id in self.stashed_nodes: + node = self.stashed_nodes.get(id) + + if index > 0: + text = data[strartIndex:index] + linkText(text) + + if not isString(node): # it's Element + for child in [node] + node.getchildren(): + if child.tail: + if child.tail.strip(): + self.__processElementText(node, child, False) + if child.text: + if child.text.strip(): + self.__processElementText(child, child) + else: # it's just a string + linkText(node) + strartIndex = phEndIndex + continue + + strartIndex = phEndIndex + result.append(node) + + else: # wrong placeholder + end = index + len(prefix) + linkText(data[strartIndex:end]) + strartIndex = end + else: + text = data[strartIndex:] + linkText(text) + data = "" + + return result + + def __applyPattern(self, pattern, data, patternIndex, startIndex=0): + """ + Check if the line fits the pattern, create the necessary + elements, add it to stashed_nodes. + + Keyword arguments: + + * data: the text to be processed + * pattern: the pattern to be checked + * patternIndex: index of current pattern + * startIndex: string index, from which we starting search + + Returns: String with placeholders instead of ElementTree elements. + + """ + match = pattern.getCompiledRegExp().match(data[startIndex:]) + leftData = data[:startIndex] + + if not match: + return data, False, 0 + + node = pattern.handleMatch(match) + + if node is None: + return data, True, len(leftData) + match.span(len(match.groups()))[0] + + if not isString(node): + if not isinstance(node.text, markdown.AtomicString): + # We need to process current node too + for child in [node] + node.getchildren(): + if not isString(node): + if child.text: + child.text = self.__handleInline(child.text, + patternIndex + 1) + if child.tail: + child.tail = self.__handleInline(child.tail, + patternIndex) + + placeholder = self.__stashNode(node, pattern.type()) + + return "%s%s%s%s" % (leftData, + match.group(1), + placeholder, match.groups()[-1]), True, 0 + + def run(self, tree): + """Apply inline patterns to a parsed Markdown tree. + + Iterate over ElementTree, find elements with inline tag, apply inline + patterns and append newly created Elements to tree. If you don't + want process your data with inline paterns, instead of normal string, + use subclass AtomicString: + + node.text = markdown.AtomicString("data won't be processed with inline patterns") + + Arguments: + + * markdownTree: ElementTree object, representing Markdown tree. + + Returns: ElementTree object with applied inline patterns. + + """ + self.stashed_nodes = {} + + stack = [tree] + + while stack: + currElement = stack.pop() + insertQueue = [] + for child in currElement.getchildren(): + if child.text and not isinstance(child.text, markdown.AtomicString): + text = child.text + child.text = None + lst = self.__processPlaceholders(self.__handleInline( + text), child) + stack += lst + insertQueue.append((child, lst)) + + if child.getchildren(): + stack.append(child) + + for element, lst in insertQueue: + if element.text: + element.text = \ + markdown.inlinepatterns.handleAttributes(element.text, + element) + i = 0 + for newChild in lst: + # Processing attributes + if newChild.tail: + newChild.tail = \ + markdown.inlinepatterns.handleAttributes(newChild.tail, + element) + if newChild.text: + newChild.text = \ + markdown.inlinepatterns.handleAttributes(newChild.text, + newChild) + element.insert(i, newChild) + i += 1 + return tree + + +class PrettifyTreeprocessor(Treeprocessor): + """ Add linebreaks to the html document. """ + + def _prettifyETree(self, elem): + """ Recursively add linebreaks to ElementTree children. """ + + i = "\n" + if markdown.isBlockLevel(elem.tag) and elem.tag not in ['code', 'pre']: + if (not elem.text or not elem.text.strip()) \ + and len(elem) and markdown.isBlockLevel(elem[0].tag): + elem.text = i + for e in elem: + if markdown.isBlockLevel(e.tag): + self._prettifyETree(e) + if not elem.tail or not elem.tail.strip(): + elem.tail = i + if not elem.tail or not elem.tail.strip(): + elem.tail = i + + def run(self, root): + """ Add linebreaks to ElementTree root object. """ + + self._prettifyETree(root) + # Do <br />'s seperately as they are often in the middle of + # inline content and missed by _prettifyETree. + brs = root.getiterator('br') + for br in brs: + if not br.tail or not br.tail.strip(): + br.tail = '\n' + else: + br.tail = '\n%s' % br.tail diff --git a/vendor/tornado/website/static/base.css b/vendor/tornado/website/static/base.css new file mode 100644 index 0000000000..543d6f24c3 --- /dev/null +++ b/vendor/tornado/website/static/base.css @@ -0,0 +1,120 @@ +body { + background: white; + color: black; + font-family: Georgia, serif; + font-size: 11pt; + margin: 10px; + margin-top: 15px; + margin-bottom: 15px; +} + +h1, +h2, +h3, +h4 { + font-family: Calibri, sans-serif; + margin: 0; +} + +img { + border: 0; +} + +pre, +code { + color: #060; +} + +a, +a code { + color: #216093; +} + +table { + border-collapse: collapse; + border: 0; +} + +td { + border: 0; + padding: 0; +} + +#body { + margin: auto; + max-width: 850px; +} + +#header { + margin-bottom: 15px; + margin-right: 30px; +} + +#content, +#footer { + margin-left: 31px; + margin-right: 31px; +} + +#content p, +#content li, +#footer { + line-height: 16pt; +} + +#content pre { + line-height: 14pt; + margin: 17pt; + padding-left: 1em; + border-left: 1px solid #ccc; +} + +#footer { + margin-top: 5em; +} + +#header .logo { + line-height: 0; + padding-bottom: 5px; + padding-right: 15px; +} + +#header .logo img { + width: 286px; + height: 72px; +} + +#header .title { + vertical-align: bottom; +} + +#header .title h1 { + font-size: 35px; + font-weight: normal; +} + +#header .title h1, +#header .title h1 a { + color: #666; +} + +#content h1, +#content h2, +#content h3 { + color: #4d8cbf; + margin-bottom: 2pt; + margin-top: 17pt; +} + +#content h2 { + font-size: 19pt; +} + +#content h3 { + font-size: 15pt; +} + +#content p { + margin: 0; + margin-bottom: 1em; +} diff --git a/vendor/tornado/website/static/facebook.png b/vendor/tornado/website/static/facebook.png Binary files differnew file mode 100755 index 0000000000..47738323ed --- /dev/null +++ b/vendor/tornado/website/static/facebook.png diff --git a/vendor/tornado/website/static/friendfeed.png b/vendor/tornado/website/static/friendfeed.png Binary files differnew file mode 100755 index 0000000000..ac09f4e453 --- /dev/null +++ b/vendor/tornado/website/static/friendfeed.png diff --git a/vendor/tornado/website/static/robots.txt b/vendor/tornado/website/static/robots.txt new file mode 100644 index 0000000000..0ad279c736 --- /dev/null +++ b/vendor/tornado/website/static/robots.txt @@ -0,0 +1,2 @@ +User-Agent: * +Disallow: diff --git a/vendor/tornado/website/static/tornado-0.1.tar.gz b/vendor/tornado/website/static/tornado-0.1.tar.gz Binary files differnew file mode 100644 index 0000000000..d06c1e04ea --- /dev/null +++ b/vendor/tornado/website/static/tornado-0.1.tar.gz diff --git a/vendor/tornado/website/static/tornado-0.2.tar.gz b/vendor/tornado/website/static/tornado-0.2.tar.gz Binary files differnew file mode 100644 index 0000000000..6aca327aee --- /dev/null +++ b/vendor/tornado/website/static/tornado-0.2.tar.gz diff --git a/vendor/tornado/website/static/tornado.png b/vendor/tornado/website/static/tornado.png Binary files differnew file mode 100644 index 0000000000..a920aa566f --- /dev/null +++ b/vendor/tornado/website/static/tornado.png diff --git a/vendor/tornado/website/static/twitter.png b/vendor/tornado/website/static/twitter.png Binary files differnew file mode 100755 index 0000000000..5099c62ee5 --- /dev/null +++ b/vendor/tornado/website/static/twitter.png diff --git a/vendor/tornado/website/templates/base.html b/vendor/tornado/website/templates/base.html new file mode 100644 index 0000000000..48da016781 --- /dev/null +++ b/vendor/tornado/website/templates/base.html @@ -0,0 +1,27 @@ +<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml"> + <head> + <meta http-equiv="Content-Type" content="text/html; charset=UTF-8"/> + <title>{% block title %}Tornado Web Server{% end %}</title> + <link rel="stylesheet" href="/static/base.css" type="text/css"/> + {% block head %}{% end %} + </head> + <body> + <div id="body"> + <div id="header"> + <table> + <tr> + <td class="logo"><a href="/"><img src="/static/tornado.png" alt="Tornado"/></a></td> + <td class="title">{% block headertitle %}{% end %}</td> + </tr> + </table> + </div> + <div id="content">{% block body %}{% end %}</div> + <div id="footer"> + <div>Tornado is one of <a href="http://developers.facebook.com/opensource.php">Facebook's open source technologies</a>. It is available under the <a href="http://www.apache.org/licenses/LICENSE-2.0.html">Apache Licence, Version 2.0</a>.</div> + <div>This web site and all documentation is licensed under <a href="http://creativecommons.org/licenses/by/3.0/">Creative Commons 3.0</a>.</div> + </div> + </div> + {% block bottom %}{% end %} + </body> +</html> diff --git a/vendor/tornado/website/templates/documentation.html b/vendor/tornado/website/templates/documentation.html new file mode 100644 index 0000000000..8c28740874 --- /dev/null +++ b/vendor/tornado/website/templates/documentation.html @@ -0,0 +1,9 @@ +{% extends "base.html" %} + +{% block title %}Tornado Web Server Documentation{% end %} + +{% block headertitle %}<h1>documentation</h1>{% end %} + +{% block body %} + {{ markdown("documentation.txt", toc=True) }} +{% end %} diff --git a/vendor/tornado/website/templates/documentation.txt b/vendor/tornado/website/templates/documentation.txt new file mode 100644 index 0000000000..81262a605a --- /dev/null +++ b/vendor/tornado/website/templates/documentation.txt @@ -0,0 +1,866 @@ +Overview +-------- +[FriendFeed](http://friendfeed.com/)'s web server is a relatively simple, +non-blocking web server written in Python. The FriendFeed application is +written using a web framework that looks a bit like +[web.py](http://webpy.org/) or Google's +[webapp](http://code.google.com/appengine/docs/python/tools/webapp/), +but with additional tools and optimizations to take advantage of the +non-blocking web server and tools. + +[Tornado](http://github.com/facebook/tornado) is an open source +version of this web server and some of the tools we use most often at +FriendFeed. The framework is distinct from most mainstream web server +frameworks (and certainly most Python frameworks) because it is +non-blocking and reasonably fast. Because it is non-blocking +and uses [epoll](http://www.kernel.org/doc/man-pages/online/pages/man4/epoll.4.html), it can handle 1000s of simultaneous standing connections, +which means the framework is ideal for real-time web services. We built the +web server specifically to handle FriendFeed's real-time features — +every active user of FriendFeed maintains an open connection to the +FriendFeed servers. (For more information on scaling servers to support +thousands of clients, see +[The C10K problem](http://www.kegel.com/c10k.html).) + +Here is the canonical "Hello, world" example app: + + import tornado.httpserver + import tornado.ioloop + import tornado.web + + class MainHandler(tornado.web.RequestHandler): + def get(self): + self.write("Hello, world") + + application = tornado.web.Application([ + (r"/", MainHandler), + ]) + + if __name__ == "__main__": + http_server = tornado.httpserver.HTTPServer(application) + http_server.listen(8888) + tornado.ioloop.IOLoop.instance().start() + +See [Tornado walkthrough](#tornado-walkthrough) below for a detailed +walkthrough of the `tornado.web` package. + +We attempted to clean up the code base to reduce interdependencies between +modules, so you should (theoretically) be able to use any of the modules +independently in your project without using the whole package. + + +Download +-------- +Download the most recent version of Tornado from GitHub: + +> [tornado-0.2.tar.gz](/static/tornado-0.2.tar.gz) + +You can also [browse the source](http://github.com/facebook/tornado) on GitHub. To install Tornado: + + tar xvzf tornado-0.2.tar.gz + cd tornado-0.2 + python setup.py build + sudo python setup.py install + +After installation, you should be able to run any of the demos in the `demos` +directory included with the Tornado package. + + ./demos/helloworld/helloworld.py + +### Prerequisites + +Tornado has been tested on Python 2.5 and 2.6. To use all of the features of Tornado, you need to have [PycURL](http://pycurl.sourceforge.net/) and a JSON library like [simplejson](http://pypi.python.org/pypi/simplejson/) installed. Complete installation instructions for Mac OS X and Ubuntu are included below for convenience. + +**Mac OS X 10.5/10.6** + + sudo easy_install setuptools pycurl==7.16.2.1 simplejson + +**Ubuntu Linux** + + sudo apt-get install python-dev python-pycurl python-simplejson + + +Module index +------------ +The most important module is [`web`](http://github.com/facebook/tornado/blob/master/tornado/web.py), which is the web framework +that includes most of the meat of the Tornado package. The other modules +are tools that make `web` more useful. See +[Tornado walkthrough](#tornado-walkthrough) below for a detailed +walkthrough of the `web` package. + +### Main modules + * [`web`](http://github.com/facebook/tornado/blob/master/tornado/web.py) - The web framework on which FriendFeed is built. `web` incorporates most of the important features of Tornado + * [`escape`](http://github.com/facebook/tornado/blob/master/tornado/escape.py) - XHTML, JSON, and URL encoding/decoding methods + * [`database`](http://github.com/facebook/tornado/blob/master/tornado/database.py) - A simple wrapper around `MySQLdb` to make MySQL easier to use + * [`template`](http://github.com/facebook/tornado/blob/master/tornado/template.py) - A Python-based web templating language + * [`httpclient`](http://github.com/facebook/tornado/blob/master/tornado/httpclient.py) - A non-blocking HTTP client designed to work with `web` and `httpserver` + * [`auth`](http://github.com/facebook/tornado/blob/master/tornado/auth.py) - Implementation of third party authentication and authorization schemes (Google OpenID/OAuth, Facebook Platform, Yahoo BBAuth, FriendFeed OpenID/OAuth, Twitter OAuth) + * [`locale`](http://github.com/facebook/tornado/blob/master/tornado/locale.py) - Localization/translation support + * [`options`](http://github.com/facebook/tornado/blob/master/tornado/options.py) - Command line and config file parsing, optimized for server environments + +### Low-level modules + * [`httpserver`](http://github.com/facebook/tornado/blob/master/tornado/httpserver.py) - A very simple HTTP server built on which `web` is built + * [`iostream`](http://github.com/facebook/tornado/blob/master/tornado/iostream.py) - A simple wrapper around non-blocking sockets to aide common reading and writing patterns + * [`ioloop`](http://github.com/facebook/tornado/blob/master/tornado/ioloop.py) - Core I/O loop + +### Random modules + * [`s3server`](http://github.com/facebook/tornado/blob/master/tornado/s3server.py) - A web server that implements most of the [Amazon S3](http://aws.amazon.com/s3/) interface, backed by local file storage + + +Tornado walkthrough +------------------- + +### Request handlers and request arguments + +A Tornado web application maps URLs or URL patterns to subclasses of +`tornado.web.RequestHandler`. Those classes define `get()` or `post()` +methods to handle HTTP `GET` or `POST` requests to that URL. + +This code maps the root URL `/` to `MainHandler` and the URL pattern +`/story/([0-9]+)` to `StoryHandler`. Regular expression groups are passed +as arguments to the `RequestHandler` methods: + + class MainHandler(tornado.web.RequestHandler): + def get(self): + self.write("You requested the main page") + + class StoryHandler(tornado.web.RequestHandler): + def get(self, story_id): + self.write("You requested the story " + story_id) + + application = tornado.web.Application([ + (r"/", MainHandler), + (r"/story/([0-9]+)", StoryHandler), + ]) + +You can get query string arguments and parse `POST` bodies with the +`get_argument()` method: + + class MainHandler(tornado.web.RequestHandler): + def get(self): + self.write('<html><body><form action="/" method="post">' + '<input type="text" name="message">' + '<input type="submit" value="Submit">' + '</form></body></html>') + + def post(self): + self.set_header("Content-Type", "text/plain") + self.write("You wrote " + self.get_argument("message")) + +If you want to send an error response to the client, e.g., 403 Unauthorized, +you can just raise a `tornado.web.HTTPError` exception: + + if not self.user_is_logged_in(): + raise tornado.web.HTTPError(403) + +The request handler can access the object representing the current request +with `self.request`. The `HTTPRequest` object includes a number of useful +attribute, including: + + * `arguments` - all of the `GET` and `POST` arguments + * `files` - all of the uploaded files (via `multipart/form-data` POST requests) + * `path` - the request path (everything before the `?`) + * `headers` - the request headers + +See the class definition for `HTTPRequest` in `httpserver` for a complete list +of attributes. + + +### Templates + +You can use any template language supported by Python, but Tornado ships +with its own templating language that is a lot faster and more flexible +than many of the most popular templating systems out there. See the +[`template`](http://github.com/facebook/tornado/blob/master/tornado/template.py) module documentation for complete documentation. + +A Tornado template is just HTML (or any other text-based format) with +Python control sequences and expressions embedded within the markup: + + <html> + <head> + <title>{{ title }}</title> + </head> + <body> + <ul> + {% for item in items %} + <li>{{ escape(item) }}</li> + {% end %} + </ul> + </body> + </html> + +If you saved this template as "template.html" and put it in the same +directory as your Python file, you could render this template with: + + class MainHandler(tornado.web.RequestHandler): + def get(self): + items = ["Item 1", "Item 2", "Item 3"] + self.render("template.html", title="My title", items=items) + +Tornado templates support *control statements* and *expressions*. Control +statements are surronded by `{%` and `%}`, e.g., `{% if len(items) > 2 %}`. +Expressions are surrounded by `{{` and `}}`, e.g., `{{ items[0] }}`. + +Control statements more or less map exactly to Python statements. We support +`if`, `for`, `while`, and `try`, all of which are terminated with `{% end %}`. +We also support *template inheritance* using the `extends` and `block` +statements, which are described in detail in the documentation for the +[`template` module](http://github.com/facebook/tornado/blob/master/tornado/template.py). + +Expressions can be any Python expression, including function calls. We +support the functions `escape`, `url_escape`, and `json_encode` by default, +and you can pass other functions into the template simply by passing them +as keyword arguments to the template render function: + + class MainHandler(tornado.web.RequestHandler): + def get(self): + self.render("template.html", add=self.add) + + def add(self, x, y): + return x + y + +When you are building a real application, you are going to want to use +all of the features of Tornado templates, especially template inheritance. +Read all about those features in the [`template` module](http://github.com/facebook/tornado/blob/master/tornado/template.py) +section. + +Under the hood, Tornado templates are translated directly to Python. +The expressions you include in your template are copied verbatim into +a Python function representing your template. We don't try to prevent +anything in the template language; we created it explicitly to provide +the flexibility that other, stricter templating systems prevent. +Consequently, if you write random stuff inside of your template expressions, +you will get random Python errors when you execute the template. + + +### Cookies and secure cookies + +You can set cookies in the user's browser with the `set_cookie` method: + + class MainHandler(tornado.web.RequestHandler): + def get(self): + if not self.get_cookie("mycookie"): + self.set_cookie("mycookie", "myvalue") + self.write("Your cookie was not set yet!") + else: + self.write("Your cookie was set!") + +Cookies are easily forged by malicious clients. If you need to set cookies +to, e.g., save the user ID of the currently logged in user, you need to +sign your cookies to prevent forgery. Tornado supports this out of the +box with the `set_secure_cookie` and `get_secure_cookie` methods. To use +these methods, you need to specify a secret key named `cookie_secret` when +you create your application. You can pass in application settings as keyword +arguments to your application: + + application = tornado.web.Application([ + (r"/", MainHandler), + ], cookie_secret="61oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=") + +Signed cookies contain the encoded value of the cookie in addition to a +timestamp and an [HMAC](http://en.wikipedia.org/wiki/HMAC) signature. If the +cookie is old or if the signature doesn't match, `get_secure_cookie` will +return `None` just as if the cookie isn't set. The secure version of the +example above: + + class MainHandler(tornado.web.RequestHandler): + def get(self): + if not self.get_secure_cookie("mycookie"): + self.set_secure_cookie("mycookie", "myvalue") + self.write("Your cookie was not set yet!") + else: + self.write("Your cookie was set!") + + +### User authentication + +The currently authenticated user is available in every request handler +as `self.current_user`, and in every template as `current_user`. By +default, `current_user` is `None`. + +To implement user authentication in your application, you need to +override the `get_current_user()` method in your request handlers to +determine the current user based on, e.g., the value of a cookie. +Here is an example that lets users log into the application simply +by specifying a nickname, which is then saved in a cookie: + + class BaseHandler(tornado.web.RequestHandler): + def get_current_user(self): + return self.get_secure_cookie("user") + + class MainHandler(BaseHandler): + def get(self): + if not self.current_user: + self.redirect("/login") + return + name = tornado.escape.xhtml_escape(self.current_user) + self.write("Hello, " + name) + + class LoginHandler(BaseHandler): + def get(self): + self.write('<html><body><form action="/login" method="post">' + 'Name: <input type="text" name="name">' + '<input type="submit" value="Sign in">' + '</form></body></html>') + + def post(self): + self.set_secure_cookie("user", self.get_argument("name")) + self.redirect("/") + + application = tornado.web.Application([ + (r"/", MainHandler), + (r"/login", LoginHandler), + ], cookie_secret="61oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=") + +You can require that the user be logged in using the +[Python decorator](http://www.python.org/dev/peps/pep-0318/) +`tornado.web.authenticated`. If a request goes to a method with this +decorator, and the user is not logged in, they will be redirected to +`login_url` (another application setting). The example above could +be rewritten: + + class MainHandler(BaseHandler): + @tornado.web.authenticated + def get(self): + name = tornado.escape.xhtml_escape(self.current_user) + self.write("Hello, " + name) + + settings = { + "cookie_secret": "61oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=", + "login_url": "/login", + } + application = tornado.web.Application([ + (r"/", MainHandler), + (r"/login", LoginHandler), + ], **settings) + +If you decorate `post()` methods with the `authenticated` decorator, and +the user is not logged in, the server will send a `403` response. + +Tornado comes with built-in support for third-party authentication schemes +like Google OAuth. See the [`auth` module](http://github.com/facebook/tornado/blob/master/tornado/auth.py) for more details. Check +out the Tornado Blog example application for a complete example that +uses authentication (and stores user data in a MySQL database). + + +### Cross-site request forgery protection + +[Cross-site request forgery](http://en.wikipedia.org/wiki/Cross-site_request_forgery), or XSRF, is a common problem for personalized web applications. See the +[Wikipedia article](http://en.wikipedia.org/wiki/Cross-site_request_forgery) +for more information on how XSRF works. + +The generally accepted solution to prevent XSRF is to cookie every user +with an unpredictable value and include that value as an additional +argument with every form submission on your site. If the cookie and the +value in the form submission do not match, then the request is likely +forged. + +Tornado comes with built-in XSRF protection. To include it in your site, +include the application setting `xsrf_cookies`: + + settings = { + "cookie_secret": "61oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=", + "login_url": "/login", + "xsrf_cookies": True, + } + application = tornado.web.Application([ + (r"/", MainHandler), + (r"/login", LoginHandler), + ], **settings) + +If `xsrf_cookies` is set, the Tornado web application will set the `_xsrf` +cookie for all users and reject all `POST` requests hat do not contain a +correct `_xsrf` value. If you turn this setting on, you need to instrument +all forms that submit via `POST` to contain this field. You can do this with +the special function `xsrf_form_html()`, available in all templates: + + <form action="/login" method="post"> + {{ xsrf_form_html() }} + <div>Username: <input type="text" name="username"/></div> + <div>Password: <input type="password" name="password"/></div> + <div><input type="submit" value="Sign in"/></div> + </form> + +If you submit AJAX `POST` requests, you will also need to instrument your +JavaScript to include the `_xsrf` value with each request. This is the +[jQuery](http://jquery.com/) function we use at FriendFeed for AJAX `POST` +requests that automatically adds the `_xsrf` value to all requests: + + function getCookie(name) { + var r = document.cookie.match("\\b" + name + "=([^;]*)\\b"); + return r ? r[1] : undefined; + } + + jQuery.postJSON = function(url, args, callback) { + args._xsrf = getCookie("_xsrf"); + $.ajax({url: url, data: $.param(args), dataType: "text", type: "POST", + success: function(response) { + callback(eval("(" + response + ")")); + }}); + }; + + +### Static files and aggressive file caching + +You can serve static files from Tornado by specifying the `static_path` +setting in your application: + + settings = { + "static_path": os.path.join(os.path.dirname(__file__), "static"), + "cookie_secret": "61oETzKXQAGaYdkL5gEmGeJJFuYh7EQnp2XdTP1o/Vo=", + "login_url": "/login", + "xsrf_cookies": True, + } + application = tornado.web.Application([ + (r"/", MainHandler), + (r"/login", LoginHandler), + ], **settings) + +This setting will automatically make all requests that start with `/static/` +serve from that static directory, e.g., [http://localhost:8888/static/foo.png](http://localhost:8888/static/foo.png) +will serve the file `foo.png` from the specified static directory. We +also automatically serve `/robots.txt` and `/favicon.ico` from the static +directory (even though they don't start with the `/static/` prefix). + +To improve performance, it is generally a good idea for browsers to +cache static resources aggressively so browsers won't send unnecessary +`If-Modified-Since` or `Etag` requests that might block the rendering of +the page. Tornado supports this out of the box with *static content +versioning*. + +To use this feature, use the `static_url()` method in your templates rather +than typing the URL of the static file directly in your HTML: + + <html> + <head> + <title>FriendFeed - {{ _("Home") }}</title> + </head> + <body> + <div><img src="{{ static_url("images/logo.png") }}"/></div> + </body> + </html> + +The `static_url()` function will translate that relative path to a URI +that looks like `/static/images/logo.png?v=aae54`. The `v` argument is +a hash of the content in `logo.png`, and its presence makes the Tornado +server send cache headers to the user's browser that will make the browser +cache the content indefinitely. + +Since the `v` argument is based on the content of the file, if you update +a file and restart your server, it will start sending a new `v` value, +so the user's browser will automatically fetch the new file. If the file's +contents don't change, the browser will continue to use a locally cached +copy without ever checking for updates on the server, significantly +improving rendering performance. + +In production, you probably want to serve static files from a more +optimized static file server like [nginx](http://nginx.net/). You can +configure most any web server to support these caching semantics. Here +is the nginx configuration we use at FriendFeed: + + location /static/ { + root /var/friendfeed/static; + if ($query_string) { + expires max; + } + } + + +### Localization + +The locale of the current user (whether they are logged in or not) is +always available as `self.locale` in the request handler and as `locale` +in templates. The name of the locale (e.g., `en_US`) is available as +`locale.name`, and you can translate strings with the `locale.translate` +method. Templates also have the global function call `_()` available +for string translation. The translate function has two forms: + + _("Translate this string") + +which translates the string directly based on the current locale, and + + _("A person liked this", "%(num)d people liked this", len(people)) % {"num": len(people)} + +which translates a string that can be singular or plural based on the value +of the third argument. In the example above, a translation of the first +string will be returned if `len(people)` is `1`, or a translation of the +second string will be returned otherwise. + +The most common pattern for translations is to use Python named placeholders +for variables (the `%(num)d` in the example above) since placeholders can +move around on translation. + +Here is a properly localized template: + + <html> + <head> + <title>FriendFeed - {{ _("Sign in") }}</title> + </head> + <body> + <form action="{{ request.path }}" method="post"> + <div>{{ _("Username") }} <input type="text" name="username"/></div> + <div>{{ _("Password") }} <input type="password" name="password"/></div> + <div><input type="submit" value="{{ _("Sign in") }}"/></div> + {{ xsrf_form_html() }} + </form> + </body> + </html> + +By default, we detect the user's locale using the `Accept-Language` header +sent by the user's browser. We choose `en_US` if we can't find an appropriate +`Accept-Language` value. If you let user's set their locale as a preference, +you can override this default locale selection by overriding `get_user_locale` +in your request handler: + + class BaseHandler(tornado.web.RequestHandler): + def get_current_user(self): + user_id = self.get_secure_cookie("user") + if not user_id: return None + return self.backend.get_user_by_id(user_id) + + def get_user_locale(self): + if "locale" not in self.current_user.prefs: + # Use the Accept-Language header + return None + return self.current_user.prefs["locale"] + +If `get_user_locale` returns `None`, we fall back on the `Accept-Language` +header. + +You can load all the translations for your application using the +`tornado.locale.load_translations` method. It takes in the name of the +directory which should contain CSV files named after the locales whose +translations they contain, e.g., `es_GT.csv` or `fr_CA.csv`. The method +loads all the translations from those CSV files and infers the list of +supported locales based on the presence of each CSV file. You typically +call this method once in the `main()` method of your server: + + def main(): + tornado.locale.load_translations( + os.path.join(os.path.dirname(__file__), "translations")) + start_server() + +You can get the list of supported locales in your application with +`tornado.locale.get_supported_locales()`. The user's locale is chosen to +be the closest match based on the supported locales. For example, if the +user's locale is `es_GT`, and the `es` locale is supported, `self.locale` +will be `es` for that request. We fall back on `en_US` if no close match +can be found. + +See the [`locale` module](http://github.com/facebook/tornado/blob/master/tornado/locale.py) documentation for detailed information +on the CSV format and other localization methods. + + +### UI modules + +Tornado supports *UI modules* to make it easy to support standard, reusable +UI widgets across your application. UI modules are like special functional +calls to render components of your page, and they can come packaged with +their own CSS and JavaScript. + +For example, if you are implementing a blog, and you want to have +blog entries appear on both the blog home page and on each blog entry page, +you can make an `Entry` module to render them on both pages. First, create +a Python module for your UI modules, e.g., `uimodules.py`: + + class Entry(tornado.web.UIModule): + def render(self, entry, show_comments=False): + return self.render_string( + "module-entry.html", show_comments=show_comments) + +Tell Tornado to use `uimodules.py` using the `ui_modules` setting in your +application: + + class HomeHandler(tornado.web.RequestHandler): + def get(self): + entries = self.db.query("SELECT * FROM entries ORDER BY date DESC") + self.render("home.html", entries=entries) + + class EntryHandler(tornado.web.RequestHandler): + def get(self, entry_id): + entry = self.db.get("SELECT * FROM entries WHERE id = %s", entry_id) + if not entry: raise tornado.web.HTTPError(404) + self.render("entry.html", entry=entry) + + settings = { + "ui_modules": uimodules, + } + application = tornado.web.Application([ + (r"/", HomeHandler), + (r"/entry/([0-9]+)", EntryHandler), + ], **settings) + +Within `home.html`, you reference the `Entry` module rather than printing +the HTML directly: + + {% for entry in entries %} + {{ modules.Entry(entry) }} + {% end %} + +Within `entry.html`, you reference the `Entry` module with the +`show_comments` argument to show the expanded form of the entry: + + {{ modules.Entry(entry, show_comments=True) }} + +Modules can include custom CSS and JavaScript functions by overriding +the `embedded_css`, `embedded_javascript`, `javascript_files`, or +`css_files` methods: + + class Entry(tornado.web.UIModule): + def embedded_css(self): + return ".entry { margin-bottom: 1em; }" + + def render(self, entry, show_comments=False): + return self.render_string( + "module-entry.html", show_comments=show_comments) + +Module CSS and JavaScript will be included once no matter how many times +a module is used on a page. CSS is always included in the `<head>` of the +page, and JavaScript is always included just before the `</body>` tag +at the end of the page. + + +### Non-blocking, asynchronous requests + +When a request handler is executed, the request is automatically finished. +Since Tornado uses a non-blocking I/O style, you can override this default +behavior if you want a request to remain open after the main request handler +method returns using the `tornado.web.asynchronous` decorator. + +When you use this decorator, it is your responsibility to call +`self.finish()` to finish the HTTP request, or the user's browser +will simply hang: + + class MainHandler(tornado.web.RequestHandler): + @tornado.web.asynchronous + def get(self): + self.write("Hello, world") + self.finish() + +Here is a real example that makes a call to the FriendFeed API using +Tornado's built-in asynchronous HTTP client: + + class MainHandler(tornado.web.RequestHandler): + @tornado.web.asynchronous + def get(self): + http = tornado.httpclient.AsyncHTTPClient() + http.fetch("http://friendfeed-api.com/v2/feed/bret", + callback=self.async_callback(self.on_response)) + + def on_response(self, response): + if response.error: raise tornado.web.HTTPError(500) + json = tornado.escape.json_decode(response.body) + self.write("Fetched " + str(len(json["entries"])) + " entries " + "from the FriendFeed API") + self.finish() + +When `get()` returns, the request has not finished. When the HTTP client +eventually calls `on_response()`, the request is still open, and the response +is finally flushed to the client with the call to `self.finish()`. + +If you make calls to asynchronous library functions that require a callback +(like the HTTP `fetch` function above), you should always wrap your +callbacks with `self.async_callback`. This simple wrapper ensures that if +your callback function raises an exception or has a programming error, +a proper HTTP error response will be sent to the browser, and the connection +will be properly closed. + +For a more advanced asynchronous example, take a look at the `chat` example +application, which implements an AJAX chat room using +[long polling](http://en.wikipedia.org/wiki/Push_technology#Long_polling). + + +### Third party authentication + +Tornado's `auth` module implements the authentication and authorization +protocols for a number of the most popular sites on the web, including +Google/Gmail, Facebook, Twitter, Yahoo, and FriendFeed. The module includes +methods to log users in via these sites and, where applicable, methods to +authorize access to the service so you can, e.g., download a user's address +book or publish a Twitter message on their behalf. + +Here is an example handler that uses Google for authentication, saving +the Google credentials in a cookie for later access: + + class GoogleHandler(tornado.web.RequestHandler, tornado.auth.GoogleMixin): + @tornado.web.asynchronous + def get(self): + if self.get_argument("openid.mode", None): + self.get_authenticated_user(self.async_callback(self._on_auth)) + return + self.authenticate_redirect() + + def _on_auth(self, user): + if not user: + self.authenticate_redirect() + return + # Save the user with, e.g., set_secure_cookie() + +See the `auth` module documentation for more details. + + +Performance +----------- +Web application performance is generally bound by architecture, not frontend +performance. That said, Tornado is pretty fast relative to most popular +Python web frameworks. + +We ran a few remedial load tests on a simple "Hello, world" application +in each of the most popular Python web frameworks +([Django](http://www.djangoproject.com/), [web.py](http://webpy.org/), and +[CherryPy](http://www.cherrypy.org/)) to get the baseline performance of +each relative to Tornado. We used Apache/mod_wsgi for Django and web.py +and ran CherryPy as a standalone server, which was our impression of how +each framework is typically run in production environments. We ran 4 +single-threaded Tornado frontends behind an [nginx](http://nginx.net/) +reverse proxy, which is how we recommend running Tornado in production +(our load test machine had four cores, and we recommend 1 frontend per +core). + +We load tested each with Apache Benchmark (`ab`) on the a separate machine +with the command + + ab -n 100000 -c 25 http://10.0.1.x/ + +The results (requests per second) on a 2.4GHz AMD Opteron processor with +4 cores: + +<div style="text-align:center;margin-top:2em;margin-bottom:2em"><img src="http://chart.apis.google.com/chart?chxt=y&chd=t%3A100%2C40%2C27%2C25%2C9&chco=609bcc&chm=t+8213%2C000000%2C0%2C0%2C11%7Ct+3353%2C000000%2C0%2C1%2C11%7Ct+2223%2C000000%2C0%2C2%2C11%7Ct+2066%2C000000%2C0%2C3%2C11%7Ct+785%2C000000%2C0%2C4%2C11&chs=600x175&cht=bhs&chtt=Web+server+requests%2Fsec+%28AMD+Opteron%2C+2.4GHz%2C+4+cores%29&chxl=0%3A%7CCherryPy+%28standalone%29%7Cweb.py+%28Apache%2Fmod_wsgi%29%7CDjango+%28Apache%2Fmod_wsgi%29%7CTornado+%281+single-threaded+frontend%29%7CTornado+%28nginx%3B+4+frontends%29%7C"/></div> + +In our tests, Tornado consistently had 4X the throughput of the next fastest +framework, and even a single standalone Tornado frontend got 33% more +throughput even though it only used one of the four cores. + +Not very scientific, but at a high level, it should give you a sense that we +have cared about performance as we built Tornado, and it shouldn't add too +much latency to your apps relative to most Python web development frameworks. + + +Running Tornado in production +----------------------------- +At FriendFeed, we use [nginx](http://nginx.net/) as a load balancer +and static file server. We run multiple instances of the Tornado web +server on multiple frontend machines. We typically run one Tornado frontend +per core on the machine (sometimes more depending on utilization). + +This is a barebones nginx config file that is structurally similar to the +one we use at FriendFeed. It assumes nginx and the Tornado servers +are running on the same machine, and the four Tornado servers +are running on ports 8000 - 8003: + + user nginx; + worker_processes 1; + + error_log /var/log/nginx/error.log; + pid /var/run/nginx.pid; + + events { + worker_connections 1024; + use epoll; + } + + http { + # Enumerate all the Tornado servers here + upstream frontends { + server 127.0.0.1:8000; + server 127.0.0.1:8001; + server 127.0.0.1:8002; + server 127.0.0.1:8003; + } + + include /etc/nginx/mime.types; + default_type application/octet-stream; + + access_log /var/log/nginx/access.log; + + keepalive_timeout 65; + proxy_read_timeout 200; + sendfile on; + tcp_nopush on; + tcp_nodelay on; + gzip on; + gzip_min_length 1000; + gzip_proxied any; + gzip_types text/plain text/html text/css text/xml + application/x-javascript application/xml + application/atom+xml text/javascript; + + # Only retry if there was a communication error, not a timeout + # on the Tornado server (to avoid propagating "queries of death" + # to all frontends) + proxy_next_upstream error; + + server { + listen 80; + + # Allow file uploads + client_max_body_size 50M; + + location ^~ /static/ { + root /var/www; + if ($query_string) { + expires max; + } + } + location = /favicon.ico { + rewrite (.*) /static/favicon.ico; + } + location = /robots.txt { + rewrite (.*) /static/robots.txt; + } + + location / { + proxy_pass_header Server; + proxy_set_header Host $http_host; + proxy_redirect false; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Scheme $scheme; + proxy_pass http://frontends; + } + } + } + + +WSGI and Google AppEngine +------------------------- +Tornado comes with limited support for [WSGI](http://wsgi.org/). However, +since WSGI does not support non-blocking requests, you cannot use any +of the asynchronous/non-blocking features of Tornado in your application +if you choose to use WSGI instead of Tornado's HTTP server. Some of the +features that are not available in WSGI applications: +`@tornado.web.asynchronous`, the `httpclient` module, and the `auth` module. + +You can create a valid WSGI application from your Tornado request handlers +by using `WSGIApplication` in the `wsgi` module instead of using +`tornado.web.Application`. Here is an example that uses the built-in WSGI +`CGIHandler` to make a valid +[Google AppEngine](http://code.google.com/appengine/) application: + + import tornado.web + import tornado.wsgi + import wsgiref.handlers + + class MainHandler(tornado.web.RequestHandler): + def get(self): + self.write("Hello, world") + + if __name__ == "__main__": + application = tornado.wsgi.WSGIApplication([ + (r"/", MainHandler), + ]) + wsgiref.handlers.CGIHandler().run(application) + +See the `appengine` example application for a full-featured AppEngine +app built on Tornado. + + +Caveats and support +------------------- +Tornado was refactored from the [FriendFeed](http://friendfeed.com/) +code base to reduce dependencies. This refactoring may have introduced +bugs. Likewise, because the FriendFeed servers have always run +[behind nginx](#running-tornado-in-production), Tornado has not been +extensively tested with HTTP/1.1 clients beyond Firefox. Tornado +currently does not attempt to handle multi-line headers and some types +of malformed input. + +You can discuss Tornado and report bugs on [the Tornado developer mailing list](http://groups.google.com/group/python-tornado). diff --git a/vendor/tornado/website/templates/index.html b/vendor/tornado/website/templates/index.html new file mode 100644 index 0000000000..4aa716598b --- /dev/null +++ b/vendor/tornado/website/templates/index.html @@ -0,0 +1,51 @@ +{% extends "base.html" %} + +{% block body %} + <p><a href="http://www.tornadoweb.org/">Tornado</a> is an open source version of the scalable, non-blocking web server and tools that power <a href="http://friendfeed.com/">FriendFeed</a>. The FriendFeed application is written using a web framework that looks a bit like <a href="http://webpy.org/">web.py</a> or <a href="http://code.google.com/appengine/docs/python/tools/webapp/">Google's webapp</a>, but with additional tools and optimizations to take advantage of the underlying non-blocking infrastructure.</p> + <p>The framework is distinct from most mainstream web server frameworks (and certainly most Python frameworks) because it is non-blocking and reasonably fast. Because it is non-blocking and uses <a href="http://www.kernel.org/doc/man-pages/online/pages/man4/epoll.4.html">epoll</a>, it can handle thousands of simultaneous standing connections, which means it is ideal for real-time web services. We built the web server specifically to handle FriendFeed's real-time features — every active user of FriendFeed maintains an open connection to the FriendFeed servers. (For more information on scaling servers to support thousands of clients, see The <a href="http://www.kegel.com/c10k.html">C10K problem</a>.)</p> + <p>See the <a href="/documentation">Tornado documentation</a> for a detailed walkthrough of the framework.</p> + + <h2>Download and install</h2> + <p><b>Download:</b> <a href="/static/tornado-0.2.tar.gz">tornado-0.2.tar.gz</a></p> + <pre><code>tar xvzf tornado-0.2.tar.gz +cd tornado-0.2 +python setup.py build +sudo python setup.py install</code></pre> + <p>The Tornado source code is <a href="http://github.com/facebook/tornado">hosted on GitHub</a>.</p> + + <h3>Prerequisites</h3> + <p>Tornado has been tested on Python 2.5 and 2.6. To use all of the features of Tornado, you need to have <a href="http://pycurl.sourceforge.net/">PycURL</a> and a JSON library like <a href="http://pypi.python.org/pypi/simplejson/">simplejson</a> installed. Complete installation instructions for Mac OS X and Ubuntu are included below for convenience.</p> + <p style="font-weight:bold">Mac OS X 10.5/10.6</p> + <pre><code>sudo easy_install setuptools pycurl==7.16.2.1 simplejson</code></pre> + + <p style="font-weight:bold">Ubuntu Linux</p> + <pre><code>sudo apt-get install python-dev python-pycurl python-simplejson</code></pre> + + <h2>Hello, world</h2> + <p>Here is the canonical "Hello, world" example app for Tornado:</p> + <pre><code>import tornado.httpserver +import tornado.ioloop +import tornado.web + +class MainHandler(tornado.web.RequestHandler): + def get(self): + self.write("Hello, world") + +application = tornado.web.Application([ + (r"/", MainHandler), +]) + +if __name__ == "__main__": + http_server = tornado.httpserver.HTTPServer(application) + http_server.listen(8888) + tornado.ioloop.IOLoop.instance().start()</code></pre> + <p>See the <a href="/documentation">Tornado documentation</a> for a detailed walkthrough of the framework.</p> + + <h2>Discussion and support</h2> + <p>You can discuss Tornado and report bugs on <a href="http://groups.google.com/group/python-tornado">the Tornado developer mailing list</a>. + + <h2>Updates</h2> + <p>Follow us on <a href="http://www.facebook.com/pages/Tornado-Web-Server/144144048921">Facebook</a>, <a href="http://twitter.com/tornadoweb">Twitter</a>, or <a href="http://friendfeed.com/tornado-web">FriendFeed</a> to get updates and announcements:</p> + <div style="margin-top:1em"><a href="http://www.facebook.com/pages/Tornado-Web-Server/144144048921" style="margin-right:10px"><img src="/static/facebook.png" style="width:64px;height:64px" alt="Facebook"/></a><a href="http://twitter.com/tornadoweb" style="margin-right:10px"><img src="/static/twitter.png" style="width:64px;height:64px" alt="Twitter"/></a><a href="http://friendfeed.com/tornado-web" style="margin-right:10px"><img src="/static/friendfeed.png" style="width:64px;height:64px" alt="Facebook"/></a></div> + +{% end %} diff --git a/vendor/tornado/website/website.py b/vendor/tornado/website/website.py new file mode 100644 index 0000000000..f073b67e6b --- /dev/null +++ b/vendor/tornado/website/website.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python +# +# Copyright 2009 Bret Taylor +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import markdown +import os +import os.path +import time +import tornado.web +import tornado.wsgi +import wsgiref.handlers + + +class ContentHandler(tornado.web.RequestHandler): + def get(self, path): + paths = ("documentation", "index") + if not path: path = "index" + if path not in paths: + raise tornado.web.HTTPError(404) + self.render(path + ".html", markdown=self.markdown) + + def markdown(self, path, toc=False): + if not hasattr(ContentHandler, "_md") or self.settings.get("debug"): + ContentHandler._md = {} + if path not in ContentHandler._md: + full_path = os.path.join(self.settings["template_path"], path) + f = open(full_path, "r") + contents = f.read().decode("utf-8") + f.close() + if toc: contents = u"[TOC]\n\n" + contents + md = markdown.Markdown(extensions=["toc"] if toc else []) + ContentHandler._md[path] = md.convert(contents).encode("utf-8") + return ContentHandler._md[path] + + +settings = { + "template_path": os.path.join(os.path.dirname(__file__), "templates"), + "xsrf_cookies": True, + "debug": os.environ.get("SERVER_SOFTWARE", "").startswith("Development/"), +} +application = tornado.wsgi.WSGIApplication([ + (r"/([a-z]*)", ContentHandler), +], **settings) + + +def main(): + wsgiref.handlers.CGIHandler().run(application) + + +if __name__ == "__main__": + main() |
