diff options
author | Jason Madden <jamadden@gmail.com> | 2017-11-03 08:03:21 -0500 |
---|---|---|
committer | Jason Madden <jamadden@gmail.com> | 2017-11-03 08:03:21 -0500 |
commit | d052e523e766b38c6ba058b7630a99fa36c35719 (patch) | |
tree | c8b1cb2f945857e075546584ab719db93583f7a0 | |
parent | c9cbf3c4f834ac838c6e14cf65a69554bf72a98e (diff) | |
download | zope-tal-d052e523e766b38c6ba058b7630a99fa36c35719.tar.gz |
Document the rest of the important parts of the API.
Some minor whitespace cleanup along the way.
-rw-r--r-- | docs/htmltalparser.rst | 5 | ||||
-rw-r--r-- | docs/index.rst | 30 | ||||
-rw-r--r-- | docs/taldefs.rst | 5 | ||||
-rw-r--r-- | docs/talgenerator.rst | 5 | ||||
-rw-r--r-- | docs/talinterpreter.rst | 5 | ||||
-rw-r--r-- | docs/talparser.rst | 7 | ||||
-rw-r--r-- | src/zope/tal/htmltalparser.py | 56 | ||||
-rw-r--r-- | src/zope/tal/taldefs.py | 47 | ||||
-rw-r--r-- | src/zope/tal/talgenerator.py | 21 | ||||
-rw-r--r-- | src/zope/tal/talinterpreter.py | 65 | ||||
-rw-r--r-- | src/zope/tal/talparser.py | 19 | ||||
-rw-r--r-- | src/zope/tal/xmlparser.py | 8 |
12 files changed, 197 insertions, 76 deletions
diff --git a/docs/htmltalparser.rst b/docs/htmltalparser.rst new file mode 100644 index 0000000..bf2ac4f --- /dev/null +++ b/docs/htmltalparser.rst @@ -0,0 +1,5 @@ +============================ + Parsing and Compiling HTML +============================ + +.. automodule:: zope.tal.htmltalparser diff --git a/docs/index.rst b/docs/index.rst index d806fdd..2b80efc 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -1,11 +1,41 @@ .. include:: ../README.rst +Using ``zope.tal`` requires three steps: choosing an expression engine +(usually :mod:`zope.tales`), creating a generator and parser, and then +interpreting the compiled program:: + + from io import StringIO + from zope.tal.talgenerator import TALGenerator + from zope.tal.htmltalparser import HTMLTALParser + from zope.tal.talinterpreter import TALInterpreter + + compiler = None # Will use a compiler for a dummy language + source_file = '<string>' + source_text = '<html><body><p>Hi</p></body></html>' + gen = TALGenerator(compiler, source_file=source_file) + parser = TALParser(gen) + parser.parseString(source_text) + program, macros = parser.getCode() + + output = StringIO() + context = None # Usually will create a zope.tales context + interpreter = TALInterpreter(self.program, macros, context, stream=output) + interpreter() + result = output.getvalue() + +These aspects are all brought together in :mod:`zope.pagetemplate`. + API Documentation: .. toctree:: :maxdepth: 2 interfaces + taldefs + talgenerator + htmltalparser + talparser + talinterpreter .. toctree:: :maxdepth: 1 diff --git a/docs/taldefs.rst b/docs/taldefs.rst new file mode 100644 index 0000000..ba2f6bd --- /dev/null +++ b/docs/taldefs.rst @@ -0,0 +1,5 @@ +==================== + Common Definitions +==================== + +.. automodule:: zope.tal.taldefs diff --git a/docs/talgenerator.rst b/docs/talgenerator.rst new file mode 100644 index 0000000..35fc115 --- /dev/null +++ b/docs/talgenerator.rst @@ -0,0 +1,5 @@ +========================== + Generating Compiled Code +========================== + +.. automodule:: zope.tal.talgenerator diff --git a/docs/talinterpreter.rst b/docs/talinterpreter.rst new file mode 100644 index 0000000..0f23220 --- /dev/null +++ b/docs/talinterpreter.rst @@ -0,0 +1,5 @@ +============================ + Interpreting Compiled Code +============================ + +.. automodule:: zope.tal.talinterpreter diff --git a/docs/talparser.rst b/docs/talparser.rst new file mode 100644 index 0000000..e7296f9 --- /dev/null +++ b/docs/talparser.rst @@ -0,0 +1,7 @@ +=========================== + Parsing and Compiling XML +=========================== + +.. automodule:: zope.tal.talparser + +.. autoclass:: zope.tal.xmlparser.XMLParser diff --git a/src/zope/tal/htmltalparser.py b/src/zope/tal/htmltalparser.py index c79bbea..1761bc7 100644 --- a/src/zope/tal/htmltalparser.py +++ b/src/zope/tal/htmltalparser.py @@ -11,7 +11,9 @@ # FOR A PARTICULAR PURPOSE. # ############################################################################## -"""Parse HTML and compile to TALInterpreter intermediate code. +""" +Parse HTML and compile to :class:`~.TALInterpreter` intermediate code, using +a :class:`~.TALGenerator`. """ # When Python 3 becomes mainstream please swap the try and except parts. @@ -28,6 +30,7 @@ except ImportError: # so here's a copy taken from Python 3.4: class HTMLParseError(Exception): def __init__(self, msg, position=(None, None)): + Exception.__init__(self) assert msg self.msg = msg self.lineno = position[0] @@ -50,30 +53,30 @@ _html_parser_extras = {} if 'convert_charrefs' in HTMLParser.__init__.__code__.co_names: _html_parser_extras['convert_charrefs'] = False # pragma: NO COVER py34 - +#: List of Boolean attributes in HTML that may be given in +#: minimized form (e.g. ``<img ismap>`` rather than ``<img ismap="">``) +#: From http://www.w3.org/TR/xhtml1/#guidelines (C.10) BOOLEAN_HTML_ATTRS = frozenset([ - # List of Boolean attributes in HTML that may be given in - # minimized form (e.g. <img ismap> rather than <img ismap="">) - # From http://www.w3.org/TR/xhtml1/#guidelines (C.10) "compact", "nowrap", "ismap", "declare", "noshade", "checked", "disabled", "readonly", "multiple", "selected", "noresize", "defer" ]) +#: List of HTML tags with an empty content model; these are +#: rendered in minimized form, e.g. ``<img />``. +#: From http://www.w3.org/TR/xhtml1/#dtds EMPTY_HTML_TAGS = frozenset([ - # List of HTML tags with an empty content model; these are - # rendered in minimized form, e.g. <img />. - # From http://www.w3.org/TR/xhtml1/#dtds "base", "meta", "link", "hr", "br", "param", "img", "area", "input", "col", "basefont", "isindex", "frame", ]) +#: List of HTML elements that close open paragraph-level elements +#: and are themselves paragraph-level. PARA_LEVEL_HTML_TAGS = frozenset([ - # List of HTML elements that close open paragraph-level elements - # and are themselves paragraph-level. "h1", "h2", "h3", "h4", "h5", "h6", "p", ]) +#: Tags that automatically close other tags. BLOCK_CLOSING_TAG_MAP = { "tr": frozenset(["tr", "td", "th"]), "td": frozenset(["td", "th"]), @@ -83,12 +86,13 @@ BLOCK_CLOSING_TAG_MAP = { "dt": frozenset(["dd", "dt"]), } +#: List of HTML tags that denote larger sections than paragraphs. BLOCK_LEVEL_HTML_TAGS = frozenset([ - # List of HTML tags that denote larger sections than paragraphs. "blockquote", "table", "tr", "th", "td", "thead", "tfoot", "tbody", "noframe", "ul", "ol", "li", "dl", "dt", "dd", "div", ]) +#: Section level HTML tags SECTION_LEVEL_HTML_TAGS = PARA_LEVEL_HTML_TAGS.union(BLOCK_LEVEL_HTML_TAGS) TIGHTEN_IMPLICIT_CLOSE_TAGS = PARA_LEVEL_HTML_TAGS.union(BLOCK_CLOSING_TAG_MAP) @@ -127,25 +131,37 @@ class OpenTagError(NestingError): HTMLParseError.__init__(self, msg, position) class HTMLTALParser(HTMLParser): + """ + Parser for HTML. + + After you call either :meth:`parseFile` and :meth:`parseString` + you can retrieve the compiled program using :meth:`getCode`. + """ # External API def __init__(self, gen=None): + """ + :keyword TALGenerator gen: The configured (with an expression compiler) + code generator to use. If one is not given, a default will be used. + """ HTMLParser.__init__(self, **_html_parser_extras) if gen is None: gen = TALGenerator(xml=0) self.gen = gen self.tagstack = [] self.nsstack = [] - self.nsdict = {'tal': ZOPE_TAL_NS, - 'metal': ZOPE_METAL_NS, - 'i18n': ZOPE_I18N_NS, - } + self.nsdict = { + 'tal': ZOPE_TAL_NS, + 'metal': ZOPE_METAL_NS, + 'i18n': ZOPE_I18N_NS, + } def parseFile(self, file): - f = open(file) - data = f.read() - f.close() + """Parse data in the given file.""" + with open(file) as f: + data = f.read() + try: self.parseString(data) except TALError as e: @@ -153,6 +169,7 @@ class HTMLTALParser(HTMLParser): raise def parseString(self, data): + """Parse data in the given string.""" self.feed(data) self.close() while self.tagstack: @@ -160,6 +177,9 @@ class HTMLTALParser(HTMLParser): assert self.nsstack == [], self.nsstack def getCode(self): + """ + After parsing, this returns ``(program, macros)``. + """ return self.gen.getCode() # Overriding HTMLParser methods diff --git a/src/zope/tal/taldefs.py b/src/zope/tal/taldefs.py index a4aaf61..539e541 100644 --- a/src/zope/tal/taldefs.py +++ b/src/zope/tal/taldefs.py @@ -17,20 +17,26 @@ import re from zope.tal.interfaces import ITALExpressionErrorInfo from zope.interface import implementer - +#: Version of the specification we implement. TAL_VERSION = "1.6" -XML_NS = "http://www.w3.org/XML/1998/namespace" # URI for XML namespace -XMLNS_NS = "http://www.w3.org/2000/xmlns/" # URI for XML NS declarations +#: URI for XML namespace +XML_NS = "http://www.w3.org/XML/1998/namespace" +#: URI for XML NS declarations +XMLNS_NS = "http://www.w3.org/2000/xmlns/" +#: TAL namespace URI ZOPE_TAL_NS = "http://xml.zope.org/namespaces/tal" +#: METAL namespace URI ZOPE_METAL_NS = "http://xml.zope.org/namespaces/metal" +#: I18N namespace URI ZOPE_I18N_NS = "http://xml.zope.org/namespaces/i18n" # This RE must exactly match the expression of the same name in the # zope.i18n.simpletranslationservice module: NAME_RE = "[a-zA-Z_][-a-zA-Z0-9_]*" +#: Known METAL attributes KNOWN_METAL_ATTRIBUTES = frozenset([ "define-macro", "extend-macro", @@ -39,6 +45,7 @@ KNOWN_METAL_ATTRIBUTES = frozenset([ "fill-slot", ]) +#: Known TAL attributes KNOWN_TAL_ATTRIBUTES = frozenset([ "define", "condition", @@ -53,6 +60,7 @@ KNOWN_TAL_ATTRIBUTES = frozenset([ # like <tal:x>, <metal:y>, <i18n:z> ]) +#: Known I18N attributes KNOWN_I18N_ATTRIBUTES = frozenset([ "translate", "domain", @@ -66,8 +74,12 @@ KNOWN_I18N_ATTRIBUTES = frozenset([ ]) class TALError(Exception): + """ + A base exception for errors raised by this implementation. + """ def __init__(self, msg, position=(None, None)): + Exception.__init__(self) assert msg != "" self.msg = msg self.lineno = position[0] @@ -88,17 +100,20 @@ class TALError(Exception): return result class METALError(TALError): - pass + """An error parsing on running METAL macros.""" class TALExpressionError(TALError): - pass + """An error parsing or running a TAL expression.""" class I18NError(TALError): - pass + """An error parsing a I18N expression.""" @implementer(ITALExpressionErrorInfo) class ErrorInfo(object): + """ + Default implementation of :class:`zope.tal.interfaces.ITALExpressionErrorInfo`. + """ def __init__(self, err, position=(None, None)): if isinstance(err, Exception): @@ -115,7 +130,7 @@ _attr_re = re.compile(r"\s*([^\s]+)\s+([^\s].*)\Z", re.S) _subst_re = re.compile(r"\s*(?:(text|structure)\s+)?(.*)\Z", re.S) def parseAttributeReplacements(arg, xml): - dict = {} + attr_dict = {} for part in splitParts(arg): m = _attr_re.match(part) if not m: @@ -123,10 +138,10 @@ def parseAttributeReplacements(arg, xml): name, expr = m.groups() if not xml: name = name.lower() - if name in dict: + if name in attr_dict: raise TALError("Duplicate attribute name in attributes: %r" % part) - dict[name] = expr - return dict + attr_dict[name] = expr + return attr_dict def parseSubstitution(arg, position=(None, None)): m = _subst_re.match(arg) @@ -151,26 +166,26 @@ def isCurrentVersion(program): version = getProgramVersion(program) return version == TAL_VERSION -def isinstance_(ob, type): +def isinstance_(ob, kind): # Proxy-friendly and faster isinstance_ check for new-style objects try: - return type in ob.__class__.__mro__ + return kind in ob.__class__.__mro__ except AttributeError: return False def getProgramMode(program): version = getProgramVersion(program) - if (version == TAL_VERSION and isinstance_(program[1], tuple) and - len(program[1]) == 2): + if (version == TAL_VERSION and isinstance_(program[1], tuple) + and len(program[1]) == 2): opcode, mode = program[1] if opcode == "mode": return mode return None def getProgramVersion(program): - if (len(program) >= 2 and - isinstance_(program[0], tuple) and len(program[0]) == 2): + if (len(program) >= 2 + and isinstance_(program[0], tuple) and len(program[0]) == 2): opcode, version = program[0] if opcode == "version": return version diff --git a/src/zope/tal/talgenerator.py b/src/zope/tal/talgenerator.py index ac77c18..ce7470c 100644 --- a/src/zope/tal/talgenerator.py +++ b/src/zope/tal/talgenerator.py @@ -11,7 +11,8 @@ # FOR A PARTICULAR PURPOSE. # ############################################################################## -"""Code generator for TALInterpreter intermediate code. +""" +Code generator for :class:`~.TALInterpreter` intermediate code. """ import re @@ -37,12 +38,20 @@ except NameError: _name_rx = re.compile(NAME_RE) class TALGenerator(object): + """ + Generate intermediate code. + """ inMacroUse = 0 inMacroDef = 0 source_file = None def __init__(self, expressionCompiler=None, xml=1, source_file=None): + """ + :keyword expressionCompiler: The implementation of + :class:`zope.tal.interfaces.ITALExpressionCompiler` to use. + If not given, we'll use a simple, undocumented, compiler. + """ if not expressionCompiler: from zope.tal.dummyengine import DummyEngine expressionCompiler = DummyEngine() @@ -96,7 +105,7 @@ class TALGenerator(object): if self.optimizeStartTag(collect, item[1], item[2], ">"): continue if opcode == "startEndTag": - endsep = self.xml and "/>" or " />" + endsep = "/>" if self.xml else " />" if self.optimizeStartTag(collect, item[1], item[2], endsep): continue if opcode in ("beginScope", "endScope"): @@ -182,9 +191,9 @@ class TALGenerator(object): output = program[:2] prev2, prev1 = output for item in program[2:]: - if ( item[0] == "beginScope" - and prev1[0] == "setPosition" - and prev2[0] == "rawtextColumn"): + if (item[0] == "beginScope" + and prev1[0] == "setPosition" + and prev2[0] == "rawtextColumn"): position = output.pop()[1] text, column = output.pop()[1] prev1 = None, None @@ -302,7 +311,7 @@ class TALGenerator(object): self.emit("condition", cexpr, program) def emitRepeat(self, arg): - m = re.match("(?s)\s*(%s)\s+(.*)\Z" % NAME_RE, arg) + m = re.match(r"(?s)\s*(%s)\s+(.*)\Z" % NAME_RE, arg) if not m: raise TALError("invalid repeat syntax: " + repr(arg), self.position) diff --git a/src/zope/tal/talinterpreter.py b/src/zope/tal/talinterpreter.py index c2ad97b..82154db 100644 --- a/src/zope/tal/talinterpreter.py +++ b/src/zope/tal/talinterpreter.py @@ -26,10 +26,9 @@ from zope.tal.translationcontext import TranslationContext try: unicode - _BLANK = unicode('') except NameError: unicode = str # Python 3.x - _BLANK = '' +_BLANK = u'' # Avoid constructing this tuple over and over @@ -43,6 +42,7 @@ BOOLEAN_HTML_ATTRS = frozenset([ # From http://www.w3.org/TR/xhtml1/#guidelines (C.10) # TODO: The problem with this is that this is not valid XML and # can't be parsed back! + # XXX: This is an exact duplicate of htmltalparser.BOOLEAN_HTML_ATTRS. Why? "compact", "nowrap", "ismap", "declare", "noshade", "checked", "disabled", "readonly", "multiple", "selected", "noresize", "defer" @@ -116,25 +116,25 @@ class TALInterpreter(object): """TAL interpreter. Some notes on source annotations. They are HTML/XML comments added to the - output whenever sourceFile is changed by a setSourceFile bytecode. Source + output whenever ``sourceFile`` is changed by a ``setSourceFile`` bytecode. Source annotations are disabled by default, but you can turn them on by passing a - sourceAnnotations argument to the constructor. You can change the format + ``sourceAnnotations`` argument to the constructor. You can change the format of the annotations by overriding formatSourceAnnotation in a subclass. The output of the annotation is delayed until some actual text is output for two reasons: - 1. setPosition bytecode follows setSourceFile, and we need position + 1. ``setPosition`` bytecode follows ``setSourceFile``, and we need position information to output the line number. - 2. Comments are not allowed in XML documents before the <?xml?> + 2. Comments are not allowed in XML documents before the ``<?xml?>`` declaration. For performance reasons (TODO: premature optimization?) instead of checking - the value of _pending_source_annotation on every write to the output - stream, the _stream_write attribute is changed to point to - _annotated_stream_write method whenever _pending_source_annotation is + the value of ``_pending_source_annotation`` on every write to the output + stream, the ``_stream_write`` attribute is changed to point to + ``_annotated_stream_write`` method whenever ``_pending_source_annotation`` is set to True, and to _stream.write when it is False. The following - invariant always holds: + invariant always holds:: if self._pending_source_annotation: assert self._stream_write is self._annotated_stream_write @@ -149,36 +149,31 @@ class TALInterpreter(object): sourceAnnotations=0): """Create a TAL interpreter. - Optional arguments: - - stream -- output stream (defaults to sys.stdout). + :param program: A compiled program, as generated + by :class:`zope.tal.talgenerator.TALGenerator` + :param macros: Namespace of macros, usually also from + :class:`~.TALGenerator` - debug -- enable debugging output to sys.stderr (off by default). + Optional arguments: - wrap -- try to wrap attributes on opening tags to this number of + :keyword stream: output stream (defaults to sys.stdout). + :keyword bool debug: enable debugging output to sys.stderr (off by default). + :keyword int wrap: try to wrap attributes on opening tags to this number of column (default: 1023). - - metal -- enable METAL macro processing (on by default). - - tal -- enable TAL processing (on by default). - - showtal -- do not strip away TAL directives. A special value of + :keyword bool metal: enable METAL macro processing (on by default). + :keyword bool tal: enable TAL processing (on by default). + :keyword int showtal: do not strip away TAL directives. A special value of -1 (which is the default setting) enables showtal when TAL processing is disabled, and disables showtal when TAL processing is enabled. Note that you must use 0, 1, or -1; true boolean values - are not supported (TODO: why?). - - strictinsert -- enable TAL processing and stricter HTML/XML + are not supported (for historical reasons). + :keyword bool strictinsert: enable TAL processing and stricter HTML/XML checking on text produced by structure inserts (on by default). Note that Zope turns this value off by default. - - stackLimit -- set macro nesting limit (default: 100). - - i18nInterpolate -- enable i18n translations (default: on). - - sourceAnnotations -- enable source annotations with HTML comments + :keyword int stackLimit: set macro nesting limit (default: 100). + :keyword bool i18nInterpolate: enable i18n translations (default: on). + :keyword bool sourceAnnotations: enable source annotations with HTML comments (default: off). - """ self.program = program self.macros = macros @@ -266,6 +261,11 @@ class TALInterpreter(object): return self.macroStack.pop() def __call__(self): + """ + Interpret the current program. + + :return: Nothing. + """ assert self.level == 0 assert self.scopeLevel == 0 assert self.i18nContext.parent is None @@ -1017,8 +1017,7 @@ class TALInterpreter(object): class FasterStringIO(list): - """Unicode-aware append-only version of StringIO. - """ + # Unicode-aware append-only version of StringIO. write = list.append def __init__(self, value=None): diff --git a/src/zope/tal/talparser.py b/src/zope/tal/talparser.py index 9adba2d..d99fc9f 100644 --- a/src/zope/tal/talparser.py +++ b/src/zope/tal/talparser.py @@ -11,7 +11,9 @@ # FOR A PARTICULAR PURPOSE. # ############################################################################## -"""Parse XML and compile to TALInterpreter intermediate code. +""" +Parse XML and compile to :class:`~.TALInterpreter` intermediate code, +using a :class:`~.TALGenerator`. """ from zope.tal.taldefs import XML_NS, ZOPE_I18N_NS, ZOPE_METAL_NS, ZOPE_TAL_NS from zope.tal.talgenerator import TALGenerator @@ -19,10 +21,22 @@ from zope.tal.xmlparser import XMLParser class TALParser(XMLParser): + """ + Parser for XML. + + After parsing with :meth:`~.XMLParser.parseFile`, + :meth:`~.XMLParser.parseString`, :meth:`~.XMLParser.parseURL` or + :meth:`~.XMLParser.parseStream`, you can call :meth:`getCode` to + retrieve the parsed program and macros. + """ ordered_attributes = 1 def __init__(self, gen=None, encoding=None): # Override + """ + :keyword TALGenerator gen: The configured (with an expression compiler) + code generator to use. If one is not given, a default will be used. + """ XMLParser.__init__(self, encoding) if gen is None: gen = TALGenerator() @@ -32,6 +46,7 @@ class TALParser(XMLParser): self.nsNew = [] def getCode(self): + """Return the compiled program and macros after parsing.""" return self.gen.getCode() def StartNamespaceDeclHandler(self, prefix, uri): @@ -117,7 +132,7 @@ class TALParser(XMLParser): def EndElementHandler(self, name): name = self.fixname(name)[0] - self.gen.emitEndElement(name, position=self.getpos()) + self.gen.emitEndElement(name, position=self.getpos()) def DefaultHandler(self, text): self.gen.emitRawText(text) diff --git a/src/zope/tal/xmlparser.py b/src/zope/tal/xmlparser.py index 9081d37..ca5c216 100644 --- a/src/zope/tal/xmlparser.py +++ b/src/zope/tal/xmlparser.py @@ -32,6 +32,9 @@ except NameError: class XMLParser(object): + """ + Parse XML using :mod:`xml.parsers.expat`. + """ ordered_attributes = 0 @@ -82,10 +85,12 @@ class XMLParser(object): return expat.ParserCreate(encoding, ' ') def parseFile(self, filename): + """Parse from the given filename.""" with open(filename, 'rb') as f: self.parseStream(f) def parseString(self, s): + """Parse the given string.""" if isinstance(s, unicode): # Expat cannot deal with unicode strings, only with # encoded ones. Also, its range of encodings is rather @@ -94,9 +99,11 @@ class XMLParser(object): self.parser.Parse(s, 1) def parseURL(self, url): + """Parse the given URL.""" self.parseStream(urlopen(url)) def parseStream(self, stream): + """Parse the given stream (open file).""" self.parser.ParseFile(stream) def parseFragment(self, s, end=0): @@ -113,4 +120,3 @@ class XMLParser(object): # [1] http://python.org/doc/current/lib/xmlparser-objects.html # [2] http://cvs.sourceforge.net/viewcvs.py/expat/expat/lib/expat.h return (self.parser.ErrorLineNumber, self.parser.ErrorColumnNumber) - |