Merge branch 'w_json_metadata'

7 files changed, 124 insertions, 43 deletions

diff --git a/doc/build/changelog.rst b/doc/build/changelog.rst
index f43b26d..8e1c9eb 100644
--- a/doc/build/changelog.rst
+++ b/doc/build/changelog.rst
@@ -17,6 +17,23 @@ Changelog
       The source base is now targeted at Python 2.6 and forwards.
 
     .. change::
+        :tags: feature
+
+      Template modules now generate a JSON "metadata" structure at the bottom
+      of the source file which includes parseable information about the
+      templates' source file, encoding etc. as well as a mapping of module
+      source lines to template lines, thus replacing the "# SOURCE LINE"
+      markers throughout the source code.  The structure also indicates those
+      lines that are explicitly not part of the template's source; the goal
+      here is to allow better integration with coverage and other tools.
+
+    .. change::
+        :tags: bug, py3k
+
+      Fixed bug in ``decode.<encoding>`` filter where a non-string object
+      would not be correctly interpreted in Python 3.
+
+    .. change::
         :tags: bug, py3k
 
       Fixed bug in ``decode.<encoding>`` filter where a non-string object
@@ -48,13 +65,6 @@ Changelog
       Pull request courtesy Derek Harland.
 
     .. change::
-        :tags: feature, py3k
-        :pullreq: github:7
-
-      Support is added for Python 3 "keyword only" arguments, as used in
-      defs.  Pull request courtesy Eevee.
-
-    .. change::
         :tags: bug
         :pullreq: bitbucket:2
 
@@ -67,6 +77,14 @@ Changelog
       template lookup directories.  Standard input for templates also works
       now too.  Pull request courtesy Derek Harland.
 
+    .. change::
+        :tags: feature, py3k
+        :pullreq: github:7
+
+      Support is added for Python 3 "keyword only" arguments, as used in
+      defs.  Pull request courtesy Eevee.
+
+
 0.9
 ===
 
diff --git a/mako/codegen.py b/mako/codegen.py
index 2240ba2..63e76a7 100644
--- a/mako/codegen.py
+++ b/mako/codegen.py
@@ -14,7 +14,7 @@ from mako import util, ast, parsetree, filters, exceptions
 from mako import compat
 
 
-MAGIC_NUMBER = 9
+MAGIC_NUMBER = 10
 
 # names which are hardwired into the
 # template and are not accessed via the
@@ -99,7 +99,6 @@ class _GenerateRenderMethod(object):
     """
     def __init__(self, printer, compiler, node):
         self.printer = printer
-        self.last_source_line = -1
         self.compiler = compiler
         self.node = node
         self.identifier_stack = [None]
@@ -146,6 +145,26 @@ class _GenerateRenderMethod(object):
             for node in defs:
                 _GenerateRenderMethod(printer, compiler, node)
 
+        if not self.in_def:
+            self.write_metadata_struct()
+
+    def write_metadata_struct(self):
+        self.printer.source_map[self.printer.lineno] = self.printer.last_source_line
+        struct = {
+            "filename": self.compiler.filename,
+            "uri": self.compiler.uri,
+            "source_encoding": self.compiler.source_encoding,
+            "line_map": self.printer.source_map,
+            "boilerplate_lines": self.printer.boilerplate_map
+        }
+        self.printer.writelines(
+            '"""',
+            '__M_BEGIN_METADATA',
+            compat.json.dumps(struct),
+            '__M_END_METADATA\n'
+            '"""'
+        )
+
     @property
     def identifiers(self):
         return self.identifier_stack[-1]
@@ -232,7 +251,7 @@ class _GenerateRenderMethod(object):
                             [n.name for n in
                             main_identifiers.topleveldefs.values()]
                         )
-        self.printer.write("\n\n")
+        self.printer.write_blanks(2)
 
         if len(module_code):
             self.write_module_code(module_code)
@@ -288,7 +307,7 @@ class _GenerateRenderMethod(object):
 
         self.write_def_finish(self.node, buffered, filtered, cached)
         self.printer.writeline(None)
-        self.printer.write("\n\n")
+        self.printer.write_blanks(2)
         if cached:
             self.write_cache_decorator(
                                 node, name,
@@ -299,7 +318,7 @@ class _GenerateRenderMethod(object):
         """write module-level template code, i.e. that which
         is enclosed in <%! %> tags in the template."""
         for n in module_code:
-            self.write_source_comment(n)
+            self.printer.start_source(n.lineno)
             self.printer.write_indented_block(n.text)
 
     def write_inherit(self, node):
@@ -330,7 +349,7 @@ class _GenerateRenderMethod(object):
         for node in namespaces.values():
             if 'import' in node.attributes:
                 self.compiler.has_ns_imports = True
-            self.write_source_comment(node)
+            self.printer.start_source(node.lineno)
             if len(node.nodes):
                 self.printer.writeline("def make_namespace():")
                 export = []
@@ -402,7 +421,7 @@ class _GenerateRenderMethod(object):
 
             self.printer.writeline(
                 "context.namespaces[(__name__, %s)] = ns" % repr(node.name))
-            self.printer.write("\n")
+            self.printer.write_blanks(1)
         if not len(namespaces):
             self.printer.writeline("pass")
         self.printer.writeline(None)
@@ -533,13 +552,6 @@ class _GenerateRenderMethod(object):
 
         self.printer.writeline("__M_writer = context.writer()")
 
-    def write_source_comment(self, node):
-        """write a source comment containing the line number of the
-        corresponding template line."""
-        if self.last_source_line != node.lineno:
-            self.printer.writeline("# SOURCE LINE %d" % node.lineno)
-            self.last_source_line = node.lineno
-
     def write_def_decl(self, node, identifiers):
         """write a locally-available callable referencing a top-level def"""
         funcname = node.funcname
@@ -757,7 +769,7 @@ class _GenerateRenderMethod(object):
         return target
 
     def visitExpression(self, node):
-        self.write_source_comment(node)
+        self.printer.start_source(node.lineno)
         if len(node.escapes) or \
                 (
                     self.compiler.pagetag is not None and
@@ -779,7 +791,7 @@ class _GenerateRenderMethod(object):
                 self.printer.writeline("loop = __M_loop._exit()")
                 self.printer.writeline(None)
         else:
-            self.write_source_comment(node)
+            self.printer.start_source(node.lineno)
             if self.compiler.enable_loop and node.keyword == 'for':
                 text = mangle_mako_loop(node, self.printer)
             else:
@@ -801,7 +813,7 @@ class _GenerateRenderMethod(object):
                 self.printer.writeline("pass")
 
     def visitText(self, node):
-        self.write_source_comment(node)
+        self.printer.start_source(node.lineno)
         self.printer.writeline("__M_writer(%s)" % repr(node.content))
 
     def visitTextTag(self, node):
@@ -827,7 +839,7 @@ class _GenerateRenderMethod(object):
 
     def visitCode(self, node):
         if not node.ismodule:
-            self.write_source_comment(node)
+            self.printer.start_source(node.lineno)
             self.printer.write_indented_block(node.text)
 
             if not self.in_def and len(self.identifiers.locally_assigned) > 0:
@@ -844,7 +856,7 @@ class _GenerateRenderMethod(object):
                     ','.join([repr(x) for x in node.declared_identifiers()]))
 
     def visitIncludeTag(self, node):
-        self.write_source_comment(node)
+        self.printer.start_source(node.lineno)
         args = node.attributes.get('args')
         if args:
             self.printer.writeline(
@@ -944,7 +956,7 @@ class _GenerateRenderMethod(object):
                 "runtime.Namespace('caller', context, "
                                 "callables=ccall(__M_caller))",
             "try:")
-        self.write_source_comment(node)
+        self.printer.start_source(node.lineno)
         self.printer.writelines(
                 "__M_writer(%s)" % self.create_filter_callable(
                                                     [], node.expression, True),
diff --git a/mako/compat.py b/mako/compat.py
index f782aa9..dea1b30 100644
--- a/mako/compat.py
+++ b/mako/compat.py
@@ -100,6 +100,7 @@ except:
         return newfunc
 
 all = all
+import json
 
 def exception_name(exc):
     return exc.__class__.__name__
diff --git a/mako/exceptions.py b/mako/exceptions.py
index a7bab8c..20b4dce 100644
--- a/mako/exceptions.py
+++ b/mako/exceptions.py
@@ -8,7 +8,6 @@
 
 import traceback
 import sys
-import re
 from mako import util, compat
 
 class MakoException(Exception):
@@ -77,7 +76,6 @@ class RichTraceback(object):
         self.records = self._init(traceback)
 
         if isinstance(self.error, (CompileException, SyntaxException)):
-            import mako.template
             self.source = self.error.source
             self.lineno = self.error.lineno
             self._has_source = True
@@ -167,14 +165,13 @@ class RichTraceback(object):
                                             None, None, None, None))
                     continue
 
-                template_ln = module_ln = 1
-                line_map = {}
-                for line in module_source.split("\n"):
-                    match = re.match(r'\s*# SOURCE LINE (\d+)', line)
-                    if match:
-                        template_ln = int(match.group(1))
-                    module_ln += 1
-                    line_map[module_ln] = template_ln
+                template_ln = 1
+
+                source_map = mako.template.ModuleInfo.\
+                                get_module_source_metadata(
+                                    module_source, full_line_map=True)
+                line_map = source_map['full_line_map']
+
                 template_lines = [line for line in
                                     template_source.split("\n")]
                 mods[filename] = (line_map, template_lines)
@@ -188,7 +185,7 @@ class RichTraceback(object):
                                 line, template_filename, template_ln,
                                 template_line, template_source))
         if not self.source:
-            for l in range(len(new_trcback)-1, 0, -1):
+            for l in range(len(new_trcback) - 1, 0, -1):
                 if new_trcback[l][5]:
                     self.source = new_trcback[l][7]
                     self.lineno = new_trcback[l][5]
diff --git a/mako/pygen.py b/mako/pygen.py
index 52e32be..dfd83d3 100644
--- a/mako/pygen.py
+++ b/mako/pygen.py
@@ -26,6 +26,9 @@ class PythonPrinter(object):
         # the stream we are writing to
         self.stream = stream
 
+        # current line number
+        self.lineno = 0
+
         # a list of lines that represents a buffered "block" of code,
         # which can be later printed relative to an indent level
         self.line_buffer = []
@@ -34,8 +37,35 @@ class PythonPrinter(object):
 
         self._reset_multi_line_flags()
 
-    def write(self, text):
-        self.stream.write(text)
+        # marker for template source lines; this
+        # is part of source/template line mapping
+        self.last_source_line = -1
+
+        self.last_boilerplate_line = -1
+
+        # mapping of generated python lines to template
+        # source lines
+        self.source_map = {}
+
+        # list of "boilerplate" lines, these are lines
+        # that precede/follow a set of template source-mapped lines
+        self.boilerplate_map = []
+
+
+    def _update_lineno(self, num):
+        if self.last_boilerplate_line <= self.last_source_line:
+            self.boilerplate_map.append(self.lineno)
+            self.last_boilerplate_line = self.lineno
+        self.lineno += num
+
+    def start_source(self, lineno):
+        if self.last_source_line != lineno:
+            self.source_map[self.lineno] = lineno
+            self.last_source_line = lineno
+
+    def write_blanks(self, num):
+        self.stream.write("\n" * num)
+        self._update_lineno(num)
 
     def write_indented_block(self, block):
         """print a line or lines of python which already contain indentation.
@@ -94,6 +124,7 @@ class PythonPrinter(object):
 
         # write the line
         self.stream.write(self._indent_line(line) + "\n")
+        self._update_lineno(1)
 
         # see if this line should increase the indentation level.
         # note that a line can both decrase (before printing) and
@@ -213,11 +244,13 @@ class PythonPrinter(object):
         for entry in self.line_buffer:
             if self._in_multi_line(entry):
                 self.stream.write(entry + "\n")
+                self._update_lineno(1)
             else:
                 entry = entry.expandtabs()
                 if stripspace is None and re.search(r"^[ \t]*[^# \t]", entry):
                     stripspace = re.match(r"^([ \t]*)", entry).group(1)
                 self.stream.write(self._indent_line(entry, stripspace) + "\n")
+                self._update_lineno(1)
 
         self.line_buffer = []
         self._reset_multi_line_flags()
diff --git a/mako/template.py b/mako/template.py
index 00783b7..c19a66a 100644
--- a/mako/template.py
+++ b/mako/template.py
@@ -596,6 +596,26 @@ class ModuleInfo(object):
         if module_filename:
             self._modules[module_filename] = self
 
+    @classmethod
+    def get_module_source_metadata(cls, module_source, full_line_map=False):
+        source_map = re.search(
+                        r"__M_BEGIN_METADATA(.+?)__M_END_METADATA",
+                        module_source, re.S).group(1)
+        source_map = compat.json.loads(source_map)
+        if full_line_map:
+            line_map = source_map['full_line_map'] = dict(
+                (int(k), v) for k, v in source_map['line_map'].items()
+            )
+
+            for mod_line in reversed(sorted(line_map)):
+                tmpl_line = line_map[mod_line]
+                while mod_line > 0:
+                    mod_line -= 1
+                    if mod_line in line_map:
+                        break
+                    line_map[mod_line] = tmpl_line
+        return source_map
+
     @property
     def code(self):
         if self.module_source is not None:
diff --git a/setup.py b/setup.py
index bbab08e..0094901 100644
--- a/setup.py
+++ b/setup.py
@@ -16,10 +16,10 @@ markupsafe_installs = (
             sys.version_info >= (2, 6) and sys.version_info < (3, 0)
         ) or sys.version_info >= (3, 3)
 
+install_requires = []
+
 if markupsafe_installs:
-    install_requires = ['MarkupSafe>=0.9.2']
-else:
-    install_requires = []
+    install_requires.append('MarkupSafe>=0.9.2')
 
 try:
     import argparse