- Template modules now generate a JSON "metadata" structure at the bottom

of the source file which includes parseable information about the
templates' source file, encoding etc. as well as a mapping of module
source lines to template lines, thus replacing the "# SOURCE LINE"
markers throughout the source code.  The structure also indicates those
lines that are explicitly not part of the template's source; the goal
here is to allow integration with coverage tools.

6 files changed, 90 insertions, 18 deletions

diff --git a/doc/build/changelog.rst b/doc/build/changelog.rst
index 2a0e5f3..7664284 100644
--- a/doc/build/changelog.rst
+++ b/doc/build/changelog.rst
@@ -10,6 +10,17 @@ Changelog
     :released:
 
     .. change::
+        :tags: feature
+
+      Template modules now generate a JSON "metadata" structure at the bottom
+      of the source file which includes parseable information about the
+      templates' source file, encoding etc. as well as a mapping of module
+      source lines to template lines, thus replacing the "# SOURCE LINE"
+      markers throughout the source code.  The structure also indicates those
+      lines that are explicitly not part of the template's source; the goal
+      here is to allow integration with coverage tools.
+
+    .. change::
         :tags: feature, py3k
         :pullreq: github:7
 
diff --git a/mako/codegen.py b/mako/codegen.py
index 045d03c..ec587ba 100644
--- a/mako/codegen.py
+++ b/mako/codegen.py
@@ -14,7 +14,7 @@ from mako import util, ast, parsetree, filters, exceptions
 from mako import compat
 
 
-MAGIC_NUMBER = 9
+MAGIC_NUMBER = 10
 
 # names which are hardwired into the
 # template and are not accessed via the
@@ -102,6 +102,8 @@ class _GenerateRenderMethod(object):
         self.last_source_line = -1
         self.compiler = compiler
         self.node = node
+        self.source_map = {}
+        self.boilerplate_map = []
         self.identifier_stack = [None]
         self.in_def = isinstance(node, (parsetree.DefTag, parsetree.BlockTag))
 
@@ -146,6 +148,27 @@ class _GenerateRenderMethod(object):
             for node in defs:
                 _GenerateRenderMethod(printer, compiler, node)
 
+        if not self.in_def:
+            self.write_metadata_struct()
+
+    def write_metadata_struct(self):
+        self.source_map[self.printer.lineno] = self.last_source_line
+        struct = {
+            "filename": self.compiler.filename,
+            "uri": self.compiler.uri,
+            "source_encoding": self.compiler.source_encoding,
+            "line_map": self.source_map,
+            "boilerplate_lines": self.boilerplate_map
+        }
+        self.mark_boilerplate()
+        self.printer.writelines(
+            '"""',
+            '__M_BEGIN_METADATA',
+            compat.json.dumps(struct),
+            '__M_END_METADATA\n'
+            '"""'
+        )
+
     @property
     def identifiers(self):
         return self.identifier_stack[-1]
@@ -232,7 +255,7 @@ class _GenerateRenderMethod(object):
                             [n.name for n in
                             main_identifiers.topleveldefs.values()]
                         )
-        self.printer.write("\n\n")
+        self.printer.write_blanks(2)
 
         if len(module_code):
             self.write_module_code(module_code)
@@ -251,6 +274,7 @@ class _GenerateRenderMethod(object):
 
         this could be the main render() method or that of a top-level def."""
 
+        self.mark_boilerplate()
         if self.in_def:
             decorator = node.decorator
             if decorator:
@@ -288,7 +312,7 @@ class _GenerateRenderMethod(object):
 
         self.write_def_finish(self.node, buffered, filtered, cached)
         self.printer.writeline(None)
-        self.printer.write("\n\n")
+        self.printer.write_blanks(2)
         if cached:
             self.write_cache_decorator(
                                 node, name,
@@ -305,6 +329,7 @@ class _GenerateRenderMethod(object):
     def write_inherit(self, node):
         """write the module-level inheritance-determination callable."""
 
+        self.mark_boilerplate()
         self.printer.writelines(
             "def _mako_inherit(template, context):",
                 "_mako_generate_namespaces(context)",
@@ -315,6 +340,7 @@ class _GenerateRenderMethod(object):
 
     def write_namespaces(self, namespaces):
         """write the module-level namespace-generating callable."""
+        self.mark_boilerplate()
         self.printer.writelines(
             "def _mako_get_namespace(context, name):",
                 "try:",
@@ -401,7 +427,7 @@ class _GenerateRenderMethod(object):
 
             self.printer.writeline(
                    "context.namespaces[(__name__, %s)] = ns" % repr(node.name))
-            self.printer.write("\n")
+            self.printer.write_blanks(1)
         if not len(namespaces):
             self.printer.writeline("pass")
         self.printer.writeline(None)
@@ -536,9 +562,12 @@ class _GenerateRenderMethod(object):
         """write a source comment containing the line number of the
         corresponding template line."""
         if self.last_source_line != node.lineno:
-            self.printer.writeline("# SOURCE LINE %d" % node.lineno)
+            self.source_map[self.printer.lineno] = node.lineno
             self.last_source_line = node.lineno
 
+    def mark_boilerplate(self):
+        self.boilerplate_map.append(self.printer.lineno)
+
     def write_def_decl(self, node, identifiers):
         """write a locally-available callable referencing a top-level def"""
         funcname = node.funcname
@@ -606,6 +635,7 @@ class _GenerateRenderMethod(object):
         writes code to retrieve captured content, apply filters, send proper
         return value."""
 
+        self.mark_boilerplate()
         if not buffered and not cached and not filtered:
             self.printer.writeline("return ''")
             if callstack:
@@ -861,6 +891,7 @@ class _GenerateRenderMethod(object):
         pass
 
     def visitBlockTag(self, node):
+        self.mark_boilerplate()
         if node.is_anonymous:
             self.printer.writeline("%s()" % node.funcname)
         else:
@@ -930,6 +961,7 @@ class _GenerateRenderMethod(object):
             n.accept_visitor(self)
         self.identifier_stack.pop()
 
+        self.mark_boilerplate()
         self.write_def_finish(node, buffered, False, False, callstack=False)
         self.printer.writelines(
             None,
diff --git a/mako/compat.py b/mako/compat.py
index c5ef84b..8e8ee70 100644
--- a/mako/compat.py
+++ b/mako/compat.py
@@ -94,6 +94,11 @@ except:
             return func(*(args + fargs), **newkeywords)
         return newfunc
 
+if py26:
+    import json
+else:
+    import simplejson as json
+
 if not py25:
     def all(iterable):
         for i in iterable:
diff --git a/mako/exceptions.py b/mako/exceptions.py
index b8f97ee..523805f 100644
--- a/mako/exceptions.py
+++ b/mako/exceptions.py
@@ -167,14 +167,24 @@ class RichTraceback(object):
                                             None, None, None, None))
                     continue
 
-                template_ln = module_ln = 1
-                line_map = {}
-                for line in module_source.split("\n"):
-                    match = re.match(r'\s*# SOURCE LINE (\d+)', line)
-                    if match:
-                        template_ln = int(match.group(1))
-                    module_ln += 1
-                    line_map[module_ln] = template_ln
+                template_ln = 1
+
+                source_map = re.search(
+                                r"__M_BEGIN_METADATA(.+?)__M_END_METADATA",
+                                module_source, re.S).group(1)
+                source_map = compat.json.loads(source_map)
+                line_map = dict(
+                    (int(k), v) for k, v in source_map['line_map'].items()
+                )
+
+                for mod_line in reversed(sorted(line_map)):
+                    tmpl_line = line_map[mod_line]
+                    while mod_line > 0:
+                        mod_line -= 1
+                        if mod_line in line_map:
+                            break
+                        line_map[mod_line] = tmpl_line
+
                 template_lines = [line for line in
                                     template_source.split("\n")]
                 mods[filename] = (line_map, template_lines)
diff --git a/mako/pygen.py b/mako/pygen.py
index cba9464..d6559e5 100644
--- a/mako/pygen.py
+++ b/mako/pygen.py
@@ -26,6 +26,9 @@ class PythonPrinter(object):
         # the stream we are writing to
         self.stream = stream
 
+        # current line number
+        self.lineno = 0
+
         # a list of lines that represents a buffered "block" of code,
         # which can be later printed relative to an indent level
         self.line_buffer = []
@@ -34,8 +37,9 @@ class PythonPrinter(object):
 
         self._reset_multi_line_flags()
 
-    def write(self, text):
-        self.stream.write(text)
+    def write_blanks(self, num=1):
+        self.stream.write("\n" * num)
+        self.lineno += num
 
     def write_indented_block(self, block):
         """print a line or lines of python which already contain indentation.
@@ -94,6 +98,7 @@ class PythonPrinter(object):
 
         # write the line
         self.stream.write(self._indent_line(line) + "\n")
+        self.lineno += 1
 
         # see if this line should increase the indentation level.
         # note that a line can both decrase (before printing) and
@@ -213,11 +218,13 @@ class PythonPrinter(object):
         for entry in self.line_buffer:
             if self._in_multi_line(entry):
                 self.stream.write(entry + "\n")
+                self.lineno += 1
             else:
                 entry = entry.expandtabs()
                 if stripspace is None and re.search(r"^[ \t]*[^# \t]", entry):
                     stripspace = re.match(r"^([ \t]*)", entry).group(1)
                 self.stream.write(self._indent_line(entry, stripspace) + "\n")
+                self.lineno += 1
 
         self.line_buffer = []
         self._reset_multi_line_flags()
diff --git a/setup.py b/setup.py
index 04d4551..9f1e8f2 100644
--- a/setup.py
+++ b/setup.py
@@ -13,10 +13,17 @@ markupsafe_installs = (
             sys.version_info >= (2, 6) and sys.version_info < (3, 0)
         ) or sys.version_info >= (3, 3)
 
+json_installs = (
+            sys.version_info < (2, 6)
+      )
+
+install_requires = []
+
 if markupsafe_installs:
-    install_requires = ['MarkupSafe>=0.9.2']
-else:
-    install_requires = []
+    install_requires.append('MarkupSafe>=0.9.2')
+
+if json_installs:
+      install_requires.append('simplejson')
 
 setup(name='Mako',
       version=VERSION,