Closes #976: Fix gettext does not extract index entries.

12 files changed, 262 insertions, 3 deletions

diff --git a/CHANGES b/CHANGES
index ba0d3848..4386b5f1 100644
--- a/CHANGES
+++ b/CHANGES
@@ -16,6 +16,8 @@ Release 1.2 (in development)
 * #869: sphinx-build now has the option :option:`-T` for printing the full
   traceback after an unhandled exception.
 
+* #976: Fix gettext does not extract index entries.
+
 * #940: Fix gettext does not extract figure caption.
 
 * #1067: Improve the ordering of the JavaScript search results: matches in titles
diff --git a/sphinx/builders/gettext.py b/sphinx/builders/gettext.py
index c07f3fc9..80a24299 100644
--- a/sphinx/builders/gettext.py
+++ b/sphinx/builders/gettext.py
@@ -15,9 +15,11 @@ from datetime import datetime
 from collections import defaultdict
 
 from sphinx.builders import Builder
-from sphinx.util.nodes import extract_messages
+from sphinx.util import split_index_msg
+from sphinx.util.nodes import extract_messages, traverse_translatable_index
 from sphinx.util.osutil import SEP, safe_relpath, ensuredir, find_catalog
 from sphinx.util.console import darkgreen
+from sphinx.locale import pairindextypes
 
 POHEADER = ur"""
 # SOME DESCRIPTIVE TITLE.
@@ -82,6 +84,16 @@ class I18nBuilder(Builder):
         for node, msg in extract_messages(doctree):
             catalog.add(msg, node)
 
+        # Extract translatable messages from index entries.
+        for node, entries in traverse_translatable_index(doctree):
+            for typ, msg, tid, main in entries:
+                for m in split_index_msg(typ, msg):
+                    if typ == 'pair' and m in pairindextypes.values():
+                        # avoid built-in translated message was incorporated
+                        # in 'sphinx.util.nodes.process_index_entry'
+                        continue
+                    catalog.add(m, node)
+
 
 class MessageCatalogBuilder(I18nBuilder):
     """
diff --git a/sphinx/directives/other.py b/sphinx/directives/other.py
index d90fb2e1..c6baf775 100644
--- a/sphinx/directives/other.py
+++ b/sphinx/directives/other.py
@@ -168,6 +168,7 @@ class Index(Directive):
         indexnode = addnodes.index()
         indexnode['entries'] = ne = []
         indexnode['inline'] = False
+        set_source_info(self, indexnode)
         for entry in arguments:
             ne.extend(process_index_entry(entry, targetid))
         return [indexnode, targetnode]
diff --git a/sphinx/environment.py b/sphinx/environment.py
index 1b904269..4d910322 100644
--- a/sphinx/environment.py
+++ b/sphinx/environment.py
@@ -38,9 +38,9 @@ from docutils.transforms.parts import ContentsFilter
 
 from sphinx import addnodes
 from sphinx.util import url_re, get_matching_docs, docname_join, split_into, \
-     FilenameUniqDict
+     split_index_msg, FilenameUniqDict
 from sphinx.util.nodes import clean_astext, make_refnode, extract_messages, \
-     WarningStream
+     traverse_translatable_index, WarningStream
 from sphinx.util.osutil import movefile, SEP, ustrftime, find_catalog, \
      fs_encoding
 from sphinx.util.matching import compile_matchers
@@ -303,6 +303,23 @@ class Locale(Transform):
                 child.parent = node
             node.children = patch.children
 
+        # Extract and translate messages for index entries.
+        for node, entries in traverse_translatable_index(self.document):
+            new_entries = []
+            for type, msg, tid, main in entries:
+                msg_parts = split_index_msg(type, msg)
+                msgstr_parts = []
+                for part in msg_parts:
+                    msgstr = catalog.gettext(part)
+                    if not msgstr:
+                        msgstr = part
+                    msgstr_parts.append(msgstr)
+
+                new_entries.append((type, ';'.join(msgstr_parts), tid, main))
+
+            node['raw_entries'] = entries
+            node['entries'] = new_entries
+
 
 class SphinxStandaloneReader(standalone.Reader):
     """
diff --git a/sphinx/roles.py b/sphinx/roles.py
index d395c372..02c5ad8f 100644
--- a/sphinx/roles.py
+++ b/sphinx/roles.py
@@ -293,6 +293,7 @@ def index_role(typ, rawtext, text, lineno, inliner, options={}, content=[]):
         entries = [('single', target, targetid, main)]
     indexnode = addnodes.index()
     indexnode['entries'] = entries
+    set_role_source_info(inliner, lineno, indexnode)
     textnode = nodes.Text(title, title)
     return [indexnode, targetnode, textnode], []
 
diff --git a/sphinx/util/__init__.py b/sphinx/util/__init__.py
index de4b14a4..8bedda12 100644
--- a/sphinx/util/__init__.py
+++ b/sphinx/util/__init__.py
@@ -360,6 +360,29 @@ def split_into(n, type, value):
     return parts
 
 
+def split_index_msg(type, value):
+    # new entry types must be listed in directives/other.py!
+    result = []
+    try:
+        if type == 'single':
+            try:
+                result = split_into(2, 'single', value)
+            except ValueError:
+                result = split_into(1, 'single', value)
+        elif type == 'pair':
+            result = split_into(2, 'pair', value)
+        elif type == 'triple':
+            result = split_into(3, 'triple', value)
+        elif type == 'see':
+            result = split_into(2, 'see', value)
+        elif type == 'seealso':
+            result = split_into(2, 'see', value)
+    except ValueError:
+        pass
+
+    return result
+
+
 def format_exception_cut_frames(x=1):
     """Format an exception with traceback, but only the last x frames."""
     typ, val, tb = sys.exc_info()
diff --git a/sphinx/util/nodes.py b/sphinx/util/nodes.py
index da055da0..65accbf2 100644
--- a/sphinx/util/nodes.py
+++ b/sphinx/util/nodes.py
@@ -74,6 +74,19 @@ def extract_messages(doctree):
             yield node, msg
 
 
+def traverse_translatable_index(doctree):
+    """Traverse translatable index node from a document tree."""
+    def is_block_index(node):
+        return isinstance(node, addnodes.index) and  \
+            node.get('inline') == False
+    for node in doctree.traverse(is_block_index):
+        if 'raw_entries' in node:
+            entries = node['raw_entries']
+        else:
+            entries = node['entries']
+        yield node, entries
+
+
 def nested_parse_with_titles(state, content, node):
     """Version of state.nested_parse() that allows titles and does not require
     titles to have the same decoration as the calling document.
diff --git a/tests/root/i18n/index.txt b/tests/root/i18n/index.txt
index dfab377a..dfacc019 100644
--- a/tests/root/i18n/index.txt
+++ b/tests/root/i18n/index.txt
@@ -8,3 +8,4 @@
    literalblock
    definition_terms
    figure_caption
+   index_entries
diff --git a/tests/root/i18n/index_entries.po b/tests/root/i18n/index_entries.po
new file mode 100644
index 00000000..6da9a813
--- /dev/null
+++ b/tests/root/i18n/index_entries.po
@@ -0,0 +1,77 @@
+# SOME DESCRIPTIVE TITLE.
+# Copyright (C) 2013, foo
+# This file is distributed under the same license as the foo package.
+# FIRST AUTHOR <EMAIL@ADDRESS>, YEAR.
+#
+#, fuzzy
+msgid ""
+msgstr ""
+"Project-Id-Version: foo foo\n"
+"Report-Msgid-Bugs-To: \n"
+"POT-Creation-Date: 2013-01-05 18:10\n"
+"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
+"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
+"Language-Team: LANGUAGE <LL@li.org>\n"
+"MIME-Version: 1.0\n"
+"Content-Type: text/plain; charset=UTF-8\n"
+"Content-Transfer-Encoding: 8bit\n"
+
+msgid "i18n with index entries"
+msgstr ""
+
+msgid "index target section"
+msgstr ""
+
+msgid "this is :index:`Newsletter` target paragraph."
+msgstr "THIS IS :index:`NEWSLETTER` TARGET PARAGRAPH."
+
+msgid "various index entries"
+msgstr ""
+
+msgid "That's all."
+msgstr ""
+
+msgid "Mailing List"
+msgstr "MAILING LIST"
+
+msgid "Newsletter"
+msgstr "NEWSLETTER"
+
+msgid "Recipients List"
+msgstr "RECIPIENTS LIST"
+
+msgid "First"
+msgstr "FIRST"
+
+msgid "Second"
+msgstr "SECOND"
+
+msgid "Third"
+msgstr "THIRD"
+
+msgid "Entry"
+msgstr "ENTRY"
+
+msgid "See"
+msgstr "SEE"
+
+msgid "Module"
+msgstr "MODULE"
+
+msgid "Keyword"
+msgstr "KEYWORD"
+
+msgid "Operator"
+msgstr "OPERATOR"
+
+msgid "Object"
+msgstr "OBJECT"
+
+msgid "Exception"
+msgstr "EXCEPTION"
+
+msgid "Statement"
+msgstr "STATEMENT"
+
+msgid "Builtin"
+msgstr "BUILTIN"
diff --git a/tests/root/i18n/index_entries.txt b/tests/root/i18n/index_entries.txt
new file mode 100644
index 00000000..c914a4b4
--- /dev/null
+++ b/tests/root/i18n/index_entries.txt
@@ -0,0 +1,31 @@
+:tocdepth: 2
+
+i18n with index entries
+=======================
+
+.. index::
+   single: Mailing List
+   pair: Newsletter; Recipients List
+
+index target section
+--------------------
+
+this is :index:`Newsletter` target paragraph.
+
+
+various index entries
+---------------------
+
+.. index::
+   triple: First; Second; Third
+   see: Entry; Mailing List
+   seealso: See; Newsletter
+   module: Module
+   keyword: Keyword
+   operator: Operator
+   object: Object
+   exception: Exception
+   statement: Statement
+   builtin: Builtin
+
+That's all.
diff --git a/tests/test_build_gettext.py b/tests/test_build_gettext.py
index ab68289e..dcbff484 100644
--- a/tests/test_build_gettext.py
+++ b/tests/test_build_gettext.py
@@ -11,6 +11,7 @@
 
 import gettext
 import os
+import re
 from subprocess import Popen, PIPE
 
 from util import *
@@ -79,3 +80,49 @@ def test_gettext(app):
 
     _ = gettext.translation('test_root', app.outdir, languages=['en']).gettext
     assert _("Testing various markup") == u"Testing various markup"
+
+
+@with_app(buildername='gettext',
+          confoverrides={'gettext_compact': False})
+def test_gettext_index_entries(app):
+    # regression test for #976
+    app.builder.build(['i18n/index_entries'])
+
+    _msgid_getter = re.compile(r'msgid "(.*)"').search
+    def msgid_getter(msgid):
+        m = _msgid_getter(msgid)
+        if m:
+            return m.groups()[0]
+        return None
+
+    pot = (app.outdir / 'i18n' / 'index_entries.pot').text(encoding='utf-8')
+    msgids = filter(None, map(msgid_getter, pot.splitlines()))
+
+    expected_msgids = [
+        "i18n with index entries",
+        "index target section",
+        "this is :index:`Newsletter` target paragraph.",
+        "various index entries",
+        "That's all.",
+        "Mailing List",
+        "Newsletter",
+        "Recipients List",
+        "First",
+        "Second",
+        "Third",
+        "Entry",
+        "See",
+        "Module",
+        "Keyword",
+        "Operator",
+        "Object",
+        "Exception",
+        "Statement",
+        "Builtin",
+    ]
+    for expect in expected_msgids:
+        assert expect in msgids
+        msgids.remove(expect)
+
+    # unexpected msgid existent
+    assert msgids == []
diff --git a/tests/test_intl.py b/tests/test_intl.py
index e93caa21..93214299 100644
--- a/tests/test_intl.py
+++ b/tests/test_intl.py
@@ -259,3 +259,37 @@ def test_i18n_figure_caption(app):
               u"\n   MY DESCRIPTION PARAGRAPH2 OF THE FIGURE.\n")
 
     assert result == expect
+
+
+@with_app(buildername='html',
+          confoverrides={'language': 'xx', 'locale_dirs': ['.'],
+                         'gettext_compact': False})
+def test_i18n_index_entries(app):
+    # regression test for #976
+    app.builder.build(['i18n/index_entries'])
+    result = (app.outdir / 'genindex.html').text(encoding='utf-8')
+
+    def wrap(tag, keyword):
+        start_tag = "<%s[^>]*>" % tag
+        end_tag = "</%s>" % tag
+        return r"%s\s*%s\s*%s" % (start_tag, keyword, end_tag)
+
+    expected_exprs = [
+        wrap('a', 'NEWSLETTER'),
+        wrap('a', 'MAILING LIST'),
+        wrap('a', 'RECIPIENTS LIST'),
+        wrap('a', 'FIRST SECOND'),
+        wrap('a', 'SECOND THIRD'),
+        wrap('a', 'THIRD, FIRST'),
+        wrap('dt', 'ENTRY'),
+        wrap('dt', 'SEE'),
+        wrap('a', 'MODULE'),
+        wrap('a', 'KEYWORD'),
+        wrap('a', 'OPERATOR'),
+        wrap('a', 'OBJECT'),
+        wrap('a', 'EXCEPTION'),
+        wrap('a', 'STATEMENT'),
+        wrap('a', 'BUILTIN'),
+    ]
+    for expr in expected_exprs:
+        assert re.search(expr, result, re.M)