diff options
| -rw-r--r-- | Makefile.am | 1 | ||||
| -rw-r--r-- | README | 1 | ||||
| -rw-r--r-- | configure.ac | 63 | ||||
| -rw-r--r-- | gtkdoc/config.py.in | 2 | ||||
| -rwxr-xr-x | gtkdoc/fixxref.py | 149 | ||||
| -rw-r--r-- | gtkdoc/highlight.py | 49 | ||||
| -rw-r--r-- | gtkdoc/mkhtml2.py | 25 | ||||
| -rw-r--r-- | requirements.txt | 1 | ||||
| -rw-r--r-- | style/style.css | 20 |
9 files changed, 82 insertions, 229 deletions
diff --git a/Makefile.am b/Makefile.am index debfdfa..134b42a 100644 --- a/Makefile.am +++ b/Makefile.am @@ -38,6 +38,7 @@ pylibdata_DATA = \ gtkdoc/common.py \ gtkdoc/config.py \ gtkdoc/fixxref.py \ + gtkdoc/highlight.py \ gtkdoc/md_to_db.py \ gtkdoc/mkdb.py \ gtkdoc/mkhtml.py \ @@ -31,6 +31,7 @@ Python 3.x Additional python modules: For the tests: unittest, parameterized For mkhtml2 (experimental): anytree, lxml and pygments + For fixxref: pygments For XML output (recommended): diff --git a/configure.ac b/configure.ac index 1503ca4..97608d2 100644 --- a/configure.ac +++ b/configure.ac @@ -58,66 +58,6 @@ JH_CHECK_XML_CATALOG([-//OASIS//DTD DocBook XML V4.3//EN], [DocBook XML DTD V4.3 JH_CHECK_XML_CATALOG([http://docbook.sourceforge.net/release/xsl/current/html/chunk.xsl], [DocBook XSL Stylesheets]) dnl -dnl Check for syntax highlighters -dnl -AC_ARG_WITH([highlight], - AS_HELP_STRING([--with-highlight], [Select source code syntax highlighter (no|source-highlight|highlight|vim|auto)]), - , [with_highlight=auto]) - -case $with_highlight in - no|source-highlight|highlight|vim|auto) ;; - *) AC_MSG_ERROR([Invalid value for syntax highlighting option.]) ;; -esac - -HIGHLIGHT_OPTIONS="" -if test "$with_highlight" = "auto"; then - AC_PATH_PROG([HIGHLIGHT], [source-highlight]) - if test -n "$HIGHLIGHT"; then - HIGHLIGHT_OPTIONS="-t4 -s\$SRC_LANG -cstyle.css --no-doc -i" - else - AC_PATH_PROG([HIGHLIGHT], [highlight]) - if test -n "$HIGHLIGHT"; then - HIGHLIGHT_OPTIONS="--syntax=\$SRC_LANG --out-format=xhtml -f --class-name=gtkdoc " - else - AC_PATH_PROG([HIGHLIGHT], [vim]) - if test -n "$HIGHLIGHT"; then - dnl vim is useless if it does not support syntax highlighting - AC_MSG_CHECKING([whether vim has +syntax feature]) - if $HIGHLIGHT --version | grep '+syntax' >/dev/null; then - AC_MSG_RESULT([yes]) - else - AC_MSG_RESULT([no]) - HIGHLIGHT= - fi - fi - fi - fi -else - if test "$with_highlight" != "no"; then - AC_PATH_PROG([HIGHLIGHT], [$with_highlight], [no]) - fi - - case $with_highlight in - source-highlight) HIGHLIGHT_OPTIONS="-t4 -s\$SRC_LANG -cstyle.css --no-doc -i";; - highlight) HIGHLIGHT_OPTIONS="--syntax=\$SRC_LANG --out-format=xhtml -f --class-name=gtkdoc ";; - vim) - AC_MSG_CHECKING([whether vim has +syntax feature]) - if $HIGHLIGHT --version | grep '+syntax' >/dev/null; then - AC_MSG_RESULT([yes]) - else - AC_MSG_RESULT([no]) - HIGHLIGHT=no - fi - ;; - esac - - if test "$HIGHLIGHT" = "no" && test "$with_highlight" != "no"; then - AC_MSG_ERROR([Could not find requested syntax highlighter]) - fi -fi -AC_SUBST([HIGHLIGHT_OPTIONS]) - -dnl dnl Set runtime package dirs so we can find the script containing common routines. dnl dnl From Autoconf Macro Archive: @@ -265,9 +205,6 @@ gtk-doc was configured with the following options: test -n "$DBLATEX$FOP" \ && AC_MSG_NOTICE([** PDF support enabled, using $DBLATEX$FOP]) \ || AC_MSG_NOTICE([ PDF support disabled, no dblatex or fop available]) -test -n "$HIGHLIGHT" \ - && AC_MSG_NOTICE([** Syntax highlighting of examples enabled, using $HIGHLIGHT]) \ - || AC_MSG_NOTICE([ Syntax highlighting of examples disabled]) test "x$build_tests" != "xno" \ && AC_MSG_NOTICE([** Building regression tests]) \ || AC_MSG_NOTICE([ Skipping regression tests]) diff --git a/gtkdoc/config.py.in b/gtkdoc/config.py.in index 472c7df..1264374 100644 --- a/gtkdoc/config.py.in +++ b/gtkdoc/config.py.in @@ -3,8 +3,6 @@ version = "@VERSION@" # tools dblatex = '@DBLATEX@' fop = '@FOP@' -highlight = '@HIGHLIGHT@' -highlight_options = '@HIGHLIGHT_OPTIONS@' pkg_config = '@PKG_CONFIG@' xsltproc = '@XSLTPROC@' diff --git a/gtkdoc/fixxref.py b/gtkdoc/fixxref.py index 21ee962..356189c 100755 --- a/gtkdoc/fixxref.py +++ b/gtkdoc/fixxref.py @@ -24,12 +24,8 @@ import logging import os import re -import shlex -import subprocess -import sys -import tempfile -from . import common, config +from . import common, highlight # This contains all the entities and their relative URLs. Links = {} @@ -55,6 +51,7 @@ def Run(options): LoadIndicies(options.module_dir, options.html_dir, options.extra_dir) ReadSections(options.module) FixCrossReferences(options.module_dir, options.module, options.src_lang) + highlight.append_style_defs(os.path.join(options.module_dir, 'style.css')) # TODO(ensonic): try to refactor so that we get a list of path's and then just @@ -227,33 +224,24 @@ def FixHTMLFile(src_lang, module, file): content = open(file, 'r', encoding='utf-8').read() - if config.highlight: - # FIXME: ideally we'd pass a clue about the example language to the highligher - # unfortunately the "language" attribute is not appearing in the html output - # we could patch the customization to have <code class="xxx"> inside of <pre> - if config.highlight.endswith('vim'): - def repl_func(m): - return HighlightSourceVim(src_lang, m.group(1), m.group(2)) - content = re.sub( - r'<div class=\"(example-contents|informalexample)\"><pre class=\"programlisting\">(.*?)</pre></div>', - repl_func, content, flags=re.DOTALL) - else: - def repl_func(m): - return HighlightSource(src_lang, m.group(1), m.group(2)) - content = re.sub( - r'<div class=\"(example-contents|informalexample)\"><pre class=\"programlisting\">(.*?)</pre></div>', - repl_func, content, flags=re.DOTALL) - - content = re.sub(r'\<GTKDOCLINK\s+HREF=\"(.*?)\"\>(.*?)\</GTKDOCLINK\>', - r'\<GTKDOCLINK\ HREF=\"\1\"\>\2\</GTKDOCLINK\>', content, flags=re.DOTALL) - - # From the highlighter we get all the functions marked up. Now we can turn them into GTKDOCLINK items - def repl_func(m): - return MakeGtkDocLink(m.group(1), m.group(2), m.group(3)) - content = re.sub(r'(<span class=\"function\">)(.*?)(</span>)', repl_func, content, flags=re.DOTALL) - # We can also try the first item in stuff marked up as 'normal' - content = re.sub( - r'(<span class=\"normal\">\s*)(.+?)((\s+.+?)?\s*</span>)', repl_func, content, flags=re.DOTALL) + # FIXME: ideally we'd pass a clue about the example language to the highligher + # unfortunately the "language" attribute is not appearing in the html output + # we could patch the customization to have <code class="xxx"> inside of <pre> + def repl_func(m): + return HighlightSourcePygments(src_lang, m.group(1), m.group(2)) + content = re.sub( + r'<div class=\"(example-contents|informalexample)\"><pre class=\"programlisting\">(.*?)</pre></div>', + repl_func, content, flags=re.DOTALL) + content = re.sub(r'\<GTKDOCLINK\s+HREF=\"(.*?)\"\>(.*?)\</GTKDOCLINK\>', + r'\<GTKDOCLINK\ HREF=\"\1\"\>\2\</GTKDOCLINK\>', content, flags=re.DOTALL) + + # From the highlighter we get all the functions marked up. Now we can turn them into GTKDOCLINK items + def repl_func(m): + return MakeGtkDocLink(m.group(1), m.group(2), m.group(3)) + content = re.sub(r'(<span class=\"function\">)(.*?)(</span>)', repl_func, content, flags=re.DOTALL) + # We can also try the first item in stuff marked up as 'normal' + content = re.sub( + r'(<span class=\"normal\">\s*)(.+?)((\s+.+?)?\s*</span>)', repl_func, content, flags=re.DOTALL) lines = content.rstrip().split('\n') @@ -373,91 +361,7 @@ def MakeGtkDocLink(pre, symbol, post): return pre + '<GTKDOCLINK HREF="' + id + '">' + symbol + '</GTKDOCLINK>' + post -def HighlightSource(src_lang, type, source): - # write source to a temp file - # FIXME: use .c for now to hint the language to the highlighter - with tempfile.NamedTemporaryFile(mode='w+', suffix='.c') as f: - temp_source_file = HighlightSourcePreProcess(f, source) - highlight_options = config.highlight_options.replace('$SRC_LANG', src_lang) - - logging.info('running %s %s %s', config.highlight, highlight_options, temp_source_file) - - # format source - highlighted_source = subprocess.check_output( - [config.highlight] + shlex.split(highlight_options) + [temp_source_file]).decode('utf-8') - logging.debug('result: [%s]', highlighted_source) - if config.highlight.endswith('/source-highlight'): - highlighted_source = re.sub(r'^<\!-- .*? -->', '', highlighted_source, flags=re.MULTILINE | re.DOTALL) - highlighted_source = re.sub( - r'<pre><tt>(.*?)</tt></pre>', r'\1', highlighted_source, flags=re.MULTILINE | re.DOTALL) - elif config.highlight.endswith('/highlight'): - # need to rewrite the stylesheet classes - highlighted_source = highlighted_source.replace('<span class="gtkdoc com">', '<span class="comment">') - highlighted_source = highlighted_source.replace('<span class="gtkdoc dir">', '<span class="preproc">') - highlighted_source = highlighted_source.replace('<span class="gtkdoc kwd">', '<span class="function">') - highlighted_source = highlighted_source.replace('<span class="gtkdoc kwa">', '<span class="keyword">') - highlighted_source = highlighted_source.replace('<span class="gtkdoc line">', '<span class="linenum">') - highlighted_source = highlighted_source.replace('<span class="gtkdoc num">', '<span class="number">') - highlighted_source = highlighted_source.replace('<span class="gtkdoc str">', '<span class="string">') - highlighted_source = highlighted_source.replace('<span class="gtkdoc sym">', '<span class="symbol">') - # maybe also do - # highlighted_source = re.sub(r'</span>(.+)<span', '</span><span class="normal">\1</span><span') - - return HighlightSourcePostprocess(type, highlighted_source) - - -def HighlightSourceVim(src_lang, type, source): - # write source to a temp file - f = tempfile.NamedTemporaryFile(mode='w+', suffix='.h', delete=False, encoding='utf-8') - try: - temp_source_file = HighlightSourcePreProcess(f, source) - if os.name == 'nt': - temp_source_file = temp_source_file.replace('\\', '/') - f.close() - - # format source - script = "echo 'let html_number_lines=0|" + \ - "let html_use_css=1|" + \ - "let html_use_xhtml=1|" + \ - "set encoding=utf-8|" \ - "e {}|".format(temp_source_file) + \ - "syn on|" + \ - "set syntax={}|".format(src_lang) + \ - "run! plugin/tohtml.vim|" + \ - "run! syntax/2html.vim|" + \ - "w! {}.html|".format(temp_source_file) + \ - "qa' | " + \ - "{} -n -e -u NONE -T xterm".format(config.highlight) - p = subprocess.Popen([os.getenv('SHELL', 'sh')], stdin=subprocess.PIPE, stdout=subprocess.PIPE) - p.communicate(script.encode('utf-8')) - if p.returncode != 0: - raise Exception("Highlighter failed. The command was: {}".format(script)) - - html_filename = temp_source_file + ".html" - try: - with open(html_filename, 'rb') as html_file: - highlighted_source = html_file.read().decode('utf-8') - highlighted_source = re.sub(r'.*<pre\b[^>]*>\n', '', highlighted_source, flags=re.DOTALL) - highlighted_source = re.sub(r'</pre>.*', '', highlighted_source, flags=re.DOTALL) - - # need to rewrite the stylesheet classes - highlighted_source = highlighted_source.replace('<span class="Comment">', '<span class="comment">') - highlighted_source = highlighted_source.replace('<span class="PreProc">', '<span class="preproc">') - highlighted_source = highlighted_source.replace('<span class="Statement">', '<span class="keyword">') - highlighted_source = highlighted_source.replace('<span class="Identifier">', '<span class="function">') - highlighted_source = highlighted_source.replace('<span class="Constant">', '<span class="number">') - highlighted_source = highlighted_source.replace('<span class="Special">', '<span class="symbol">') - highlighted_source = highlighted_source.replace('<span class="Type">', '<span class="type">') - finally: - # remove temp files - os.unlink(html_filename) - finally: - os.unlink(f.name) - - return HighlightSourcePostprocess(type, highlighted_source) - - -def HighlightSourcePreProcess(f, source): +def HighlightSourcePygments(src_lang, div_class, source): # chop of leading and trailing empty lines, leave leading space in first real line source = source.strip(' ') source = source.strip('\n') @@ -471,14 +375,11 @@ def HighlightSourcePreProcess(f, source): source = source.replace('<', '<') source = source.replace('>', '>') source = source.replace('&', '&') - if sys.version_info < (3,): - source = source.encode('utf-8') - f.write(source) - f.flush() - return f.name + highlighted_source = highlight.highlight_code(source, src_lang) + if not highlighted_source: + highlighted_source = source -def HighlightSourcePostprocess(type, highlighted_source): # chop of leading and trailing empty lines highlighted_source = highlighted_source.strip() @@ -501,4 +402,4 @@ def HighlightSourcePostprocess(type, highlighted_source): </tbody> </table> </div> -""" % (type, source_lines, highlighted_source) +""" % (div_class, source_lines, highlighted_source) diff --git a/gtkdoc/highlight.py b/gtkdoc/highlight.py new file mode 100644 index 0000000..6023d33 --- /dev/null +++ b/gtkdoc/highlight.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +# -*- python; coding: utf-8 -*- +# +# gtk-doc - GTK DocBook documentation generator. +# Copyright (C) 2018 Stefan Sauer +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; either version 2 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. +# + +""" +Highlight sourcecode snippets. +""" + +from pygments import highlight +from pygments.lexers import CLexer +from pygments.lexers import get_lexer_by_name +from pygments.formatters import HtmlFormatter + +# lazily constructed lexer cache +LEXERS = { + 'c': CLexer() +} +HTML_FORMATTER = HtmlFormatter(nowrap=True) + + +def highlight_code(code, lang='c'): + if lang not in LEXERS: + LEXERS[lang] = get_lexer_by_name(lang) + lexer = LEXERS.get(lang, None) + if not lexer: + return None + return highlight(code, lexer, HTML_FORMATTER) + + +def append_style_defs(css_file_name): + with open(css_file_name, 'at', newline='\n', encoding='utf-8') as css: + css.write(HTML_FORMATTER.get_style_defs()) diff --git a/gtkdoc/mkhtml2.py b/gtkdoc/mkhtml2.py index c663202..91860a7 100644 --- a/gtkdoc/mkhtml2.py +++ b/gtkdoc/mkhtml2.py @@ -105,20 +105,9 @@ from anytree import Node, PreOrderIter from copy import deepcopy from glob import glob from lxml import etree -from pygments import highlight -from pygments.lexers import CLexer -from pygments.lexers import get_lexer_by_name -from pygments.formatters import HtmlFormatter from timeit import default_timer as timer -from . import config, fixxref - -# pygments setup -# lazily constructed lexer cache -LEXERS = { - 'c': CLexer() -} -HTML_FORMATTER = HtmlFormatter(nowrap=True) +from . import config, highlight, fixxref class ChunkParams(object): @@ -876,12 +865,8 @@ def convert_programlisting(ctx, xml): if xml.attrib.get('role', '') == 'example': if xml.text: lang = xml.attrib.get('language', ctx['src-lang']).lower() - if lang not in LEXERS: - LEXERS[lang] = get_lexer_by_name(lang) - lexer = LEXERS.get(lang, None) - if lexer: - highlighted = highlight(xml.text, lexer, HTML_FORMATTER) - + highlighted = highlight.highlight_code(xml.text, lang) + if highlighted: # we do own line-numbering line_count = highlighted.count('\n') source_lines = '\n'.join([str(i) for i in range(1, line_count + 1)]) @@ -1805,9 +1790,7 @@ def main(module, index_file, out_dir, uninstalled, src_lang, paths): css_file = os.path.join(styledir, 'style.css') for f in glob(os.path.join(styledir, '*.png')) + [css_file]: shutil.copy(f, out_dir) - css_file = os.path.join(out_dir, 'style.css') - with open(css_file, 'at', newline='\n', encoding='utf-8') as css: - css.write(HTML_FORMATTER.get_style_defs()) + highlight.append_style_defs(os.path.join(out_dir, 'style.css')) logging.warning("2: %7.3lf: copy datafiles", timer() - _t) # 3) load xref targets diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..a9f49e0 --- /dev/null +++ b/requirements.txt @@ -0,0 +1 @@ +pygments diff --git a/style/style.css b/style/style.css index 4be4ede..6f0bc25 100644 --- a/style/style.css +++ b/style/style.css @@ -293,29 +293,11 @@ h2 .extralinks, h3 .extralinks font-weight: normal; } -acronym,abbr +acronym,abbr { border-bottom: 1px dotted gray; } -/* code listings */ - -.listing_code .programlisting .normal, -.listing_code .programlisting .normal a, -.listing_code .programlisting .number, -.listing_code .programlisting .cbracket, -.listing_code .programlisting .symbol { color: #555753; } -.listing_code .programlisting .comment, -.listing_code .programlisting .linenum { color: #babdb6; } /* tango: aluminium 3 */ -.listing_code .programlisting .function, -.listing_code .programlisting .function a, -.listing_code .programlisting .preproc { color: #204a87; } /* tango: sky blue 3 */ -.listing_code .programlisting .string { color: #ad7fa8; } /* tango: plum */ -.listing_code .programlisting .keyword, -.listing_code .programlisting .usertype, -.listing_code .programlisting .type, -.listing_code .programlisting .type a { color: #4e9a06; } /* tango: chameleon 3 */ - .listing_frame { /* tango:sky blue 1 */ border: solid 1px #729fcf; |