path: root/libyelp/yelp-man-parser.c

/* -*- Mode: C; tab-width: 8; indent-tabs-mode: t; c-basic-offset: 4 -*- */
/*
 * Copyright (C) 2003-2010 Shaun McCance <shaunm@gnome.org>
 *
 * This program is free software; you can redistribute it and/or
 * modify it under the terms of the GNU General Public License as
 * published by the Free Software Foundation; either version 2 of the
 * License, or (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
 * General Public License for more details.
 *
 * You should have received a copy of the GNU General Public
 * License along with this program; if not, write to the
 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
 * Boston, MA 02111-1307, USA.
 *
 * Author: Shaun McCance <shaunm@gnome.org>
 */

#ifdef HAVE_CONFIG_H
#include <config.h>
#endif

#include <glib.h>
#include <glib/gi18n.h>
#include <libxml/tree.h>
#include <gio/gio.h>
#include <string.h>
#include <math.h>

#include "yelp-error.h"
#include "yelp-man-parser.h"

#define MAN_FONTS 8

/* The format has two copies of the title like MAN(1) at the top,
 * possibly with a string of text in between for the collection.
 *
 * Start with the parser on START, then HAVE_TITLE when we've read the
 * first word with parentheses. At that point, stick new words into
 * the "collection" tag. Then finally switch to BODY when we've seen
 * the second copy of the one with parentheses.
 */
typedef enum ManParserState
{
    START,
    HAVE_TITLE,
    BODY
} ManParserState;

/* See parse_body_text for how this is used. */
typedef enum ManParserSectionState
{
    SECTION_TITLE,
    SECTION_BODY
} ManParserSectionState;

struct _YelpManParser {
    xmlDocPtr     doc;           /* The top-level XML document */
    xmlNodePtr    header;        /* The header node */
    xmlNodePtr    section_node;  /* The current section */
    xmlNodePtr    sheet_node;    /* The current sheet */

    GDataInputStream *stream;    /* The GIO input stream to read from */
    gchar            *buffer;    /* The buffer, line at a time */
    gsize             length;    /* The buffer length */

    /* The width and height of a character according to troff. */
    guint char_width;
    guint char_height;

    /* Count the number of lines we've parsed (needed to get prologue) */
    guint line_no;

    /* The x f k name command sets the k'th register to be name. */
    gchar* font_registers[MAN_FONTS];

    /* The current font. Should be the index of one of the
     * font_registers. Starts at 0 (of course!)
     */
    guint current_font;

    /* See description of ManParserState above */
    ManParserState state;

    /* Vertical and horizontal position as far as the troff output is
     * concerned. (Measured from top-left).
     */
    guint vpos, hpos;

    /* Text accumulator (needed since it comes through in dribs &
     * drabs...) */
    GString *accumulator;

    /* See parse_body_text for how this is used. */
    ManParserSectionState section_state;

    /* The indent of the current sheet */
    guint sheet_indent;

    /* Set to TRUE if there's been a newline since the last text was
     * parsed. */
    gboolean newline;

    /* Count the number of 'N' lines we've seen since the last h
     * command. This is because for some reason N doesn't
     * automatically move the position forward. Thus immediately after
     * one, you see a h24 or the like. Unless there's a space. Then it
     * might be wh48. This is set in parse_N (obviously) and used in
     * parse_h.
     */
    guint N_count;

    /* Keep track of whether the last character was a space. We can't
     * just do this by looking at the last char of accumulator,
     * because if there's a font change, it gets zeroed. This gets set
     * to TRUE by parse_w and is FALSE the rest of the time.
     */
    gboolean last_char_was_space;

    /* Keep track of the size of the last vertical jump - used to tell
     * whether we need to insert extra space above a line.
     */
    gint last_vertical_jump;

    /* The title we read earlier (eg 'Foo(2)') */
    gchar *title_str;
};

static gboolean parser_parse_line (YelpManParser *parser, GError **error);
static gboolean parse_prologue_line (YelpManParser *parser, GError **error);

/* Parsers for different types of line */
typedef gboolean (*LineParser)(YelpManParser *, GError **);
#define DECLARE_LINE_PARSER(name) \
    static gboolean (name) (YelpManParser *parser, GError **error);

DECLARE_LINE_PARSER (parse_xf);
DECLARE_LINE_PARSER (parse_f);
DECLARE_LINE_PARSER (parse_V);
DECLARE_LINE_PARSER (parse_H);
DECLARE_LINE_PARSER (parse_v);
DECLARE_LINE_PARSER (parse_h);
DECLARE_LINE_PARSER (parse_text);
DECLARE_LINE_PARSER (parse_w);
DECLARE_LINE_PARSER (parse_body_text);
DECLARE_LINE_PARSER (parse_n);
DECLARE_LINE_PARSER (parse_N);
DECLARE_LINE_PARSER (parse_C);
DECLARE_LINE_PARSER (parse_p);

/* Declare a sort of alist registry of parsers for different lines. */
struct LineParsePair
{
    const gchar *prefix;
    LineParser handler;
};
static struct LineParsePair line_parsers[] = {
    { "x f", parse_xf }, { "f", parse_f },
    { "V", parse_V }, { "H", parse_H },
    { "v", parse_v }, { "h", parse_h },
    { "t", parse_text },
    { "w", parse_w },
    { "n", parse_n },
    { "N", parse_N },
    { "C", parse_C },
    { "p", parse_p },
    { NULL, NULL }
};

/******************************************************************************/
/* Parser helper functions (managing the state of the various parsing
 * bits) */
static void finish_span (YelpManParser *parser);
static guint dx_to_em_count (YelpManParser *parser, guint dx);
static void append_nbsps (YelpManParser *parser, guint k);
static void deal_with_newlines (YelpManParser *parser);
static void new_sheet (YelpManParser *parser);
static void register_title (YelpManParser *parser,
                            const gchar* name, const gchar* section);
static void right_truncate_common (gchar *dst, const gchar *src);
static gboolean cheeky_call_parse_line (YelpManParser *parser,
                                        GError **error,
                                        gchar first_char,
                                        const gchar *text);
static void cleanup_parsed_page (YelpManParser *parser);
static gboolean parse_last_line (YelpManParser *parser, gchar* line);
static void unicode_strstrip (gchar *str);

/******************************************************************************/
/* Translations for the 'C' command. This is indeed hackish, but the
 * -Tutf8 output doesn't seem to give include files so we can do this
 * at runtime :-(
 *
 * On my machine, this data's at /usr/share/groff/current/tmac/ in
 * latin1.tmac, unicode.tmac and I worked out the lq and rq from
 * running man: I'm not sure where that comes from!
 */
struct StringPair
{
    const gchar *from;
    gunichar to;
};
static const struct StringPair char_translations[] = {
    { "r!", 161 },
    { "ct", 162 },
    { "Po", 163 },
    { "Cs", 164 },
    { "Ye", 165 },
    { "bb", 166 },
    { "sc", 167 },
    { "ad", 168 },
    { "co", 169 },
    { "Of", 170 },
    { "Fo", 171 },
    { "tno", 172 },
    { "%", 173 },
    { "rg", 174 },
    { "a-", 175 },
    { "de", 176 },
    { "t+-", 177 },
    { "S2", 178 },
    { "S3", 179 },
    { "aa", 180 },
    { "mc", 181 },
    { "ps", 182 },
    { "pc", 183 },
    { "ac", 184 },
    { "S1", 185 },
    { "Om", 186 },
    { "Fc", 187 },
    { "14", 188 },
    { "12", 189 },
    { "34", 190 },
    { "r?", 191 },
    { "`A", 192 },
    { "'A", 193 },
    { "^A", 194 },
    { "~A", 195 },
    { ":A", 196 },
    { "oA", 197 },
    { "AE", 198 },
    { ",C", 199 },
    { "`E", 200 },
    { "'E", 201 },
    { "^E", 202 },
    { ":E", 203 },
    { "`I", 204 },
    { "'I", 205 },
    { "^I", 206 },
    { ":I", 207 },
    { "-D", 208 },
    { "~N", 209 },
    { "`O", 210 },
    { "'O", 211 },
    { "^O", 212 },
    { "~O", 213 },
    { ":O", 214 },
    { "tmu", 215 },
    { "/O", 216 },
    { "`U", 217 },
    { "'U", 218 },
    { "^U", 219 },
    { ":U", 220 },
    { "'Y", 221 },
    { "TP", 222 },
    { "ss", 223 },
    { "`a", 224 },
    { "'a", 225 },
    { "^a", 226 },
    { "~a", 227 },
    { ":a", 228 },
    { "oa", 229 },
    { "ae", 230 },
    { ",c", 231 },
    { "`e", 232 },
    { "'e", 233 },
    { "^e", 234 },
    { ":e", 235 },
    { "`i", 236 },
    { "'i", 237 },
    { "^i", 238 },
    { ":i", 239 },
    { "Sd", 240 },
    { "~n", 241 },
    { "`o", 242 },
    { "'o", 243 },
    { "^o", 244 },
    { "~o", 245 },
    { ":o", 246 },
    { "tdi", 247 },
    { "/o", 248 },
    { "`u", 249 },
    { "'u", 250 },
    { "^u", 251 },
    { ":u", 252 },
    { "'y", 253 },
    { "Tp", 254 },
    { ":y", 255 },
    { "hy", '-' },
    { "oq", '`' },
    { "cq", '\'' },
    { "lq", 8220 }, // left smart quotes
    { "rq", 8221 }, // right smart quotes
    { "en", 8211 }, // en-dash
    { "em", 8212 }, // em-dash
    { "la", 10216 }, // left angle bracket
    { "ra", 10217 }, // left angle bracket
    { "rs", '\\' },
    { "<=", 8804 }, // < or equal to sign
    { ">=", 8805 }, // > or equal to sign
    { "aq", '\'' },
    { "tm", 8482 }, // trademark symbol
    { NULL, 0 }
};

/******************************************************************************/

YelpManParser *
yelp_man_parser_new (void)
{
    YelpManParser *parser = g_new0 (YelpManParser, 1);
    parser->accumulator = g_string_sized_new (1024);
    return parser;
}

/*
  This function is responsible for taking a path to a man file and
  returning something in the groff intermediate output format for us
  to use.

  If something goes wrong, we return NULL and set error to be a
  YelpError describing the problem.
*/
static GInputStream*
get_troff (gchar *path, GError **error)
{
    gint stdout;
    GError *err = NULL;
    gchar *argv[] = { "man", "-Z", "-Tutf8", "-EUTF-8", NULL, NULL };

    argv[4] = path;

    if (!g_spawn_async_with_pipes (NULL, argv, NULL,
                                   G_SPAWN_SEARCH_PATH, NULL, NULL,
                                   NULL, NULL, &stdout, NULL, &err)) {
        /* We failed to run the man program. Return a "Huh?" error. */
        *error = g_error_new (YELP_ERROR, YELP_ERROR_UNKNOWN,
                              err->message);
        g_error_free (err);
        return NULL;
    }

    return (GInputStream*) g_unix_input_stream_new (stdout, TRUE);
}

xmlDocPtr
yelp_man_parser_parse_file (YelpManParser *parser,
                            gchar *path,
                            GError **error)
{
    GInputStream *troff_stream;
    gchar *line;
    gsize len;
    gboolean ret;
    xmlNodePtr root;

    troff_stream = get_troff (path, error);
    if (!troff_stream) return NULL;

    parser->stream = g_data_input_stream_new (troff_stream);

    parser->doc = xmlNewDoc (BAD_CAST "1.0");
    root = xmlNewNode (NULL, BAD_CAST "Man");
    xmlDocSetRootElement (parser->doc, root);

    parser->header = xmlNewNode (NULL, BAD_CAST "header");
    xmlAddChild (root, parser->header);

    while (1) {
       parser->buffer =
       g_data_input_stream_read_line (parser->stream,
                                      &(parser->length),
                                      NULL, NULL);
       if (parser->buffer == NULL) break;

       parser->line_no++;
       ret = parser_parse_line (parser, error);

       g_free (parser->buffer);

       if (!ret) {
           xmlFreeDoc (parser->doc);
           parser->doc = NULL;
           break;
       }
    }

    cleanup_parsed_page (parser);

    g_object_unref (parser->stream);

    return parser->doc;
}

void
yelp_man_parser_free (YelpManParser *parser)
{
    guint k;
    if (parser) {
        for (k=0; k<MAN_FONTS; k++)
            g_free (parser->font_registers[k]);
    }
    g_string_free (parser->accumulator, TRUE);
    g_free (parser->title_str);
    g_free (parser);
}

/******************************************************************************/

/* Sets the k'th font register to be name. Copies name, so free it
 * afterwards. k should be in [0,MAN_FONTS). It seems that man always
 * gives us ones at least 1, but groff_out(5) says non-negative.
 */
static void
set_font_register (YelpManParser *parser, guint k, const gchar* name)
{
    if (k > MAN_FONTS) {
        g_warning ("Tried to set nonexistant font register %d to %s",
                   k, name);
        return;
    }
    g_free (parser->font_registers[k]);
    parser->font_registers[k] = g_strdup (name);
}

static const gchar*
get_font (const YelpManParser *parser)
{
    guint k = parser->current_font;
    if (k > MAN_FONTS ||
        parser->font_registers[k] == NULL) {

        g_warning ("Tried to get nonexistant font register %d", k);

        return "";
    }

    return parser->font_registers[k];
}

/******************************************************************************/

/*
  Convenience macros to scan a string, checking for the correct number
  of things read.

  Also to raise an error. Add an %s to the end of the format string,
  which automatically gets given parser->buffer.
 */
#define SSCANF(fmt,num,...)                                 \
    (sscanf (parser->buffer, (fmt), __VA_ARGS__) != (num))

#define PARSE_ERROR(...)                                    \
    g_error_new (YELP_ERROR, YELP_ERROR_PROCESSING,         \
                 __VA_ARGS__, parser->buffer)
#define RAISE_PARSE_ERROR(...)                              \
    { *error = PARSE_ERROR (__VA_ARGS__); return FALSE; }

static gboolean
parser_parse_line (YelpManParser *parser, GError **error)
{
    if (parser->line_no <= 3)
        return parse_prologue_line (parser, error);

    const struct LineParsePair *p = line_parsers;
    while (p->handler != NULL) {
        if (g_str_has_prefix (parser->buffer, p->prefix)) {
            return p->handler(parser, error);
        }
        p++;
    }
    return TRUE;
}

static gboolean
parse_prologue_line (YelpManParser *parser, GError **error)
{
    if (parser->line_no != 2) return TRUE;

    /* This is the interesting line, which should look like
              x res 240 24 40
       The interesting bits are the 24 and the 40, which are the
       width and height of a character as far as -Tutf8 is
       concerned.
    */
    if (SSCANF ("x %*s %*u %u %u", 2,
                &parser->char_width, &parser->char_height)) {
        RAISE_PARSE_ERROR ("Wrong 'x res' line from troff: %s");
    }

    return TRUE;
}

static gboolean
parse_xf (YelpManParser *parser, GError **error)
{
    gchar name[10];
    guint k;

    if (SSCANF ("x f%*s %u %10s", 2, &k, name)) {
        RAISE_PARSE_ERROR ("Invalid 'x f' line from troff: %s");
    }
    set_font_register (parser, k, name);
    return TRUE;
}

static gboolean
parse_f (YelpManParser *parser, GError **error)
{
    guint k;
    if (SSCANF ("f%u", 1, &k)) {
        RAISE_PARSE_ERROR ("Invalid font line from troff: %s");
    }
    finish_span (parser);

    parser->current_font = k;

    return TRUE;
}

static gboolean
parse_v (YelpManParser *parser, GError **error)
{
    guint dy;
    if (SSCANF ("v%u", 1, &dy)) {
        RAISE_PARSE_ERROR ("Invalid v line from troff: %s");
    }
    parser->last_vertical_jump += dy;
    parser->vpos += dy;
    return TRUE;
}

static gboolean
parse_h (YelpManParser *parser, GError **error)
{
    guint dx;
    int k;

    if (SSCANF ("h%u", 1, &dx)) {
        RAISE_PARSE_ERROR ("Invalid h line from troff: %s");
    }
    parser->hpos += dx;

    /* This is a bit hackish to be honest but... if we're in something
     * that'll end up in a span, a spacing h command means that a gap
     * should appear. It seems that the easiest way to get this is to
     * insert nonbreaking spaces (eugh!)
     *
     * Of course we don't want to do this when chained from wh24 or
     * whatever, so use the last_char_was_space flag
     * but... unfortunately some documents actually use stuff like
     * wh96 for spacing (eg the lists in perl(1)). So (very hackish!),
     * ignore double spaces, since that's probably just been put in to
     * make the text justified (eugh), but allow bigger jumps.
     *
     * Incidentally, the perl manual here has bizarre gaps in the
     * synopsis section. God knows why, but man displays them too so
     * it's not our fault! :-)
     */
    k = dx_to_em_count (parser, dx);

    if ((!parser->last_char_was_space) || (k > 2)) {

        k -= parser->N_count;
        if (k < 0) k = 0;

        append_nbsps (parser, k);
    }

    parser->N_count = 0;

    return TRUE;
}

static gboolean
parse_V (YelpManParser *parser, GError **error)
{
    guint y;
    if (SSCANF ("V%u", 1, &y)) {
        RAISE_PARSE_ERROR ("Invalid V line from troff: %s");
    }
    parser->last_vertical_jump += y - parser->vpos;
    parser->vpos = y;
    return TRUE;
}

static gboolean
parse_H (YelpManParser *parser, GError **error)
{
    guint x;
    if (SSCANF ("H%u", 1, &x)) {
        RAISE_PARSE_ERROR ("Invalid H line from troff: %s");
    }
    parser->hpos = x;
    return TRUE;
}

static gboolean
parse_text (YelpManParser *parser, GError **error)
{
    gchar *text, *section, *tmp;
    const gchar *acc;

    /*
      Sneakily, this might get called with something other than t
      starting the buffer: see parse_C and parse_N.
    */
    if (parser->buffer[0] == 't') {
        parser->N_count = 0;
    }

    if (parser->state == START) {
        /* This should be the 'Title String(1)' line. It might come in
         * chunks (for example, it might be more than one line
         * long!). So just read bits until we get a (blah) bit: stick
         * everything in the accumulator and check for
         * parentheses. When we've got some, stick the parsed title in
         * the header and switch to HAVE_TITLE.
         *
         * The parse_n code will error out if we didn't manage to get
         * a title before the first newline and otherwise is in charge
         * of switching to body-parsing mode.
         */
        g_string_append (parser->accumulator, parser->buffer+1);

        acc = parser->accumulator->str;

        section = strchr (acc, '(');

        if (section) {
            section++;
            tmp = strchr (section, ')');
        }

        if (section && tmp) {
            /* We've got 'Blah (3)' or the like in the accumulator */
            if (*(tmp+1) != '\0') {
                RAISE_PARSE_ERROR ("Don't understand title line: '%s'");
            }
            parser->state = HAVE_TITLE;
            parser->title_str = g_strdup (acc);

            text = g_strndup (acc, (section - 1) - acc);
            section = g_strndup (section, tmp - section);

            register_title (parser, text, section);

            g_string_truncate (parser->accumulator, 0);

            g_free (text);
            g_free (section);
        }

        return TRUE;
    }

    if (parser->state == BODY)
        return parse_body_text (parser, error);

    /* In state HAVE_TITLE */
    else {
        /* We expect (maybe!) to get some lines in between the two
         * occurrences of the title itself. So collect up all the text
         * we get and then we'll remove the copy of the title at the
         * end (hopefully) when we find a newline in parse_n.
         */
        g_string_append (parser->accumulator, parser->buffer+1);
        return TRUE;
    }
}

static gboolean
parse_body_text (YelpManParser *parser, GError **error)
{
    /*
      It's this function which is responsible for trying to get *some*
      semantic information back out of the manual page.

      The highest-level chopping up is into sections. We use the
      heuristic that if either
        (1) We haven't got a section yet or
        (2) text starts a line (hpos=0)
      then it's a section title.

      It's possible to have spaces in section titles, so we carry on
      accumulating the section title until the next newline.
    */
    if (parser->section_state == SECTION_BODY &&
        (!parser->section_node || (parser->hpos == 0))) {
        g_string_truncate (parser->accumulator, 0);
        /* End the current sheet & section */
        parser->section_state = SECTION_TITLE;
        parser->sheet_node = NULL;

        parser->section_node =
            xmlAddChild (xmlDocGetRootElement (parser->doc),
                         xmlNewNode (NULL, BAD_CAST "section"));
    }

    if (parser->section_state != SECTION_TITLE) {
        deal_with_newlines (parser);
    }

    g_string_append (parser->accumulator, parser->buffer+1);

    /* Move hpos forward per char */
    parser->hpos += strlen (parser->buffer+1) * parser->char_width;

    parser->last_char_was_space = FALSE;

    return TRUE;
}

/*
  w is a sort of prefix argument. It indicates a space, so we register
  that here, then call parser_parse_line again on the rest of the
  string to deal with that.
 */
static gboolean
parse_w (YelpManParser *parser, GError **error)
{
    gboolean ret;

    if (parser->state != START) {
        g_string_append_c (parser->accumulator, ' ');
    }

    parser->buffer++;
    parser->last_char_was_space = TRUE;

    ret = parser_parse_line (parser, error);

    parser->buffer--;
    return ret;
}

static gboolean
parse_n (YelpManParser *parser, GError **error)
{
    xmlNodePtr node;

    /* When we're in the header, the parse_n is responsible for
     * switching to body text. (See the body of parse_text() for more
     * of an explanation).
     */
    if (parser->state == START) {
        /* Oh no! We've not got a proper title yet! Ho hum, let's
           stick whatever's going into a 'title title' and have a null
           section. Sob.
        */
        register_title (parser,
                        parser->accumulator->str,
                        "unknown section");
        g_string_truncate (parser->accumulator, 0);
        parser->state = BODY;
        return TRUE;
    }

    if (parser->state == HAVE_TITLE) {
        /* What we've got so far is the manual's collection, followed
           by the title again. So we want to get rid of the latter if
           possible...
        */
        right_truncate_common (parser->accumulator->str,
                               parser->title_str);
        unicode_strstrip (parser->accumulator->str);

        xmlNewTextChild (parser->header,
                         NULL, BAD_CAST "collection",
                         parser->accumulator->str);
        g_string_truncate (parser->accumulator, 0);
        parser->state = BODY;
        parser->section_state = SECTION_BODY;
        return TRUE;
    }

    /* parser->state == BODY */
    if (parser->section_state == SECTION_TITLE) {

        g_strchomp (parser->accumulator->str);
        xmlNewTextChild (parser->section_node, NULL,
                         BAD_CAST "title", parser->accumulator->str);
        g_string_truncate (parser->accumulator, 0);

        parser->section_state = SECTION_BODY;
    }
    else if (parser->sheet_node != NULL) {
        /*
          In the body of a section, when we get to a newline we should
          have an accumulator with text in it and a non-null sheet
          (hopefully!).

          We know the current font, so add a span for that font
          containing the relevant text. Then add a <br/> tag.
        */
        finish_span (parser);
        node = xmlNewNode (NULL, BAD_CAST "br");
        xmlAddChild (parser->sheet_node, node);
    }

    parser->newline = TRUE;
    parser->last_char_was_space = FALSE;

    return TRUE;
}

static void
finish_span (YelpManParser *parser)
{
    xmlNodePtr node;

    if (parser->accumulator->str[0] != '\0') {
        node = xmlNewTextChild (parser->sheet_node, NULL,
                                BAD_CAST "span",
                                parser->accumulator->str);
        xmlNewProp (node, BAD_CAST "class", get_font (parser));
        g_string_truncate (parser->accumulator, 0);
    }
}

static guint
dx_to_em_count (YelpManParser *parser, guint dx)
{
    return (int)(dx / ((float)parser->char_width));
}

static gboolean
parse_N (YelpManParser *parser, GError **error)
{
    gint n;
    gchar tmp[2];

    if (SSCANF ("N%i", 1, &n)) {
        RAISE_PARSE_ERROR ("Strange format for N line: %s");
    }
    if (n > 127) {
        RAISE_PARSE_ERROR ("N line has non-7-bit character: %s");
    }
    if (n < -200) {
        RAISE_PARSE_ERROR ("Bizarrely many nbsps in N line: %s");
    }

    if (n < 0) {
        append_nbsps (parser, -n);
        parser->N_count += -n;
        return TRUE;
    }

    parser->N_count++;

    tmp[0] = (gchar)n;
    tmp[1] = '\0';

    return cheeky_call_parse_line (parser, error, 'N', tmp);
}

static void
append_nbsps (YelpManParser *parser, guint k)
{
    for (; k > 0; k--) {
        /* 0xc2 0xa0 is nonbreaking space in utf8 */
        g_string_append_c (parser->accumulator, 0xc2);
        g_string_append_c (parser->accumulator, 0xa0);
    }
}

static gboolean
parse_C (YelpManParser *parser, GError **error)
{
    gchar name[16];
    gunichar code = 0;
    guint k;
    gint len;

    if (SSCANF ("C%16s", 1, name)) {
        RAISE_PARSE_ERROR ("Can't understand special character: %s");
    }

    for (k=0; char_translations[k].from; k++) {
        if (g_str_equal (char_translations[k].from, name)) {
            code = char_translations[k].to;
            break;
        }
    }
    if (sscanf (name, "u%x", &k) == 1) {
        code = k;
    }

    if (!code) {
        g_warning ("Couldn't parse troff special character: '%s'",
                   name);
        code = 65533; /* Unicode replacement character */
    }

    /* Output buffer must be length >= 6. 16 >= 6, so we're ok. */
    len = g_unichar_to_utf8 (code, name);
    name[len] = '\0';

    parser->N_count++;

    return cheeky_call_parse_line (parser, error, 'C', name);
}

static void
deal_with_newlines (YelpManParser *parser)
{
    /*
      If newline is true, this is the first word on a line.

      In which case, we check to see whether hpos agrees with the
      current sheet's indent. If so (or if there isn't a sheet yet!),
      we just add to the accumulator. If not, start a new sheet with
      the correct indent.

      If we aren't the first word on the line, just add to the
      accumulator.
    */
    gchar tmp[64];
    guint jump_lines;
    gboolean made_sheet = FALSE, dont_jump = FALSE;

    /* This only happens at the start of a section, where there's
       already a gap
    */
    if (!parser->sheet_node) {
        dont_jump = TRUE;
    }

    if ((!parser->sheet_node) ||
        (parser->newline && (parser->hpos != parser->sheet_indent))) {
        new_sheet (parser);
        made_sheet = TRUE;
    }

    if (parser->newline) {
        append_nbsps (parser, dx_to_em_count (parser, parser->hpos));

        if ((parser->last_vertical_jump > 0) && (!dont_jump)) {
            jump_lines =
                parser->last_vertical_jump/parser->char_height;
        } else {
            jump_lines = 1;
        }

        if (jump_lines > 1) {
            if (!made_sheet) new_sheet (parser);
            made_sheet = TRUE;
        }

        if (made_sheet) {
            snprintf (tmp, 64, "%u", jump_lines-1);
            xmlNewProp (parser->sheet_node, BAD_CAST "jump", tmp);
        }
    }

    parser->newline = FALSE;
    parser->last_vertical_jump = 0;
}

static gboolean
parse_p (YelpManParser *parser, GError **error)
{
    parser->vpos = 0;
    parser->hpos = 0;
    return TRUE;
}

static void
new_sheet (YelpManParser *parser)
{
    /* We don't need to worry about finishing the current sheet,
       since the accumulator etc. get cleared on newlines and we
       know we're at the start of a line.
    */
    parser->sheet_node =
        xmlAddChild (parser->section_node,
                     xmlNewNode (NULL, BAD_CAST "sheet"));
    parser->sheet_indent = parser->hpos;
}

static void
register_title (YelpManParser *parser,
                const gchar* name, const gchar* section)
{
    xmlNewTextChild (parser->header,
                     NULL, BAD_CAST "title", name);
    xmlNewTextChild (parser->header,
                     NULL, BAD_CAST "section", section);
}

static void
right_truncate_common (gchar *dst, const gchar *src)
{
    guint len_src = strlen (src);
    guint len_dst = strlen (dst);

    guint k = (len_src < len_dst) ? len_src - 1 : len_dst - 1;

    dst += len_dst - 1;
    src += len_src - 1;

    while (k >= 0) {
        if (*dst != *src) break;
        *dst = '\0';

        k--;
        dst--;
        src--;
    }
}

static gboolean
cheeky_call_parse_line (YelpManParser *parser, GError **error,
                        gchar first_char, const gchar* text)
{
    /* Do a cunning trick. There's all sorts of code that parse_text
     * does, which we don't want to duplicate in parse_N and
     * parse_C. So feed a buffer back to parse_text. Tada! Start it
     * with "C" or "N" rather than "t" so clever stuff in parse_text
     * can tell the difference.
     */
    gchar *tmp;
    gboolean ret;
    guint len = strlen (text);

    tmp = parser->buffer;
    parser->buffer = g_new (gchar, 2 + len);
    parser->buffer[0] = first_char;
    strncpy (parser->buffer + 1, text, len + 1);

    ret = parse_text (parser, error);

    g_free (parser->buffer);
    parser->buffer = tmp;

    return ret;
}

static void
cleanup_parsed_page (YelpManParser *parser)
{
    /* First job: the last line usually has the version, date and
     * title (again!). The code above misunderstands and parses this
     * as a section, so we need to "undo" this and stick the data in
     * the header where it belongs.
     *
     * parser->section_node should still point to it. We assume this
     * has happened if it has exactly one child element (the <title>
     * tag)
     */
    gchar *lastline;

    if (xmlChildElementCount (parser->section_node) == 1) {
        lastline = xmlNodeGetContent (parser->section_node);

        /* If parse_last_line works, it sets the data from it in the
           <header> tag, so delete the final section. */
        if (parse_last_line (parser, lastline)) {
            xmlUnlinkNode (parser->section_node);
            xmlFreeNode (parser->section_node);
        }
        else {
            /* Oh dear. This would be unexpected and doesn't seem to
               happen with man on my system. But we probably shouldn't
               ditch the info, so let's leave the <section> tag and
               print a warning message to the console.
            */
            g_warning ("Unexpected final line in man document (%s)\n",
                       lastline);
        }

        xmlFree (lastline);
    }
}

static gchar *
skip_whitespace (gchar *text)
{
    while (g_unichar_isspace (g_utf8_get_char (text))) {
        text = g_utf8_next_char (text);
    }
    return text;
}

static gchar *
last_non_whitespace (gchar *text)
{
    gchar *end = text + strlen(text);
    gchar *prev;

    prev = g_utf8_find_prev_char (text, end);
    if (!prev) {
        /* The string must have been zero-length. */
        return NULL;
    }

    while (g_unichar_isspace (g_utf8_get_char (prev))) {
        end = prev;
        prev = g_utf8_find_prev_char (text, prev);
        if (!prev) return NULL;
    }
    return end;
}

static gchar *
find_contiguous_whitespace (gchar *text, guint ws_len)
{
    guint counter = 0;
    gchar *ws_start;
    while (*text) {
        if (g_unichar_isspace (g_utf8_get_char (text))) {
            if (!counter) ws_start = text;
            counter++;
        }
        else counter = 0;

        if (counter == ws_len) return ws_start;

        text = g_utf8_next_char (text);
    }
    return NULL;
}

static gboolean
parse_last_line (YelpManParser *parser, gchar* line)
{
    /* We expect a line of the form
           '1.2.3      blah 2009       libfoo(1)'
       where the spaces are all nbsp's.

       Look for a gap of at least 3 in a row. If we find that, expand
       either side and declare the stuff before to be the version
       number and then the stuff afterwards to be the start of the
       date. Then do the same thing on the next gap, if there is one.
    */
    gchar *gap, *date_start;

    gchar *version;
    gchar *date;

    gap = find_contiguous_whitespace (line, 3);
    if (!gap) return FALSE;

    version = g_strndup (line, gap - line);

    date_start = skip_whitespace (gap);

    gap = find_contiguous_whitespace (date_start, 3);
    if (!gap) return FALSE;

    date = g_strndup (date_start, gap - date_start);

    xmlNewProp (parser->header, BAD_CAST "version", version);
    xmlNewProp (parser->header, BAD_CAST "date", date);

    g_free (version);
    g_free (date);

    return TRUE;
}

/* This should work like g_strstrip, but that's an ASCII-only version
 * and I want to strip the nbsp's that I so thoughtfully plaster
 * stuff with...
 */
static void
unicode_strstrip (gchar *str)
{
    gchar *start, *end;

    if (str == NULL) return;

    end = last_non_whitespace (str);

    if (!end) {
        /* String is zero-length or entirely whitespace */
        *str = '\0';
        return;
    }
    start = skip_whitespace (str);
    g_utf8_next_char (end);

    g_memmove (str, start, end - start);
    *(str + (end - start)) = '\0';
}