Merge valadoc 0.36.0

Consider valadoc a part of vala's toolchain and therefore let it live in
the main repository.

With this merge there is no need to maintain multiple driver sources since
only one is required from now on.

There is no dependency on gee-0.8 and vala's internal gee copy has made to
be sufficient.

The libvaladoc library will be suffixed with vala's version suffix too.
Besides this renaming the rest of the valadoc file layout is kept the same.

https://bugzilla.gnome.org/show_bug.cgi?id=782782

12 files changed, 1451 insertions, 0 deletions

diff --git a/libvaladoc/parser/manyrule.vala b/libvaladoc/parser/manyrule.vala
new file mode 100644
index 000000000..00d928ae0
--- /dev/null
+++ b/libvaladoc/parser/manyrule.vala
@@ -0,0 +1,111 @@
+/* manyrule.vala
+ *
+ * Copyright (C) 2008-2009 Florian Brosch, Didier Villevalois
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+ *
+ * Author:
+ * 	Didier 'Ptitjes Villevalois <ptitjes@free.fr>
+ */
+
+
+internal class Valadoc.ManyRule : Rule {
+
+	public ManyRule (Object scheme) {
+		_scheme = scheme;
+	}
+
+	private Object _scheme;
+
+	private class State : Object {
+		public bool started = false;
+		public bool done_one = false;
+	}
+
+	public override bool is_optional () {
+		return is_optional_rule (_scheme);
+	}
+
+	public override bool starts_with_token (Token token) {
+		if (has_start_token (_scheme, token)) {
+			return true;
+		}
+		return false;
+	}
+
+	public override bool accept_token (Token token, ParserCallback parser, Rule.Forward forward)
+									   throws ParserError
+	{
+		var state = parser.get_rule_state () as State;
+		if (state == null) {
+			state = new State ();
+			parser.set_rule_state (state);
+		}
+
+		if (!state.started) {
+			do_start (parser);
+			state.started = true;
+		}
+
+		if (state.done_one && parser.would_parent_accept_token (token)) {
+			do_reduce (parser);
+			return false;
+		}
+		if (parser.would_parent_reduce_to_rule (token, this)) {
+			do_reduce (parser);
+			return false;
+		}
+
+		bool handled;
+		if (try_to_apply (_scheme, token, parser, out handled)) {
+			state.done_one = true;
+			return handled;
+		}
+		if (parser.would_parent_accept_token (token)) {
+			do_reduce (parser);
+			return false;
+		}
+
+		if (_scheme is TokenType) {
+			parser.error (null, "expected %s".printf (((TokenType) _scheme).to_pretty_string ()));
+		} else {
+			parser.error (token, "unexpected token");
+		}
+		assert_not_reached ();
+	}
+
+	public override bool would_accept_token (Token token, Object? state) {
+		if (has_start_token (_scheme, token)) {
+			return true;
+		}
+		return false;
+	}
+
+	public override bool would_reduce (Token token, Object? rule_state) {
+		var state = rule_state as State;
+		return state.done_one || is_optional_rule (_scheme);
+	}
+
+	public override string to_string (Object? rule_state) {
+		var state = rule_state as State;
+		if (state == null) {
+			state = new State ();
+		}
+		return "%-15s%-15s(started=%s;done_one=%s)".printf (name != null ? name : " ",
+															"[many]",
+															state.started.to_string (),
+															state.done_one.to_string ());
+	}
+}
diff --git a/libvaladoc/parser/oneofrule.vala b/libvaladoc/parser/oneofrule.vala
new file mode 100644
index 000000000..4e0144df4
--- /dev/null
+++ b/libvaladoc/parser/oneofrule.vala
@@ -0,0 +1,97 @@
+/* oneofrule.vala
+ *
+ * Copyright (C) 2008-2009 Florian Brosch, Didier Villevalois
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+ *
+ * Author:
+ * 	Didier 'Ptitjes Villevalois <ptitjes@free.fr>
+ */
+
+
+internal class Valadoc.OneOfRule : Rule {
+
+	public OneOfRule (Object[] scheme) {
+		_scheme = scheme;
+	}
+
+	private Object[] _scheme;
+
+	private class State : Object {
+		public int selected = -1;
+	}
+
+	public override bool is_optional () {
+		return false;
+	}
+
+	public override bool starts_with_token (Token token) {
+		foreach (Object? scheme_element in _scheme) {
+			if (has_start_token (scheme_element, token)) {
+				return true;
+			}
+		}
+		return false;
+	}
+
+	public override bool accept_token (Token token, ParserCallback parser, Rule.Forward forward)
+									   throws ParserError
+	{
+		var state = parser.get_rule_state () as State;
+		if (state == null) {
+			state = new State ();
+			parser.set_rule_state (state);
+		}
+
+		if (state.selected == -1) {
+			do_start (parser);
+
+			bool handled;
+			for (int i = 0; i < _scheme.length; i++) {
+				Object? scheme_element = _scheme[i];
+				if (try_to_apply (scheme_element, token, parser, out handled)) {
+					state.selected = i;
+					return handled;
+				}
+			}
+		} else {
+			do_reduce (parser);
+			return false;
+		}
+
+		parser.error (token, "unexpected token");
+		assert_not_reached ();
+	}
+
+	public override bool would_accept_token (Token token, Object? state) {
+		return false;
+	}
+
+	public override bool would_reduce (Token token, Object? rule_state) {
+		var state = rule_state as State;
+		return state.selected != -1;
+	}
+
+	public override string to_string (Object? rule_state) {
+		var state = rule_state as State;
+		if (state == null) {
+			state = new State ();
+		}
+		return "%-15s%-15s(selected=%d/%d)".printf (name != null ? name : " ",
+													"[one-of]",
+													state.selected,
+													_scheme.length);
+	}
+}
diff --git a/libvaladoc/parser/optionalrule.vala b/libvaladoc/parser/optionalrule.vala
new file mode 100644
index 000000000..aeef1da04
--- /dev/null
+++ b/libvaladoc/parser/optionalrule.vala
@@ -0,0 +1,87 @@
+/* optionalrule.vala
+ *
+ * Copyright (C) 2008-2009 Florian Brosch, Didier Villevalois
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+ *
+ * Author:
+ * 	Didier 'Ptitjes Villevalois <ptitjes@free.fr>
+ */
+
+
+internal class Valadoc.OptionalRule : Rule {
+
+	public OptionalRule (Object scheme) {
+		_scheme = scheme;
+	}
+
+	private Object _scheme;
+
+	private class State : Object {
+		public bool started = false;
+	}
+
+	public override bool is_optional () {
+		return true;
+	}
+
+	public override bool starts_with_token (Token token) {
+		return has_start_token (_scheme, token);
+	}
+
+	public override bool accept_token (Token token, ParserCallback parser, Rule.Forward forward)
+									   throws ParserError
+	{
+		var state = parser.get_rule_state () as State;
+		if (state == null) {
+			state = new State ();
+			parser.set_rule_state (state);
+		}
+
+		if (!state.started) {
+			do_start (parser);
+			state.started = true;
+
+			bool handled;
+			if (try_to_apply (_scheme, token, parser, out handled)) {
+				return handled;
+			} else {
+				do_skip (parser);
+				return false;
+			}
+		} else {
+			do_reduce (parser);
+			return false;
+		}
+	}
+
+	public override bool would_accept_token (Token token, Object? state) {
+		return false;
+	}
+
+	public override bool would_reduce (Token token, Object? state) {
+		return true;
+	}
+
+	public override string to_string (Object? rule_state) {
+		var state = rule_state as State;
+		if (state == null) {
+			state = new State ();
+		}
+		return "%-15s%-15s(started=%s)".printf (name != null ? name : " ",
+												"[option]",
+												state.started.to_string ());
+	}
+}
diff --git a/libvaladoc/parser/parser.vala b/libvaladoc/parser/parser.vala
new file mode 100644
index 000000000..acf1d82be
--- /dev/null
+++ b/libvaladoc/parser/parser.vala
@@ -0,0 +1,305 @@
+/* parser.vala
+ *
+ * Copyright (C) 2008-2009 Florian Brosch, Didier Villevalois
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+ *
+ * Author:
+ * 	Didier 'Ptitjes Villevalois <ptitjes@free.fr>
+ */
+
+
+public errordomain Valadoc.ParserError {
+	INTERNAL_ERROR,
+	UNEXPECTED_TOKEN
+}
+
+public class Valadoc.Parser : ParserCallback {
+
+	public Parser (Settings settings, Scanner scanner, ErrorReporter reporter) {
+		_settings = settings;
+		_scanner = scanner;
+		_reporter = reporter;
+
+		TokenType.init_token_types ();
+	}
+
+	private Settings _settings;
+	private Scanner _scanner;
+	private ErrorReporter _reporter;
+	private Rule _root_rule;
+
+	private string _filename;
+	private int _first_line;
+	private int _first_column;
+	private Token _current_token;
+
+	private Vala.ArrayList<Rule> rule_stack = new Vala.ArrayList<Rule> ();
+	private Vala.ArrayList<Object?> rule_state_stack = new Vala.ArrayList<Object?> ();
+
+	public void set_root_rule (Rule root_rule) {
+		_root_rule = root_rule;
+	}
+
+	public void parse (string content, string filename, int first_line, int first_column)
+					   throws ParserError
+	{
+		_filename = filename;
+		_first_line = first_line;
+		_first_column = first_column;
+
+		rule_stack.clear ();
+		rule_state_stack.clear ();
+
+		try {
+			push_rule (_root_rule);
+			_scanner.reset ();
+			_scanner.scan (content);
+			_scanner.end ();
+
+			if (rule_stack.size != 0) {
+				error (null, "Rule stack is not empty!");
+			}
+		} catch (ParserError e) {
+			#if DEBUG
+				log_error (e.message);
+			#endif
+			// Set an in_error boolean, try to recover
+			// And only throw the error at the end of parse ?
+			throw e;
+		}
+	}
+
+	public void accept_token (Token token) throws ParserError {
+		#if HARD_DEBUG
+			debug ("Incomming token: %s", token.to_pretty_string ());
+		#endif
+
+		_current_token = token;
+		int rule_depth = rule_stack.size;
+		Rule.Forward forward = Rule.Forward.NONE;
+		Rule? rule = peek_rule ();
+		if (rule == null) {
+			throw new ParserError.INTERNAL_ERROR ("Rule stack is empty!");
+		}
+		while (rule != null) {
+			if (rule.accept_token (token, this, forward)) {
+				break;
+			}
+
+			// Check for invalid recursion
+			if (rule_depth != rule_stack.size && peek_rule () == rule) {
+				error (null, "Parser state error");
+				break;
+			}
+			rule = peek_rule ();
+
+			// Rule stack size have changed
+			// Check for propagation
+			forward = rule_depth > rule_stack.size ? Rule.Forward.CHILD
+			                                       : Rule.Forward.PARENT;
+			rule_depth = rule_stack.size;
+		}
+	}
+
+	private Rule? peek_rule (int offset = -1) {
+		assert (offset < 0);
+		if (rule_stack.size + offset < 0) {
+			return null;
+		}
+		return rule_stack.get (rule_stack.size + offset);
+	}
+
+	private Rule pop_rule () {
+		int last_index = rule_stack.size - 1;
+		Rule rule = rule_stack.get (last_index);
+		rule_stack.remove_at (last_index);
+		rule_state_stack.remove_at (last_index);
+		return rule;
+	}
+
+	public void push_rule (Rule rule) {
+		rule_stack.add (rule);
+		rule_state_stack.add (null);
+
+		#if HARD_DEBUG
+			debug ("Pushed at  %2d: %s", rule_stack.size - 1, rule.to_string (null));
+		#endif
+	}
+
+	private Object? peek_state (int offset = -1) {
+		assert (offset < 0);
+		if (rule_state_stack.size + offset < 0) {
+			return null;
+		}
+		return rule_state_stack.get (rule_state_stack.size + offset);
+	}
+
+	public Object? get_rule_state () {
+		return peek_state ();
+	}
+
+	public void set_rule_state (Object state) {
+		int last_index = rule_stack.size - 1;
+		rule_state_stack.set (last_index, state);
+	}
+
+	public void reduce () {
+		pop_rule ();
+
+		#if HARD_DEBUG
+			Rule? parent_rule = peek_rule ();
+			if (parent_rule != null) {
+				debug ("Reduced to %2d: %s", rule_stack.size - 1,
+					   parent_rule.to_string (peek_state ()));
+			}
+		#endif
+	}
+
+	public bool would_parent_accept_token (Token token) {
+		int offset = -2;
+		Rule? parent_rule = peek_rule (offset);
+		Object? state = peek_state (offset);
+		while (parent_rule != null) {
+			#if VERY_HARD_DEBUG
+				debug ("WouldAccept - Offset %d; Index %d: %s", offset,
+					rule_stack.size + offset, parent_rule.to_string (state));
+			#endif
+			if (parent_rule.would_accept_token (token, state)) {
+				#if VERY_HARD_DEBUG
+					debug ("WouldAccept - Yes");
+				#endif
+				return true;
+			}
+			if (!parent_rule.would_reduce (token, state)) {
+				#if VERY_HARD_DEBUG
+					debug ("WouldAccept - No");
+				#endif
+				return false;
+			}
+			offset--;
+			parent_rule = peek_rule (offset);
+			state = peek_state (offset);
+		}
+		#if VERY_HARD_DEBUG
+			debug ("WouldAccept - No");
+		#endif
+		return false;
+	}
+
+	public bool would_parent_reduce_to_rule (Token token, Rule rule) {
+		int offset = -2;
+		Rule? parent_rule = peek_rule (offset);
+		Object? state = peek_state (offset);
+		while (parent_rule != null) {
+			#if VERY_HARD_DEBUG
+				debug ("WouldReduce - Offset %d; Index %d: %s", offset,
+					rule_stack.size + offset, parent_rule.to_string (state));
+			#endif
+			if (!parent_rule.would_reduce (token, state)) {
+				break;
+			}
+			offset--;
+			parent_rule = peek_rule (offset);
+			state = peek_state (offset);
+		}
+		if ((parent_rule != null && parent_rule.would_accept_token (token, state))
+			|| (parent_rule == null && TokenType.EOF.matches (token))) {
+			#if VERY_HARD_DEBUG
+				debug ("WouldReduce - Yes");
+			#endif
+			return true;
+		}
+		#if VERY_HARD_DEBUG
+			debug ("WouldReduce - No");
+		#endif
+		return false;
+	}
+
+	public void warning (Token? token, string message) {
+		string error_message;
+
+		if (token != null) {
+			error_message = message + ": " + token.to_pretty_string ();
+		} else {
+			error_message = message;
+		}
+
+		_reporter.warning (_filename,
+		                   get_line (token),
+		                   get_start_column (token),
+		                   get_end_column (token),
+		                   _scanner.get_line_content (),
+		                   error_message);
+	}
+
+	public void error (Token? token, string message) throws ParserError {
+		string error_message;
+
+		if (token != null) {
+			error_message = message + ": " + token.to_pretty_string ();
+		} else {
+			error_message = message;
+		}
+
+		_reporter.error (_filename,
+						 get_line (token),
+		                 get_start_column (token),
+		                 get_end_column (token),
+		                 _scanner.get_line_content (),
+		                 error_message);
+
+		throw new ParserError.UNEXPECTED_TOKEN (error_message);
+	}
+
+	private int get_line (Token? token) {
+		if (token == null) {
+			token = _current_token;
+		}
+		return token.begin.line + _first_line;
+	}
+
+	private int get_start_column (Token? token) {
+		if (token == null) {
+			token = _current_token;
+		}
+		if (token.begin.line == 0) {
+			return token.begin.column + _first_column + 1;
+		} else {
+			return token.begin.column + 1;
+		}
+	}
+
+	private int get_end_column (Token? token) {
+		if (token == null) {
+			token = _current_token;
+		}
+		if (token.end.line == 0) {
+			return token.end.column + _first_column + 1;
+		} else {
+			return token.end.column + 1;
+		}
+	}
+
+#if DEBUG
+	private void log_error (string message) {
+		stderr.printf ("An error occured while parsing: %s\n", message);
+		stderr.printf ("\nDumping rule stack:\n");
+		for (int i = 0; i < rule_stack.size; i++) {
+			stderr.printf ("\t%2d: %s\n", i, rule_stack[i].to_string (rule_state_stack[i]));
+		}
+	}
+#endif
+}
diff --git a/libvaladoc/parser/parsercallback.vala b/libvaladoc/parser/parsercallback.vala
new file mode 100644
index 000000000..8014ee92a
--- /dev/null
+++ b/libvaladoc/parser/parsercallback.vala
@@ -0,0 +1,36 @@
+/* parsercallback.vala
+ *
+ * Copyright (C) 2008-2009 Florian Brosch, Didier Villevalois
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+ *
+ * Author:
+ * 	Didier 'Ptitjes Villevalois <ptitjes@free.fr>
+ */
+
+
+public interface Valadoc.ParserCallback {
+	public abstract Object? get_rule_state ();
+	public abstract void set_rule_state (Object state);
+
+	public abstract void push_rule (Rule rule);
+	public abstract void reduce ();
+
+	public abstract bool would_parent_accept_token (Token token);
+	public abstract bool would_parent_reduce_to_rule (Token token, Rule rule);
+
+	public abstract void warning (Token? token, string message);
+	public abstract void error (Token? token, string message) throws ParserError;
+}
diff --git a/libvaladoc/parser/rule.vala b/libvaladoc/parser/rule.vala
new file mode 100644
index 000000000..88be423c7
--- /dev/null
+++ b/libvaladoc/parser/rule.vala
@@ -0,0 +1,172 @@
+/* rule.vala
+ *
+ * Copyright (C) 2008-2009 Florian Brosch, Didier Villevalois
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+ *
+ * Author:
+ * 	Didier 'Ptitjes Villevalois <ptitjes@free.fr>
+ */
+
+
+public abstract class Valadoc.Rule : Object {
+
+	public static Rule seq (Object[] scheme) {
+		return new SequenceRule (scheme);
+	}
+
+	public static Rule one_of (Object[] scheme) {
+		return new OneOfRule (scheme);
+	}
+
+	public static Rule many (Object[] scheme) {
+		if (scheme.length == 1) {
+			return new ManyRule (scheme[0]);
+		} else {
+			return new ManyRule (new SequenceRule (scheme));
+		}
+	}
+
+	public static Rule option (Object[] scheme) {
+		if (scheme.length == 1) {
+			return new OptionalRule (scheme[0]);
+		} else {
+			return new OptionalRule (new SequenceRule (scheme));
+		}
+	}
+
+	protected Rule () {
+	}
+
+	private string? _name = null;
+	private Action _start_action;
+	private Action _reduce_action;
+	private Action _skip_action;
+
+	public string name { get { return _name; } }
+
+	public Rule set_name (string name) {
+		_name = name;
+		return this;
+	}
+
+	public delegate void Action () throws ParserError;
+
+	public Rule set_start (Action action) {
+		//TODO: Ownership Transfer
+		_start_action = () => { action (); };
+		return this;
+	}
+
+	public Rule set_reduce (Action action) {
+		//TODO: Ownership Transfer
+		_reduce_action = () => { action (); };
+		return this;
+	}
+
+	public Rule set_skip (Action action) {
+		//TODO: Ownership Transfer
+		_skip_action = () => { action (); };
+		return this;
+	}
+
+	public enum Forward {
+		NONE,
+		PARENT,
+		CHILD
+	}
+
+	public abstract bool is_optional ();
+	public abstract bool starts_with_token (Token token);
+	public abstract bool accept_token (Token token, ParserCallback parser, Rule.Forward forward) throws ParserError;
+	public abstract bool would_accept_token (Token token, Object? state);
+	public abstract bool would_reduce (Token token, Object? state);
+
+	public abstract string to_string (Object? state);
+
+	protected bool is_optional_rule (Object? scheme_element) {
+		Rule? scheme_rule = scheme_element as Rule;
+		if (scheme_rule != null) {
+			return scheme_rule.is_optional ();
+		}
+		return false;
+	}
+
+	protected bool has_start_token (Object? scheme_element, Token token) {
+		TokenType? scheme_token_type = scheme_element as TokenType;
+		if (scheme_token_type != null) {
+			return scheme_token_type.matches (token);
+		}
+		Rule? scheme_rule = scheme_element as Rule;
+		if (scheme_rule != null) {
+			return scheme_rule.starts_with_token (token);
+		}
+		return false;
+	}
+
+	protected bool try_to_apply (Object? scheme_element, Token token, ParserCallback parser,
+								 out bool handled) throws ParserError
+	{
+		#if VERY_HARD_DEBUG
+			{
+				TokenType? scheme_token = scheme_element as TokenType;
+				Rule? scheme_rule = scheme_element as Rule;
+				if (scheme_token != null) {
+					message ("TryToApply: token='%s'; scheme_token='%s'", token.to_string (),
+						scheme_token.to_string ());
+				} else if (scheme_rule != null) {
+					message ("TryToApply: token='%s'; scheme_rule='%s'", token.to_string (),
+						scheme_rule.to_string (parser.get_rule_state ()));
+				} else {
+					assert (scheme_element != null);
+				}
+			}
+		#endif
+		TokenType? scheme_token_type = scheme_element as TokenType;
+		if (scheme_token_type != null && scheme_token_type.matches (token)) {
+			scheme_token_type.do_action (token);
+			handled = true;
+			return true;
+		}
+		Rule? scheme_rule = scheme_element as Rule;
+		if (scheme_rule != null && scheme_rule.starts_with_token (token)) {
+			parser.push_rule (scheme_rule);
+			handled = false;
+			return true;
+		}
+
+		handled = false;
+		return false;
+	}
+
+	protected void do_start (ParserCallback parser) throws ParserError {
+		if (_start_action != null) {
+			_start_action ();
+		}
+	}
+
+	protected void do_reduce (ParserCallback parser) throws ParserError {
+		if (_reduce_action != null) {
+			_reduce_action ();
+		}
+		parser.reduce ();
+	}
+
+	protected void do_skip (ParserCallback parser) throws ParserError {
+		if (_skip_action != null) {
+			_skip_action ();
+		}
+	}
+}
diff --git a/libvaladoc/parser/scanner.vala b/libvaladoc/parser/scanner.vala
new file mode 100644
index 000000000..fd148efab
--- /dev/null
+++ b/libvaladoc/parser/scanner.vala
@@ -0,0 +1,37 @@
+/* scanner.vala
+ *
+ * Copyright (C) 2008-2009 Florian Brosch, Didier Villevalois
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+ *
+ * Author:
+ * 	Didier 'Ptitjes Villevalois <ptitjes@free.fr>
+ */
+
+
+public interface Valadoc.Scanner : Object {
+
+	public abstract void set_parser (Parser parser);
+
+	public abstract void reset ();
+
+	public abstract void scan (string content) throws ParserError;
+
+	public abstract void end () throws ParserError;
+
+	public abstract void stop ();
+
+	public abstract string get_line_content ();
+}
diff --git a/libvaladoc/parser/sequencerule.vala b/libvaladoc/parser/sequencerule.vala
new file mode 100644
index 000000000..e1e6797a4
--- /dev/null
+++ b/libvaladoc/parser/sequencerule.vala
@@ -0,0 +1,131 @@
+/* sequencerule.vala
+ *
+ * Copyright (C) 2008-2009 Florian Brosch, Didier Villevalois
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+ *
+ * Author:
+ * 	Didier 'Ptitjes Villevalois <ptitjes@free.fr>
+ */
+
+
+internal class Valadoc.SequenceRule : Rule {
+
+	public SequenceRule (Object[] scheme) {
+		_scheme = scheme;
+	}
+
+	private Object[] _scheme;
+
+	private class State : Object {
+		public int index = 0;
+	}
+
+	public override bool is_optional () {
+		return false;
+	}
+
+	public override bool starts_with_token (Token token) {
+		return test_token (0, token);
+	}
+
+	private bool test_token (int from_index, Token token) {
+		int i = from_index;
+		while (i < _scheme.length) {
+			if (has_start_token (_scheme[i], token)) {
+				return true;
+			}
+			if (!is_optional_rule (_scheme[i])) {
+				break;
+			}
+			i++;
+		}
+		return false;
+	}
+
+	private bool test_reduce (int from_index, Token token) {
+		int i = from_index;
+		while (i < _scheme.length) {
+			if (!is_optional_rule (_scheme[i])) {
+				return false;
+			}
+			i++;
+		}
+		return true;
+	}
+
+	public override bool accept_token (Token token, ParserCallback parser,
+									   Rule.Forward forward) throws ParserError
+	{
+		var state = parser.get_rule_state () as State;
+		if (state == null) {
+			state = new State ();
+			parser.set_rule_state (state);
+		}
+
+		if (state.index == 0) {
+			do_start (parser);
+		} else if (state.index == _scheme.length) {
+			do_reduce (parser);
+			return false;
+		}
+
+		Object? scheme_element = null;
+		bool handled;
+		do {
+			scheme_element = _scheme[state.index];
+			if (try_to_apply (scheme_element, token, parser, out handled)) {
+				state.index++;
+				return handled;
+			}
+			if (!is_optional_rule (scheme_element)) {
+				break;
+			} else {
+				((Rule) scheme_element).do_skip (parser);
+			}
+			state.index++;
+		} while (state.index < _scheme.length);
+
+		if (state.index == _scheme.length) {
+			do_reduce (parser);
+			return false;
+		}
+
+		if (scheme_element is TokenType) {
+			parser.error (token, "expected %s".printf (((TokenType) scheme_element).to_pretty_string ()));
+		} else {
+			parser.error (token, "unexpected token");
+		}
+		assert_not_reached ();
+	}
+
+	public override bool would_accept_token (Token token, Object? rule_state) {
+		var state = rule_state as State;
+		return test_token (state.index, token);
+	}
+
+	public override bool would_reduce (Token token, Object? rule_state) {
+		var state = rule_state as State;
+		return state.index == _scheme.length || test_reduce (state.index, token);
+	}
+
+	public override string to_string (Object? rule_state) {
+		var state = rule_state as State;
+		if (state == null) {
+			state = new State ();
+		}
+		return "%-15s%-15s(index=%d/%d)".printf (name != null ? name : " ", "[seq]", state.index, _scheme.length);
+	}
+}
diff --git a/libvaladoc/parser/sourcelocation.vala b/libvaladoc/parser/sourcelocation.vala
new file mode 100644
index 000000000..51b5bc1f9
--- /dev/null
+++ b/libvaladoc/parser/sourcelocation.vala
@@ -0,0 +1,35 @@
+/* sourcelocation.vala
+ *
+ * Copyright (C) 2008  Jürg Billeter
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+ *
+ * Author:
+ * 	Jürg Billeter <j@bitron.ch>
+ */
+
+
+/**
+ * Represents a position in a source file.
+ */
+public struct Valadoc.SourceLocation {
+	public int line;
+	public int column;
+
+	public SourceLocation (int _line, int _column) {
+		line = _line;
+		column = _column;
+	}
+}
diff --git a/libvaladoc/parser/stubrule.vala b/libvaladoc/parser/stubrule.vala
new file mode 100644
index 000000000..73f06655a
--- /dev/null
+++ b/libvaladoc/parser/stubrule.vala
@@ -0,0 +1,61 @@
+/* stubrule.vala
+ *
+ * Copyright (C) 2008-2009 Florian Brosch, Didier Villevalois
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+ *
+ * Author:
+ * 	Didier 'Ptitjes Villevalois <ptitjes@free.fr>
+ */
+
+
+public class Valadoc.StubRule : Rule {
+
+	public StubRule () {
+	}
+
+	private Rule _rule;
+
+	public Rule set_rule (Rule rule) {
+		_rule = rule;
+		return this;
+	}
+
+	public override bool is_optional () {
+		return _rule.is_optional ();
+	}
+
+	public override bool starts_with_token (Token token) {
+		return _rule.starts_with_token (token);
+	}
+
+	public override bool accept_token (Token token, ParserCallback parser, Rule.Forward forward)
+									   throws ParserError
+	{
+		return _rule.accept_token (token, parser, forward);
+	}
+
+	public override bool would_accept_token (Token token, Object? state) {
+		return _rule.would_accept_token (token, state);
+	}
+
+	public override bool would_reduce (Token token, Object? state) {
+		return _rule.would_reduce (token, state);
+	}
+
+	public override string to_string (Object? state) {
+		return _rule.to_string (state);
+	}
+}
diff --git a/libvaladoc/parser/token.vala b/libvaladoc/parser/token.vala
new file mode 100644
index 000000000..52833b24e
--- /dev/null
+++ b/libvaladoc/parser/token.vala
@@ -0,0 +1,109 @@
+/* token.vala
+ *
+ * Copyright (C) 2008-2009 Florian Brosch, Didier Villevalois
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+ *
+ * Author:
+ * 	Didier 'Ptitjes Villevalois <ptitjes@free.fr>
+ */
+
+
+public class Valadoc.Token : Object {
+
+	public Token.from_type (TokenType type, SourceLocation begin, SourceLocation end, string? val = null) {
+		_type = type;
+		_begin = begin;
+		_end = end;
+		_value = val;
+	}
+
+	public Token.from_word (string word, SourceLocation begin, SourceLocation end) {
+		_word = word;
+		_begin = begin;
+		_end = end;
+	}
+
+	private TokenType? _type = null;
+	private string? _word = null;
+	private SourceLocation _begin;
+	private SourceLocation _end;
+	private string? _value;
+
+	public bool is_word {
+		get {
+			return _word != null;
+		}
+	}
+
+	public bool is_number {
+		get {
+			if (_word == null || _word.length == 0) {
+				return false;
+			} else if (_word[0] == '0' && _word.length > 1) {
+				return false;
+			}
+			for (int i = 0; i < _word.length; i++) {
+				if (_word[i] < '0' || _word[i] > '9') {
+					return false;
+				}
+			}
+			return true;
+		}
+	}
+
+	public string? word {
+		get {
+			return _word;
+		}
+	}
+
+	public string? value {
+		get {
+			return _value;
+		}
+	}
+
+	public TokenType? token_type {
+		get {
+			return _type;
+		}
+	}
+
+	public SourceLocation begin {
+		get {
+			return _begin;
+		}
+	}
+
+	public SourceLocation end {
+		get {
+			return _end;
+		}
+	}
+
+	public string to_string () {
+		return _word == null ? _type.to_string () : _word;
+	}
+
+	public string to_pretty_string () {
+		return _word == null ? _type.to_pretty_string () : _word;
+	}
+
+	public int to_int () {
+		assert (is_number);
+		return int.parse (_word);
+	}
+}
diff --git a/libvaladoc/parser/tokentype.vala b/libvaladoc/parser/tokentype.vala
new file mode 100644
index 000000000..4a51d02ad
--- /dev/null
+++ b/libvaladoc/parser/tokentype.vala
@@ -0,0 +1,270 @@
+/* tokentype.vala
+ *
+ * Copyright (C) 2008-2009 Florian Brosch, Didier Villevalois
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Lesser General Public
+ * License as published by the Free Software Foundation; either
+ * version 2.1 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+ * Lesser General Public License for more details.
+ *
+ * You should have received a copy of the GNU Lesser General Public
+ * License along with this library; if not, write to the Free Software
+ * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301  USA
+ *
+ * Author:
+ * 	Didier 'Ptitjes Villevalois <ptitjes@free.fr>
+ */
+
+
+public class Valadoc.TokenType : Object {
+	// valadoc-comments:
+	public static TokenType ANY;
+	public static TokenType ANY_WORD;
+	public static TokenType ANY_NUMBER;
+	public static TokenType EOF;
+	public static TokenType EOL;
+	public static TokenType BREAK;
+ 	public static TokenType AROBASE;
+	public static TokenType SPACE;
+	public static TokenType TAB;
+	public static TokenType EQUAL_1;
+	public static TokenType EQUAL_2;
+	public static TokenType EQUAL_3;
+	public static TokenType EQUAL_4;
+	public static TokenType EQUAL_5;
+	public static TokenType MINUS;
+	public static TokenType LESS_THAN;
+	public static TokenType GREATER_THAN;
+	public static TokenType ALIGN_TOP;
+	public static TokenType ALIGN_BOTTOM;
+	public static TokenType SINGLE_QUOTE_2;
+	public static TokenType SLASH_2;
+	public static TokenType UNDERSCORE_2;
+	public static TokenType BACK_QUOTE_2;
+	public static TokenType OPEN_BRACE;
+	public static TokenType CLOSED_BRACE;
+	public static TokenType DOUBLE_OPEN_BRACE;
+	public static TokenType DOUBLE_CLOSED_BRACE;
+	public static TokenType TRIPLE_OPEN_BRACE;
+	public static TokenType TRIPLE_CLOSED_BRACE;
+	public static TokenType DOUBLE_OPEN_BRACKET;
+	public static TokenType DOUBLE_CLOSED_BRACKET;
+	public static TokenType PIPE;
+	public static TokenType DOUBLE_PIPE;
+	public static TokenType ALIGN_RIGHT;
+	public static TokenType ALIGN_CENTER;
+
+
+	// .valadoc (importer)
+	public static TokenType VALADOC_COMMENT_START;
+	public static TokenType VALADOC_COMMENT_END;
+	public static TokenType VALADOC_ANY_WORD;
+	public static TokenType VALADOC_SPACE;
+	public static TokenType VALADOC_TAB;
+	public static TokenType VALADOC_EOL;
+
+
+	// markdown:
+	public static TokenType MARKDOWN_PARAGRAPH;
+	public static TokenType MARKDOWN_ANY_WORD;
+	public static TokenType MARKDOWN_SPACE;
+	public static TokenType MARKDOWN_SOURCE;
+	public static TokenType MARKDOWN_PARAMETER;
+	public static TokenType MARKDOWN_CONSTANT;
+	public static TokenType MARKDOWN_SYMBOL;
+	public static TokenType MARKDOWN_LOCAL_GMEMBER;
+	public static TokenType MARKDOWN_FUNCTION;
+	public static TokenType MARKDOWN_MAIL;
+	public static TokenType MARKDOWN_LINK;
+	public static TokenType MARKDOWN_GREATER_THAN;
+	public static TokenType MARKDOWN_LESS_THAN;
+	public static TokenType MARKDOWN_OPEN_BRACKET;
+	public static TokenType MARKDOWN_CLOSE_BRACKET;
+	public static TokenType MARKDOWN_OPEN_PARENS;
+	public static TokenType MARKDOWN_CLOSE_PARENS;
+	public static TokenType MARKDOWN_EXCLAMATION_MARK;
+	public static TokenType MARKDOWN_HEADLINE_1;
+	public static TokenType MARKDOWN_HEADLINE_2;
+	public static TokenType MARKDOWN_HEADLINE_HASH;
+	public static TokenType MARKDOWN_HEADLINE_END;
+	public static TokenType MARKDOWN_UNORDERED_LIST_ITEM_START;
+	public static TokenType MARKDOWN_UNORDERED_LIST_ITEM_END;
+	public static TokenType MARKDOWN_ORDERED_LIST_ITEM_START;
+	public static TokenType MARKDOWN_ORDERED_LIST_ITEM_END;
+	public static TokenType MARKDOWN_BLOCK_START;
+	public static TokenType MARKDOWN_BLOCK_END;
+	public static TokenType MARKDOWN_EOC;
+
+
+	private static bool initialized = false;
+
+	internal static void init_token_types () {
+		if (!initialized) {
+			// valadoc-comments:
+			ANY = new TokenType.basic ("<any>");
+			ANY_WORD = new TokenType.basic ("<any-word>");
+			ANY_NUMBER = new TokenType.basic ("<any-number>");
+			EOF = new TokenType.basic ("\0", "<end-of-file>");
+			EOL = new TokenType.basic ("\n", "<end-of-line>");
+			BREAK = new TokenType.basic ("<<BR>>");
+			AROBASE = new TokenType.basic ("@");
+			SPACE = new TokenType.basic (" ", "<space>");
+			TAB = new TokenType.basic ("\t", "<tab>");
+			EQUAL_1 = new TokenType.basic ("=");
+			EQUAL_2 = new TokenType.basic ("==");
+			EQUAL_3 = new TokenType.basic ("====");
+			EQUAL_4 = new TokenType.basic ("=====");
+			EQUAL_5 = new TokenType.basic ("======");
+			MINUS = new TokenType.basic ("-");
+			LESS_THAN = new TokenType.basic ("<");
+			GREATER_THAN = new TokenType.basic (">");
+			ALIGN_TOP = new TokenType.basic ("^");
+			ALIGN_BOTTOM = new TokenType.basic ("v");
+			SINGLE_QUOTE_2 = new TokenType.basic ("''");
+			SLASH_2 = new TokenType.basic ("//");
+			UNDERSCORE_2 = new TokenType.basic ("__");
+			BACK_QUOTE_2 = new TokenType.basic ("``");
+			OPEN_BRACE = new TokenType.basic ("{");
+			CLOSED_BRACE = new TokenType.basic ("}");
+			DOUBLE_OPEN_BRACE = new TokenType.basic ("{{");
+			DOUBLE_CLOSED_BRACE = new TokenType.basic ("}}");
+			TRIPLE_OPEN_BRACE = new TokenType.basic ("{{{");
+			TRIPLE_CLOSED_BRACE = new TokenType.basic ("}}}");
+			DOUBLE_OPEN_BRACKET = new TokenType.basic ("[[");
+			DOUBLE_CLOSED_BRACKET = new TokenType.basic ("]]");
+			PIPE = new TokenType.basic ("|");
+			DOUBLE_PIPE = new TokenType.basic ("||");
+			ALIGN_RIGHT = new TokenType.basic ("))");
+			ALIGN_CENTER = new TokenType.basic (")(");
+
+			// .valadoc (importer)
+			VALADOC_COMMENT_START = new TokenType.basic ("/*");
+			VALADOC_COMMENT_END = new TokenType.basic ("*/");
+			VALADOC_ANY_WORD = ANY_WORD;
+			VALADOC_SPACE = SPACE;
+			VALADOC_TAB = TAB;
+			VALADOC_EOL = EOL;
+
+			initialized = true;
+
+
+			// Markdown: (importer)
+			MARKDOWN_PARAGRAPH = new TokenType.basic ("<paragraph>");
+			MARKDOWN_BLOCK_START = new TokenType.basic ("<block>");
+			MARKDOWN_BLOCK_END = new TokenType.basic ("</block>");
+			MARKDOWN_UNORDERED_LIST_ITEM_START = new TokenType.basic ("<unordered-list>");
+			MARKDOWN_UNORDERED_LIST_ITEM_END = new TokenType.basic ("</unordered-list>");
+			MARKDOWN_ORDERED_LIST_ITEM_START = new TokenType.basic ("<ordered-list>");
+			MARKDOWN_ORDERED_LIST_ITEM_END  = new TokenType.basic ("</ordered-list>");
+
+			MARKDOWN_HEADLINE_1 = new TokenType.basic ("<headline-1>");
+			MARKDOWN_HEADLINE_2 = new TokenType.basic ("<headline-2>");
+			MARKDOWN_HEADLINE_HASH = new TokenType.basic ("<hash>");
+			MARKDOWN_HEADLINE_END = new TokenType.basic ("</headline>");
+			MARKDOWN_SOURCE = new TokenType.basic ("<source>");
+			MARKDOWN_PARAMETER = new TokenType.basic ("<parameter>");
+			MARKDOWN_CONSTANT = new TokenType.basic ("<constant>");
+			MARKDOWN_FUNCTION = new TokenType.basic ("<function>");
+			MARKDOWN_SYMBOL = new TokenType.basic ("<symbol>");
+			MARKDOWN_LOCAL_GMEMBER = new TokenType.basic ("<local-gmember>");
+			MARKDOWN_MAIL = new TokenType.basic ("<mail>");
+			MARKDOWN_LINK = new TokenType.basic ("<link>");
+
+			MARKDOWN_OPEN_BRACKET = new TokenType.basic ("[");
+			MARKDOWN_CLOSE_BRACKET = new TokenType.basic ("]");
+			MARKDOWN_OPEN_PARENS = new TokenType.basic ("(");
+			MARKDOWN_CLOSE_PARENS = new TokenType.basic (")");
+			MARKDOWN_EXCLAMATION_MARK = new TokenType.basic ("!");
+			MARKDOWN_GREATER_THAN = GREATER_THAN;
+			MARKDOWN_LESS_THAN = LESS_THAN;
+
+			MARKDOWN_ANY_WORD = ANY_WORD;
+			MARKDOWN_SPACE = SPACE;
+			MARKDOWN_EOC = EOL;
+		}
+	}
+
+	private static int EXACT_WORD = -1;
+
+	public static TokenType str (string str) {
+		return new TokenType (str, EXACT_WORD, null);
+	}
+
+	public static TokenType any () {
+		return ANY;
+	}
+
+	public static TokenType any_word () {
+		return ANY_WORD;
+	}
+
+	public static TokenType any_number () {
+		return ANY_NUMBER;
+	}
+
+	private TokenType (string string_value, int basic_value, Action? __action) {
+		_string_value = string_value;
+		_basic_value = basic_value;
+		if (__action != null) {
+			_action = (token) => { __action (token); };
+		} else {
+			_action = null;
+		}
+	}
+
+	private TokenType.basic (string string_value, string? pretty_string = null) {
+		_string_value = string_value;
+		_pretty_string = pretty_string;
+		_basic_value = ++_last_basic_value;
+	}
+
+	private static int _last_basic_value = -1;
+
+	private string _string_value;
+	private string? _pretty_string;
+	private int _basic_value = -1;
+	private Action? _action = null;
+
+	public delegate void Action (Token token) throws ParserError;
+
+	public TokenType action (Action action) {
+		return new TokenType (_string_value, _basic_value, action);
+	}
+
+	public void do_action (Token matched_token) throws ParserError {
+		if (_action != null) {
+			_action (matched_token);
+		}
+	}
+
+	public bool matches (Token token) {
+		if (_basic_value == ANY._basic_value) {
+			return true;
+		} else if (_basic_value == ANY_WORD._basic_value && token.is_word) {
+			return true;
+		} else if (_basic_value == ANY_NUMBER._basic_value && token.is_number) {
+			return true;
+		} else if (_basic_value == EXACT_WORD && token.is_word && token.word == _string_value) {
+			return true;
+		} else if (token.token_type != null && token.token_type._basic_value == _basic_value) {
+			return true;
+		}
+		return false;
+	}
+
+	public string to_string () {
+		return _string_value;
+	}
+
+	public string to_pretty_string () {
+		if (_pretty_string != null) {
+			return _pretty_string;
+		}
+		return _string_value;
+	}
+}