testing a token redeftoken-redef

7 files changed, 51 insertions, 13 deletions

diff --git a/src/colm.lm b/src/colm.lm
index 16a9eeb5..bab4f77a 100644
--- a/src/colm.lm
+++ b/src/colm.lm
@@ -214,6 +214,7 @@ def root_item
 	[rl_def] :Rl commit
 |	[literal_def] :Literal commit
 |	[token_def] :Token commit
+|	[token_redef] :TokenRedef commit
 |	[ic_def] :IgnoreCollector commit
 |	[ignore_def] :Ignore commit
 |	[cfl_def] :CflDef commit
@@ -467,6 +468,13 @@ def token_def
 		no_ignore_right
 		opt_translate]
 
+def token_redef
+	[REDEF TOKEN id VarDefList: var_def<*
+		no_ignore_left
+		LEX_FSLASH opt_lex_expr LEX_FSLASH
+		no_ignore_right
+		opt_translate]
+
 def ic_def
 	[TOKEN id MINUS]
 
diff --git a/src/consinit.cc b/src/consinit.cc
index 50775084..bed6a636 100644
--- a/src/consinit.cc
+++ b/src/consinit.cc
@@ -229,7 +229,7 @@ void ConsInit::wsIgnore()
 	LexExpression *expr = LexExpression::cons( term );
 	LexJoin *join = LexJoin::cons( expr );
 
-	defineToken( internal, String(), join, objectDef, 0, true, false, false );
+	defineToken( internal, false, String(), join, objectDef, 0, true, false, false );
 }
 
 void ConsInit::commentIgnore()
@@ -249,7 +249,7 @@ void ConsInit::commentIgnore()
 
 	LexJoin *join = LexJoin::cons( expr );
 
-	defineToken( internal, String(), join, objectDef, 0, true, false, false );
+	defineToken( internal, false, String(), join, objectDef, 0, true, false, false );
 }
 
 void ConsInit::idToken()
@@ -275,7 +275,7 @@ void ConsInit::idToken()
 	LexExpression *expr = LexExpression::cons( concat );
 	LexJoin *join = LexJoin::cons( expr );
 
-	defineToken( internal, hello, join, objectDef, 0, false, false, false );
+	defineToken( internal, false, hello, join, objectDef, 0, false, false, false );
 }
 
 void ConsInit::literalToken()
@@ -308,7 +308,7 @@ void ConsInit::literalToken()
 	LexExpression *expr = LexExpression::cons( concat );
 	LexJoin *join = LexJoin::cons( expr );
 
-	defineToken( internal, hello, join, objectDef, 0, false, false, false );
+	defineToken( internal, false, hello, join, objectDef, 0, false, false, false );
 }
 
 void ConsInit::keyword( const String &name, const String &lit )
@@ -317,7 +317,7 @@ void ConsInit::keyword( const String &name, const String &lit )
 	LexTerm *term = litTerm( lit );
 	LexExpression *expr = LexExpression::cons( term );
 	LexJoin *join = LexJoin::cons( expr );
-	defineToken( internal, name, join, objectDef, 0, false, false, false );
+	defineToken( internal, false, name, join, objectDef, 0, false, false, false );
 }
 
 void ConsInit::keyword( const String &kw )
diff --git a/src/loadfinal.cc b/src/loadfinal.cc
index d478a446..84f85368 100644
--- a/src/loadfinal.cc
+++ b/src/loadfinal.cc
@@ -397,10 +397,33 @@ struct LoadColm
 
 		CodeBlock *translate = walkOptTranslate( TokenDef.opt_translate() );
 
-		defineToken( TokenDef.id().loc(), name, join, objectDef,
+		defineToken( TokenDef.id().loc(), false, name, join, objectDef,
 				translate, false, niLeft, niRight );
 	}
 
+	void walkTokenRedef( token_redef TokenRedef )
+	{
+		String name = TokenRedef.id().data();
+
+		bool niLeft = walkNoIgnoreLeft( TokenRedef.no_ignore_left() );
+		bool niRight = walkNoIgnoreRight( TokenRedef.no_ignore_right() );
+
+		ObjectDef *objectDef = walkVarDefList( TokenRedef.VarDefList() );
+		objectDef->name = name;
+
+		LexJoin *join = 0;
+		if ( TokenRedef.opt_lex_expr().lex_expr() != 0 ) {
+			LexExpression *expr = walkLexExpr( TokenRedef.opt_lex_expr().lex_expr() );
+			join = LexJoin::cons( expr );
+		}
+
+		CodeBlock *translate = walkOptTranslate( TokenRedef.opt_translate() );
+
+		defineToken( TokenRedef.id().loc(), true, name, join, objectDef,
+				translate, false, niLeft, niRight );
+	}
+
+
 	void walkIgnoreCollector( ic_def IgnoreCollector )
 	{
 		String id = IgnoreCollector.id().data();
@@ -452,7 +475,7 @@ struct LoadColm
 			join = LexJoin::cons( expr );
 		}
 
-		defineToken( IgnoreDef.IGNORE().loc(), name, join, objectDef,
+		defineToken( IgnoreDef.IGNORE().loc(), false, name, join, objectDef,
 				0, true, false, false );
 	}
 
@@ -2771,6 +2794,9 @@ struct LoadColm
 		case root_item::Token:
 			walkTokenDef( rootItem.token_def() );
 			break;
+		case root_item::TokenRedef:
+			walkTokenRedef( rootItem.token_redef() );
+			break;
 		case root_item::IgnoreCollector:
 			walkIgnoreCollector( rootItem.ic_def() );
 			break;
diff --git a/src/loadinit.cc b/src/loadinit.cc
index d9a047d3..045345f4 100644
--- a/src/loadinit.cc
+++ b/src/loadinit.cc
@@ -237,7 +237,7 @@ void LoadInit::walkTokenList( token_list &tokenList )
 		bool leftNi = walkNoIgnore( tokenDef.LeftNi() );
 		bool rightNi = walkNoIgnore( tokenDef.RightNi() );
 
-		defineToken( internal, name, join, objectDef, 0, false, leftNi, rightNi );
+		defineToken( internal, false, name, join, objectDef, 0, false, leftNi, rightNi );
 	}
 
 	if ( tokenList.IgnoreDef() != 0 ) {
@@ -249,7 +249,7 @@ void LoadInit::walkTokenList( token_list &tokenList )
 		LexExpression *expr = walkLexExpr( LexExpr );
 		LexJoin *join = LexJoin::cons( expr );
 
-		defineToken( internal, String(), join, objectDef, 0, true, false, false );
+		defineToken( internal, false, String(), join, objectDef, 0, true, false, false );
 	}
 }
 
diff --git a/src/parser.cc b/src/parser.cc
index ce598e38..9009c0e9 100644
--- a/src/parser.cc
+++ b/src/parser.cc
@@ -297,8 +297,8 @@ LexJoin *BaseParser::literalJoin( const InputLoc &loc, const String &data )
 	return join;
 }
 
-void BaseParser::defineToken( const InputLoc &loc, String name, LexJoin *join,
-		ObjectDef *objectDef, CodeBlock *transBlock, bool ignore,
+void BaseParser::defineToken( const InputLoc &loc, bool isRedef, String name,
+		LexJoin *join, ObjectDef *objectDef, CodeBlock *transBlock, bool ignore,
 		bool noPreIgnore, bool noPostIgnore )
 {
 	bool pushedRegion = false;
@@ -323,6 +323,8 @@ void BaseParser::defineToken( const InputLoc &loc, String name, LexJoin *join,
 
 	TokenDef *tokenDef = TokenDef::cons( name, String(), false, ignore, join, 
 			transBlock, loc, 0, nspace, regionSet, objectDef, curStruct() );
+	
+	tokenDef->isRedef = isRedef;
 
 	regionSet->tokenDefList.append( tokenDef );
 	nspace->tokenDefList.append( tokenDef );
diff --git a/src/parser.h b/src/parser.h
index 98956c0a..b55cc7bf 100644
--- a/src/parser.h
+++ b/src/parser.h
@@ -92,8 +92,8 @@ struct BaseParser
 
 	Reduction *createReduction( const InputLoc loc, const String &name );
 
-	void defineToken( const InputLoc &loc, String name, LexJoin *join,
-			ObjectDef *objectDef, CodeBlock *transBlock,
+	void defineToken( const InputLoc &loc, bool isRedef, String name,
+			LexJoin *join, ObjectDef *objectDef, CodeBlock *transBlock,
 			bool ignore, bool noPreIgnore, bool noPostIgnore );
 
 	void zeroDef( const InputLoc &loc, const String &name );
diff --git a/src/parsetree.h b/src/parsetree.h
index 1890bbae..bb63b7a2 100644
--- a/src/parsetree.h
+++ b/src/parsetree.h
@@ -404,6 +404,7 @@ struct TokenDef
 		t->noPostIgnore = false;
 		t->noPreIgnore = false;
 		t->isZero = false;
+		t->isRedef = false;
 
 		return t;
 	}
@@ -436,6 +437,7 @@ struct TokenDef
 	bool noPostIgnore;
 	bool noPreIgnore;
 	bool isZero;
+	bool isRedef;
 };
 
 struct TokenInstancePtr