moved common regular language definitions to host-specific grammars

There are common regular language definitions in ragel.lm that are used in the
host-specific lexers. These need to move out into the host code so we can begin
to specialize the lexers for the different host languages.

1 files changed, 32 insertions, 2 deletions

diff --git a/src/rlparse.lm b/src/rlparse.lm
index 02a937c5..f0003be0 100644
--- a/src/rlparse.lm
+++ b/src/rlparse.lm
@@ -1,6 +1,38 @@
 include 'ragel.lm'
 include 'rlreduce.lm'
 
+rl ident
+	/( alpha | '_' ) ( alpha | digit | '_' )*/
+
+rl number
+	/ digit+ /
+
+rl hex_number
+	/ '0x' [0-9a-fA-F]+ /
+
+rl hex_char
+	/ '0x' [0-9a-fA-F]{2}  /
+
+rl NL / '\n' /
+
+rl c_comment 
+	/ '/*' ( any | NL )* :>> '*/' /
+
+rl cpp_comment
+	/ '//' [^\n]* NL /
+
+rl ruby_comment
+	/ '#' [^\n]* NL /
+
+rl s_literal
+	/ "'" ([^'\\\n] | '\\' (any | NL))* "'" /
+
+rl d_literal
+	/ '"' ([^"\\] | NL | '\\' (any | NL))* '"' /
+
+rl host_re_literal
+	/ '/' ([^/\\] | NL | '\\' (any | NL))* '/' /
+
 namespace inline
 	lex 
 		literal `fpc `fc  `fcurs `ftargs
@@ -44,10 +76,8 @@ namespace inline
 		token c_any
 			/ any /
 	end
-
 end
 
-
 namespace host
 	lex 
 		literal `%%{