moved common regular language definitions to host-specific grammars

There are common regular language definitions in ragel.lm that are used in the
host-specific lexers. These need to move out into the host code so we can begin
to specialize the lexers for the different host languages.

14 files changed, 404 insertions, 34 deletions

diff --git a/src/host-asm/rlparse.lm b/src/host-asm/rlparse.lm
index 911e3ea6..1951ac0f 100644
--- a/src/host-asm/rlparse.lm
+++ b/src/host-asm/rlparse.lm
@@ -1,6 +1,32 @@
 include 'ragel.lm'
 include 'rlreduce.lm'
 
+rl ident
+	/( alpha | '_' ) ( alpha | digit | '_' )*/
+
+rl number
+	/ digit+ /
+
+rl hex_number
+	/ '0x' [0-9a-fA-F]+ /
+
+rl hex_char
+	/ '0x' [0-9a-fA-F]{2}  /
+
+rl NL / '\n' /
+
+rl c_comment 
+	/ '/*' ( any | NL )* :>> '*/' /
+
+rl cpp_comment
+	/ '//' [^\n]* NL /
+
+rl s_literal
+	/ "'" ([^'\\\n] | '\\' (any | NL))* "'" /
+
+rl d_literal
+	/ '"' ([^"\\] | NL | '\\' (any | NL))* '"' /
+
 namespace inline
 	lex 
 		literal `fpc `fc  `fcurs `ftargs
diff --git a/src/host-c/rlparse.lm b/src/host-c/rlparse.lm
index 0fd61bbf..ee07d736 100644
--- a/src/host-c/rlparse.lm
+++ b/src/host-c/rlparse.lm
@@ -1,6 +1,32 @@
 include 'ragel.lm'
 include 'rlreduce.lm'
 
+rl ident
+	/( alpha | '_' ) ( alpha | digit | '_' )*/
+
+rl number
+	/ digit+ /
+
+rl hex_number
+	/ '0x' [0-9a-fA-F]+ /
+
+rl hex_char
+	/ '0x' [0-9a-fA-F]{2}  /
+
+rl NL / '\n' /
+
+rl c_comment 
+	/ '/*' ( any | NL )* :>> '*/' /
+
+rl cpp_comment
+	/ '//' [^\n]* NL /
+
+rl s_literal
+	/ "'" ([^'\\\n] | '\\' (any | NL))* "'" /
+
+rl d_literal
+	/ '"' ([^"\\] | NL | '\\' (any | NL))* '"' /
+
 namespace inline
 	lex 
 		literal `fpc `fc  `fcurs `ftargs
diff --git a/src/host-crack/rlparse.lm b/src/host-crack/rlparse.lm
index 100a9223..098cb269 100644
--- a/src/host-crack/rlparse.lm
+++ b/src/host-crack/rlparse.lm
@@ -1,6 +1,32 @@
 include 'ragel.lm'
 include 'rlreduce.lm'
 
+rl ident
+	/( alpha | '_' ) ( alpha | digit | '_' )*/
+
+rl number
+	/ digit+ /
+
+rl hex_number
+	/ '0x' [0-9a-fA-F]+ /
+
+rl hex_char
+	/ '0x' [0-9a-fA-F]{2}  /
+
+rl NL / '\n' /
+
+rl c_comment 
+	/ '/*' ( any | NL )* :>> '*/' /
+
+rl cpp_comment
+	/ '//' [^\n]* NL /
+
+rl s_literal
+	/ "'" ([^'\\\n] | '\\' (any | NL))* "'" /
+
+rl d_literal
+	/ '"' ([^"\\] | NL | '\\' (any | NL))* '"' /
+
 namespace inline
 	lex 
 		literal `fpc `fc  `fcurs `ftargs
diff --git a/src/host-csharp/rlparse.lm b/src/host-csharp/rlparse.lm
index 100a9223..098cb269 100644
--- a/src/host-csharp/rlparse.lm
+++ b/src/host-csharp/rlparse.lm
@@ -1,6 +1,32 @@
 include 'ragel.lm'
 include 'rlreduce.lm'
 
+rl ident
+	/( alpha | '_' ) ( alpha | digit | '_' )*/
+
+rl number
+	/ digit+ /
+
+rl hex_number
+	/ '0x' [0-9a-fA-F]+ /
+
+rl hex_char
+	/ '0x' [0-9a-fA-F]{2}  /
+
+rl NL / '\n' /
+
+rl c_comment 
+	/ '/*' ( any | NL )* :>> '*/' /
+
+rl cpp_comment
+	/ '//' [^\n]* NL /
+
+rl s_literal
+	/ "'" ([^'\\\n] | '\\' (any | NL))* "'" /
+
+rl d_literal
+	/ '"' ([^"\\] | NL | '\\' (any | NL))* '"' /
+
 namespace inline
 	lex 
 		literal `fpc `fc  `fcurs `ftargs
diff --git a/src/host-d/rlparse.lm b/src/host-d/rlparse.lm
index 05b21ea2..80ef040d 100644
--- a/src/host-d/rlparse.lm
+++ b/src/host-d/rlparse.lm
@@ -1,6 +1,32 @@
 include 'ragel.lm'
 include 'rlreduce.lm'
 
+rl ident
+	/( alpha | '_' ) ( alpha | digit | '_' )*/
+
+rl number
+	/ digit+ /
+
+rl hex_number
+	/ '0x' [0-9a-fA-F]+ /
+
+rl hex_char
+	/ '0x' [0-9a-fA-F]{2}  /
+
+rl NL / '\n' /
+
+rl c_comment 
+	/ '/*' ( any | NL )* :>> '*/' /
+
+rl cpp_comment
+	/ '//' [^\n]* NL /
+
+rl s_literal
+	/ "'" ([^'\\\n] | '\\' (any | NL))* "'" /
+
+rl d_literal
+	/ '"' ([^"\\] | NL | '\\' (any | NL))* '"' /
+
 namespace inline
 	lex 
 		literal `fpc `fc  `fcurs `ftargs
diff --git a/src/host-go/rlparse.lm b/src/host-go/rlparse.lm
index 100a9223..098cb269 100644
--- a/src/host-go/rlparse.lm
+++ b/src/host-go/rlparse.lm
@@ -1,6 +1,32 @@
 include 'ragel.lm'
 include 'rlreduce.lm'
 
+rl ident
+	/( alpha | '_' ) ( alpha | digit | '_' )*/
+
+rl number
+	/ digit+ /
+
+rl hex_number
+	/ '0x' [0-9a-fA-F]+ /
+
+rl hex_char
+	/ '0x' [0-9a-fA-F]{2}  /
+
+rl NL / '\n' /
+
+rl c_comment 
+	/ '/*' ( any | NL )* :>> '*/' /
+
+rl cpp_comment
+	/ '//' [^\n]* NL /
+
+rl s_literal
+	/ "'" ([^'\\\n] | '\\' (any | NL))* "'" /
+
+rl d_literal
+	/ '"' ([^"\\] | NL | '\\' (any | NL))* '"' /
+
 namespace inline
 	lex 
 		literal `fpc `fc  `fcurs `ftargs
diff --git a/src/host-java/rlparse.lm b/src/host-java/rlparse.lm
index 100a9223..163f44f9 100644
--- a/src/host-java/rlparse.lm
+++ b/src/host-java/rlparse.lm
@@ -1,6 +1,32 @@
 include 'ragel.lm'
 include 'rlreduce.lm'
 
+rl ident
+	/( alpha | '_' ) ( alpha | digit | '_' )*/
+
+rl number
+	/ digit+ /
+
+rl hex_number
+	/ '0x' [0-9a-fA-F]+ /
+
+rl hex_char
+	/ '0x' [0-9a-fA-F]{2}  /
+
+rl NL / '\n' /
+
+rl c_comment 
+	/ '/*' ( any | NL )* :>> '*/' /
+
+rl cpp_comment
+	/ '//' [^\n]* NL /
+
+rl s_literal
+	/ "'" ([^'\\\n] | '\\' (any | NL))* "'" /
+
+rl d_literal
+	/ '"' ([^"\\] | NL | '\\' (any | NL))* '"' /
+
 namespace inline
 	lex 
 		literal `fpc `fc  `fcurs `ftargs
@@ -44,7 +70,6 @@ namespace inline
 		token c_any
 			/ any /
 	end
-
 end
 
 namespace host
diff --git a/src/host-js/rlparse.lm b/src/host-js/rlparse.lm
index 100a9223..163f44f9 100644
--- a/src/host-js/rlparse.lm
+++ b/src/host-js/rlparse.lm
@@ -1,6 +1,32 @@
 include 'ragel.lm'
 include 'rlreduce.lm'
 
+rl ident
+	/( alpha | '_' ) ( alpha | digit | '_' )*/
+
+rl number
+	/ digit+ /
+
+rl hex_number
+	/ '0x' [0-9a-fA-F]+ /
+
+rl hex_char
+	/ '0x' [0-9a-fA-F]{2}  /
+
+rl NL / '\n' /
+
+rl c_comment 
+	/ '/*' ( any | NL )* :>> '*/' /
+
+rl cpp_comment
+	/ '//' [^\n]* NL /
+
+rl s_literal
+	/ "'" ([^'\\\n] | '\\' (any | NL))* "'" /
+
+rl d_literal
+	/ '"' ([^"\\] | NL | '\\' (any | NL))* '"' /
+
 namespace inline
 	lex 
 		literal `fpc `fc  `fcurs `ftargs
@@ -44,7 +70,6 @@ namespace inline
 		token c_any
 			/ any /
 	end
-
 end
 
 namespace host
diff --git a/src/host-julia/rlparse.lm b/src/host-julia/rlparse.lm
index 100a9223..163f44f9 100644
--- a/src/host-julia/rlparse.lm
+++ b/src/host-julia/rlparse.lm
@@ -1,6 +1,32 @@
 include 'ragel.lm'
 include 'rlreduce.lm'
 
+rl ident
+	/( alpha | '_' ) ( alpha | digit | '_' )*/
+
+rl number
+	/ digit+ /
+
+rl hex_number
+	/ '0x' [0-9a-fA-F]+ /
+
+rl hex_char
+	/ '0x' [0-9a-fA-F]{2}  /
+
+rl NL / '\n' /
+
+rl c_comment 
+	/ '/*' ( any | NL )* :>> '*/' /
+
+rl cpp_comment
+	/ '//' [^\n]* NL /
+
+rl s_literal
+	/ "'" ([^'\\\n] | '\\' (any | NL))* "'" /
+
+rl d_literal
+	/ '"' ([^"\\] | NL | '\\' (any | NL))* '"' /
+
 namespace inline
 	lex 
 		literal `fpc `fc  `fcurs `ftargs
@@ -44,7 +70,6 @@ namespace inline
 		token c_any
 			/ any /
 	end
-
 end
 
 namespace host
diff --git a/src/host-ocaml/rlparse.lm b/src/host-ocaml/rlparse.lm
index 911e3ea6..1951ac0f 100644
--- a/src/host-ocaml/rlparse.lm
+++ b/src/host-ocaml/rlparse.lm
@@ -1,6 +1,32 @@
 include 'ragel.lm'
 include 'rlreduce.lm'
 
+rl ident
+	/( alpha | '_' ) ( alpha | digit | '_' )*/
+
+rl number
+	/ digit+ /
+
+rl hex_number
+	/ '0x' [0-9a-fA-F]+ /
+
+rl hex_char
+	/ '0x' [0-9a-fA-F]{2}  /
+
+rl NL / '\n' /
+
+rl c_comment 
+	/ '/*' ( any | NL )* :>> '*/' /
+
+rl cpp_comment
+	/ '//' [^\n]* NL /
+
+rl s_literal
+	/ "'" ([^'\\\n] | '\\' (any | NL))* "'" /
+
+rl d_literal
+	/ '"' ([^"\\] | NL | '\\' (any | NL))* '"' /
+
 namespace inline
 	lex 
 		literal `fpc `fc  `fcurs `ftargs
diff --git a/src/host-ruby/rlparse.lm b/src/host-ruby/rlparse.lm
index 99811877..fcd6d96a 100644
--- a/src/host-ruby/rlparse.lm
+++ b/src/host-ruby/rlparse.lm
@@ -1,6 +1,38 @@
 include 'ragel.lm'
 include 'rlreduce.lm'
 
+rl ident
+	/( alpha | '_' ) ( alpha | digit | '_' )*/
+
+rl number
+	/ digit+ /
+
+rl hex_number
+	/ '0x' [0-9a-fA-F]+ /
+
+rl hex_char
+	/ '0x' [0-9a-fA-F]{2}  /
+
+rl NL / '\n' /
+
+rl c_comment 
+	/ '/*' ( any | NL )* :>> '*/' /
+
+rl cpp_comment
+	/ '//' [^\n]* NL /
+
+rl ruby_comment
+	/ '#' [^\n]* NL /
+
+rl s_literal
+	/ "'" ([^'\\\n] | '\\' (any | NL))* "'" /
+
+rl d_literal
+	/ '"' ([^"\\] | NL | '\\' (any | NL))* '"' /
+
+rl host_re_literal
+	/ '/' ([^/\\] | NL | '\\' (any | NL))* '/' /
+
 namespace inline
 	lex 
 		literal `fpc `fc  `fcurs `ftargs
diff --git a/src/host-rust/rlparse.lm b/src/host-rust/rlparse.lm
index 100a9223..f872fd61 100644
--- a/src/host-rust/rlparse.lm
+++ b/src/host-rust/rlparse.lm
@@ -1,6 +1,38 @@
 include 'ragel.lm'
 include 'rlreduce.lm'
 
+rl ident
+	/( alpha | '_' ) ( alpha | digit | '_' )*/
+
+rl number
+	/ digit+ /
+
+rl hex_number
+	/ '0x' [0-9a-fA-F]+ /
+
+rl hex_char
+	/ '0x' [0-9a-fA-F]{2}  /
+
+rl NL / '\n' /
+
+rl c_comment 
+	/ '/*' ( any | NL )* :>> '*/' /
+
+rl cpp_comment
+	/ '//' [^\n]* NL /
+
+rl ruby_comment
+	/ '#' [^\n]* NL /
+
+rl s_literal
+	/ "'" ([^'\\\n] | '\\' (any | NL))* "'" /
+
+rl d_literal
+	/ '"' ([^"\\] | NL | '\\' (any | NL))* '"' /
+
+rl host_re_literal
+	/ '/' ([^/\\] | NL | '\\' (any | NL))* '/' /
+
 namespace inline
 	lex 
 		literal `fpc `fc  `fcurs `ftargs
diff --git a/src/ragel.lm b/src/ragel.lm
index 1325bbb3..4105de46 100644
--- a/src/ragel.lm
+++ b/src/ragel.lm
@@ -77,43 +77,37 @@ struct machine
 end
 
 
-rl ident
-	/( alpha | '_' ) ( alpha | digit | '_' )*/
-
-rl number
-	/ digit+ /
-
-rl hex_number
-	/ '0x' [0-9a-fA-F]+ /
+#
+# Consuming ragel defintions without parsing. Used for included sections we
+# don't want and for import (TODO).
+#
+namespace consume
+	rl ident
+		/( alpha | '_' ) ( alpha | digit | '_' )*/
 
-rl hex_char
-	/ '0x' [0-9a-fA-F]{2}  /
+	rl number
+		/ digit+ /
 
-rl NL / '\n' /
+	rl hex_number
+		/ '0x' [0-9a-fA-F]+ /
 
-rl c_comment 
-	/ '/*' ( any | NL )* :>> '*/' /
+	rl hex_char
+		/ '0x' [0-9a-fA-F]{2}  /
 
-rl cpp_comment
-	/ '//' [^\n]* NL /
+	rl NL / '\n' /
 
-rl ruby_comment
-	/ '#' [^\n]* NL /
+	rl c_comment 
+		/ '/*' ( any | NL )* :>> '*/' /
 
-rl s_literal
-	/ "'" ([^'\\\n] | '\\' (any | NL))* "'" /
+	rl cpp_comment
+		/ '//' [^\n]* NL /
 
-rl d_literal
-	/ '"' ([^"\\] | NL | '\\' (any | NL))* '"' /
+	rl s_literal
+		/ "'" ([^'\\\n] | '\\' (any | NL))* "'" /
 
-rl host_re_literal
-	/ '/' ([^/\\] | NL | '\\' (any | NL))* '/' /
+	rl d_literal
+		/ '"' ([^"\\] | NL | '\\' (any | NL))* '"' /
 
-#
-# Consuming ragel defintions without parsing. Used for included sections we
-# don't want and for import (TODO).
-#
-namespace consume
 	lex 
 		token h_word / [a-zA-Z_][a-zA-Z0-9_]* /
 
@@ -162,7 +156,6 @@ namespace consume
 			#"/" ( [^\/\\] | '\\' any )* "/" 'i'?
 		/
 
-
 		token open  /'{'/ -ni
 		token close ni- /'}'/
 		token c_any / any /
@@ -269,6 +262,32 @@ namespace inline
 end
 
 namespace ragel
+	rl ident
+		/( alpha | '_' ) ( alpha | digit | '_' )*/
+
+	rl number
+		/ digit+ /
+
+	rl hex_number
+		/ '0x' [0-9a-fA-F]+ /
+
+	rl hex_char
+		/ '0x' [0-9a-fA-F]{2}  /
+
+	rl NL / '\n' /
+
+	rl c_comment 
+		/ '/*' ( any | NL )* :>> '*/' /
+
+	rl cpp_comment
+		/ '//' [^\n]* NL /
+
+	rl s_literal
+		/ "'" ([^'\\\n] | '\\' (any | NL))* "'" /
+
+	rl d_literal
+		/ '"' ([^"\\] | NL | '\\' (any | NL))* '"' /
+
 	lex
 		literal `}%% -ni
 
diff --git a/src/rlparse.lm b/src/rlparse.lm
index 02a937c5..f0003be0 100644
--- a/src/rlparse.lm
+++ b/src/rlparse.lm
@@ -1,6 +1,38 @@
 include 'ragel.lm'
 include 'rlreduce.lm'
 
+rl ident
+	/( alpha | '_' ) ( alpha | digit | '_' )*/
+
+rl number
+	/ digit+ /
+
+rl hex_number
+	/ '0x' [0-9a-fA-F]+ /
+
+rl hex_char
+	/ '0x' [0-9a-fA-F]{2}  /
+
+rl NL / '\n' /
+
+rl c_comment 
+	/ '/*' ( any | NL )* :>> '*/' /
+
+rl cpp_comment
+	/ '//' [^\n]* NL /
+
+rl ruby_comment
+	/ '#' [^\n]* NL /
+
+rl s_literal
+	/ "'" ([^'\\\n] | '\\' (any | NL))* "'" /
+
+rl d_literal
+	/ '"' ([^"\\] | NL | '\\' (any | NL))* '"' /
+
+rl host_re_literal
+	/ '/' ([^/\\] | NL | '\\' (any | NL))* '/' /
+
 namespace inline
 	lex 
 		literal `fpc `fc  `fcurs `ftargs
@@ -44,10 +76,8 @@ namespace inline
 		token c_any
 			/ any /
 	end
-
 end
 
-
 namespace host
 	lex 
 		literal `%%{