Run tokenizer_data_gen.sh to reorganize tokens

This will help us in future PRs, reducing the numbers of changed lines

1 files changed, 70 insertions, 70 deletions

diff --git a/ext/tokenizer/tokenizer_data.c b/ext/tokenizer/tokenizer_data.c
index 67426bd02e..11ee044a59 100644
--- a/ext/tokenizer/tokenizer_data.c
+++ b/ext/tokenizer/tokenizer_data.c
@@ -27,11 +27,11 @@
 
 
 void tokenizer_register_constants(INIT_FUNC_ARGS) {
-	REGISTER_LONG_CONSTANT("T_REQUIRE_ONCE", T_REQUIRE_ONCE, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_REQUIRE", T_REQUIRE, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_EVAL", T_EVAL, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_INCLUDE_ONCE", T_INCLUDE_ONCE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_INCLUDE", T_INCLUDE, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_INCLUDE_ONCE", T_INCLUDE_ONCE, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_EVAL", T_EVAL, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_REQUIRE", T_REQUIRE, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_REQUIRE_ONCE", T_REQUIRE_ONCE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_LOGICAL_OR", T_LOGICAL_OR, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_LOGICAL_XOR", T_LOGICAL_XOR, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_LOGICAL_AND", T_LOGICAL_AND, CONST_CS | CONST_PERSISTENT);
@@ -39,52 +39,52 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
 	REGISTER_LONG_CONSTANT("T_YIELD", T_YIELD, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_DOUBLE_ARROW", T_DOUBLE_ARROW, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_YIELD_FROM", T_YIELD_FROM, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_POW_EQUAL", T_POW_EQUAL, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_SR_EQUAL", T_SR_EQUAL, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_SL_EQUAL", T_SL_EQUAL, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_XOR_EQUAL", T_XOR_EQUAL, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_OR_EQUAL", T_OR_EQUAL, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_AND_EQUAL", T_AND_EQUAL, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_MOD_EQUAL", T_MOD_EQUAL, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_CONCAT_EQUAL", T_CONCAT_EQUAL, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_DIV_EQUAL", T_DIV_EQUAL, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_MUL_EQUAL", T_MUL_EQUAL, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_MINUS_EQUAL", T_MINUS_EQUAL, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_PLUS_EQUAL", T_PLUS_EQUAL, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_MINUS_EQUAL", T_MINUS_EQUAL, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_MUL_EQUAL", T_MUL_EQUAL, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_DIV_EQUAL", T_DIV_EQUAL, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_CONCAT_EQUAL", T_CONCAT_EQUAL, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_MOD_EQUAL", T_MOD_EQUAL, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_AND_EQUAL", T_AND_EQUAL, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_OR_EQUAL", T_OR_EQUAL, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_XOR_EQUAL", T_XOR_EQUAL, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_SL_EQUAL", T_SL_EQUAL, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_SR_EQUAL", T_SR_EQUAL, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_POW_EQUAL", T_POW_EQUAL, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_COALESCE", T_COALESCE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_BOOLEAN_OR", T_BOOLEAN_OR, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_BOOLEAN_AND", T_BOOLEAN_AND, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_SPACESHIP", T_SPACESHIP, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_IS_NOT_IDENTICAL", T_IS_NOT_IDENTICAL, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_IS_IDENTICAL", T_IS_IDENTICAL, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_IS_NOT_EQUAL", T_IS_NOT_EQUAL, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_IS_EQUAL", T_IS_EQUAL, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_IS_GREATER_OR_EQUAL", T_IS_GREATER_OR_EQUAL, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_IS_NOT_EQUAL", T_IS_NOT_EQUAL, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_IS_IDENTICAL", T_IS_IDENTICAL, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_IS_NOT_IDENTICAL", T_IS_NOT_IDENTICAL, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_SPACESHIP", T_SPACESHIP, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_IS_SMALLER_OR_EQUAL", T_IS_SMALLER_OR_EQUAL, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_SR", T_SR, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_IS_GREATER_OR_EQUAL", T_IS_GREATER_OR_EQUAL, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_SL", T_SL, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_SR", T_SR, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_INSTANCEOF", T_INSTANCEOF, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_UNSET_CAST", T_UNSET_CAST, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_BOOL_CAST", T_BOOL_CAST, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_OBJECT_CAST", T_OBJECT_CAST, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_ARRAY_CAST", T_ARRAY_CAST, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_STRING_CAST", T_STRING_CAST, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_DOUBLE_CAST", T_DOUBLE_CAST, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_INT_CAST", T_INT_CAST, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_DEC", T_DEC, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_INC", T_INC, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_DEC", T_DEC, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_INT_CAST", T_INT_CAST, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_DOUBLE_CAST", T_DOUBLE_CAST, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_STRING_CAST", T_STRING_CAST, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_ARRAY_CAST", T_ARRAY_CAST, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_OBJECT_CAST", T_OBJECT_CAST, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_BOOL_CAST", T_BOOL_CAST, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_UNSET_CAST", T_UNSET_CAST, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_POW", T_POW, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_CLONE", T_CLONE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_NEW", T_NEW, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_CLONE", T_CLONE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_ELSEIF", T_ELSEIF, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_ELSE", T_ELSE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_ENDIF", T_ENDIF, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_PUBLIC", T_PUBLIC, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_PROTECTED", T_PROTECTED, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_PRIVATE", T_PRIVATE, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_FINAL", T_FINAL, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_ABSTRACT", T_ABSTRACT, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_STATIC", T_STATIC, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_ABSTRACT", T_ABSTRACT, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_FINAL", T_FINAL, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_PRIVATE", T_PRIVATE, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_PROTECTED", T_PROTECTED, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_PUBLIC", T_PUBLIC, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_LNUMBER", T_LNUMBER, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_DNUMBER", T_DNUMBER, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_STRING", T_STRING, CONST_CS | CONST_PERSISTENT);
@@ -167,11 +167,11 @@ char *get_token_type_name(int token_type)
 {
 	switch (token_type) {
 
-		case T_REQUIRE_ONCE: return "T_REQUIRE_ONCE";
-		case T_REQUIRE: return "T_REQUIRE";
-		case T_EVAL: return "T_EVAL";
-		case T_INCLUDE_ONCE: return "T_INCLUDE_ONCE";
 		case T_INCLUDE: return "T_INCLUDE";
+		case T_INCLUDE_ONCE: return "T_INCLUDE_ONCE";
+		case T_EVAL: return "T_EVAL";
+		case T_REQUIRE: return "T_REQUIRE";
+		case T_REQUIRE_ONCE: return "T_REQUIRE_ONCE";
 		case T_LOGICAL_OR: return "T_LOGICAL_OR";
 		case T_LOGICAL_XOR: return "T_LOGICAL_XOR";
 		case T_LOGICAL_AND: return "T_LOGICAL_AND";
@@ -179,52 +179,52 @@ char *get_token_type_name(int token_type)
 		case T_YIELD: return "T_YIELD";
 		case T_DOUBLE_ARROW: return "T_DOUBLE_ARROW";
 		case T_YIELD_FROM: return "T_YIELD_FROM";
-		case T_POW_EQUAL: return "T_POW_EQUAL";
-		case T_SR_EQUAL: return "T_SR_EQUAL";
-		case T_SL_EQUAL: return "T_SL_EQUAL";
-		case T_XOR_EQUAL: return "T_XOR_EQUAL";
-		case T_OR_EQUAL: return "T_OR_EQUAL";
-		case T_AND_EQUAL: return "T_AND_EQUAL";
-		case T_MOD_EQUAL: return "T_MOD_EQUAL";
-		case T_CONCAT_EQUAL: return "T_CONCAT_EQUAL";
-		case T_DIV_EQUAL: return "T_DIV_EQUAL";
-		case T_MUL_EQUAL: return "T_MUL_EQUAL";
-		case T_MINUS_EQUAL: return "T_MINUS_EQUAL";
 		case T_PLUS_EQUAL: return "T_PLUS_EQUAL";
+		case T_MINUS_EQUAL: return "T_MINUS_EQUAL";
+		case T_MUL_EQUAL: return "T_MUL_EQUAL";
+		case T_DIV_EQUAL: return "T_DIV_EQUAL";
+		case T_CONCAT_EQUAL: return "T_CONCAT_EQUAL";
+		case T_MOD_EQUAL: return "T_MOD_EQUAL";
+		case T_AND_EQUAL: return "T_AND_EQUAL";
+		case T_OR_EQUAL: return "T_OR_EQUAL";
+		case T_XOR_EQUAL: return "T_XOR_EQUAL";
+		case T_SL_EQUAL: return "T_SL_EQUAL";
+		case T_SR_EQUAL: return "T_SR_EQUAL";
+		case T_POW_EQUAL: return "T_POW_EQUAL";
 		case T_COALESCE: return "T_COALESCE";
 		case T_BOOLEAN_OR: return "T_BOOLEAN_OR";
 		case T_BOOLEAN_AND: return "T_BOOLEAN_AND";
-		case T_SPACESHIP: return "T_SPACESHIP";
-		case T_IS_NOT_IDENTICAL: return "T_IS_NOT_IDENTICAL";
-		case T_IS_IDENTICAL: return "T_IS_IDENTICAL";
-		case T_IS_NOT_EQUAL: return "T_IS_NOT_EQUAL";
 		case T_IS_EQUAL: return "T_IS_EQUAL";
-		case T_IS_GREATER_OR_EQUAL: return "T_IS_GREATER_OR_EQUAL";
+		case T_IS_NOT_EQUAL: return "T_IS_NOT_EQUAL";
+		case T_IS_IDENTICAL: return "T_IS_IDENTICAL";
+		case T_IS_NOT_IDENTICAL: return "T_IS_NOT_IDENTICAL";
+		case T_SPACESHIP: return "T_SPACESHIP";
 		case T_IS_SMALLER_OR_EQUAL: return "T_IS_SMALLER_OR_EQUAL";
-		case T_SR: return "T_SR";
+		case T_IS_GREATER_OR_EQUAL: return "T_IS_GREATER_OR_EQUAL";
 		case T_SL: return "T_SL";
+		case T_SR: return "T_SR";
 		case T_INSTANCEOF: return "T_INSTANCEOF";
-		case T_UNSET_CAST: return "T_UNSET_CAST";
-		case T_BOOL_CAST: return "T_BOOL_CAST";
-		case T_OBJECT_CAST: return "T_OBJECT_CAST";
-		case T_ARRAY_CAST: return "T_ARRAY_CAST";
-		case T_STRING_CAST: return "T_STRING_CAST";
-		case T_DOUBLE_CAST: return "T_DOUBLE_CAST";
-		case T_INT_CAST: return "T_INT_CAST";
-		case T_DEC: return "T_DEC";
 		case T_INC: return "T_INC";
+		case T_DEC: return "T_DEC";
+		case T_INT_CAST: return "T_INT_CAST";
+		case T_DOUBLE_CAST: return "T_DOUBLE_CAST";
+		case T_STRING_CAST: return "T_STRING_CAST";
+		case T_ARRAY_CAST: return "T_ARRAY_CAST";
+		case T_OBJECT_CAST: return "T_OBJECT_CAST";
+		case T_BOOL_CAST: return "T_BOOL_CAST";
+		case T_UNSET_CAST: return "T_UNSET_CAST";
 		case T_POW: return "T_POW";
-		case T_CLONE: return "T_CLONE";
 		case T_NEW: return "T_NEW";
+		case T_CLONE: return "T_CLONE";
 		case T_ELSEIF: return "T_ELSEIF";
 		case T_ELSE: return "T_ELSE";
 		case T_ENDIF: return "T_ENDIF";
-		case T_PUBLIC: return "T_PUBLIC";
-		case T_PROTECTED: return "T_PROTECTED";
-		case T_PRIVATE: return "T_PRIVATE";
-		case T_FINAL: return "T_FINAL";
-		case T_ABSTRACT: return "T_ABSTRACT";
 		case T_STATIC: return "T_STATIC";
+		case T_ABSTRACT: return "T_ABSTRACT";
+		case T_FINAL: return "T_FINAL";
+		case T_PRIVATE: return "T_PRIVATE";
+		case T_PROTECTED: return "T_PROTECTED";
+		case T_PUBLIC: return "T_PUBLIC";
 		case T_LNUMBER: return "T_LNUMBER";
 		case T_DNUMBER: return "T_DNUMBER";
 		case T_STRING: return "T_STRING";