- Fixed ext/tokenizer to work with the new tokens (<, >, @, !, etc)

2 files changed, 103 insertions, 73 deletions

diff --git a/ext/tokenizer/tokenizer.c b/ext/tokenizer/tokenizer.c
index 0ffe83d23a..b36ab4e134 100644
--- a/ext/tokenizer/tokenizer.c
+++ b/ext/tokenizer/tokenizer.c
@@ -126,25 +126,55 @@ static void tokenize(zval *return_value TSRMLS_DC)
 				break;
 		}
 
-		if (token_type >= 256) {
-			MAKE_STD_ZVAL(keyword);
-			array_init(keyword);
-			add_next_index_long(keyword, token_type);
-			if (token_type == T_END_HEREDOC) {
-				if (CG(increment_lineno)) {
-					token_line = ++CG(zend_lineno);
-					CG(increment_lineno) = 0;
+		switch (token_type) {
+			case T_DOT:
+			case T_AT:
+			case T_BOOL_NOT:
+			case T_BW_AND:
+			case T_BW_OR:
+			case T_BW_XOR:
+			case T_BW_NOT:
+			case T_PLUS:
+			case T_MINUS:
+			case T_MOD:
+			case T_DIV:
+			case T_MULT:
+			case T_SEMICOLON:
+			case T_COLON:
+			case T_COMMA:
+			case T_LBRACE:
+			case T_RBRACE:
+			case T_LBRACKET:
+			case T_RBRACKET:
+			case T_LPAREN:
+			case T_RPAREN:
+			case T_EQUAL:
+			case T_QUOTE:
+			case T_BACKQUOTE:
+			case T_QUESTION_MARK:
+			case T_IS_GREATER:
+			case T_IS_SMALLER:
+				 add_next_index_stringl(return_value, (char *)zendtext, zendleng, 1);
+				 break;
+			default:
+				MAKE_STD_ZVAL(keyword);
+				array_init(keyword);
+				add_next_index_long(keyword, token_type);
+				if (token_type == T_END_HEREDOC) {
+					if (CG(increment_lineno)) {
+						token_line = ++CG(zend_lineno);
+						CG(increment_lineno) = 0;
+					}
+					add_next_index_stringl(keyword, Z_STRVAL(token), Z_STRLEN(token), 1);
+					efree(Z_STRVAL(token));
+				} else {
+					add_next_index_stringl(keyword, (char *)zendtext, zendleng, 1);
 				}
-				add_next_index_stringl(keyword, Z_STRVAL(token), Z_STRLEN(token), 1);
-				efree(Z_STRVAL(token));
-			} else {
-				add_next_index_stringl(keyword, (char *)zendtext, zendleng, 1);
-			}
-			add_next_index_long(keyword, token_line);
-			add_next_index_zval(return_value, keyword);
-		} else {
-			add_next_index_stringl(return_value, (char *)zendtext, zendleng, 1);
+				add_next_index_long(keyword, token_line);
+				add_next_index_zval(return_value, keyword);
+				break;
 		}
+
 		if (destroy && Z_TYPE(token) != IS_NULL) {
 			zval_dtor(&token);
 		}
diff --git a/ext/tokenizer/tokenizer_data.c b/ext/tokenizer/tokenizer_data.c
index 5cea1bc2e0..7001ca654f 100644
--- a/ext/tokenizer/tokenizer_data.c
+++ b/ext/tokenizer/tokenizer_data.c
@@ -103,41 +103,55 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
 	REGISTER_LONG_CONSTANT("T_PRIVATE", T_PRIVATE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_PROTECTED", T_PROTECTED, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_PUBLIC", T_PUBLIC, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_OPEN_TAG", T_OPEN_TAG, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_OPEN_TAG_WITH_ECHO", T_OPEN_TAG_WITH_ECHO, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_WHITESPACE", T_WHITESPACE, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_COMMENT", T_COMMENT, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_CLOSE_TAG", T_CLOSE_TAG, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_DOC_COMMENT", T_DOC_COMMENT, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_STRING", T_STRING, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_NS_SEPARATOR", T_NS_SEPARATOR, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_NAMESPACE", T_NAMESPACE, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_LBRACE", T_LBRACE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_HALT_COMPILER", T_HALT_COMPILER, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_LPAREN", T_LPAREN, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_RPAREN", T_RPAREN, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_SEMICOLON", T_SEMICOLON, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_NAMESPACE", T_NAMESPACE, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_LBRACE", T_LBRACE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_RBRACE", T_RBRACE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_USE", T_USE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_AS", T_AS, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_CONST", T_CONST, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_ENDWHILE", T_ENDWHILE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_WHILE", T_WHILE, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_DO", T_DO, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_ENDFOR", T_ENDFOR, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_FOR", T_FOR, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_ENDSWITCH", T_ENDSWITCH, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_CASE", T_CASE, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_DEFAULT", T_DEFAULT, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_SWITCH", T_SWITCH, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_ENDDECLARE", T_ENDDECLARE, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_DECLARE", T_DECLARE, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_DOUBLE_ARROW", T_DOUBLE_ARROW, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_ENDFOREACH", T_ENDFOREACH, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_FOREACH", T_FOREACH, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_CATCH", T_CATCH, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_VARIABLE", T_VARIABLE, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_TRY", T_TRY, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_GLOBAL", T_GLOBAL, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_ECHO", T_ECHO, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_UNSET", T_UNSET, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_BREAK", T_BREAK, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_CONTINUE", T_CONTINUE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_RETURN", T_RETURN, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_GLOBAL", T_GLOBAL, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_ECHO", T_ECHO, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_INLINE_HTML", T_INLINE_HTML, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_UNSET", T_UNSET, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_FOREACH", T_FOREACH, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_THROW", T_THROW, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_GOTO", T_GOTO, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_CATCH", T_CATCH, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_VARIABLE", T_VARIABLE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_CLASS", T_CLASS, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_TRAIT", T_TRAIT, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_EXTENDS", T_EXTENDS, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_INTERFACE", T_INTERFACE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_IMPLEMENTS", T_IMPLEMENTS, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_DOUBLE_ARROW", T_DOUBLE_ARROW, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_ENDFOR", T_ENDFOR, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_ENDFOREACH", T_ENDFOREACH, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_ENDDECLARE", T_ENDDECLARE, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_ENDWHILE", T_ENDWHILE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_ARRAY", T_ARRAY, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_BOOL_HINT", T_BOOL_HINT, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_STRING_HINT", T_STRING_HINT, CONST_CS | CONST_PERSISTENT);
@@ -151,6 +165,7 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
 	REGISTER_LONG_CONSTANT("T_INSTEADOF", T_INSTEADOF, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_PAAMAYIM_NEKUDOTAYIM", T_PAAMAYIM_NEKUDOTAYIM, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_VAR", T_VAR, CONST_CS | CONST_PERSISTENT);
+	REGISTER_LONG_CONSTANT("T_LIST", T_LIST, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_BACKQUOTE", T_BACKQUOTE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_FUNCTION", T_FUNCTION, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_OBJECT_OPERATOR", T_OBJECT_OPERATOR, CONST_CS | CONST_PERSISTENT);
@@ -170,26 +185,11 @@ void tokenizer_register_constants(INIT_FUNC_ARGS) {
 	REGISTER_LONG_CONSTANT("T_STRING_VARNAME", T_STRING_VARNAME, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_QUOTE", T_QUOTE, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_LBRACKET", T_LBRACKET, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_LIST", T_LIST, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_DOLLAR_OPEN_CURLY_BRACES", T_DOLLAR_OPEN_CURLY_BRACES, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_CURLY_OPEN", T_CURLY_OPEN, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_NUM_STRING", T_NUM_STRING, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_ISSET", T_ISSET, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_EMPTY", T_EMPTY, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_OPEN_TAG", T_OPEN_TAG, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_OPEN_TAG_WITH_ECHO", T_OPEN_TAG_WITH_ECHO, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_WHITESPACE", T_WHITESPACE, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_COMMENT", T_COMMENT, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_CLOSE_TAG", T_CLOSE_TAG, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_DOC_COMMENT", T_DOC_COMMENT, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_DEFAULT", T_DEFAULT, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_DECLARE", T_DECLARE, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_DO", T_DO, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_FOR", T_FOR, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_CASE", T_CASE, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_ENDSWITCH", T_ENDSWITCH, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_TRY", T_TRY, CONST_CS | CONST_PERSISTENT);
-	REGISTER_LONG_CONSTANT("T_SWITCH", T_SWITCH, CONST_CS | CONST_PERSISTENT);
 	REGISTER_LONG_CONSTANT("T_DOUBLE_COLON", T_PAAMAYIM_NEKUDOTAYIM, CONST_CS | CONST_PERSISTENT);
 }
 
@@ -271,41 +271,55 @@ char *get_token_type_name(int token_type)
 		case T_PRIVATE: return "T_PRIVATE";
 		case T_PROTECTED: return "T_PROTECTED";
 		case T_PUBLIC: return "T_PUBLIC";
+		case T_OPEN_TAG: return "T_OPEN_TAG";
+		case T_OPEN_TAG_WITH_ECHO: return "T_OPEN_TAG_WITH_ECHO";
+		case T_WHITESPACE: return "T_WHITESPACE";
+		case T_COMMENT: return "T_COMMENT";
+		case T_CLOSE_TAG: return "T_CLOSE_TAG";
+		case T_DOC_COMMENT: return "T_DOC_COMMENT";
 		case T_STRING: return "T_STRING";
 		case T_NS_SEPARATOR: return "T_NS_SEPARATOR";
+		case T_NAMESPACE: return "T_NAMESPACE";
+		case T_LBRACE: return "T_LBRACE";
 		case T_HALT_COMPILER: return "T_HALT_COMPILER";
 		case T_LPAREN: return "T_LPAREN";
 		case T_RPAREN: return "T_RPAREN";
 		case T_SEMICOLON: return "T_SEMICOLON";
-		case T_NAMESPACE: return "T_NAMESPACE";
-		case T_LBRACE: return "T_LBRACE";
 		case T_RBRACE: return "T_RBRACE";
 		case T_USE: return "T_USE";
 		case T_AS: return "T_AS";
 		case T_CONST: return "T_CONST";
+		case T_ENDWHILE: return "T_ENDWHILE";
 		case T_WHILE: return "T_WHILE";
+		case T_DO: return "T_DO";
+		case T_ENDFOR: return "T_ENDFOR";
+		case T_FOR: return "T_FOR";
+		case T_ENDSWITCH: return "T_ENDSWITCH";
+		case T_CASE: return "T_CASE";
+		case T_DEFAULT: return "T_DEFAULT";
+		case T_SWITCH: return "T_SWITCH";
+		case T_ENDDECLARE: return "T_ENDDECLARE";
+		case T_DECLARE: return "T_DECLARE";
+		case T_DOUBLE_ARROW: return "T_DOUBLE_ARROW";
+		case T_ENDFOREACH: return "T_ENDFOREACH";
+		case T_FOREACH: return "T_FOREACH";
+		case T_CATCH: return "T_CATCH";
+		case T_VARIABLE: return "T_VARIABLE";
+		case T_TRY: return "T_TRY";
+		case T_GLOBAL: return "T_GLOBAL";
+		case T_ECHO: return "T_ECHO";
+		case T_UNSET: return "T_UNSET";
 		case T_BREAK: return "T_BREAK";
 		case T_CONTINUE: return "T_CONTINUE";
 		case T_RETURN: return "T_RETURN";
-		case T_GLOBAL: return "T_GLOBAL";
-		case T_ECHO: return "T_ECHO";
 		case T_INLINE_HTML: return "T_INLINE_HTML";
-		case T_UNSET: return "T_UNSET";
-		case T_FOREACH: return "T_FOREACH";
 		case T_THROW: return "T_THROW";
 		case T_GOTO: return "T_GOTO";
-		case T_CATCH: return "T_CATCH";
-		case T_VARIABLE: return "T_VARIABLE";
 		case T_CLASS: return "T_CLASS";
 		case T_TRAIT: return "T_TRAIT";
 		case T_EXTENDS: return "T_EXTENDS";
 		case T_INTERFACE: return "T_INTERFACE";
 		case T_IMPLEMENTS: return "T_IMPLEMENTS";
-		case T_DOUBLE_ARROW: return "T_DOUBLE_ARROW";
-		case T_ENDFOR: return "T_ENDFOR";
-		case T_ENDFOREACH: return "T_ENDFOREACH";
-		case T_ENDDECLARE: return "T_ENDDECLARE";
-		case T_ENDWHILE: return "T_ENDWHILE";
 		case T_ARRAY: return "T_ARRAY";
 		case T_BOOL_HINT: return "T_BOOL_HINT";
 		case T_STRING_HINT: return "T_STRING_HINT";
@@ -319,6 +333,7 @@ char *get_token_type_name(int token_type)
 		case T_INSTEADOF: return "T_INSTEADOF";
 		case T_PAAMAYIM_NEKUDOTAYIM: return "T_DOUBLE_COLON";
 		case T_VAR: return "T_VAR";
+		case T_LIST: return "T_LIST";
 		case T_BACKQUOTE: return "T_BACKQUOTE";
 		case T_FUNCTION: return "T_FUNCTION";
 		case T_OBJECT_OPERATOR: return "T_OBJECT_OPERATOR";
@@ -338,26 +353,11 @@ char *get_token_type_name(int token_type)
 		case T_STRING_VARNAME: return "T_STRING_VARNAME";
 		case T_QUOTE: return "T_QUOTE";
 		case T_LBRACKET: return "T_LBRACKET";
-		case T_LIST: return "T_LIST";
 		case T_DOLLAR_OPEN_CURLY_BRACES: return "T_DOLLAR_OPEN_CURLY_BRACES";
 		case T_CURLY_OPEN: return "T_CURLY_OPEN";
 		case T_NUM_STRING: return "T_NUM_STRING";
 		case T_ISSET: return "T_ISSET";
 		case T_EMPTY: return "T_EMPTY";
-		case T_OPEN_TAG: return "T_OPEN_TAG";
-		case T_OPEN_TAG_WITH_ECHO: return "T_OPEN_TAG_WITH_ECHO";
-		case T_WHITESPACE: return "T_WHITESPACE";
-		case T_COMMENT: return "T_COMMENT";
-		case T_CLOSE_TAG: return "T_CLOSE_TAG";
-		case T_DOC_COMMENT: return "T_DOC_COMMENT";
-		case T_DEFAULT: return "T_DEFAULT";
-		case T_DECLARE: return "T_DECLARE";
-		case T_DO: return "T_DO";
-		case T_FOR: return "T_FOR";
-		case T_CASE: return "T_CASE";
-		case T_ENDSWITCH: return "T_ENDSWITCH";
-		case T_TRY: return "T_TRY";
-		case T_SWITCH: return "T_SWITCH";
 
 	}
 	return "UNKNOWN";