vala: Add support for verbatim template string

Fixes https://gitlab.gnome.org/GNOME/vala/issues/1373

6 files changed, 90 insertions, 22 deletions

diff --git a/vala/valagenieparser.vala b/vala/valagenieparser.vala
index 7c7c48309..9781cd339 100644
--- a/vala/valagenieparser.vala
+++ b/vala/valagenieparser.vala
@@ -394,6 +394,11 @@ public class Vala.Genie.Parser : CodeVisitor {
 			string raw_string = get_last_string ();
 			string escaped_string = raw_string.substring (3, raw_string.length - 6).escape ("");
 			return new StringLiteral ("\"%s\"".printf (escaped_string), get_src (begin));
+		case TokenType.VERBATIM_TEMPLATE_STRING_LITERAL:
+			next ();
+			string raw_string = get_last_string ();
+			string escaped_string = raw_string.escape ("");
+			return new StringLiteral ("\"%s\"".printf (escaped_string), get_src (begin));
 		case TokenType.NULL:
 			next ();
 			return new NullLiteral (get_src (begin));
@@ -701,6 +706,7 @@ public class Vala.Genie.Parser : CodeVisitor {
 		case TokenType.STRING_LITERAL:
 		case TokenType.TEMPLATE_STRING_LITERAL:
 		case TokenType.VERBATIM_STRING_LITERAL:
+		case TokenType.VERBATIM_TEMPLATE_STRING_LITERAL:
 		case TokenType.NULL:
 			expr = parse_literal ();
 			break;
@@ -1285,6 +1291,7 @@ public class Vala.Genie.Parser : CodeVisitor {
 					case TokenType.STRING_LITERAL:
 					case TokenType.TEMPLATE_STRING_LITERAL:
 					case TokenType.VERBATIM_STRING_LITERAL:
+					case TokenType.VERBATIM_TEMPLATE_STRING_LITERAL:
 					case TokenType.NULL:
 					case TokenType.SELF:
 					case TokenType.SUPER:
diff --git a/vala/valageniescanner.vala b/vala/valageniescanner.vala
index ef54ae7d3..286992206 100644
--- a/vala/valageniescanner.vala
+++ b/vala/valageniescanner.vala
@@ -67,7 +67,8 @@ public class Vala.Genie.Scanner {
 		BRACKET,
 		REGEX_LITERAL,
 		TEMPLATE,
-		TEMPLATE_PART
+		TEMPLATE_PART,
+		VERBATIM_TEMPLATE
 	}
 
 	public Scanner (SourceFile source_file) {
@@ -97,6 +98,10 @@ public class Vala.Genie.Scanner {
 		return (state_stack.length > 0 && state_stack[state_stack.length - 1] == State.TEMPLATE);
 	}
 
+	bool in_verbatim_template () {
+		return (state_stack.length > 0 && state_stack[state_stack.length - 1] == State.VERBATIM_TEMPLATE);
+	}
+
 	bool in_template_part () {
 		return (state_stack.length > 0 && state_stack[state_stack.length - 1] == State.TEMPLATE_PART);
 	}
@@ -695,6 +700,7 @@ public class Vala.Genie.Scanner {
 
 
 	public TokenType read_template_token (out SourceLocation token_begin, out SourceLocation token_end) {
+		bool is_verbatim = in_verbatim_template ();
 		TokenType type;
 		char* begin = current;
 		token_begin = SourceLocation (begin, line, column);
@@ -706,9 +712,22 @@ public class Vala.Genie.Scanner {
 		} else {
 			switch (current[0]) {
 			case '"':
-				type = TokenType.CLOSE_TEMPLATE;
-				current++;
-				state_stack.length--;
+				if (is_verbatim) {
+					if (current < end -2 && current[1] == '"' && current[2] == '"' && current[3] != '"') {
+						type = TokenType.CLOSE_TEMPLATE;
+						current += 3;
+						state_stack.length--;
+					} else {
+						type = TokenType.VERBATIM_TEMPLATE_STRING_LITERAL;
+						current++;
+						token_length_in_chars++;
+						state_stack += State.TEMPLATE_PART;
+					}
+				} else {
+					type = TokenType.CLOSE_TEMPLATE;
+					current++;
+					state_stack.length--;
+				}
 				break;
 			case '$':
 				token_begin.pos++; // $ is not part of following token
@@ -727,7 +746,7 @@ public class Vala.Genie.Scanner {
 					state_stack += State.PARENS;
 					return read_token (out token_begin, out token_end);
 				} else if (current[0] == '$') {
-					type = TokenType.TEMPLATE_STRING_LITERAL;
+					type = is_verbatim ? TokenType.VERBATIM_TEMPLATE_STRING_LITERAL : TokenType.TEMPLATE_STRING_LITERAL;
 					current++;
 					state_stack += State.TEMPLATE_PART;
 				} else {
@@ -736,10 +755,10 @@ public class Vala.Genie.Scanner {
 				}
 				break;
 			default:
-				type = TokenType.TEMPLATE_STRING_LITERAL;
+				type = is_verbatim ? TokenType.VERBATIM_TEMPLATE_STRING_LITERAL : TokenType.TEMPLATE_STRING_LITERAL;
 				token_length_in_chars = 0;
 				while (current < end && current[0] != '"' && current[0] != '$') {
-					if (current[0] == '\\') {
+					if (current[0] == '\\' && !is_verbatim) {
 						current++;
 						token_length_in_chars++;
 						if (current >= end) {
@@ -842,7 +861,7 @@ public class Vala.Genie.Scanner {
 			return TokenType.EOF;
 		}
 
-		if (in_template ()) {
+		if (in_template () || in_verbatim_template ()) {
 			return read_template_token (out token_begin, out token_end);
 		} else if (in_template_part ()) {
 			state_stack.length--;
@@ -960,9 +979,15 @@ public class Vala.Genie.Scanner {
 			type = get_identifier_or_keyword (begin, len);
 		} else if (current[0] == '@') {
 			if (current < end - 1 && current[1] == '"') {
+				current += 1;
+				if (current < end - 5 && current[1] == '"' && current[2] == '"') {
+					current += 3;
+					state_stack += State.VERBATIM_TEMPLATE;
+				} else {
+					current += 1;
+					state_stack += State.TEMPLATE;
+				}
 				type = TokenType.OPEN_TEMPLATE;
-				current += 2;
-				state_stack += State.TEMPLATE;
 			} else {
 				token_begin.pos++; // @ is not part of the identifier
 				current++;
@@ -1053,7 +1078,7 @@ public class Vala.Genie.Scanner {
 				if (state_stack.length > 0) {
 					state_stack.length--;
 				}
-				if (in_template ()) {
+				if (in_template () || in_verbatim_template ()) {
 					type = TokenType.COMMA;
 				}
 				break;
diff --git a/vala/valagenietokentype.vala b/vala/valagenietokentype.vala
index 08bef53fd..fc46724b4 100644
--- a/vala/valagenietokentype.vala
+++ b/vala/valagenietokentype.vala
@@ -163,6 +163,7 @@ public enum Vala.Genie.TokenType {
 	USES,
 	VAR,
 	VERBATIM_STRING_LITERAL,
+	VERBATIM_TEMPLATE_STRING_LITERAL,
 	VIRTUAL,
 	VOID,
 	VOLATILE,
@@ -311,6 +312,7 @@ public enum Vala.Genie.TokenType {
 		case USES: return "`uses'";
 		case VAR: return "`var'";
 		case VERBATIM_STRING_LITERAL: return "verbatim string literal";
+		case VERBATIM_TEMPLATE_STRING_LITERAL: return "verbatim template string literal";
 		case VIRTUAL: return "`virtual'";
 		case VOID: return "`void'";
 		case VOLATILE: return "`volatile'";
diff --git a/vala/valaparser.vala b/vala/valaparser.vala
index f703e9c86..23b1e7db4 100644
--- a/vala/valaparser.vala
+++ b/vala/valaparser.vala
@@ -384,6 +384,11 @@ public class Vala.Parser : CodeVisitor {
 			string raw_string = get_last_string ();
 			string escaped_string = raw_string.substring (3, raw_string.length - 6).escape ("");
 			return new StringLiteral ("\"%s\"".printf (escaped_string), get_src (begin));
+		case TokenType.VERBATIM_TEMPLATE_STRING_LITERAL:
+			next ();
+			string raw_string = get_last_string ();
+			string escaped_string = raw_string.escape ("");
+			return new StringLiteral ("\"%s\"".printf (escaped_string), get_src (begin));
 		case TokenType.NULL:
 			next ();
 			return new NullLiteral (get_src (begin));
@@ -719,6 +724,7 @@ public class Vala.Parser : CodeVisitor {
 		case TokenType.REGEX_LITERAL:
 		case TokenType.TEMPLATE_STRING_LITERAL:
 		case TokenType.VERBATIM_STRING_LITERAL:
+		case TokenType.VERBATIM_TEMPLATE_STRING_LITERAL:
 		case TokenType.NULL:
 			expr = parse_literal ();
 			break;
@@ -1269,6 +1275,7 @@ public class Vala.Parser : CodeVisitor {
 						case TokenType.STRING_LITERAL:
 						case TokenType.TEMPLATE_STRING_LITERAL:
 						case TokenType.VERBATIM_STRING_LITERAL:
+						case TokenType.VERBATIM_TEMPLATE_STRING_LITERAL:
 						case TokenType.REGEX_LITERAL:
 						case TokenType.NULL:
 						case TokenType.THIS:
diff --git a/vala/valascanner.vala b/vala/valascanner.vala
index daeb97feb..13a4698b6 100644
--- a/vala/valascanner.vala
+++ b/vala/valascanner.vala
@@ -54,7 +54,8 @@ public class Vala.Scanner {
 		BRACKET,
 		TEMPLATE,
 		TEMPLATE_PART,
-		REGEX_LITERAL
+		REGEX_LITERAL,
+		VERBATIM_TEMPLATE
 	}
 
 	public Scanner (SourceFile source_file) {
@@ -82,6 +83,10 @@ public class Vala.Scanner {
 		return (state_stack.length > 0 && state_stack[state_stack.length - 1] == State.TEMPLATE);
 	}
 
+	bool in_verbatim_template () {
+		return (state_stack.length > 0 && state_stack[state_stack.length - 1] == State.VERBATIM_TEMPLATE);
+	}
+
 	bool in_template_part () {
 		return (state_stack.length > 0 && state_stack[state_stack.length - 1] == State.TEMPLATE_PART);
 	}
@@ -686,6 +691,7 @@ public class Vala.Scanner {
 	}
 
 	public TokenType read_template_token (out SourceLocation token_begin, out SourceLocation token_end) {
+		bool is_verbatim = in_verbatim_template ();
 		TokenType type;
 		char* begin = current;
 		token_begin = SourceLocation (begin, line, column);
@@ -697,9 +703,22 @@ public class Vala.Scanner {
 		} else {
 			switch (current[0]) {
 			case '"':
-				type = TokenType.CLOSE_TEMPLATE;
-				current++;
-				state_stack.length--;
+				if (is_verbatim) {
+					if (current < end -2 && current[1] == '"' && current[2] == '"' && current[3] != '"') {
+						type = TokenType.CLOSE_TEMPLATE;
+						current += 3;
+						state_stack.length--;
+					} else {
+						type = TokenType.VERBATIM_TEMPLATE_STRING_LITERAL;
+						current++;
+						token_length_in_chars++;
+						state_stack += State.TEMPLATE_PART;
+					}
+				} else {
+					type = TokenType.CLOSE_TEMPLATE;
+					current++;
+					state_stack.length--;
+				}
 				break;
 			case '$':
 				token_begin.pos++; // $ is not part of following token
@@ -718,7 +737,7 @@ public class Vala.Scanner {
 					state_stack += State.PARENS;
 					return read_token (out token_begin, out token_end);
 				} else if (current[0] == '$') {
-					type = TokenType.TEMPLATE_STRING_LITERAL;
+					type = is_verbatim ? TokenType.VERBATIM_TEMPLATE_STRING_LITERAL : TokenType.TEMPLATE_STRING_LITERAL;
 					current++;
 					state_stack += State.TEMPLATE_PART;
 				} else {
@@ -727,10 +746,10 @@ public class Vala.Scanner {
 				}
 				break;
 			default:
-				type = TokenType.TEMPLATE_STRING_LITERAL;
+				type = is_verbatim ? TokenType.VERBATIM_TEMPLATE_STRING_LITERAL : TokenType.TEMPLATE_STRING_LITERAL;
 				token_length_in_chars = 0;
 				while (current < end && current[0] != '"' && current[0] != '$') {
-					if (current[0] == '\\') {
+					if (current[0] == '\\' && !is_verbatim) {
 						current++;
 						token_length_in_chars++;
 						if (current >= end) {
@@ -829,7 +848,7 @@ public class Vala.Scanner {
 	}
 
 	public TokenType read_token (out SourceLocation token_begin, out SourceLocation token_end) {
-		if (in_template ()) {
+		if (in_template () || in_verbatim_template ()) {
 			return read_template_token (out token_begin, out token_end);
 		} else if (in_template_part ()) {
 			state_stack.length--;
@@ -861,9 +880,15 @@ public class Vala.Scanner {
 			type = get_identifier_or_keyword (begin, len);
 		} else if (current[0] == '@') {
 			if (current < end - 1 && current[1] == '"') {
+				current += 1;
+				if (current < end - 5 && current[1] == '"' && current[2] == '"') {
+					current += 3;
+					state_stack += State.VERBATIM_TEMPLATE;
+				} else {
+					current += 1;
+					state_stack += State.TEMPLATE;
+				}
 				type = TokenType.OPEN_TEMPLATE;
-				current += 2;
-				state_stack += State.TEMPLATE;
 			} else {
 				token_begin.pos++; // @ is not part of the identifier
 				current++;
@@ -901,7 +926,7 @@ public class Vala.Scanner {
 				if (state_stack.length > 0) {
 					state_stack.length--;
 				}
-				if (in_template ()) {
+				if (in_template () || in_verbatim_template ()) {
 					type = TokenType.COMMA;
 				}
 				break;
diff --git a/vala/valatokentype.vala b/vala/valatokentype.vala
index 2c64ec1b0..6097beaff 100644
--- a/vala/valatokentype.vala
+++ b/vala/valatokentype.vala
@@ -149,6 +149,7 @@ public enum Vala.TokenType {
 	USING,
 	VAR,
 	VERBATIM_STRING_LITERAL,
+	VERBATIM_TEMPLATE_STRING_LITERAL,
 	VIRTUAL,
 	VOID,
 	VOLATILE,
@@ -284,6 +285,7 @@ public enum Vala.TokenType {
 		case USING: return "`using'";
 		case VAR: return "`var'";
 		case VERBATIM_STRING_LITERAL: return "verbatim string literal";
+		case VERBATIM_TEMPLATE_STRING_LITERAL: return "verbatim template string literal";
 		case VIRTUAL: return "`virtual'";
 		case VOID: return "`void'";
 		case VOLATILE: return "`volatile'";