Incorporate support for raw string literals (#485)

parser/lexer/lexer.go

@@ -219,3 +219,17 @@ func (l *lexer) scanString(quote rune) (n int) {
 	}
 	return
 }
+
+func (l *lexer) scanRawString(quote rune) (n int) {
+	ch := l.next() // read character after back tick
+	for ch != quote {
+		if ch == eof {
+			l.error("literal not terminated")
+			return
+		}
+		ch = l.next()
+		n++
+	}
+	l.emitValue(String, l.input[l.start+1:l.end-1])
+	return
+}

parser/lexer/lexer_test.go

@@ -49,6 +49,19 @@ func TestLex(t *testing.T) {
 				{Kind: EOF},
 			},
 		},
+		{
+			"`backtick` `hello\u263Aworld` `hello\n\tworld` `hello\"world'`  `\xC3\xBF\u263A\U000003A8` `❤️`",
+			[]Token{
+				{Kind: String, Value: `backtick`},
+				{Kind: String, Value: `hello☺world`},
+				{Kind: String, Value: `hello
+	world`},
+				{Kind: String, Value: `hello"world'`},
+				{Kind: String, Value: `ÿ☺Ψ`},
+				{Kind: String, Value: "❤️"},
+				{Kind: EOF},
+			},
+		},
 		{
 			"a and orb().val #.",
 			[]Token{

parser/lexer/state.go

@@ -23,6 +23,8 @@ func root(l *lexer) stateFn {
 			l.error("%v", err)
 		}
 		l.emitValue(String, str)
+	case r == '`':
+		l.scanRawString(r)
 	case '0' <= r && r <= '9':
 		l.backup()
 		return number

parser/parser_test.go

@@ -24,6 +24,11 @@ func TestParse(t *testing.T) {
 			`"str"`,
 			&StringNode{Value: "str"},
 		},
+		{
+			"`hello\nworld`",
+			&StringNode{Value: `hello
+world`},
+		},
 		{
 			"3",
 			&IntegerNode{Value: 3},