change regex rule format

compiler/generator/generator_test.go

@@ -199,9 +199,9 @@ func TestDot(t *testing.T) {
 			"[ \t\n\r]*",
 			"\\(",
 			"\\)",
-			"{",
-			"}",
-			"\\+|\\-|\\*|/|:=|==|!=",
+			"\\{",
+			"\\}",
+			"\\+|\\-|\\*|\\/|:=|==|!=",
 			".",
 		},
 	).

compiler/generator/parser.go

@@ -34,9 +34,15 @@ func (p *Parser) Parse() (def string, rules [][]string, userCode string) {
 }
 
 func (p *Parser) parseRules(ruleStr string) [][]string {
-	buf := bytes.NewBufferString("\n" + ruleStr)
+	buf := bytes.NewBufferString(ruleStr)
 	rules := make([][]string, 0)
 	for {
+		if err := skipWhitespace(buf); err != nil {
+			if errors.Is(err, io.EOF) {
+				return rules
+			}
+			panic(err)
+		}
 		rule := p.readRule(buf)
 		blk := p.readBlock(buf)
 		if blk == "" {
@@ -48,72 +54,86 @@ func (p *Parser) parseRules(ruleStr string) [][]string {
 	return rules
 }
 
-func (p *Parser) readRule(reader io.RuneReader) string {
-	var prev rune
+func skipWhitespace(reader io.RuneScanner) error {
 	for {
 		r, _, err := reader.ReadRune()
 		if err != nil {
-			if errors.Is(err, io.EOF) {
-				return ""
-			}
-			panic(err)
+			return err
 		}
-		if prev == '\n' && r == '"' {
-			break
+		switch r {
+		case '\n', '\r', '\t', ' ':
+			continue
+		}
+		if err := reader.UnreadRune(); err != nil {
+			return err
 		}
-		prev = r
+		break
 	}
 
+	return nil
+}
+
+func (p *Parser) readRule(reader io.RuneScanner) string {
+	inRange := false
 	rs := make([]rune, 0)
 	for {
 		r, _, err := reader.ReadRune()
 		if err != nil {
-			if errors.Is(err, io.EOF) {
-				break
-			}
 			panic(err)
 		}
 
 		switch r {
-		case '\\':
-			if prev == '\\' {
-				rs = append(rs, r)
-				prev = 0
-				continue
-			}
-			prev = r
-			continue
-		case 'n':
-			if prev == '\\' {
-				rs = append(rs, '\n')
-				prev = 0
-				continue
-			}
-		case 'r':
-			if prev == '\\' {
-				rs = append(rs, '\r')
-				prev = 0
-				continue
-			}
-		case 't':
-			if prev == '\\' {
-				rs = append(rs, '\t')
-				prev = 0
-				continue
-			}
 		case '"':
-			if prev == '\\' {
-				prev = r
-				rs = append(rs, r)
-				continue
+			rs = append(rs, r)
+			for {
+				r, _, err = reader.ReadRune()
+				if err != nil {
+					panic(err)
+				}
+
+				switch r {
+				case '\\':
+					nr, err := nextRune(reader)
+					if err != nil {
+						panic(err)
+					}
+					reader.ReadRune()
+					if nr == '"' {
+						rs = append(rs, '\\')
+						rs = append(rs, '"')
+					}
+				case '"':
+					rs = append(rs, r)
+					return string(rs)
+				default:
+					rs = append(rs, r)
+				}
 			}
-
+		case '[':
+			inRange = true
+		case ']':
+			inRange = false
+		case ' ':
+			if !inRange {
+				return string(rs)
+			}
+		case '\t':
 			return string(rs)
 		}
-		prev = r
+
 		rs = append(rs, r)
 	}
-	return ""
+}
+
+func nextRune(reader io.RuneScanner) (rune, error) {
+	r, _, err := reader.ReadRune()
+	if err != nil {
+		return 0, err
+	}
+	if err := reader.UnreadRune(); err != nil {
+		return 0, err
+	}
+	return r, nil
 }
 
 func (p *Parser) readBlock(reader io.RuneReader) string {
@@ -163,7 +183,7 @@ func (p *Parser) Split() (def string, rules string, userCode string) {
 
 	def = p.readUntil(p.r, "%}\n")
 
-	p.readUntil(p.r, "%%\n")
+	_ = p.readUntil(p.r, "%%\n")
 	rules = p.readUntil(p.r, "%%\n")
 
 	var buf bytes.Buffer

compiler/generator/parser_test.go

@@ -0,0 +1,79 @@
+package generator_test
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"testing"
+
+	"github.com/goropikari/tlex/compiler/generator"
+)
+
+func TestParser(t *testing.T) {
+	tests := []struct {
+		name  string
+		given string
+	}{
+		{
+			name: "parser test",
+			given: `
+%{
+
+import (
+	"fmt"
+	"log"
+	"bytes"
+)
+
+// generated lexer returned types are (int, error).
+const (
+	Keyword int = iota + 1
+	Type
+	Identifier
+	Digit
+	Whitespace
+	LParen
+	RParen
+	LBracket
+	RBracket
+	Operator
+	Hiragana
+)
+
+%}
+
+%%
+if|for|while|func|return { return Keyword, nil }
+int|float64 { return Type, nil }
+[a-zA-Z][a-zA-Z0-9]* { return Identifier, nil }
+[1-9][0-9]* { return Digit, nil }
+[ \t\n\r]* { }
+"(" { return LParen, nil }
+")" { return RParen, nil }
+"{" { return LBracket, nil }
+"}" { return RBracket, nil }
+"+" { return Operator, nil }
+"-" { return Operator, nil }
+"*" { return Operator, nil }
+"/" { return Operator, nil }
+":=" { return Operator, nil }
+"==" { return Operator, nil }
+"!=" { return Operator, nil }
+[ぁ-ゔ]* { return Hiragana, nil }
+. {}
+%%
+
+user defined code
+`,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			r := bytes.NewBufferString(tt.given)
+			p := generator.NewParser(bufio.NewReader(r))
+			def, rules, userCode := p.Parse()
+			fmt.Println(def, rules, userCode)
+		})
+	}
+}