improve spaces check

Author: wmentor

.github/workflows/ci.yaml

@@ -6,7 +6,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        go-version: [1.15.x]
+        go-version: [1.20.x]
         platform: [ubuntu-latest]
     runs-on: ${{ matrix.platform }}
     steps:

go.mod

@@ -1,5 +1,5 @@
 module github.com/wmentor/tokens
 
-go 1.15
+go 1.20
 
-require github.com/wmentor/tbuf v1.0.0 // indirect
+require github.com/wmentor/tbuf v1.0.1

go.sum

@@ -1,2 +1,6 @@
-github.com/wmentor/tbuf v1.0.0 h1:KHfiIdOTWor7a/5dSoLovxgGuipLAAU1X4+U3jzdIZ4=
-github.com/wmentor/tbuf v1.0.0/go.mod h1:YvYY3BMph/UVPSIMbQoraxgr7+7DCAvYSSJHZk2gsBQ=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/wmentor/tbuf v1.0.1 h1:IonHpWwR0Wyh3Jfu0AbGSqzVDzUZ1zU61ML5F1CdBno=
+github.com/wmentor/tbuf v1.0.1/go.mod h1:1lO+hvrkqqjEcR74vrNfBL3jg0NnpGHDWHeFxRsk7js=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=

runes/const.go

@@ -1,3 +1,5 @@
+// Copyright (c) 2023, Mikhail Kirillov <mikkirillov@yandex.ru>
+
 package runes
 
 const (
@@ -47,4 +49,8 @@ const (
 	TRADE    rune = '™'
 	UML      rune = '¨'
 	YEN      rune = '¥'
+	ZWSP     rune = '\u200B'
+	ZWNBSP   rune = '\uFEFF'
+	ZWJ      rune = '\u200D'
+	ZWNJ     rune = '\u200C'
 )

tokenizer.go

@@ -1,3 +1,5 @@
+// Copyright (c) 2023, Mikhail Kirillov <mikkirillov@yandex.ru>
+
 package tokens
 
 import (
@@ -7,6 +9,7 @@ import (
 	"unicode"
 
 	buffer "github.com/wmentor/tbuf"
+
 	"github.com/wmentor/tokens/runes"
 )
 
@@ -176,7 +179,7 @@ func (t *Tokenizer) onRune(r rune) {
 
 func (t *Tokenizer) state0(r rune) {
 	switch {
-	case unicode.IsSpace(r):
+	case isSpace(r):
 		t.mode = 0
 
 	case t.isAlNum(r):
@@ -235,7 +238,7 @@ func (t *Tokenizer) state1(r rune) {
 	case t.isAlNum(r):
 		t.mkr1.WriteRune(r)
 
-	case unicode.IsSpace(r):
+	case isSpace(r):
 		t.onToken(t.mkr1.String())
 		t.mode = 0
 
@@ -268,7 +271,7 @@ func (t *Tokenizer) state2(r rune) {
 		t.onToken(string(r))
 		t.mode = 0
 
-	case unicode.IsSpace(r):
+	case isSpace(r):
 		t.onToken(t.mkr1.String())
 		t.onToken(string(t.prevRune))
 		t.mode = 0
@@ -305,7 +308,7 @@ func (t *Tokenizer) state4(r rune) {
 }
 
 func (t *Tokenizer) state5(r rune) {
-	if unicode.IsSpace(r) {
+	if isSpace(r) {
 		t.onToken(t.mkr1.String())
 		t.mode = 0
 	} else {
@@ -323,7 +326,7 @@ func (t *Tokenizer) state6(r rune) {
 		t.mkr1.Reset()
 		t.mkr1.WriteRune('#')
 
-	case unicode.IsSpace(r):
+	case isSpace(r):
 		t.onToken(t.mkr1.String())
 		t.mode = 0
 

tokenizer_test.go

@@ -1,3 +1,5 @@
+// Copyright (c) 2023, Mikhail Kirillov <mikkirillov@yandex.ru>
+
 package tokens_test
 
 import (

util.go

@@ -0,0 +1,13 @@
+// Copyright (c) 2023, Mikhail Kirillov <mikkirillov@yandex.ru>
+
+package tokens
+
+import (
+	"unicode"
+
+	"github.com/wmentor/tokens/runes"
+)
+
+func isSpace(r rune) bool {
+	return unicode.IsSpace(r) || r == runes.ZWSP || r == runes.ZWNBSP || r == runes.ZWJ || r == runes.ZWNJ
+}

vars.go

@@ -1,3 +1,5 @@
+// Copyright (c) 2023, Mikhail Kirillov <mikkirillov@yandex.ru>
+
 package tokens
 
 import (