Skip to content

Commit d0e3caf

Browse files
minhduc140583minhduc140583
committed
Optimize performance for reader and add sql
1 parent a757478 commit d0e3caf

17 files changed

+1905
-187
lines changed

‎formatter/delimiter_formatter.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -31,5 +31,5 @@ func NewDelimiterFormatter[T any](opts ...string) (*DelimiterFormatter[T], error
3131
}
3232

3333
func (f *DelimiterFormatter[T]) Format(ctx context.Context, model *T) string {
34-
return writer.ToTextWithDelimiter(ctx, model, f.Delimiter, f.formatCols)
34+
return writer.ToTextWithDelimiter(model, f.Delimiter, f.formatCols)
3535
}

‎reader/common.go

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
package reader
2+
3+
import (
4+
"math/big"
5+
"reflect"
6+
"strconv"
7+
"time"
8+
)
9+
10+
const DateLayout string = "2006-01-02 15:04:05 +0700 +07"
11+
12+
var funcMap = map[string]func(f reflect.Value, line string, format string, scale int) error{
13+
"string": HandleString,
14+
"*string": HandleString,
15+
"time.Time": HandleTime,
16+
"*time.Time": HandleTime,
17+
"bool": HandleBool,
18+
"*bool": HandleBool,
19+
"int": HandleInt,
20+
"*int": HandleInt,
21+
"int64": HandleInt,
22+
"*int64": HandleInt64,
23+
"int32": HandleInt32,
24+
"*int32": HandleInt32,
25+
"big.Int": HandleBigInt,
26+
"*big.Int": HandleBigInt,
27+
"float64": HandleFloat64,
28+
"*float64": HandleFloat64,
29+
"big.Float": HandleBigFloat,
30+
"*big.Float": HandleBigFloat,
31+
}
32+
33+
func HandleUnknown(f reflect.Value, line string, format string, scale int) error {
34+
return nil
35+
}
36+
func HandleString(f reflect.Value, line string, format string, scale int) error {
37+
if f.Kind() == reflect.Ptr {
38+
f.Set(reflect.ValueOf(&line))
39+
} else {
40+
f.SetString(line)
41+
}
42+
return nil
43+
}
44+
func HandleTime(f reflect.Value, line string, format string, scale int) error {
45+
var fieldDate time.Time
46+
var err error
47+
if len(format) > 0 {
48+
fieldDate, err = time.Parse(format, line)
49+
} else {
50+
fieldDate, err = time.Parse(DateLayout, line)
51+
}
52+
if err != nil {
53+
return err
54+
}
55+
if f.Kind() == reflect.Ptr {
56+
f.Set(reflect.ValueOf(&fieldDate))
57+
} else {
58+
f.Set(reflect.ValueOf(fieldDate))
59+
}
60+
return nil
61+
}
62+
func HandleFloat64(f reflect.Value, line string, format string, scale int) error {
63+
floatValue, err := strconv.ParseFloat(line, 64)
64+
if err != nil {
65+
return err
66+
}
67+
if f.Kind() == reflect.Ptr {
68+
f.Set(reflect.ValueOf(&floatValue))
69+
} else {
70+
f.SetFloat(floatValue)
71+
}
72+
return nil
73+
}
74+
func HandleInt32(f reflect.Value, line string, format string, scale int) error {
75+
value, err := strconv.Atoi(line)
76+
if err != nil {
77+
return err
78+
}
79+
var i int32 = int32(value)
80+
if f.Kind() == reflect.Ptr {
81+
f.Set(reflect.ValueOf(&i))
82+
} else {
83+
f.Set(reflect.ValueOf(i))
84+
}
85+
return nil
86+
}
87+
func HandleInt64(f reflect.Value, line string, format string, scale int) error {
88+
value, err := strconv.ParseInt(line, 10, 64)
89+
if err != nil {
90+
return err
91+
}
92+
if f.Kind() == reflect.Ptr {
93+
f.Set(reflect.ValueOf(&value))
94+
} else {
95+
f.SetInt(value)
96+
}
97+
return nil
98+
}
99+
func HandleInt(f reflect.Value, line string, format string, scale int) error {
100+
value, err := strconv.Atoi(line)
101+
if err != nil {
102+
return err
103+
}
104+
if f.Kind() == reflect.Ptr {
105+
f.Set(reflect.ValueOf(&value))
106+
} else {
107+
f.Set(reflect.ValueOf(value))
108+
}
109+
return nil
110+
}
111+
func HandleBool(f reflect.Value, line string, format string, scale int) error {
112+
boolValue, err := strconv.ParseBool(line)
113+
if err != nil {
114+
return err
115+
}
116+
if f.Kind() == reflect.Ptr {
117+
f.Set(reflect.ValueOf(&boolValue))
118+
} else {
119+
f.SetBool(boolValue)
120+
}
121+
return nil
122+
}
123+
func HandleBigFloat(f reflect.Value, line string, format string, scale int) error {
124+
bf := new(big.Float)
125+
if bfv, ok := bf.SetString(line); ok {
126+
if scale >= 0 && bfv != nil {
127+
k := Round(*bf, scale)
128+
bf = &k
129+
}
130+
if f.Kind() == reflect.Ptr {
131+
f.Set(reflect.ValueOf(bfv))
132+
} else {
133+
if bfv != nil {
134+
f.Set(reflect.ValueOf(*bfv))
135+
}
136+
}
137+
}
138+
return nil
139+
}
140+
func HandleBigInt(f reflect.Value, line string, format string, scale int) error {
141+
bf := new(big.Int)
142+
if bfv, oki := bf.SetString(line, 10); oki {
143+
if f.Kind() == reflect.Ptr {
144+
f.Set(reflect.ValueOf(bfv))
145+
} else {
146+
if bfv != nil {
147+
f.Set(reflect.ValueOf(*bfv))
148+
}
149+
}
150+
}
151+
return nil
152+
}

‎reader/delimiter_transformer.go

Lines changed: 15 additions & 93 deletions
Original file line numberDiff line numberDiff line change
@@ -3,15 +3,9 @@ package reader
33
import (
44
"context"
55
"errors"
6-
"math/big"
76
"reflect"
87
"strconv"
98
"strings"
10-
"time"
11-
)
12-
13-
const (
14-
DateLayout string = "2006-01-02 15:04:05 +0700 +07"
159
)
1610

1711
func GetIndexesByTag(modelType reflect.Type, tagName string) (map[int]Delimiter, error) {
@@ -44,6 +38,13 @@ func GetIndexesByTag(modelType reflect.Type, tagName string) (map[int]Delimiter,
4438
}
4539
v.Format = tagValue
4640
}
41+
v.TypeName = field.Type.String()
42+
fn, ok := funcMap[v.TypeName]
43+
if ok {
44+
v.Handle = fn
45+
} else {
46+
v.Handle = HandleUnknown
47+
}
4748
ma[i] = v
4849
}
4950
}
@@ -70,8 +71,10 @@ type DelimiterFormatter struct {
7071
}
7172

7273
type Delimiter struct {
73-
Format string
74-
Scale int
74+
TypeName string
75+
Format string
76+
Scale int
77+
Handle func(f reflect.Value, line string, format string, scale int) error
7578
}
7679

7780
func (f DelimiterFormatter) ToStruct(ctx context.Context, lineStr string, res interface{}) error {
@@ -86,99 +89,18 @@ func Min(n1 int, n2 int) int {
8689
return n2
8790
}
8891
func ScanLine(lines []string, record interface{}, formatCols map[int]Delimiter) error {
89-
modelType := reflect.TypeOf(record).Elem()
9092
s := reflect.Indirect(reflect.ValueOf(record))
9193
numFields := s.NumField()
9294
l := len(formatCols)
9395
le := Min(numFields, l)
9496
for i := 0; i < le; i++ {
95-
field := modelType.Field(i)
96-
typef := field.Type.String()
9797
line := lines[i]
9898
f := s.Field(i)
9999
if f.CanSet() {
100-
switch typef {
101-
case "string", "*string":
102-
if f.Kind() == reflect.Ptr {
103-
f.Set(reflect.ValueOf(&line))
104-
} else {
105-
f.SetString(line)
106-
}
107-
case "time.Time", "*time.Time":
108-
if format, ok := formatCols[i]; ok {
109-
var fieldDate time.Time
110-
var err error
111-
if len(format.Format) > 0 {
112-
fieldDate, err = time.Parse(format.Format, line)
113-
} else {
114-
fieldDate, err = time.Parse(DateLayout, line)
115-
}
116-
if err != nil {
117-
return err
118-
}
119-
if f.Kind() == reflect.Ptr {
120-
f.Set(reflect.ValueOf(&fieldDate))
121-
} else {
122-
f.Set(reflect.ValueOf(fieldDate))
123-
}
124-
}
125-
case "float64", "*float64":
126-
floatValue, _ := strconv.ParseFloat(line, 64)
127-
if f.Kind() == reflect.Ptr {
128-
f.Set(reflect.ValueOf(&floatValue))
129-
} else {
130-
f.SetFloat(floatValue)
131-
}
132-
case "int64", "*int64":
133-
value, _ := strconv.ParseInt(line, 10, 64)
134-
if f.Kind() == reflect.Ptr {
135-
f.Set(reflect.ValueOf(&value))
136-
} else {
137-
f.SetInt(value)
138-
}
139-
case "int", "*int":
140-
value, _ := strconv.Atoi(line)
141-
if f.Kind() == reflect.Ptr {
142-
f.Set(reflect.ValueOf(&value))
143-
} else {
144-
f.Set(reflect.ValueOf(value))
145-
}
146-
case "bool", "*bool":
147-
boolValue, _ := strconv.ParseBool(line)
148-
if f.Kind() == reflect.Ptr {
149-
f.Set(reflect.ValueOf(&boolValue))
150-
} else {
151-
f.SetBool(boolValue)
152-
}
153-
case "big.Float", "*big.Float":
154-
if formatf, ok := formatCols[i]; ok {
155-
bf := new(big.Float)
156-
if bfv, ok := bf.SetString(line); ok {
157-
if formatf.Scale >= 0 && bfv != nil {
158-
k := Round(*bf, formatf.Scale)
159-
bf = &k
160-
}
161-
if f.Kind() == reflect.Ptr {
162-
f.Set(reflect.ValueOf(bfv))
163-
} else {
164-
if bfv != nil {
165-
f.Set(reflect.ValueOf(*bfv))
166-
}
167-
}
168-
}
169-
}
170-
case "big.Int", "*big.Int":
171-
if _, ok := formatCols[i]; ok {
172-
bf := new(big.Int)
173-
if bfv, oki := bf.SetString(line, 10); oki {
174-
if f.Kind() == reflect.Ptr {
175-
f.Set(reflect.ValueOf(bfv))
176-
} else {
177-
if bfv != nil {
178-
f.Set(reflect.ValueOf(*bfv))
179-
}
180-
}
181-
}
100+
if format, ok := formatCols[i]; ok {
101+
err := format.Handle(f, line, format.Format, format.Scale)
102+
if err != nil {
103+
return err
182104
}
183105
}
184106
}

0 commit comments

Comments
 (0)