Skip to content

Commit 8999c64

Browse files
sunya-chsunya-ch
authored
Merge pull request #18 from sunya-ch/v1.0.0
Add mlperf and fmwork parser
2 parents 9bfe550 + 5fe37a4 commit 8999c64

File tree

7 files changed

+355
-5
lines changed

7 files changed

+355
-5
lines changed

‎cpe-parser/api.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ var gromacsParser parser.Parser = parser.NewGromacsParser()
3232
var linpackParser parser.Parser = parser.NewLinpackParser()
3333
var timeParser parser.Parser = parser.NewTimeParser()
3434
var stressParser parser.Parser = parser.NewStressParser()
35+
var mlperfParser parser.Parser = parser.NewMlPerfParser()
36+
var fmworkParser parser.Parser = parser.NewFMWorkParser()
3537

3638
var parserMap map[string]parser.Parser = map[string]parser.Parser{
3739
"codait": codaitParser,
@@ -47,6 +49,8 @@ var parserMap map[string]parser.Parser = map[string]parser.Parser{
4749
"linpack": linpackParser,
4850
"time": timeParser,
4951
"stress": stressParser,
52+
"mlperf": mlperfParser,
53+
"fmwork": fmworkParser,
5054
}
5155

5256
/////////////////////////////////////////////

‎cpe-parser/parser/common.go

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
/*
2+
* Copyright 2022- IBM Inc. All rights reserved
3+
* SPDX-License-Identifier: Apache2.0
4+
*/
5+
6+
package parser
7+
8+
const (
9+
BIGLINE = "===================================="
10+
)

‎cpe-parser/parser/fmwork.go

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
/*
2+
* Copyright 2022- IBM Inc. All rights reserved
3+
* SPDX-License-Identifier: Apache2.0
4+
*/
5+
6+
package parser
7+
8+
import (
9+
"bufio"
10+
"bytes"
11+
"fmt"
12+
"io"
13+
"strconv"
14+
"strings"
15+
)
16+
17+
type FMWorkParser struct {
18+
*BaseParser
19+
}
20+
21+
/*
22+
Tokens per iteration: 2
23+
24+
s/iter seqs/s tokens/s ms/token
25+
MIN 0.031715076 31.530745819 63.061491639 15.857538000
26+
MAX 1.158101110 0.863482464 1.726964928 579.050555000
27+
AVG 0.034738420 28.786571393 57.573142786 17.369209871
28+
MED 0.031836234 31.410750898 62.821501796 15.918116750
29+
P95 0.032091481 31.160917510 62.321835020 16.045740625
30+
STD 0.056241701
31+
*/
32+
33+
const (
34+
FMWorkPerfKey = "Tokens per iteration"
35+
)
36+
37+
func NewFMWorkParser() *FMWorkParser {
38+
fmworkParser := &FMWorkParser{}
39+
abs := &BaseParser{
40+
Parser: fmworkParser,
41+
}
42+
fmworkParser.BaseParser = abs
43+
return fmworkParser
44+
}
45+
46+
func (p *FMWorkParser) ParseValue(body []byte) (map[string]interface{}, error) {
47+
values := make(map[string]interface{})
48+
49+
bytesReader := bytes.NewReader(body)
50+
bufReader := bufio.NewReader(bytesReader)
51+
for {
52+
line, _, err := bufReader.ReadLine()
53+
if err == io.EOF {
54+
break
55+
} else if err != nil {
56+
return nil, err
57+
}
58+
linestr := string(line)
59+
if strings.Contains(linestr, FMWorkPerfKey) {
60+
// start from here // Tokens per iteration
61+
splitedLine := strings.Split(linestr, ":")
62+
if len(splitedLine) == 2 {
63+
valueStr := strings.TrimSpace(splitedLine[1])
64+
key := strings.TrimSpace(splitedLine[0])
65+
value, err := strconv.ParseFloat(valueStr, 64)
66+
if err == nil {
67+
values[key] = value
68+
}
69+
}
70+
// skip one line
71+
_, _, err := bufReader.ReadLine()
72+
if err == io.EOF {
73+
break
74+
} else if err != nil {
75+
return nil, err
76+
}
77+
line, _, err := bufReader.ReadLine()
78+
if err == io.EOF {
79+
break
80+
} else if err != nil {
81+
return nil, err
82+
}
83+
headers_line := string(line)
84+
headers := strings.Fields(headers_line)
85+
for {
86+
line, _, err := bufReader.ReadLine()
87+
if err == io.EOF {
88+
break
89+
} else if err != nil {
90+
return nil, err
91+
}
92+
linestr := string(line)
93+
valueStrs := strings.Fields(linestr)
94+
if len(valueStrs) > 1 {
95+
for index, valueStr := range valueStrs[1:] {
96+
key := fmt.Sprintf("%s %s", strings.ToLower(valueStrs[0]), headers[index])
97+
value, err := strconv.ParseFloat(valueStr, 64)
98+
if err == nil {
99+
values[key] = value
100+
}
101+
}
102+
}
103+
}
104+
// end
105+
break
106+
}
107+
}
108+
return values, nil
109+
}
110+
111+
func (p *FMWorkParser) GetPerformanceValue(values map[string]interface{}) (string, float64) {
112+
candidatePerformanceStatKeys := []string{"p95", "p90", "p99", "avg"}
113+
for _, performanceStatKey := range candidatePerformanceStatKeys {
114+
performanceKey := fmt.Sprintf("%s s/iter", performanceStatKey)
115+
if value, ok := values[performanceKey]; ok {
116+
return performanceKey, value.(float64)
117+
}
118+
}
119+
120+
return "NoKey", -1
121+
}

‎cpe-parser/parser/fmwork_test.go

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
/*
2+
* Copyright 2022- IBM Inc. All rights reserved
3+
* SPDX-License-Identifier: Apache2.0
4+
*/
5+
6+
// Run go test -v parser/fmwork_test.go
7+
8+
package parser
9+
10+
import (
11+
"fmt"
12+
"io/ioutil"
13+
"testing"
14+
15+
"github.com/IBM/cpe-operator/cpe-parser/parser"
16+
"github.com/stretchr/testify/assert"
17+
)
18+
19+
// update log key here
20+
const (
21+
LOG_KEY string = "fmwork"
22+
)
23+
24+
func getFileName() string {
25+
return fmt.Sprintf("sample/%s_pod_log.log", LOG_KEY)
26+
}
27+
28+
var generalParser parser.Parser
29+
30+
// update parser init function
31+
var testParser = parser.NewFMWorkParser()
32+
33+
func TestParseValue(t *testing.T) {
34+
fileName := getFileName()
35+
bytes, err := ioutil.ReadFile(fileName)
36+
generalParser = testParser
37+
assert.Nil(t, err)
38+
values, err := generalParser.ParseValue(bytes)
39+
fmt.Printf("Values: %v\n", values)
40+
assert.Nil(t, err)
41+
// update assert value length
42+
assert.Equal(t, len(values), 22)
43+
}
44+
45+
func TestGetPerformanceValue(t *testing.T) {
46+
fileName := getFileName()
47+
bytes, err := ioutil.ReadFile(fileName)
48+
generalParser = testParser
49+
assert.Nil(t, err)
50+
values, err := generalParser.ParseValue(bytes)
51+
key, pvalue := testParser.GetPerformanceValue(values)
52+
fmt.Printf("PKey: %s, Pvalue: %.2f\n", key, pvalue)
53+
// update assert performance value
54+
assert.Equal(t, pvalue, 0.032091481)
55+
}

‎cpe-parser/parser/gloo.go

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
* SPDX-License-Identifier: Apache2.0
44
*/
55

6-
package parser
6+
package parser
77

88
import (
99
"bufio"
@@ -18,12 +18,11 @@ type GlooParser struct {
1818
}
1919

2020
/*
21-
# OSU MPI
22-
# Size Latency (us)
21+
# OSU MPI
22+
# Size Latency (us)
2323
*/
2424

2525
const (
26-
GLOOLINE = "===================================================================================================="
2726
GLOO_PERFORMANCE_KEY = "bandwidth_GBps"
2827
)
2928

@@ -53,7 +52,7 @@ func (p *GlooParser) ParseValue(body []byte) (map[string]interface{}, error) {
5352
return nil, err
5453
}
5554
if linecount < 2 {
56-
if strings.Contains(linestr, GLOOLINE) {
55+
if strings.Contains(linestr, BIGLINE) {
5756
linecount += 1
5857
}
5958
} else {

‎cpe-parser/parser/mlperf.go

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
/*
2+
* Copyright 2022- IBM Inc. All rights reserved
3+
* SPDX-License-Identifier: Apache2.0
4+
*/
5+
6+
package parser
7+
8+
import (
9+
"bufio"
10+
"bytes"
11+
"io"
12+
"strconv"
13+
"strings"
14+
)
15+
16+
type MlPerfParser struct {
17+
*BaseParser
18+
}
19+
20+
/*
21+
================================================
22+
MLPerf Results Summary
23+
================================================
24+
SUT name : PySUT
25+
Scenario : Offline
26+
Mode : PerformanceOnly
27+
Samples per second: 1196.99
28+
Result is : INVALID
29+
Min duration satisfied : NO
30+
Min queries satisfied : Yes
31+
Early stopping satisfied: Yes
32+
Recommendations:
33+
* Increase expected QPS so the loadgen pre-generates a larger (coalesced) query.
34+
35+
*/
36+
37+
const (
38+
MlPerfSummarySession = "MLPerf Results Summary"
39+
AdditionalStatSession = "Additional Stats"
40+
)
41+
42+
func NewMlPerfParser() *MlPerfParser {
43+
mlperfParser := &MlPerfParser{}
44+
abs := &BaseParser{
45+
Parser: mlperfParser,
46+
}
47+
mlperfParser.BaseParser = abs
48+
return mlperfParser
49+
}
50+
51+
func (p *MlPerfParser) ParseValue(body []byte) (map[string]interface{}, error) {
52+
values := make(map[string]interface{})
53+
54+
session := ""
55+
56+
bytesReader := bytes.NewReader(body)
57+
bufReader := bufio.NewReader(bytesReader)
58+
for {
59+
line, _, err := bufReader.ReadLine()
60+
linestr := string(line)
61+
if err == io.EOF {
62+
break
63+
} else if err != nil {
64+
return nil, err
65+
}
66+
if session == "" {
67+
if strings.Contains(linestr, MlPerfSummarySession) {
68+
session = MlPerfSummarySession
69+
} else if strings.Contains(linestr, AdditionalStatSession) {
70+
session = AdditionalStatSession
71+
}
72+
if session != "" {
73+
_, _, err := bufReader.ReadLine()
74+
if err == io.EOF {
75+
break
76+
} else if err != nil {
77+
return nil, err
78+
}
79+
}
80+
} else {
81+
if strings.Contains(linestr, BIGLINE) {
82+
session = ""
83+
continue
84+
}
85+
splitedLine := strings.Split(linestr, ":")
86+
if len(splitedLine) != 2 {
87+
continue
88+
}
89+
valueStr := strings.TrimSpace(splitedLine[1])
90+
key := strings.TrimSpace(splitedLine[0])
91+
value, err := strconv.ParseFloat(valueStr, 64)
92+
if err == nil {
93+
values[key] = value
94+
}
95+
}
96+
}
97+
return values, nil
98+
}
99+
100+
func (p *MlPerfParser) GetPerformanceValue(values map[string]interface{}) (string, float64) {
101+
performanceKey := "Samples per second"
102+
if value, ok := values[performanceKey]; ok {
103+
return performanceKey, value.(float64)
104+
}
105+
return "NoKey", -1
106+
}

0 commit comments

Comments
 (0)