Merge pull request #18 from sunya-ch/v1.0.0

Add mlperf and fmwork parser

cpe-parser/api.go

@@ -32,6 +32,8 @@ var gromacsParser parser.Parser = parser.NewGromacsParser()
 var linpackParser parser.Parser = parser.NewLinpackParser()
 var timeParser parser.Parser = parser.NewTimeParser()
 var stressParser parser.Parser = parser.NewStressParser()
+var mlperfParser parser.Parser = parser.NewMlPerfParser()
+var fmworkParser parser.Parser = parser.NewFMWorkParser()
 
 var parserMap map[string]parser.Parser = map[string]parser.Parser{
  "codait": codaitParser,
@@ -47,6 +49,8 @@ var parserMap map[string]parser.Parser = map[string]parser.Parser{
  "linpack": linpackParser,
  "time": timeParser,
  "stress": stressParser,
+ "mlperf": mlperfParser,
+ "fmwork": fmworkParser,
 }
 
 /////////////////////////////////////////////

cpe-parser/parser/common.go

@@ -0,0 +1,10 @@
+/*
+ * Copyright 2022- IBM Inc. All rights reserved
+ * SPDX-License-Identifier: Apache2.0
+ */
+
+package parser
+
+const (
+ BIGLINE = "===================================="
+)

cpe-parser/parser/fmwork.go

@@ -0,0 +1,121 @@
+/*
+ * Copyright 2022- IBM Inc. All rights reserved
+ * SPDX-License-Identifier: Apache2.0
+ */
+
+package parser
+
+import (
+ "bufio"
+ "bytes"
+ "fmt"
+ "io"
+ "strconv"
+ "strings"
+)
+
+type FMWorkParser struct {
+ *BaseParser
+}
+
+/*
+Tokens per iteration: 2
+
+ s/iter seqs/s tokens/s ms/token
+MIN 0.031715076 31.530745819 63.061491639 15.857538000
+MAX 1.158101110 0.863482464 1.726964928 579.050555000
+AVG 0.034738420 28.786571393 57.573142786 17.369209871
+MED 0.031836234 31.410750898 62.821501796 15.918116750
+P95 0.032091481 31.160917510 62.321835020 16.045740625
+STD 0.056241701
+*/
+
+const (
+ FMWorkPerfKey = "Tokens per iteration"
+)
+
+func NewFMWorkParser() *FMWorkParser {
+ fmworkParser := &FMWorkParser{}
+ abs := &BaseParser{
+ Parser: fmworkParser,
+ }
+ fmworkParser.BaseParser = abs
+ return fmworkParser
+}
+
+func (p *FMWorkParser) ParseValue(body []byte) (map[string]interface{}, error) {
+ values := make(map[string]interface{})
+
+ bytesReader := bytes.NewReader(body)
+ bufReader := bufio.NewReader(bytesReader)
+ for {
+ line, _, err := bufReader.ReadLine()
+ if err == io.EOF {
+ break
+ } else if err != nil {
+ return nil, err
+ }
+ linestr := string(line)
+ if strings.Contains(linestr, FMWorkPerfKey) {
+ // start from here // Tokens per iteration
+ splitedLine := strings.Split(linestr, ":")
+ if len(splitedLine) == 2 {
+ valueStr := strings.TrimSpace(splitedLine[1])
+ key := strings.TrimSpace(splitedLine[0])
+ value, err := strconv.ParseFloat(valueStr, 64)
+ if err == nil {
+ values[key] = value
+ }
+ }
+ // skip one line
+ _, _, err := bufReader.ReadLine()
+ if err == io.EOF {
+ break
+ } else if err != nil {
+ return nil, err
+ }
+ line, _, err := bufReader.ReadLine()
+ if err == io.EOF {
+ break
+ } else if err != nil {
+ return nil, err
+ }
+ headers_line := string(line)
+ headers := strings.Fields(headers_line)
+ for {
+ line, _, err := bufReader.ReadLine()
+ if err == io.EOF {
+ break
+ } else if err != nil {
+ return nil, err
+ }
+ linestr := string(line)
+ valueStrs := strings.Fields(linestr)
+ if len(valueStrs) > 1 {
+ for index, valueStr := range valueStrs[1:] {
+ key := fmt.Sprintf("%s %s", strings.ToLower(valueStrs[0]), headers[index])
+ value, err := strconv.ParseFloat(valueStr, 64)
+ if err == nil {
+ values[key] = value
+ }
+ }
+ }
+ }
+ // end
+ break
+ }
+ }
+ return values, nil
+}
+
+func (p *FMWorkParser) GetPerformanceValue(values map[string]interface{}) (string, float64) {
+ candidatePerformanceStatKeys := []string{"p95", "p90", "p99", "avg"}
+ for _, performanceStatKey := range candidatePerformanceStatKeys {
+ performanceKey := fmt.Sprintf("%s s/iter", performanceStatKey)
+ if value, ok := values[performanceKey]; ok {
+ return performanceKey, value.(float64)
+ }
+ }
+
+ return "NoKey", -1
+}

cpe-parser/parser/fmwork_test.go

@@ -0,0 +1,55 @@
+/*
+ * Copyright 2022- IBM Inc. All rights reserved
+ * SPDX-License-Identifier: Apache2.0
+ */
+
+// Run go test -v parser/fmwork_test.go
+
+package parser
+
+import (
+ "fmt"
+ "io/ioutil"
+ "testing"
+
+ "github.com/IBM/cpe-operator/cpe-parser/parser"
+ "github.com/stretchr/testify/assert"
+)
+
+// update log key here
+const (
+ LOG_KEY string = "fmwork"
+)
+
+func getFileName() string {
+ return fmt.Sprintf("sample/%s_pod_log.log", LOG_KEY)
+}
+
+var generalParser parser.Parser
+
+// update parser init function
+var testParser = parser.NewFMWorkParser()
+
+func TestParseValue(t *testing.T) {
+ fileName := getFileName()
+ bytes, err := ioutil.ReadFile(fileName)
+ generalParser = testParser
+ assert.Nil(t, err)
+ values, err := generalParser.ParseValue(bytes)
+ fmt.Printf("Values: %v\n", values)
+ assert.Nil(t, err)
+ // update assert value length
+ assert.Equal(t, len(values), 22)
+}
+
+func TestGetPerformanceValue(t *testing.T) {
+ fileName := getFileName()
+ bytes, err := ioutil.ReadFile(fileName)
+ generalParser = testParser
+ assert.Nil(t, err)
+ values, err := generalParser.ParseValue(bytes)
+ key, pvalue := testParser.GetPerformanceValue(values)
+ fmt.Printf("PKey: %s, Pvalue: %.2f\n", key, pvalue)
+ // update assert performance value
+ assert.Equal(t, pvalue, 0.032091481)
+}

cpe-parser/parser/gloo.go

@@ -3,7 +3,7 @@
  * SPDX-License-Identifier: Apache2.0
  */
 
- package parser
+package parser
 
 import (
  "bufio"
@@ -18,12 +18,11 @@ type GlooParser struct {
 }
 
 /*
-# OSU MPI
-# Size Latency (us)
+ # OSU MPI
+ # Size Latency (us)
 */
 
 const (
- GLOOLINE = "===================================================================================================="
  GLOO_PERFORMANCE_KEY = "bandwidth_GBps"
 )
 
@@ -53,7 +52,7 @@ func (p *GlooParser) ParseValue(body []byte) (map[string]interface{}, error) {
  return nil, err
  }
  if linecount < 2 {
- if strings.Contains(linestr, GLOOLINE) {
+ if strings.Contains(linestr, BIGLINE) {
  linecount += 1
  }
  } else {

cpe-parser/parser/mlperf.go

@@ -0,0 +1,106 @@
+/*
+ * Copyright 2022- IBM Inc. All rights reserved
+ * SPDX-License-Identifier: Apache2.0
+ */
+
+package parser
+
+import (
+ "bufio"
+ "bytes"
+ "io"
+ "strconv"
+ "strings"
+)
+
+type MlPerfParser struct {
+ *BaseParser
+}
+
+/*
+================================================
+MLPerf Results Summary
+================================================
+SUT name : PySUT
+Scenario : Offline
+Mode : PerformanceOnly
+Samples per second: 1196.99
+Result is : INVALID
+Min duration satisfied : NO
+Min queries satisfied : Yes
+Early stopping satisfied: Yes
+Recommendations:
+* Increase expected QPS so the loadgen pre-generates a larger (coalesced) query.
+
+*/
+
+const (
+ MlPerfSummarySession = "MLPerf Results Summary"
+ AdditionalStatSession = "Additional Stats"
+)
+
+func NewMlPerfParser() *MlPerfParser {
+ mlperfParser := &MlPerfParser{}
+ abs := &BaseParser{
+ Parser: mlperfParser,
+ }
+ mlperfParser.BaseParser = abs
+ return mlperfParser
+}
+
+func (p *MlPerfParser) ParseValue(body []byte) (map[string]interface{}, error) {
+ values := make(map[string]interface{})
+
+ session := ""
+
+ bytesReader := bytes.NewReader(body)
+ bufReader := bufio.NewReader(bytesReader)
+ for {
+ line, _, err := bufReader.ReadLine()
+ linestr := string(line)
+ if err == io.EOF {
+ break
+ } else if err != nil {
+ return nil, err
+ }
+ if session == "" {
+ if strings.Contains(linestr, MlPerfSummarySession) {
+ session = MlPerfSummarySession
+ } else if strings.Contains(linestr, AdditionalStatSession) {
+ session = AdditionalStatSession
+ }
+ if session != "" {
+ _, _, err := bufReader.ReadLine()
+ if err == io.EOF {
+ break
+ } else if err != nil {
+ return nil, err
+ }
+ }
+ } else {
+ if strings.Contains(linestr, BIGLINE) {
+ session = ""
+ continue
+ }
+ splitedLine := strings.Split(linestr, ":")
+ if len(splitedLine) != 2 {
+ continue
+ }
+ valueStr := strings.TrimSpace(splitedLine[1])
+ key := strings.TrimSpace(splitedLine[0])
+ value, err := strconv.ParseFloat(valueStr, 64)
+ if err == nil {
+ values[key] = value
+ }
+ }
+ }
+ return values, nil
+}
+
+func (p *MlPerfParser) GetPerformanceValue(values map[string]interface{}) (string, float64) {
+ performanceKey := "Samples per second"
+ if value, ok := values[performanceKey]; ok {
+ return performanceKey, value.(float64)
+ }
+ return "NoKey", -1
+}